ikawrakow/ik_llama.cpp
1
0
Fork 0
mirror of https://github.com/ikawrakow/ik_llama.cpp.git synced 2026-05-01 03:41:53 +00:00
Code Issues Packages Projects Releases Wiki Activity

server: keep logit bias unchanged when client does not set it (#1144)

Browse Source 
Co-authored-by: firecoperana <firecoperana>
This commit is contained in:
firecoperana
2026-01-13 10:08:09 -06:00
committed by GitHub
parent 0adff91363
commit 672df48ed1
1 changed files with 7 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
examples/server/server-context.cpp
View File

@@ -1059,15 +1059,10 @@ bool server_context::launch_slot_with_task(server_slot& slot, server_task& task)
} }
} }
{ { // apply logit bias
slot.sparams.logit_bias.clear();
if (json_value(data, "ignore_eos", false) && has_eos_token) {
slot.sparams.logit_bias[llama_token_eos(model)] = -INFINITY;
}
const auto& logit_bias = data.find("logit_bias"); const auto& logit_bias = data.find("logit_bias");
if (logit_bias != data.end() && logit_bias->is_array()) { if (logit_bias != data.end() && logit_bias->is_array()) {
slot.sparams.logit_bias.clear(); // only clear if user sets it
const int n_vocab = llama_n_vocab(model); const int n_vocab = llama_n_vocab(model);
for (const auto& el : *logit_bias) { for (const auto& el : *logit_bias) {
// TODO: we may want to throw errors here, in case "el" is incorrect // TODO: we may want to throw errors here, in case "el" is incorrect
@@ -1098,13 +1093,16 @@ bool server_context::launch_slot_with_task(server_slot& slot, server_task& task)
} }
} }
} }
if (json_value(data, "ignore_eos", false) && has_eos_token) {
slot.sparams.logit_bias[llama_token_eos(model)] = -INFINITY;
}
} }
{ {
slot.params.antiprompt.clear();
const auto& stop = data.find("stop"); const auto& stop = data.find("stop");
if (stop != data.end() && stop->is_array()) { if (stop != data.end() && stop->is_array()) {
slot.params.antiprompt.clear();
for (const auto& word : *stop) { for (const auto& word : *stop) {
if (!word.empty()) { if (!word.empty()) {
slot.params.antiprompt.push_back(word); slot.params.antiprompt.push_back(word);