mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-10-20 11:00:40 +00:00
allow more samplers in batched mode
This commit is contained in:
Concedo
parent
6ef0d37bdb
commit
15fa0165b8
3 changed files with 235 additions and 60 deletions
|
|
@ -12,7 +12,7 @@ Current version indicated by LITEVER below.
|
|||
-->
|
||||
<head>
|
||||
<script id="init-config">
|
||||
const LITEVER = 347;
|
||||
const LITEVER = 348;
|
||||
const urlParams = new URLSearchParams(window.location.search);
|
||||
var localflag = urlParams.get('local'); //this will be replaced automatically in embedded kcpp
|
||||
const STORAGE_PREFIX = (localflag?"e_":"")+"kaihordewebui_";
|
||||
|
|
@ -7306,6 +7306,61 @@ Current version indicated by LITEVER below.
|
|||
render_gametext();
|
||||
});
|
||||
}
|
||||
function normalize_oai_response_content(content)
|
||||
{
|
||||
if(content==null)
|
||||
{
|
||||
return "";
|
||||
}
|
||||
if(typeof content==="string")
|
||||
{
|
||||
return content;
|
||||
}
|
||||
if(Array.isArray(content))
|
||||
{
|
||||
let result = "";
|
||||
for(let part of content)
|
||||
{
|
||||
if(part && part.type==="thinking")
|
||||
{
|
||||
let thinking = normalize_oai_response_content(part.thinking!=null?part.thinking:part.text);
|
||||
if(thinking!="")
|
||||
{
|
||||
result += `${localsettings.start_thinking_tag}${thinking}${localsettings.stop_thinking_tag}`;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
result += normalize_oai_response_content(part);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
if(typeof content==="object")
|
||||
{
|
||||
if(typeof content.text==="string")
|
||||
{
|
||||
return content.text;
|
||||
}
|
||||
if(content.text && typeof content.text.value==="string")
|
||||
{
|
||||
return content.text.value;
|
||||
}
|
||||
if(typeof content.value==="string")
|
||||
{
|
||||
return content.value;
|
||||
}
|
||||
if(typeof content.refusal==="string")
|
||||
{
|
||||
return content.refusal;
|
||||
}
|
||||
if(content.content!=null)
|
||||
{
|
||||
return normalize_oai_response_content(content.content);
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
function oai_api_sync_req(targetep,oai_payload,oaiheaders)
|
||||
{
|
||||
fetch(targetep, {
|
||||
|
|
@ -7324,7 +7379,7 @@ Current version indicated by LITEVER below.
|
|||
synchro_polled_response = dch.text;
|
||||
}
|
||||
else if (dch.message) {
|
||||
synchro_polled_response = dch.message.content;
|
||||
synchro_polled_response = normalize_oai_response_content(dch.message.content);
|
||||
let reasoningtxt = "";
|
||||
let has_called_tools = (dch.message.tool_calls && dch.message.tool_calls.length>0);
|
||||
if(dch.message.reasoning_content!=null && dch.message.reasoning_content!="") //vllm compat
|
||||
|
|
@ -7337,13 +7392,13 @@ Current version indicated by LITEVER below.
|
|||
}
|
||||
if(reasoningtxt!="")
|
||||
{
|
||||
if(dch.message.content==null && !has_called_tools)
|
||||
if(synchro_polled_response=="" && !has_called_tools)
|
||||
{
|
||||
synchro_polled_response = reasoningtxt;
|
||||
}
|
||||
else
|
||||
{
|
||||
synchro_polled_response = `${localsettings.start_thinking_tag}${reasoningtxt}${localsettings.stop_thinking_tag}${dch.message.content?dch.message.content:''}`;
|
||||
synchro_polled_response = `${localsettings.start_thinking_tag}${reasoningtxt}${localsettings.stop_thinking_tag}${synchro_polled_response}`;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -7547,32 +7602,52 @@ Current version indicated by LITEVER below.
|
|||
{
|
||||
let cnt = event.data.choices[0].delta.content;
|
||||
|
||||
if(cnt && cnt.length>0 && cnt[0].type=="thinking") //mistral
|
||||
if(Array.isArray(cnt)) //mistral can include thinking and text blocks in the same delta
|
||||
{
|
||||
if(cnt[0].thinking && cnt[0].thinking.length>0 && cnt[0].thinking[0].text)
|
||||
for(let content_part of cnt)
|
||||
{
|
||||
streaming_was_thinking = true;
|
||||
cnt = cnt[0].thinking[0].text;
|
||||
}
|
||||
else
|
||||
{
|
||||
cnt = "";
|
||||
if(content_part.type=="thinking")
|
||||
{
|
||||
if(Array.isArray(content_part.thinking))
|
||||
{
|
||||
for(let thinking_part of content_part.thinking)
|
||||
{
|
||||
if(thinking_part.text)
|
||||
{
|
||||
streaming_was_thinking = true;
|
||||
synchro_pending_stream += thinking_part.text;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
else if(content_part.type=="text" && content_part.text)
|
||||
{
|
||||
if(streaming_was_thinking)
|
||||
{
|
||||
streaming_was_thinking = false;
|
||||
synchro_pending_stream = `${localsettings.start_thinking_tag}${synchro_pending_stream}${localsettings.stop_thinking_tag}`;
|
||||
}
|
||||
synchro_pending_stream += content_part.text;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if(streaming_was_thinking)
|
||||
else
|
||||
{
|
||||
streaming_was_thinking = false;
|
||||
//if there is some final reasoning, add it
|
||||
if(event.data.choices[0].delta.reasoning_content!=null&& event.data.choices[0].delta.reasoning_content!="")
|
||||
if(streaming_was_thinking)
|
||||
{
|
||||
synchro_pending_stream += event.data.choices[0].delta.reasoning_content;
|
||||
}else if(event.data.choices[0].delta.reasoning!=null&& event.data.choices[0].delta.reasoning!="")
|
||||
{
|
||||
synchro_pending_stream += event.data.choices[0].delta.reasoning;
|
||||
streaming_was_thinking = false;
|
||||
//if there is some final reasoning, add it
|
||||
if(event.data.choices[0].delta.reasoning_content!=null&& event.data.choices[0].delta.reasoning_content!="")
|
||||
{
|
||||
synchro_pending_stream += event.data.choices[0].delta.reasoning_content;
|
||||
}else if(event.data.choices[0].delta.reasoning!=null&& event.data.choices[0].delta.reasoning!="")
|
||||
{
|
||||
synchro_pending_stream += event.data.choices[0].delta.reasoning;
|
||||
}
|
||||
synchro_pending_stream = `${localsettings.start_thinking_tag}${synchro_pending_stream}${localsettings.stop_thinking_tag}`;
|
||||
}
|
||||
synchro_pending_stream = `${localsettings.start_thinking_tag}${synchro_pending_stream}${localsettings.stop_thinking_tag}`;
|
||||
synchro_pending_stream += cnt;
|
||||
}
|
||||
synchro_pending_stream += cnt;
|
||||
}
|
||||
else if(event.data.choices[0].delta && !event.data.choices[0].delta.content && event.data.choices[0].delta.reasoning_content!=null&& event.data.choices[0].delta.reasoning_content!="")
|
||||
{
|
||||
|
|
@ -14804,9 +14879,40 @@ Current version indicated by LITEVER below.
|
|||
localsettings.saved_oai_favorites = localsettings.saved_oai_favorites.filter(favorite =>
|
||||
favorite && typeof favorite.name === "string" && typeof favorite.endpoint === "string" && typeof favorite.key === "string");
|
||||
}
|
||||
|
||||
let favorites = localsettings.saved_oai_favorites;
|
||||
let default_index = favorites.findIndex(favorite => favorite.is_default === true);
|
||||
if(default_index < 0)
|
||||
{
|
||||
// Migrate the old single endpoint/key setting when it belongs to official OpenAI.
|
||||
let default_key = (oai_favorite_endpoints_match(localsettings.saved_oai_addr, default_oai_base) && localsettings.saved_oai_key !== dummy_api_key)
|
||||
? localsettings.saved_oai_key
|
||||
: "";
|
||||
favorites.unshift({name:"[Default]", endpoint:default_oai_base, key:default_key, is_default:true});
|
||||
}
|
||||
else
|
||||
{
|
||||
let default_favorite = favorites[default_index];
|
||||
default_favorite.name = "[Default]";
|
||||
default_favorite.endpoint = default_oai_base;
|
||||
// Keep one canonical built-in entry and place it first.
|
||||
favorites = favorites.filter((favorite, index) => favorite.is_default !== true || index === default_index);
|
||||
default_index = favorites.indexOf(default_favorite);
|
||||
if(default_index > 0)
|
||||
{
|
||||
favorites.splice(default_index, 1);
|
||||
favorites.unshift(default_favorite);
|
||||
}
|
||||
localsettings.saved_oai_favorites = favorites;
|
||||
}
|
||||
return localsettings.saved_oai_favorites;
|
||||
}
|
||||
|
||||
function oai_favorite_endpoints_match(first, second)
|
||||
{
|
||||
return transform_oai_ep(first) === transform_oai_ep(second);
|
||||
}
|
||||
|
||||
function refresh_oai_favorites(selected_index = -1)
|
||||
{
|
||||
let dropdown = document.getElementById("custom_oai_favorites");
|
||||
|
|
@ -14817,10 +14923,6 @@ Current version indicated by LITEVER below.
|
|||
|
||||
let favorites = get_oai_favorites();
|
||||
dropdown.replaceChildren();
|
||||
let default_option = document.createElement("option");
|
||||
default_option.value = "default";
|
||||
default_option.textContent = "[Default]";
|
||||
dropdown.appendChild(default_option);
|
||||
let placeholder = document.createElement("option");
|
||||
placeholder.value = "";
|
||||
placeholder.textContent = "[Custom]";
|
||||
|
|
@ -14843,72 +14945,81 @@ Current version indicated by LITEVER below.
|
|||
}
|
||||
else
|
||||
{
|
||||
selected_index = favorites.findIndex(favorite => favorite && favorite.endpoint === endpoint && favorite.key === key);
|
||||
// A named favorite is more specific than the built-in default if both pairs match.
|
||||
selected_index = favorites.findIndex(favorite => !favorite.is_default && oai_favorite_endpoints_match(favorite.endpoint, endpoint) && favorite.key === key);
|
||||
if(selected_index < 0)
|
||||
{
|
||||
selected_index = favorites.findIndex(favorite => oai_favorite_endpoints_match(favorite.endpoint, endpoint) && favorite.key === key);
|
||||
}
|
||||
if(selected_index >= 0)
|
||||
{
|
||||
dropdown.value = selected_index.toString();
|
||||
}
|
||||
else if(endpoint === default_oai_base)
|
||||
{
|
||||
dropdown.value = "default";
|
||||
}
|
||||
else
|
||||
{
|
||||
dropdown.value = "";
|
||||
}
|
||||
}
|
||||
dropdown.dataset.currentValue = dropdown.value;
|
||||
update_oai_favorite_btns();
|
||||
}
|
||||
|
||||
function update_oai_favorite_btns()
|
||||
{
|
||||
let selected_value = document.getElementById("custom_oai_favorites").value;
|
||||
let favorite = selected_value === "" ? null : get_oai_favorites()[Number.parseInt(selected_value, 10)];
|
||||
document.getElementById("btn_save_oai_favorite").classList.toggle("hidden", selected_value !== "");
|
||||
document.getElementById("btn_delete_oai_favorite").classList.toggle("hidden", selected_value === "" || selected_value === "default");
|
||||
document.getElementById("btn_delete_oai_favorite").classList.toggle("hidden", !favorite || favorite.is_default === true);
|
||||
}
|
||||
|
||||
function oai_favorite_fields_changed()
|
||||
{
|
||||
// The same inputs are reused by the dedicated providers; their keys must not affect these favorites.
|
||||
if(document.getElementById("customapidropdown").value != 2)
|
||||
{
|
||||
return;
|
||||
}
|
||||
let endpoint = document.getElementById("custom_oai_endpoint").value.trim();
|
||||
let key = document.getElementById("custom_oai_key").value.trim();
|
||||
let dropdown = document.getElementById("custom_oai_favorites");
|
||||
let selected_value = dropdown.value;
|
||||
if(selected_value === "default")
|
||||
{
|
||||
if(endpoint !== default_oai_base)
|
||||
{
|
||||
dropdown.value = "";
|
||||
}
|
||||
}
|
||||
else if(selected_value !== "")
|
||||
if(selected_value !== "")
|
||||
{
|
||||
let favorite = get_oai_favorites()[Number.parseInt(selected_value, 10)];
|
||||
if(!favorite || favorite.endpoint !== endpoint || favorite.key !== key)
|
||||
if(favorite && favorite.is_default === true)
|
||||
{
|
||||
favorite.key = key;
|
||||
}
|
||||
if(!favorite || !oai_favorite_endpoints_match(favorite.endpoint, endpoint) || favorite.key !== key)
|
||||
{
|
||||
dropdown.value = "";
|
||||
}
|
||||
}
|
||||
dropdown.dataset.currentValue = dropdown.value;
|
||||
update_oai_favorite_btns();
|
||||
}
|
||||
|
||||
function load_oai_favorite()
|
||||
{
|
||||
let dropdown = document.getElementById("custom_oai_favorites");
|
||||
if(dropdown.value === "default")
|
||||
let favorites = get_oai_favorites();
|
||||
let previous_favorite = favorites[Number.parseInt(dropdown.dataset.currentValue, 10)];
|
||||
if(previous_favorite && previous_favorite.is_default === true)
|
||||
{
|
||||
document.getElementById("custom_oai_endpoint").value = default_oai_base;
|
||||
update_oai_favorite_btns();
|
||||
return;
|
||||
// Capture autofill and other changes that may not have fired the input handler.
|
||||
previous_favorite.key = document.getElementById("custom_oai_key").value.trim();
|
||||
}
|
||||
let index = Number.parseInt(dropdown.value, 10);
|
||||
let favorite = get_oai_favorites()[index];
|
||||
let favorite = favorites[index];
|
||||
if(!favorite)
|
||||
{
|
||||
dropdown.dataset.currentValue = dropdown.value;
|
||||
update_oai_favorite_btns();
|
||||
return;
|
||||
}
|
||||
document.getElementById("custom_oai_endpoint").value = favorite.endpoint;
|
||||
document.getElementById("custom_oai_key").value = favorite.key;
|
||||
dropdown.dataset.currentValue = dropdown.value;
|
||||
update_oai_favorite_btns();
|
||||
}
|
||||
|
||||
|
|
@ -14921,8 +15032,10 @@ Current version indicated by LITEVER below.
|
|||
msgbox("Enter an OpenAI API URL before adding a favorite.", "Missing API URL");
|
||||
return;
|
||||
}
|
||||
endpoint = transform_oai_ep(endpoint);
|
||||
document.getElementById("custom_oai_endpoint").value = endpoint;
|
||||
|
||||
let default_name = "Endpoint " + (get_oai_favorites().length + 1);
|
||||
let default_name = "Endpoint " + (get_oai_favorites().filter(favorite => !favorite.is_default).length + 1);
|
||||
try
|
||||
{
|
||||
default_name = new URL(endpoint).host || default_name;
|
||||
|
|
@ -14952,7 +15065,7 @@ Current version indicated by LITEVER below.
|
|||
let dropdown = document.getElementById("custom_oai_favorites");
|
||||
let index = Number.parseInt(dropdown.value, 10);
|
||||
let favorite = get_oai_favorites()[index];
|
||||
if(!favorite)
|
||||
if(!favorite || favorite.is_default === true)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
|
@ -15579,6 +15692,12 @@ Current version indicated by LITEVER below.
|
|||
custom_oai_key = desired_oai_key;
|
||||
if(epchoice==2)
|
||||
{
|
||||
let oai_favorite_index = Number.parseInt(document.getElementById("custom_oai_favorites").value, 10);
|
||||
let selected_oai_favorite = get_oai_favorites()[oai_favorite_index];
|
||||
if(selected_oai_favorite && selected_oai_favorite.is_default === true && custom_oai_endpoint === default_oai_base)
|
||||
{
|
||||
selected_oai_favorite.key = custom_oai_key;
|
||||
}
|
||||
if(localsettings.saved_oai_addr!=custom_oai_endpoint || localsettings.saved_oai_key!=custom_oai_key)
|
||||
{
|
||||
//if not reconnecting exact same key and API, overwrite custom eps
|
||||
|
|
@ -25630,7 +25749,7 @@ Current version indicated by LITEVER below.
|
|||
console.log("sync request: handle recv reply");
|
||||
pending_response_id = "";
|
||||
poll_in_progress = false;
|
||||
let resp = synchro_polled_response;
|
||||
let resp = normalize_oai_response_content(synchro_polled_response);
|
||||
if(waiting_for_summarycall==0)
|
||||
{
|
||||
last_reply_was_empty = (resp=="" || resp.trim()=="");
|
||||
|
|
@ -33680,7 +33799,7 @@ Current version indicated by LITEVER below.
|
|||
<div id="custom_oai_favorites_container" class="hidden">
|
||||
<div style="display:flex; gap:4px; margin-bottom:4px; align-items:center; justify-content:flex-end;">
|
||||
Favorites: <select title="Saved OpenAI API URL and key pairs" class="form-control" id="custom_oai_favorites" style="min-width:0;flex:1;height: 25px;font-size: 12px;padding: 3px;display: inline;max-width: 140px;" onchange="load_oai_favorite()">
|
||||
<option value="default">[Default]</option>
|
||||
<option value="0">[Default]</option>
|
||||
<option value="">[Custom]</option>
|
||||
</select>
|
||||
<button title="Save the current API URL and key as a favorite" type="button" class="btn btn-primary" id="btn_save_oai_favorite" style="padding:3px; font-size:12px;" onclick="save_oai_favorite()">Save</button>
|
||||
|
|
|
|||
|
|
@ -4347,10 +4347,20 @@ struct BatchGenerateRequest
|
|||
float top_p = 1.0f;
|
||||
float min_p = 0.0f;
|
||||
float typical_p = 1.0f;
|
||||
float nsigma = 0.0f;
|
||||
float rep_pen = 1.0f;
|
||||
float rep_pen_slope = 1.0f;
|
||||
int rep_pen_range = 0;
|
||||
float presence_penalty = 0.0f;
|
||||
int mirostat = 0;
|
||||
float mirostat_tau = 5.0f;
|
||||
float mirostat_eta = 0.1f;
|
||||
float xtc_threshold = 0.2f;
|
||||
float xtc_probability = 0.0f;
|
||||
float dynatemp_range = 0.0f;
|
||||
float dynatemp_exponent = 1.0f;
|
||||
float adaptive_target = -1.0f;
|
||||
float adaptive_decay = 0.9f;
|
||||
bool allow_eos_token = true;
|
||||
bool bypass_eos_token = false;
|
||||
bool render_special = false;
|
||||
|
|
@ -4505,11 +4515,11 @@ static bool batch_inputs_eligible(const generation_inputs & inputs)
|
|||
{
|
||||
return false;
|
||||
}
|
||||
if(inputs.mirostat != 0 || inputs.xtc_probability > 0.0f || inputs.nsigma > 0.0f || inputs.smoothing_factor > 0.0f || inputs.adaptive_target > 0.0f)
|
||||
if(inputs.smoothing_factor > 0.0f)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
if(inputs.top_a > 0.0f || inputs.tfs != 1.0f || inputs.dynatemp_range > 0.0f)
|
||||
if(inputs.top_a > 0.0f || inputs.tfs != 1.0f || inputs.dynatemp_range < 0.0f)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
|
@ -4663,6 +4673,8 @@ static llama_sampler * batch_build_sampler(const BatchGenerateRequest & req)
|
|||
{
|
||||
llama_sampler_chain_params params = llama_sampler_chain_default_params();
|
||||
llama_sampler * chain = llama_sampler_chain_init(params);
|
||||
const uint32_t sampler_seed = req.seed < 0 ? LLAMA_DEFAULT_SEED : (uint32_t) req.seed;
|
||||
const int32_t sampler_n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(llama_get_model(llama_ctx_v4)));
|
||||
llama_sampler_chain_add(chain, batch_rep_pen_init(
|
||||
req.rep_pen_range,
|
||||
req.rep_pen,
|
||||
|
|
@ -4670,8 +4682,20 @@ static llama_sampler * batch_build_sampler(const BatchGenerateRequest & req)
|
|||
req.presence_penalty));
|
||||
if(req.logit_biases.size()>0)
|
||||
{
|
||||
int32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(llama_get_model(llama_ctx_v4)));
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_logit_bias(n_vocab, req.logit_biases.size(), req.logit_biases.data()));
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_logit_bias(sampler_n_vocab, req.logit_biases.size(), req.logit_biases.data()));
|
||||
}
|
||||
if(req.mirostat == 1 || req.mirostat == 2)
|
||||
{
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_temp(req.temperature));
|
||||
if(req.mirostat == 1)
|
||||
{
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_mirostat(sampler_n_vocab, sampler_seed, req.mirostat_tau, req.mirostat_eta, 100));
|
||||
}
|
||||
else
|
||||
{
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_mirostat_v2(sampler_seed, req.mirostat_tau, req.mirostat_eta));
|
||||
}
|
||||
return chain;
|
||||
}
|
||||
if(req.top_k > 0)
|
||||
{
|
||||
|
|
@ -4689,10 +4713,32 @@ static llama_sampler * batch_build_sampler(const BatchGenerateRequest & req)
|
|||
{
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_typical(req.typical_p, 1));
|
||||
}
|
||||
if(req.temperature > 0.0f)
|
||||
if(req.temperature > 0.0f || req.dynatemp_range > 0.0f)
|
||||
{
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_temp(req.temperature));
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_dist(req.seed < 0 ? LLAMA_DEFAULT_SEED : (uint32_t) req.seed));
|
||||
if(req.dynatemp_range > 0.0f)
|
||||
{
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_temp_ext(req.temperature, req.dynatemp_range, req.dynatemp_exponent));
|
||||
}
|
||||
else
|
||||
{
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_temp(req.temperature));
|
||||
}
|
||||
if(req.nsigma > 0.0f)
|
||||
{
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_top_n_sigma(req.nsigma));
|
||||
}
|
||||
if(req.xtc_probability > 0.0f)
|
||||
{
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_xtc(req.xtc_probability, req.xtc_threshold, 1, sampler_seed));
|
||||
}
|
||||
if(req.adaptive_target > 0.0f)
|
||||
{
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_adaptive_p(req.adaptive_target, req.adaptive_decay, sampler_seed));
|
||||
}
|
||||
else
|
||||
{
|
||||
llama_sampler_chain_add(chain, llama_sampler_init_dist(sampler_seed));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
|
@ -5004,10 +5050,20 @@ int gpttype_batch_generate_submit(const generation_inputs inputs)
|
|||
req->top_p = inputs.top_p;
|
||||
req->min_p = inputs.min_p;
|
||||
req->typical_p = inputs.typical_p;
|
||||
req->nsigma = inputs.nsigma;
|
||||
req->rep_pen = inputs.rep_pen;
|
||||
req->rep_pen_slope = inputs.rep_pen_slope;
|
||||
req->rep_pen_range = inputs.rep_pen_range;
|
||||
req->presence_penalty = inputs.presence_penalty;
|
||||
req->mirostat = inputs.mirostat;
|
||||
req->mirostat_tau = inputs.mirostat_tau;
|
||||
req->mirostat_eta = inputs.mirostat_eta;
|
||||
req->xtc_threshold = inputs.xtc_threshold;
|
||||
req->xtc_probability = inputs.xtc_probability;
|
||||
req->dynatemp_range = inputs.dynatemp_range;
|
||||
req->dynatemp_exponent = std::max(inputs.dynatemp_exponent, 0.0f);
|
||||
req->adaptive_target = inputs.adaptive_target;
|
||||
req->adaptive_decay = inputs.adaptive_decay;
|
||||
req->allow_eos_token = inputs.allow_eos_token;
|
||||
req->bypass_eos_token = inputs.bypass_eos_token;
|
||||
req->render_special = inputs.render_special;
|
||||
|
|
|
|||
|
|
@ -2538,13 +2538,13 @@ def continuous_batching_python_eligible(genparams, api_format):
|
|||
if genparams.get("grammar") or genparams.get("grammar_retain_state") or genparams.get("banned_tokens") or genparams.get("banned_strings"):
|
||||
utfprint("Batching disabled due to grammar or bans",2)
|
||||
return False
|
||||
if tryparsefloat(genparams.get("dry_multiplier", 0), 0) or tryparseint(genparams.get("mirostat", 0), 0) or tryparsefloat(genparams.get("xtc_probability", 0), 0) or tryparsefloat(genparams.get("nsigma", 0), 0):
|
||||
if tryparsefloat(genparams.get("dry_multiplier", 0), 0):
|
||||
utfprint("Batching disabled due to samplers set 1",2)
|
||||
return False
|
||||
if tryparsefloat(genparams.get("smoothing_factor", 0), 0) or tryparsefloat(genparams.get("adaptive_target", -1), -1) > 0 or genparams.get("using_openai_tools", False):
|
||||
if tryparsefloat(genparams.get("smoothing_factor", 0), 0) or genparams.get("using_openai_tools", False):
|
||||
utfprint("Batching disabled due to samplers set 2",2)
|
||||
return False
|
||||
if tryparsefloat(genparams.get("top_a", 0), 0) or tryparsefloat(genparams.get("tfs", 1), 1) != 1 or tryparsefloat(genparams.get("dynatemp_range", 0), 0):
|
||||
if tryparsefloat(genparams.get("top_a", 0), 0) or tryparsefloat(genparams.get("tfs", 1), 1) != 1 or tryparsefloat(genparams.get("dynatemp_range", 0), 0) < 0:
|
||||
utfprint("Batching disabled due to samplers set 3",2)
|
||||
return False
|
||||
if genparams.get("sampler_order") and genparams.get("sampler_order") != [6, 0, 1, 3, 4, 2, 5]:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue