mirror of
https://github.com/LostRuins/koboldcpp.git
synced 2026-10-22 06:09:59 +00:00
model : add H2O-Lightning-4B decision model (#30361)
* model : add H2O-Lightning-4B decision model Adds the h2o-lightning decision type for h2oai/h2o-lightning-4b (Qwen3.5-4B with a merged decision LoRA). None of the existing types match this model. It answers with " A" rather than "A", uses up to 255 labels, and was trained on Python json.dumps text, so floats like 12.0 must render as 12.0. The converter writes the decision metadata and the systemone template from the model's serve_config.json and refuses a config that doesn't match. Tests skip unless the tiny test model is available locally. Assisted-by: Claude Opus 5.5 * rm redundant code * put labels to gguf --------- Co-authored-by: Jonathan C. McKinney <jon.mckinney@h2o.ai> Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
This commit is contained in:
PSEUDOTENSOR / Jonathan McKinney
GitHub
parent
5ba3b9fd3d
commit
7e3eeee048
16 changed files with 478 additions and 15 deletions
|
|
@ -1175,6 +1175,7 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
|
|||
{ COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" },
|
||||
{ COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" },
|
||||
{ COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
|
||||
{ COMMON_DECISION_TYPE_H2O_LIGHTNING, "h2o-lightning" },
|
||||
// for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -975,6 +975,7 @@ enum common_decision_type {
|
|||
COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type
|
||||
COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
|
||||
COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
|
||||
COMMON_DECISION_TYPE_H2O_LIGHTNING, // same as openjev, the labels are read after an answer prefill
|
||||
COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -226,6 +226,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
|
|||
"MuseSparkForConditionalGeneration": "muse_spark",
|
||||
"OpenELMForCausalLM": "openelm",
|
||||
"OpenJevModel": "qwen",
|
||||
"H2OLightningModel": "h2o_lightning",
|
||||
"OrionForCausalLM": "orion",
|
||||
"PLMForCausalLM": "plm",
|
||||
"PLaMo2ForCausalLM": "plamo",
|
||||
|
|
@ -375,6 +376,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
|
|||
"Qwen3VLForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3VLMoeForConditionalGeneration": "qwen3vl",
|
||||
"OpenJevModel": "qwen3vl",
|
||||
"H2OLightningModel": "h2o_lightning",
|
||||
"Qwen3_5ForConditionalGeneration": "qwen3vl",
|
||||
"Qwen3_5MoeForConditionalGeneration": "qwen3vl",
|
||||
"Qwen4ExpForConditionalGeneration": "qwen4exp",
|
||||
|
|
|
|||
96
conversion/h2o_lightning.py
Normal file
96
conversion/h2o_lightning.py
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .base import ModelBase, gguf, logger
|
||||
from .qwen import Qwen3_5TextModel
|
||||
from .qwen3vl import Qwen3VLVisionModel
|
||||
|
||||
|
||||
def _is_h2o_lightning_checkpoint(dir_model: Path) -> bool:
|
||||
return (dir_model / "h2o_lightning_shim.py").is_file() and (dir_model / "serve_config.json").is_file()
|
||||
|
||||
|
||||
@ModelBase.register_hparams_loader(_is_h2o_lightning_checkpoint)
|
||||
def _load_h2o_lightning_hparams(dir_model: Path) -> dict[str, Any]:
|
||||
logger.info("gguf: detected H2O-Lightning checkpoint")
|
||||
hparams = ModelBase.load_hparams(dir_model, False, guess=False)
|
||||
hparams["architectures"] = ["H2OLightningModel"]
|
||||
return hparams
|
||||
|
||||
|
||||
@ModelBase.register("H2OLightningModel")
|
||||
@ModelBase.example("h2oai/h2o-lightning-4b")
|
||||
class H2OLightningModel(Qwen3_5TextModel):
|
||||
model_arch = gguf.MODEL_ARCH.QWEN35
|
||||
|
||||
# prompt and calibration follow serve_config.json and h2o_lightning_shim.py of the model repo
|
||||
# the state budget is hard-coded in server-decision.cpp
|
||||
_MAX_STATE_TOKENS = 32000
|
||||
_DEFAULT_INSTRUCTIONS = "Answer the question below."
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
with open(self.dir_model / "serve_config.json", "r", encoding="utf-8") as f:
|
||||
self._serve_config = json.load(f)
|
||||
prompt = self._serve_config["prompt"]
|
||||
expected = {
|
||||
"state_compact": (prompt.get("state_compact", True), True),
|
||||
"max_state_tokens": (prompt.get("max_state_tokens"), self._MAX_STATE_TOKENS),
|
||||
"layout": (prompt.get("layout", "plain"), "plain"),
|
||||
"chat_template": (prompt.get("chat_template", True), True),
|
||||
"enable_thinking": (prompt.get("enable_thinking", False), False),
|
||||
}
|
||||
for name, (got, want) in expected.items():
|
||||
if got != want:
|
||||
raise ValueError(f"serve_config.json: {name} is {got!r}, the h2o-lightning decision type expects {want!r}")
|
||||
|
||||
def get_vocab_base_pre(self, tokenizer) -> str:
|
||||
# transformers loads this checkpoint as Qwen2Tokenizer: the model was trained and is served with the Qwen2 split
|
||||
pre = super().get_vocab_base_pre(tokenizer)
|
||||
if pre != "qwen35":
|
||||
raise ValueError(f"expected the Qwen3.5 tokenizer, got pre-tokenizer {pre!r}")
|
||||
return "qwen2"
|
||||
|
||||
def set_vocab(self):
|
||||
super().set_vocab()
|
||||
self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
|
||||
|
||||
def _systemone_template(self) -> str:
|
||||
# the server gives the template texts rendered as the shim renders them, it only lays them out
|
||||
prompt = self._serve_config["prompt"]
|
||||
nl = "{{ '\\n' }}"
|
||||
description = (
|
||||
"{% if type == 'noul' %}"
|
||||
"{% if o.description %}{{ o.description }}{% elif o.key == 'true' %}the statement holds{% else %}it does not{% endif %}"
|
||||
"{% elif o.description is none %}{{ o.key }}{% else %}{{ o.description }}{% endif %}"
|
||||
)
|
||||
user = (
|
||||
"{% set user %}record: {{ state }}" + nl
|
||||
+ "question: {{ instructions if instructions else '" + self._DEFAULT_INSTRUCTIONS + "' }}" + nl
|
||||
+ "options:{% for o in options %}" + nl + "{{ o.label | trim }}) {{ o.key }}: " + description + "{% endfor %}{% endset %}"
|
||||
)
|
||||
return (
|
||||
"<|im_start|>system" + nl + prompt["system"].strip() + "<|im_end|>" + nl
|
||||
+ "<|im_start|>user" + nl + "{% for image in images %}{{ image }}{% endfor %}" + user + "{{ user | trim }}<|im_end|>" + nl
|
||||
+ "<|im_start|>assistant" + nl + "<think>{{ '\\n\\n' }}</think>{{ '\\n\\n' }}" + prompt["prefill"]
|
||||
)
|
||||
|
||||
def set_gguf_parameters(self):
|
||||
super().set_gguf_parameters()
|
||||
self.gguf_writer.add_decision_type(gguf.DecisionType.H2O_LIGHTNING)
|
||||
# the answer follows "Answer:", so the token of a label has a leading space
|
||||
cfg = self._serve_config
|
||||
self.gguf_writer.add_decision_labels([" " + label for label in cfg["labels"][:cfg.get("max_options", len(cfg["labels"]))]])
|
||||
for name in ("choice", "noul", "score"):
|
||||
self.gguf_writer.add_decision_temperature(name, self._serve_config["temperature"][name])
|
||||
# a request with images has this temperature for all question types
|
||||
self.gguf_writer.add_decision_temperature("image", self._serve_config["image"]["temperature"])
|
||||
|
||||
|
||||
@ModelBase.register("H2OLightningModel")
|
||||
class H2OLightningVisionModel(Qwen3VLVisionModel):
|
||||
pass
|
||||
|
|
@ -344,6 +344,7 @@ class Keys:
|
|||
ROUTING_BLOCK_COUNT = "{arch}.decision.routing_block_count"
|
||||
HEAD_COUNT = "{arch}.decision.head_count"
|
||||
MAX_HEAD_TOKENS = "{arch}.decision.max_head_tokens"
|
||||
LABELS = "{arch}.decision.labels" # the text of the token read for each option, in order
|
||||
TEMPERATURE = "{arch}.decision.temperature.{name}" # name: "<type>" or "<type>.<n_opt bucket>"
|
||||
SCORE_BIAS = "{arch}.decision.score_bias.{n}.{i}" # added to the logit of level i of a score question with n levels
|
||||
|
||||
|
|
@ -6339,6 +6340,7 @@ class DecisionType:
|
|||
LFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question type
|
||||
LFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout
|
||||
GRANITE_SWITCH = "granite-switch" # same as openjev, run by the decision aLoRA of granite switch
|
||||
H2O_LIGHTNING = "h2o-lightning" # same as openjev, labels read after an answer prefill
|
||||
|
||||
|
||||
class VisionProjectorType:
|
||||
|
|
|
|||
|
|
@ -1401,6 +1401,9 @@ class GGUFWriter:
|
|||
def add_decision_max_head_tokens(self, value: int) -> None:
|
||||
self.add_uint32(Keys.Decision.MAX_HEAD_TOKENS.format(arch=self.arch), value)
|
||||
|
||||
def add_decision_labels(self, labels: Sequence[str]) -> None:
|
||||
self.add_array(Keys.Decision.LABELS.format(arch=self.arch), labels)
|
||||
|
||||
def add_decision_temperature(self, name: str, value: float) -> None:
|
||||
self.add_float32(Keys.Decision.TEMPERATURE.format(arch=self.arch, name=name), value)
|
||||
|
||||
|
|
|
|||
|
|
@ -383,6 +383,7 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
|
|||
{ LLM_KV_DECISION_BLOCK_COUNT, "%s.decision.block_count" },
|
||||
{ LLM_KV_DECISION_ROUTING_BLOCK_COUNT, "%s.decision.routing_block_count" },
|
||||
{ LLM_KV_DECISION_HEAD_COUNT, "%s.decision.head_count" },
|
||||
{ LLM_KV_DECISION_LABELS, "%s.decision.labels" },
|
||||
|
||||
{ LLM_KV_TARGET_LAYERS, "%s.target_layers" },
|
||||
{ LLM_KV_TARGET_HIDDEN_SIZE, "%s.target_hidden_size" },
|
||||
|
|
|
|||
|
|
@ -431,6 +431,7 @@ enum llm_kv {
|
|||
LLM_KV_DECISION_BLOCK_COUNT,
|
||||
LLM_KV_DECISION_ROUTING_BLOCK_COUNT,
|
||||
LLM_KV_DECISION_HEAD_COUNT,
|
||||
LLM_KV_DECISION_LABELS,
|
||||
|
||||
LLM_KV_TARGET_LAYERS,
|
||||
LLM_KV_TARGET_HIDDEN_SIZE,
|
||||
|
|
|
|||
|
|
@ -122,6 +122,9 @@ enum llama_decision_order {
|
|||
// The embeddings output has one value per entry: row i is the score of option i
|
||||
LLAMA_API bool llama_batch_ext_set_decision_order(struct llama_batch_ext * batch, int32_t idx, enum llama_decision_order order);
|
||||
|
||||
// Label of option i of a decision model ("<arch>.decision.labels"), nullptr past the last one
|
||||
LLAMA_API const char * llama_model_decision_label(const struct llama_model * model, uint32_t i);
|
||||
|
||||
// mirrors:
|
||||
// LLAMA_API float * llama_get_embeddings(struct llama_context * ctx);
|
||||
LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx);
|
||||
|
|
|
|||
|
|
@ -1549,6 +1549,9 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
|
|||
hparams.n_cls_out = classifier_labels.size();
|
||||
}
|
||||
|
||||
// for decision models
|
||||
ml.get_arr(LLM_KV_DECISION_LABELS, decision_labels, false);
|
||||
|
||||
// per-arch hparams
|
||||
load_arch_hparams(ml);
|
||||
|
||||
|
|
@ -3438,6 +3441,14 @@ const char * llama_model_cls_label(const struct llama_model * model, uint32_t i)
|
|||
return nullptr;
|
||||
}
|
||||
|
||||
const char * llama_model_decision_label(const struct llama_model * model, uint32_t i) {
|
||||
if (i < model->decision_labels.size()) {
|
||||
return model->decision_labels[i].c_str();
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// deprecated
|
||||
int32_t llama_n_ctx_train(const llama_model * model) {
|
||||
return llama_model_n_ctx_train(model);
|
||||
|
|
|
|||
|
|
@ -724,6 +724,7 @@ struct llama_model {
|
|||
|
||||
// for classifier models
|
||||
std::vector<std::string> classifier_labels;
|
||||
std::vector<std::string> decision_labels;
|
||||
|
||||
struct ggml_tensor * tok_embd = nullptr;
|
||||
struct ggml_tensor * type_embd = nullptr;
|
||||
|
|
|
|||
|
|
@ -1725,13 +1725,13 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo
|
|||
|
||||
The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly.
|
||||
|
||||
The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with.
|
||||
The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1, lfm2-d1-omni and h2o-lightning. For laya, long questions and options are truncated to the token budget the model was trained with.
|
||||
|
||||
For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
|
||||
For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. An h2o-lightning state longer than 32000 tokens keeps its first and last 16000 tokens. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`.
|
||||
|
||||
*Image input:*
|
||||
|
||||
Image input needs a model that supports it (for example: openjev, clef, pplx-decider, vllm-d3, lfm2-d1, lfm2-d1-omni) and its multimodal projector, see `--mmproj`.
|
||||
Image input needs a model that supports it (for example: openjev, clef, pplx-decider, vllm-d3, lfm2-d1, lfm2-d1-omni, h2o-lightning) and its multimodal projector, see `--mmproj`.
|
||||
|
||||
Images can be given in two ways, and both can be used in the same request:
|
||||
|
||||
|
|
@ -1760,6 +1760,8 @@ All the images are placed before the state in the prompt, the ones from `files`
|
|||
|
||||
The probabilities are scaled with the temperatures stored in the model file. They are not guaranteed to be calibrated for your data.
|
||||
|
||||
For h2o-lightning, `score` questions can have 1 to 255 levels, and the `instructions` and `type` of a question can be left out (`choice` by default; `"instructions": null` is written as `null`).
|
||||
|
||||
*Examples:*
|
||||
|
||||
```shell
|
||||
|
|
|
|||
|
|
@ -215,6 +215,7 @@ std::vector<std::string> server_model_output_modalities(common_decision_type dec
|
|||
case COMMON_DECISION_TYPE_VLLM_D3:
|
||||
case COMMON_DECISION_TYPE_LFM2_D1:
|
||||
case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
|
||||
case COMMON_DECISION_TYPE_H2O_LIGHTNING:
|
||||
return {"decisions"};
|
||||
case COMMON_DECISION_TYPE_GRANITE_SWITCH:
|
||||
return {"text", "decisions"};
|
||||
|
|
|
|||
|
|
@ -5890,7 +5890,7 @@ void server_routes::init_routes() {
|
|||
const auto questions = decision.parse_questions(body);
|
||||
|
||||
std::vector<raw_buffer> files;
|
||||
const json state = decision.parse_state(body, files);
|
||||
const json state = decision.prepare_state(decision.parse_state(body, files));
|
||||
if (!files.empty() && (!decision.can_use_images() || !meta->has_inp_image)) {
|
||||
res->error(format_error_response("This server does not support image input for decisions. For a model that supports it, start it with `--mmproj`", ERROR_TYPE_NOT_SUPPORTED));
|
||||
return res;
|
||||
|
|
|
|||
|
|
@ -19,6 +19,10 @@ static const char * decision_question_type_name(server_decision_question_type ty
|
|||
// lev reads noul from a rating scale: 0 = certainly no, 8 = certainly yes
|
||||
static const size_t DECISION_LEV_N_RATINGS = 9;
|
||||
|
||||
// h2o-lightning follows serve_config.json and h2o_lightning_shim.py of the model repo
|
||||
// a longer state keeps its head and its tail
|
||||
static const size_t DECISION_H2O_MAX_STATE_TOKENS = 32000;
|
||||
|
||||
static std::string decision_meta_str(const llama_model * model, const std::string & key) {
|
||||
char buf[256];
|
||||
const int32_t n = llama_model_meta_val_str(model, key.c_str(), buf, sizeof(buf));
|
||||
|
|
@ -164,6 +168,39 @@ void server_decision_context::init(const llama_model * model) {
|
|||
throw std::runtime_error("decision model has no mask token");
|
||||
}
|
||||
n_options_max = 255;
|
||||
} else if (model_type == COMMON_DECISION_TYPE_H2O_LIGHTNING) {
|
||||
// the answer is read at the token of a label, its text is given to the template
|
||||
for (uint32_t i = 0; llama_model_decision_label(model, i) != nullptr; i++) {
|
||||
const std::string text = llama_model_decision_label(model, i);
|
||||
const auto toks = common_tokenize(vocab, text, false, false);
|
||||
if (toks.size() != 1 || std::find(labels.begin(), labels.end(), toks[0]) != labels.end()) {
|
||||
throw std::runtime_error(string_format("decision label \"%s\" is not a single distinct token", text.c_str()));
|
||||
}
|
||||
labels.push_back(toks[0]);
|
||||
label_texts.push_back(text);
|
||||
}
|
||||
if (labels.empty()) {
|
||||
throw std::runtime_error("decision model has no labels");
|
||||
}
|
||||
// the texts of the control and user-defined tokens are quoted in the input, so that they stay text
|
||||
for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) {
|
||||
if (llama_vocab_get_attr(vocab, id) & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
|
||||
const std::string text = llama_vocab_get_text(vocab, id);
|
||||
if (!text.empty()) {
|
||||
quoted_spellings.push_back(text);
|
||||
}
|
||||
}
|
||||
}
|
||||
std::stable_sort(quoted_spellings.begin(), quoted_spellings.end(), [](const std::string & a, const std::string & b) {
|
||||
return a.size() > b.size();
|
||||
});
|
||||
for (const auto & spelling : quoted_spellings) {
|
||||
if (quoted_first.find(spelling[0]) == std::string::npos) {
|
||||
quoted_first += spelling[0];
|
||||
}
|
||||
}
|
||||
n_options_max = labels.size();
|
||||
noul_true_first = true;
|
||||
} else {
|
||||
throw std::runtime_error("unsupported decision model type: " + type_name);
|
||||
}
|
||||
|
|
@ -201,8 +238,10 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
|
|||
throw err("must be an object");
|
||||
}
|
||||
// the vllm-d3 prompt has a default question
|
||||
const json instructions = q.contains("instructions") ? q.at("instructions") : json();
|
||||
if (instructions.is_null() && type != COMMON_DECISION_TYPE_VLLM_D3) {
|
||||
// h2o-lightning: the instructions can be left out, the template has a default, and the type defaults to choice
|
||||
const bool is_h2o = type == COMMON_DECISION_TYPE_H2O_LIGHTNING;
|
||||
const json instructions = q.contains("instructions") ? q.at("instructions") : (is_h2o ? json(std::string()) : json());
|
||||
if (instructions.is_null() && type != COMMON_DECISION_TYPE_VLLM_D3 && !is_h2o) {
|
||||
throw err("\"instructions\" must be provided");
|
||||
}
|
||||
if (strict_payload && (!decision_is_payload(instructions) || instructions == "")) {
|
||||
|
|
@ -213,7 +252,7 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
|
|||
question.id = id;
|
||||
question.instructions = instructions;
|
||||
|
||||
const std::string type_name = json_value(q, "type", std::string());
|
||||
const std::string type_name = json_value(q, "type", std::string(is_h2o ? "choice" : ""));
|
||||
const json criteria = q.contains("criteria") ? q.at("criteria") : json();
|
||||
|
||||
if (type_name == "choice") {
|
||||
|
|
@ -237,7 +276,11 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
|
|||
}
|
||||
} else if (type_name == "score") {
|
||||
question.type = SERVER_DECISION_QUESTION_SCORE;
|
||||
if (!criteria.is_array() || criteria.size() < 2 || criteria.size() > 10) {
|
||||
// h2o-lightning: from 1 level, the maximum is n_options_max
|
||||
if (is_h2o && (!criteria.is_array() || criteria.empty())) {
|
||||
throw err("\"criteria\" must be a non-empty array");
|
||||
}
|
||||
if (!is_h2o && (!criteria.is_array() || criteria.size() < 2 || criteria.size() > 10)) {
|
||||
throw err("\"criteria\" must be an array of 2 to 10 levels");
|
||||
}
|
||||
for (size_t i = 0; i < criteria.size(); i++) {
|
||||
|
|
@ -262,8 +305,8 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
|
|||
json description;
|
||||
if (criteria.is_object() && criteria.contains(key)) {
|
||||
description = criteria.at(key);
|
||||
} else if (criteria.is_object() && type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
|
||||
// lfm2-d1-omni also reads the descriptions under "no" and "yes"
|
||||
} else if (criteria.is_object() && (type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || is_h2o)) {
|
||||
// lfm2-d1-omni and h2o-lightning also read the descriptions under "no" and "yes"
|
||||
const char * alias = std::string(key) == "true" ? "yes" : "no";
|
||||
description = criteria.contains(alias) ? criteria.at(alias) : json();
|
||||
}
|
||||
|
|
@ -506,6 +549,280 @@ static json decision_d1omni_escape(const json & val) {
|
|||
return val;
|
||||
}
|
||||
|
||||
//
|
||||
// h2o-lightning: the texts are rendered the way the reference shim renders them with Python (json.dumps, str.lstrip)
|
||||
//
|
||||
|
||||
// one code point of UTF-8 text at s[i], i moves past it; an invalid byte is read as is
|
||||
static uint32_t decision_utf8_next(const std::string & s, size_t & i) {
|
||||
const unsigned char c = s[i];
|
||||
size_t n = c < 0x80 ? 1 : (c >> 5) == 0x6 ? 2 : (c >> 4) == 0xe ? 3 : (c >> 3) == 0x1e ? 4 : 1;
|
||||
if (i + n > s.size()) {
|
||||
n = 1;
|
||||
}
|
||||
uint32_t cp = n == 1 ? c : n == 2 ? (c & 0x1f) : n == 3 ? (c & 0x0f) : (c & 0x07);
|
||||
for (size_t k = 1; k < n; k++) {
|
||||
cp = (cp << 6) | (s[i + k] & 0x3f);
|
||||
}
|
||||
i += n;
|
||||
return cp;
|
||||
}
|
||||
|
||||
// str.isspace() of Python
|
||||
static bool decision_py_isspace(uint32_t cp) {
|
||||
return (cp >= 0x09 && cp <= 0x0d) || (cp >= 0x1c && cp <= 0x20) || cp == 0x85 || cp == 0xa0 || cp == 0x1680 ||
|
||||
(cp >= 0x2000 && cp <= 0x200a) || cp == 0x2028 || cp == 0x2029 || cp == 0x202f || cp == 0x205f || cp == 0x3000;
|
||||
}
|
||||
|
||||
static std::string decision_py_rstrip(const std::string & s) {
|
||||
size_t end = 0;
|
||||
for (size_t i = 0; i < s.size();) {
|
||||
if (!decision_py_isspace(decision_utf8_next(s, i))) {
|
||||
end = i;
|
||||
}
|
||||
}
|
||||
return s.substr(0, end);
|
||||
}
|
||||
|
||||
static std::string decision_py_lstrip(const std::string & s) {
|
||||
size_t i = 0;
|
||||
while (i < s.size()) {
|
||||
size_t next = i;
|
||||
if (!decision_py_isspace(decision_utf8_next(s, next))) {
|
||||
break;
|
||||
}
|
||||
i = next;
|
||||
}
|
||||
return s.substr(i);
|
||||
}
|
||||
|
||||
// a code point as JSON escapes, a surrogate pair above the BMP
|
||||
static std::string decision_py_escape_cp(uint32_t cp) {
|
||||
if (cp < 0x10000) {
|
||||
return string_format("\\u%04x", cp);
|
||||
}
|
||||
cp -= 0x10000;
|
||||
return string_format("\\u%04x\\u%04x", 0xd800 + (cp >> 10), 0xdc00 + (cp & 0x3ff));
|
||||
}
|
||||
|
||||
// repr() of a Python float: the shortest digits that read back the same, in exponent form below 1e-4 and from 1e16
|
||||
static std::string decision_py_float(double v) {
|
||||
char buf[64];
|
||||
// the fewest digits that read back the same
|
||||
for (int prec = 0; prec < 17; prec++) {
|
||||
snprintf(buf, sizeof(buf), "%.*e", prec, v);
|
||||
if (std::strtod(buf, nullptr) == v) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
const std::string sci(buf); // for example "-1.25e+03"
|
||||
const size_t pos_e = sci.find('e');
|
||||
const int exp = std::atoi(sci.c_str() + pos_e + 1);
|
||||
std::string sign;
|
||||
std::string digits;
|
||||
for (size_t i = 0; i < pos_e; i++) {
|
||||
if (sci[i] == '-') {
|
||||
sign = "-";
|
||||
} else if (sci[i] != '.') {
|
||||
digits += sci[i];
|
||||
}
|
||||
}
|
||||
const int n = (int) digits.size();
|
||||
if (exp < -4 || exp >= 16) {
|
||||
return sign + digits.substr(0, 1) + (n > 1 ? "." + digits.substr(1) : "") + string_format("e%c%02d", exp < 0 ? '-' : '+', std::abs(exp));
|
||||
}
|
||||
if (exp >= n - 1) {
|
||||
return sign + digits + std::string(exp - (n - 1), '0') + ".0";
|
||||
}
|
||||
if (exp >= 0) {
|
||||
return sign + digits.substr(0, exp + 1) + "." + digits.substr(exp + 1);
|
||||
}
|
||||
return sign + "0." + std::string(-exp - 1, '0') + digits;
|
||||
}
|
||||
|
||||
static void decision_py_str(std::string & out, const std::string & s, bool ensure_ascii) {
|
||||
out += '"';
|
||||
for (size_t i = 0; i < s.size();) {
|
||||
const unsigned char c = s[i];
|
||||
switch (c) {
|
||||
case '"': out += "\\\""; i++; continue;
|
||||
case '\\': out += "\\\\"; i++; continue;
|
||||
case '\b': out += "\\b"; i++; continue;
|
||||
case '\f': out += "\\f"; i++; continue;
|
||||
case '\n': out += "\\n"; i++; continue;
|
||||
case '\r': out += "\\r"; i++; continue;
|
||||
case '\t': out += "\\t"; i++; continue;
|
||||
default: break;
|
||||
}
|
||||
if (c < 0x20 || (ensure_ascii && c == 0x7f)) {
|
||||
out += string_format("\\u%04x", c);
|
||||
i++;
|
||||
} else if (c < 0x80 || !ensure_ascii) {
|
||||
out += (char) c;
|
||||
i++;
|
||||
} else {
|
||||
out += decision_py_escape_cp(decision_utf8_next(s, i));
|
||||
}
|
||||
}
|
||||
out += '"';
|
||||
}
|
||||
|
||||
// json.dumps() of Python: indent < 0 is no indent, keys stay in their order
|
||||
static void decision_py_dump(std::string & out, const json & val, int indent, int level, const std::string & item_sep, const std::string & key_sep, bool ensure_ascii) {
|
||||
const auto newline = [&](int lvl) {
|
||||
return indent < 0 ? std::string() : "\n" + std::string(indent * lvl, ' ');
|
||||
};
|
||||
if (val.is_null()) {
|
||||
out += "null";
|
||||
} else if (val.is_boolean()) {
|
||||
out += val.get<bool>() ? "true" : "false";
|
||||
} else if (val.is_number_float()) {
|
||||
out += decision_py_float(val.get<double>());
|
||||
} else if (val.is_number()) {
|
||||
out += val.dump();
|
||||
} else if (val.is_string()) {
|
||||
decision_py_str(out, val.get<std::string>(), ensure_ascii);
|
||||
} else if (val.is_array()) {
|
||||
if (val.empty()) {
|
||||
out += "[]";
|
||||
return;
|
||||
}
|
||||
out += "[" + newline(level + 1);
|
||||
for (size_t i = 0; i < val.size(); i++) {
|
||||
if (i > 0) {
|
||||
out += item_sep + newline(level + 1);
|
||||
}
|
||||
decision_py_dump(out, val.at(i), indent, level + 1, item_sep, key_sep, ensure_ascii);
|
||||
}
|
||||
out += newline(level) + "]";
|
||||
} else if (val.is_object()) {
|
||||
if (val.empty()) {
|
||||
out += "{}";
|
||||
return;
|
||||
}
|
||||
out += "{" + newline(level + 1);
|
||||
bool first = true;
|
||||
for (const auto & [key, item] : val.items()) {
|
||||
if (!first) {
|
||||
out += item_sep + newline(level + 1);
|
||||
}
|
||||
first = false;
|
||||
decision_py_str(out, key, ensure_ascii);
|
||||
out += key_sep;
|
||||
decision_py_dump(out, item, indent, level + 1, item_sep, key_sep, ensure_ascii);
|
||||
}
|
||||
out += newline(level) + "}";
|
||||
}
|
||||
}
|
||||
|
||||
static std::string decision_py_dumps(const json & val, int indent, const std::string & item_sep, const std::string & key_sep, bool ensure_ascii) {
|
||||
std::string out;
|
||||
decision_py_dump(out, val, indent, 0, item_sep, key_sep, ensure_ascii);
|
||||
return out;
|
||||
}
|
||||
|
||||
// a state longer than max_state_tokens keeps its head and its tail around a "..." line
|
||||
std::string server_decision_context::h2o_cut_state(const std::string & text) const {
|
||||
// a byte-level BPE never has more tokens than bytes
|
||||
const size_t max_state_tokens = DECISION_H2O_MAX_STATE_TOKENS;
|
||||
if (text.size() <= max_state_tokens) {
|
||||
return text;
|
||||
}
|
||||
const llama_tokens tokens = common_tokenize(vocab, text, false, true);
|
||||
if (tokens.size() <= max_state_tokens) {
|
||||
return text;
|
||||
}
|
||||
// each half is the text of its token pieces, a cut inside a character gives U+FFFD as in a JSON response of the server
|
||||
const size_t half = max_state_tokens / 2;
|
||||
auto text_of = [&](llama_tokens::const_iterator begin, llama_tokens::const_iterator end) {
|
||||
std::string out;
|
||||
for (auto it = begin; it != end; ++it) {
|
||||
out += common_token_to_piece(vocab, *it, true);
|
||||
}
|
||||
return json::parse(json::make(out).dump_safe()).get<std::string>();
|
||||
};
|
||||
return text_of(tokens.begin(), tokens.begin() + half) + "\n...\n" + text_of(tokens.end() - half, tokens.end());
|
||||
}
|
||||
|
||||
// the special token texts written in the input are quoted with JSON escapes, so they stay text
|
||||
std::string server_decision_context::h2o_quote(const std::string & text) const {
|
||||
std::string out;
|
||||
for (size_t i = 0; i < text.size();) {
|
||||
bool is_quoted = false;
|
||||
if (quoted_first.find(text[i]) == std::string::npos) {
|
||||
out += text[i++];
|
||||
continue;
|
||||
}
|
||||
for (const auto & spelling : quoted_spellings) {
|
||||
if (text.compare(i, spelling.size(), spelling) == 0) {
|
||||
std::string quoted = spelling;
|
||||
string_replace_all(quoted, "<", "\\u003c");
|
||||
string_replace_all(quoted, ">", "\\u003e");
|
||||
out += quoted;
|
||||
i += spelling.size();
|
||||
is_quoted = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!is_quoted) {
|
||||
out += text[i++];
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
json server_decision_context::prepare_state(const json & state) const {
|
||||
if (type != COMMON_DECISION_TYPE_H2O_LIGHTNING) {
|
||||
return state;
|
||||
}
|
||||
// done once per request: compact JSON, the cut, the quoting
|
||||
return h2o_quote(h2o_cut_state(state.is_string() ? state.get<std::string>() : decision_py_dumps(state, -1, ",", ":", false)));
|
||||
}
|
||||
|
||||
json server_decision_context::render_h2o(const json & state, const server_decision_question & question) const {
|
||||
// the instructions are JSON with an indent of 1, the descriptions are default JSON, the state is from prepare_state()
|
||||
const std::string instructions = decision_py_lstrip(question.instructions.is_string() ?
|
||||
question.instructions.get<std::string>() : decision_py_dumps(question.instructions, 1, ",", ": ", true));
|
||||
|
||||
json options = json::array();
|
||||
for (size_t i = 0; i < question.options.size(); i++) {
|
||||
const auto & opt = question.options[i];
|
||||
json description; // null if not provided, the template has a default, except a score level that is shown as JSON null
|
||||
if (question.type == SERVER_DECISION_QUESTION_SCORE && opt.description.is_null()) {
|
||||
description = "null";
|
||||
} else if (!opt.description.is_null()) {
|
||||
description = h2o_quote(opt.description.is_string() ? opt.description.get<std::string>() : decision_py_dumps(opt.description, -1, ", ", ": ", false));
|
||||
}
|
||||
const std::string key = h2o_quote(opt.key);
|
||||
|
||||
// the user turn is stripped like Python's str.strip(), the template only trims ASCII whitespace
|
||||
// so the text that ends the turn is stripped here, a space keeps a description that is only whitespace
|
||||
if (i + 1 == question.options.size()) {
|
||||
const bool is_key = question.type == SERVER_DECISION_QUESTION_CHOICE && description.is_null();
|
||||
if (is_key || description.is_string()) {
|
||||
const std::string text = is_key ? key : description.get<std::string>();
|
||||
const std::string stripped = decision_py_rstrip(text);
|
||||
if (stripped != text) {
|
||||
description = stripped.empty() ? " " : stripped;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
json option = json::object();
|
||||
option["label"] = label_texts[i];
|
||||
option["key"] = key;
|
||||
option["description"] = description;
|
||||
options.push_back(option);
|
||||
}
|
||||
|
||||
json inp = json::object();
|
||||
inp["type"] = decision_question_type_name(question.type);
|
||||
inp["state"] = state;
|
||||
inp["instructions"] = h2o_quote(instructions);
|
||||
inp["options"] = options;
|
||||
return inp;
|
||||
}
|
||||
|
||||
// given to the lfm2-d1-omni template: text between the pieces of the prompt, and at the start of the pieces that are cut to a token budget
|
||||
static const std::string D1OMNI_MARKER = "<<d1omni:";
|
||||
static const std::string D1OMNI_SEP = "<<d1omni:sep>>";
|
||||
|
|
@ -672,7 +989,8 @@ std::string server_decision_context::render(
|
|||
size_t n_images,
|
||||
bool is_audio) const {
|
||||
// the template is given raw JSON values, it serializes the ones that are not strings
|
||||
json inp = json{
|
||||
// h2o-lightning: the template is given texts
|
||||
json inp = type == COMMON_DECISION_TYPE_H2O_LIGHTNING ? render_h2o(state, question) : json{
|
||||
{"id", question.id},
|
||||
{"type", decision_question_type_name(question.type)},
|
||||
{"instructions", question.instructions},
|
||||
|
|
@ -756,7 +1074,8 @@ void server_decision_context::fill_task(
|
|||
|
||||
const std::string prompt = render(state, questions, question, variant, files.size());
|
||||
|
||||
if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3) {
|
||||
if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 ||
|
||||
type == COMMON_DECISION_TYPE_H2O_LIGHTNING) {
|
||||
// lev reads the ratings of a noul question at its first labels, not at the digits
|
||||
task.decision.labels.assign(labels.begin(), labels.begin() + n_outputs(question));
|
||||
}
|
||||
|
|
@ -769,7 +1088,8 @@ void server_decision_context::fill_task(
|
|||
task.decision.label_groups.push_back(group.size());
|
||||
}
|
||||
}
|
||||
if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 || type == COMMON_DECISION_TYPE_LFM2_D1) {
|
||||
if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 || type == COMMON_DECISION_TYPE_LFM2_D1 ||
|
||||
type == COMMON_DECISION_TYPE_H2O_LIGHTNING) {
|
||||
if (!files.empty()) {
|
||||
task.tokens = process_mtmd_prompt(mctx, prompt, files, init_opt);
|
||||
return;
|
||||
|
|
@ -1181,7 +1501,12 @@ json server_decision_context::format_answer(const server_decision_question & que
|
|||
|
||||
// softmax over the outputs of each variant, then the average of the variants
|
||||
// lfm2-d1-omni: image and audio answers are not calibrated
|
||||
const float temperature = has_media && type == COMMON_DECISION_TYPE_LFM2_D1_OMNI ? 1.0f : get_temperature(question);
|
||||
float temperature = has_media && type == COMMON_DECISION_TYPE_LFM2_D1_OMNI ? 1.0f : get_temperature(question);
|
||||
// a model can have its own temperature for requests with media
|
||||
const auto it_image = temperatures.find("image");
|
||||
if (has_media && it_image != temperatures.end()) {
|
||||
temperature = it_image->second;
|
||||
}
|
||||
std::vector<double> probs(n, 0.0);
|
||||
for (size_t v = 0; v < scores.size(); v++) {
|
||||
const auto & s = scores[v];
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ struct server_decision_context {
|
|||
case COMMON_DECISION_TYPE_PPLX_DECIDER:
|
||||
case COMMON_DECISION_TYPE_VLLM_DECISION2:
|
||||
case COMMON_DECISION_TYPE_LFM2_D1:
|
||||
case COMMON_DECISION_TYPE_H2O_LIGHTNING:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
|
|
@ -66,6 +67,7 @@ struct server_decision_context {
|
|||
case COMMON_DECISION_TYPE_VLLM_D3:
|
||||
case COMMON_DECISION_TYPE_LFM2_D1:
|
||||
case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
|
||||
case COMMON_DECISION_TYPE_H2O_LIGHTNING:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
|
|
@ -75,6 +77,9 @@ struct server_decision_context {
|
|||
// throw std::invalid_argument on bad input
|
||||
std::vector<server_decision_question> parse_questions(const json & body) const;
|
||||
|
||||
// the state as the model reads it, done once per request; returns the state for most models
|
||||
json prepare_state(const json & state) const;
|
||||
|
||||
// returns the state without its images, they are appended to files in order
|
||||
// images come from "files" (alias "images") and from the image_url and input_audio parts of a state made of chat messages
|
||||
// an image can be an audio clip if the model supports it
|
||||
|
|
@ -121,7 +126,7 @@ private:
|
|||
bool strict_payload = false; // state, instructions and descriptions must be text, an object or an array
|
||||
bool canonical_legend = false; // non-string score descriptions are returned as canonical JSON text
|
||||
|
||||
// OPENJEV, LEV, NIMBLE, PPLX_DECIDER, VLLM_D3
|
||||
// OPENJEV, LEV, NIMBLE, PPLX_DECIDER, VLLM_D3, H2O_LIGHTNING
|
||||
std::vector<llama_token> labels;
|
||||
std::vector<std::string> label_texts; // only if the label of an option is given to the template
|
||||
|
||||
|
|
@ -132,6 +137,10 @@ private:
|
|||
size_t max_head_tokens = 0; // question + options
|
||||
size_t max_option_tokens = 48;
|
||||
|
||||
// H2O_LIGHTNING
|
||||
std::vector<std::string> quoted_spellings; // texts of the special tokens, quoted in the input, longest first
|
||||
std::string quoted_first; // the first bytes of these texts
|
||||
|
||||
std::string render(
|
||||
const json & state,
|
||||
const std::vector<server_decision_question> & questions,
|
||||
|
|
@ -143,6 +152,10 @@ private:
|
|||
size_t n_outputs(const server_decision_question & question) const;
|
||||
// LFM2_D1: label text and tokens of each option
|
||||
void d1_labels(const server_decision_question & question, std::vector<std::string> & texts, std::vector<llama_tokens> & groups) const;
|
||||
// H2O_LIGHTNING: the input of the template, its texts are rendered as the reference shim does
|
||||
json render_h2o(const json & state, const server_decision_question & question) const;
|
||||
std::string h2o_cut_state(const std::string & text) const;
|
||||
std::string h2o_quote(const std::string & text) const;
|
||||
void fill_task_laya(llama_tokens & tokens, const server_decision_question & question, server_task & task) const;
|
||||
void fill_task_d1omni(
|
||||
const json & state,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue