diff --git a/common/common.cpp b/common/common.cpp index b59e66e3b..b0ddfcb91 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1175,6 +1175,7 @@ static const std::map COMMON_DECISION_TYPE_NA { COMMON_DECISION_TYPE_LFM2_D1, "lfm2-d1" }, { COMMON_DECISION_TYPE_LFM2_D1_OMNI, "lfm2-d1-omni" }, { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"}, + { COMMON_DECISION_TYPE_H2O_LIGHTNING, "h2o-lightning" }, // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname" }; diff --git a/common/common.h b/common/common.h index b172b4efe..b089abd7e 100644 --- a/common/common.h +++ b/common/common.h @@ -975,6 +975,7 @@ enum common_decision_type { COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch + COMMON_DECISION_TYPE_H2O_LIGHTNING, // same as openjev, the labels are read after an answer prefill COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported }; diff --git a/conversion/__init__.py b/conversion/__init__.py index e3c10d5fc..efa537b0f 100644 --- a/conversion/__init__.py +++ b/conversion/__init__.py @@ -226,6 +226,7 @@ TEXT_MODEL_MAP: dict[str, str] = { "MuseSparkForConditionalGeneration": "muse_spark", "OpenELMForCausalLM": "openelm", "OpenJevModel": "qwen", + "H2OLightningModel": "h2o_lightning", "OrionForCausalLM": "orion", "PLMForCausalLM": "plm", "PLaMo2ForCausalLM": "plamo", @@ -375,6 +376,7 @@ MMPROJ_MODEL_MAP: dict[str, str] = { "Qwen3VLForConditionalGeneration": "qwen3vl", "Qwen3VLMoeForConditionalGeneration": "qwen3vl", "OpenJevModel": "qwen3vl", + "H2OLightningModel": "h2o_lightning", "Qwen3_5ForConditionalGeneration": "qwen3vl", "Qwen3_5MoeForConditionalGeneration": "qwen3vl", "Qwen4ExpForConditionalGeneration": "qwen4exp", diff --git a/conversion/h2o_lightning.py b/conversion/h2o_lightning.py new file mode 100644 index 000000000..764da4b97 --- /dev/null +++ b/conversion/h2o_lightning.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import json + +from pathlib import Path +from typing import Any + +from .base import ModelBase, gguf, logger +from .qwen import Qwen3_5TextModel +from .qwen3vl import Qwen3VLVisionModel + + +def _is_h2o_lightning_checkpoint(dir_model: Path) -> bool: + return (dir_model / "h2o_lightning_shim.py").is_file() and (dir_model / "serve_config.json").is_file() + + +@ModelBase.register_hparams_loader(_is_h2o_lightning_checkpoint) +def _load_h2o_lightning_hparams(dir_model: Path) -> dict[str, Any]: + logger.info("gguf: detected H2O-Lightning checkpoint") + hparams = ModelBase.load_hparams(dir_model, False, guess=False) + hparams["architectures"] = ["H2OLightningModel"] + return hparams + + +@ModelBase.register("H2OLightningModel") +@ModelBase.example("h2oai/h2o-lightning-4b") +class H2OLightningModel(Qwen3_5TextModel): + model_arch = gguf.MODEL_ARCH.QWEN35 + + # prompt and calibration follow serve_config.json and h2o_lightning_shim.py of the model repo + # the state budget is hard-coded in server-decision.cpp + _MAX_STATE_TOKENS = 32000 + _DEFAULT_INSTRUCTIONS = "Answer the question below." + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + with open(self.dir_model / "serve_config.json", "r", encoding="utf-8") as f: + self._serve_config = json.load(f) + prompt = self._serve_config["prompt"] + expected = { + "state_compact": (prompt.get("state_compact", True), True), + "max_state_tokens": (prompt.get("max_state_tokens"), self._MAX_STATE_TOKENS), + "layout": (prompt.get("layout", "plain"), "plain"), + "chat_template": (prompt.get("chat_template", True), True), + "enable_thinking": (prompt.get("enable_thinking", False), False), + } + for name, (got, want) in expected.items(): + if got != want: + raise ValueError(f"serve_config.json: {name} is {got!r}, the h2o-lightning decision type expects {want!r}") + + def get_vocab_base_pre(self, tokenizer) -> str: + # transformers loads this checkpoint as Qwen2Tokenizer: the model was trained and is served with the Qwen2 split + pre = super().get_vocab_base_pre(tokenizer) + if pre != "qwen35": + raise ValueError(f"expected the Qwen3.5 tokenizer, got pre-tokenizer {pre!r}") + return "qwen2" + + def set_vocab(self): + super().set_vocab() + self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}]) + + def _systemone_template(self) -> str: + # the server gives the template texts rendered as the shim renders them, it only lays them out + prompt = self._serve_config["prompt"] + nl = "{{ '\\n' }}" + description = ( + "{% if type == 'noul' %}" + "{% if o.description %}{{ o.description }}{% elif o.key == 'true' %}the statement holds{% else %}it does not{% endif %}" + "{% elif o.description is none %}{{ o.key }}{% else %}{{ o.description }}{% endif %}" + ) + user = ( + "{% set user %}record: {{ state }}" + nl + + "question: {{ instructions if instructions else '" + self._DEFAULT_INSTRUCTIONS + "' }}" + nl + + "options:{% for o in options %}" + nl + "{{ o.label | trim }}) {{ o.key }}: " + description + "{% endfor %}{% endset %}" + ) + return ( + "<|im_start|>system" + nl + prompt["system"].strip() + "<|im_end|>" + nl + + "<|im_start|>user" + nl + "{% for image in images %}{{ image }}{% endfor %}" + user + "{{ user | trim }}<|im_end|>" + nl + + "<|im_start|>assistant" + nl + "{{ '\\n\\n' }}{{ '\\n\\n' }}" + prompt["prefill"] + ) + + def set_gguf_parameters(self): + super().set_gguf_parameters() + self.gguf_writer.add_decision_type(gguf.DecisionType.H2O_LIGHTNING) + # the answer follows "Answer:", so the token of a label has a leading space + cfg = self._serve_config + self.gguf_writer.add_decision_labels([" " + label for label in cfg["labels"][:cfg.get("max_options", len(cfg["labels"]))]]) + for name in ("choice", "noul", "score"): + self.gguf_writer.add_decision_temperature(name, self._serve_config["temperature"][name]) + # a request with images has this temperature for all question types + self.gguf_writer.add_decision_temperature("image", self._serve_config["image"]["temperature"]) + + +@ModelBase.register("H2OLightningModel") +class H2OLightningVisionModel(Qwen3VLVisionModel): + pass diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index b2d8d59e0..2840348cb 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -344,6 +344,7 @@ class Keys: ROUTING_BLOCK_COUNT = "{arch}.decision.routing_block_count" HEAD_COUNT = "{arch}.decision.head_count" MAX_HEAD_TOKENS = "{arch}.decision.max_head_tokens" + LABELS = "{arch}.decision.labels" # the text of the token read for each option, in order TEMPERATURE = "{arch}.decision.temperature.{name}" # name: "" or "." SCORE_BIAS = "{arch}.decision.score_bias.{n}.{i}" # added to the logit of level i of a score question with n levels @@ -6339,6 +6340,7 @@ class DecisionType: LFM2_D1 = "lfm2-d1" # same as openjev, the labels depend on the question type LFM2_D1_OMNI = "lfm2-d1-omni" # same head as laya on a bidirectional LFM2 trunk, other prompt layout GRANITE_SWITCH = "granite-switch" # same as openjev, run by the decision aLoRA of granite switch + H2O_LIGHTNING = "h2o-lightning" # same as openjev, labels read after an answer prefill class VisionProjectorType: diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index 2943e0186..e8a96e571 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -1401,6 +1401,9 @@ class GGUFWriter: def add_decision_max_head_tokens(self, value: int) -> None: self.add_uint32(Keys.Decision.MAX_HEAD_TOKENS.format(arch=self.arch), value) + def add_decision_labels(self, labels: Sequence[str]) -> None: + self.add_array(Keys.Decision.LABELS.format(arch=self.arch), labels) + def add_decision_temperature(self, name: str, value: float) -> None: self.add_float32(Keys.Decision.TEMPERATURE.format(arch=self.arch, name=name), value) diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index 961011a2c..1b954f176 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -383,6 +383,7 @@ static const std::map LLM_KV_NAMES = { { LLM_KV_DECISION_BLOCK_COUNT, "%s.decision.block_count" }, { LLM_KV_DECISION_ROUTING_BLOCK_COUNT, "%s.decision.routing_block_count" }, { LLM_KV_DECISION_HEAD_COUNT, "%s.decision.head_count" }, + { LLM_KV_DECISION_LABELS, "%s.decision.labels" }, { LLM_KV_TARGET_LAYERS, "%s.target_layers" }, { LLM_KV_TARGET_HIDDEN_SIZE, "%s.target_hidden_size" }, diff --git a/src/llama-arch.h b/src/llama-arch.h index 3ea86c601..5a8ca4250 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -431,6 +431,7 @@ enum llm_kv { LLM_KV_DECISION_BLOCK_COUNT, LLM_KV_DECISION_ROUTING_BLOCK_COUNT, LLM_KV_DECISION_HEAD_COUNT, + LLM_KV_DECISION_LABELS, LLM_KV_TARGET_LAYERS, LLM_KV_TARGET_HIDDEN_SIZE, diff --git a/src/llama-ext.h b/src/llama-ext.h index 8b3932bd7..933cb8013 100644 --- a/src/llama-ext.h +++ b/src/llama-ext.h @@ -122,6 +122,9 @@ enum llama_decision_order { // The embeddings output has one value per entry: row i is the score of option i LLAMA_API bool llama_batch_ext_set_decision_order(struct llama_batch_ext * batch, int32_t idx, enum llama_decision_order order); +// Label of option i of a decision model (".decision.labels"), nullptr past the last one +LLAMA_API const char * llama_model_decision_label(const struct llama_model * model, uint32_t i); + // mirrors: // LLAMA_API float * llama_get_embeddings(struct llama_context * ctx); LLAMA_API float * llama_get_embeddings_nextn(struct llama_context * ctx); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 803a8329e..6712f0f22 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1549,6 +1549,9 @@ void llama_model_base::load_hparams(llama_model_loader & ml) { hparams.n_cls_out = classifier_labels.size(); } + // for decision models + ml.get_arr(LLM_KV_DECISION_LABELS, decision_labels, false); + // per-arch hparams load_arch_hparams(ml); @@ -3438,6 +3441,14 @@ const char * llama_model_cls_label(const struct llama_model * model, uint32_t i) return nullptr; } +const char * llama_model_decision_label(const struct llama_model * model, uint32_t i) { + if (i < model->decision_labels.size()) { + return model->decision_labels[i].c_str(); + } + + return nullptr; +} + // deprecated int32_t llama_n_ctx_train(const llama_model * model) { return llama_model_n_ctx_train(model); diff --git a/src/llama-model.h b/src/llama-model.h index a187c2749..9fe9d77a2 100644 --- a/src/llama-model.h +++ b/src/llama-model.h @@ -724,6 +724,7 @@ struct llama_model { // for classifier models std::vector classifier_labels; + std::vector decision_labels; struct ggml_tensor * tok_embd = nullptr; struct ggml_tensor * type_embd = nullptr; diff --git a/tools/server/README.md b/tools/server/README.md index e1c5804da..8d9f53569 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -1725,13 +1725,13 @@ Follows the [TypeSafe API](https://docs.typesafe.ai/api), streaming is not suppo The questions of a request are answered independently, an answer does not depend on the other questions. The exception is clef: it reads all the questions in one prompt and decides them jointly. -The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1 and lfm2-d1-omni. For laya, long questions and options are truncated to the token budget the model was trained with. +The number of options of a `choice` question is limited by the model, for example: 52 for openjev, 255 for granite-switch, laya, clef, pplx-decider, vllm-decision2, vllm-d3, lfm2-d1, lfm2-d1-omni and h2o-lightning. For laya, long questions and options are truncated to the token budget the model was trained with. -For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`. +For laya, clef, vllm-d3 and lfm2-d1-omni, the whole prompt is evaluated in one batch: it must fit in `--ubatch-size`. An lfm2-d1-omni prompt is cut to 16384 tokens. For vllm-decision2, the options of a question and the end of the prompt must fit in one batch. An h2o-lightning state longer than 32000 tokens keeps its first and last 16000 tokens. A server that runs clef only serves this endpoint, text generation is not available. A decision model that is also a text model, such as granite-switch, serves both and reports `["text", "decisions"]`. *Image input:* -Image input needs a model that supports it (for example: openjev, clef, pplx-decider, vllm-d3, lfm2-d1, lfm2-d1-omni) and its multimodal projector, see `--mmproj`. +Image input needs a model that supports it (for example: openjev, clef, pplx-decider, vllm-d3, lfm2-d1, lfm2-d1-omni, h2o-lightning) and its multimodal projector, see `--mmproj`. Images can be given in two ways, and both can be used in the same request: @@ -1760,6 +1760,8 @@ All the images are placed before the state in the prompt, the ones from `files` The probabilities are scaled with the temperatures stored in the model file. They are not guaranteed to be calibrated for your data. +For h2o-lightning, `score` questions can have 1 to 255 levels, and the `instructions` and `type` of a question can be left out (`choice` by default; `"instructions": null` is written as `null`). + *Examples:* ```shell diff --git a/tools/server/server-common.cpp b/tools/server/server-common.cpp index 3ea20c3d2..075687602 100644 --- a/tools/server/server-common.cpp +++ b/tools/server/server-common.cpp @@ -215,6 +215,7 @@ std::vector server_model_output_modalities(common_decision_type dec case COMMON_DECISION_TYPE_VLLM_D3: case COMMON_DECISION_TYPE_LFM2_D1: case COMMON_DECISION_TYPE_LFM2_D1_OMNI: + case COMMON_DECISION_TYPE_H2O_LIGHTNING: return {"decisions"}; case COMMON_DECISION_TYPE_GRANITE_SWITCH: return {"text", "decisions"}; diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 74812189c..84cd67768 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -5890,7 +5890,7 @@ void server_routes::init_routes() { const auto questions = decision.parse_questions(body); std::vector files; - const json state = decision.parse_state(body, files); + const json state = decision.prepare_state(decision.parse_state(body, files)); if (!files.empty() && (!decision.can_use_images() || !meta->has_inp_image)) { res->error(format_error_response("This server does not support image input for decisions. For a model that supports it, start it with `--mmproj`", ERROR_TYPE_NOT_SUPPORTED)); return res; diff --git a/tools/server/server-decision.cpp b/tools/server/server-decision.cpp index be9bc8ef4..ce4cb4d09 100644 --- a/tools/server/server-decision.cpp +++ b/tools/server/server-decision.cpp @@ -19,6 +19,10 @@ static const char * decision_question_type_name(server_decision_question_type ty // lev reads noul from a rating scale: 0 = certainly no, 8 = certainly yes static const size_t DECISION_LEV_N_RATINGS = 9; +// h2o-lightning follows serve_config.json and h2o_lightning_shim.py of the model repo +// a longer state keeps its head and its tail +static const size_t DECISION_H2O_MAX_STATE_TOKENS = 32000; + static std::string decision_meta_str(const llama_model * model, const std::string & key) { char buf[256]; const int32_t n = llama_model_meta_val_str(model, key.c_str(), buf, sizeof(buf)); @@ -164,6 +168,39 @@ void server_decision_context::init(const llama_model * model) { throw std::runtime_error("decision model has no mask token"); } n_options_max = 255; + } else if (model_type == COMMON_DECISION_TYPE_H2O_LIGHTNING) { + // the answer is read at the token of a label, its text is given to the template + for (uint32_t i = 0; llama_model_decision_label(model, i) != nullptr; i++) { + const std::string text = llama_model_decision_label(model, i); + const auto toks = common_tokenize(vocab, text, false, false); + if (toks.size() != 1 || std::find(labels.begin(), labels.end(), toks[0]) != labels.end()) { + throw std::runtime_error(string_format("decision label \"%s\" is not a single distinct token", text.c_str())); + } + labels.push_back(toks[0]); + label_texts.push_back(text); + } + if (labels.empty()) { + throw std::runtime_error("decision model has no labels"); + } + // the texts of the control and user-defined tokens are quoted in the input, so that they stay text + for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) { + if (llama_vocab_get_attr(vocab, id) & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) { + const std::string text = llama_vocab_get_text(vocab, id); + if (!text.empty()) { + quoted_spellings.push_back(text); + } + } + } + std::stable_sort(quoted_spellings.begin(), quoted_spellings.end(), [](const std::string & a, const std::string & b) { + return a.size() > b.size(); + }); + for (const auto & spelling : quoted_spellings) { + if (quoted_first.find(spelling[0]) == std::string::npos) { + quoted_first += spelling[0]; + } + } + n_options_max = labels.size(); + noul_true_first = true; } else { throw std::runtime_error("unsupported decision model type: " + type_name); } @@ -201,8 +238,10 @@ std::vector server_decision_context::parse_questions(c throw err("must be an object"); } // the vllm-d3 prompt has a default question - const json instructions = q.contains("instructions") ? q.at("instructions") : json(); - if (instructions.is_null() && type != COMMON_DECISION_TYPE_VLLM_D3) { + // h2o-lightning: the instructions can be left out, the template has a default, and the type defaults to choice + const bool is_h2o = type == COMMON_DECISION_TYPE_H2O_LIGHTNING; + const json instructions = q.contains("instructions") ? q.at("instructions") : (is_h2o ? json(std::string()) : json()); + if (instructions.is_null() && type != COMMON_DECISION_TYPE_VLLM_D3 && !is_h2o) { throw err("\"instructions\" must be provided"); } if (strict_payload && (!decision_is_payload(instructions) || instructions == "")) { @@ -213,7 +252,7 @@ std::vector server_decision_context::parse_questions(c question.id = id; question.instructions = instructions; - const std::string type_name = json_value(q, "type", std::string()); + const std::string type_name = json_value(q, "type", std::string(is_h2o ? "choice" : "")); const json criteria = q.contains("criteria") ? q.at("criteria") : json(); if (type_name == "choice") { @@ -237,7 +276,11 @@ std::vector server_decision_context::parse_questions(c } } else if (type_name == "score") { question.type = SERVER_DECISION_QUESTION_SCORE; - if (!criteria.is_array() || criteria.size() < 2 || criteria.size() > 10) { + // h2o-lightning: from 1 level, the maximum is n_options_max + if (is_h2o && (!criteria.is_array() || criteria.empty())) { + throw err("\"criteria\" must be a non-empty array"); + } + if (!is_h2o && (!criteria.is_array() || criteria.size() < 2 || criteria.size() > 10)) { throw err("\"criteria\" must be an array of 2 to 10 levels"); } for (size_t i = 0; i < criteria.size(); i++) { @@ -262,8 +305,8 @@ std::vector server_decision_context::parse_questions(c json description; if (criteria.is_object() && criteria.contains(key)) { description = criteria.at(key); - } else if (criteria.is_object() && type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) { - // lfm2-d1-omni also reads the descriptions under "no" and "yes" + } else if (criteria.is_object() && (type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || is_h2o)) { + // lfm2-d1-omni and h2o-lightning also read the descriptions under "no" and "yes" const char * alias = std::string(key) == "true" ? "yes" : "no"; description = criteria.contains(alias) ? criteria.at(alias) : json(); } @@ -506,6 +549,280 @@ static json decision_d1omni_escape(const json & val) { return val; } +// +// h2o-lightning: the texts are rendered the way the reference shim renders them with Python (json.dumps, str.lstrip) +// + +// one code point of UTF-8 text at s[i], i moves past it; an invalid byte is read as is +static uint32_t decision_utf8_next(const std::string & s, size_t & i) { + const unsigned char c = s[i]; + size_t n = c < 0x80 ? 1 : (c >> 5) == 0x6 ? 2 : (c >> 4) == 0xe ? 3 : (c >> 3) == 0x1e ? 4 : 1; + if (i + n > s.size()) { + n = 1; + } + uint32_t cp = n == 1 ? c : n == 2 ? (c & 0x1f) : n == 3 ? (c & 0x0f) : (c & 0x07); + for (size_t k = 1; k < n; k++) { + cp = (cp << 6) | (s[i + k] & 0x3f); + } + i += n; + return cp; +} + +// str.isspace() of Python +static bool decision_py_isspace(uint32_t cp) { + return (cp >= 0x09 && cp <= 0x0d) || (cp >= 0x1c && cp <= 0x20) || cp == 0x85 || cp == 0xa0 || cp == 0x1680 || + (cp >= 0x2000 && cp <= 0x200a) || cp == 0x2028 || cp == 0x2029 || cp == 0x202f || cp == 0x205f || cp == 0x3000; +} + +static std::string decision_py_rstrip(const std::string & s) { + size_t end = 0; + for (size_t i = 0; i < s.size();) { + if (!decision_py_isspace(decision_utf8_next(s, i))) { + end = i; + } + } + return s.substr(0, end); +} + +static std::string decision_py_lstrip(const std::string & s) { + size_t i = 0; + while (i < s.size()) { + size_t next = i; + if (!decision_py_isspace(decision_utf8_next(s, next))) { + break; + } + i = next; + } + return s.substr(i); +} + +// a code point as JSON escapes, a surrogate pair above the BMP +static std::string decision_py_escape_cp(uint32_t cp) { + if (cp < 0x10000) { + return string_format("\\u%04x", cp); + } + cp -= 0x10000; + return string_format("\\u%04x\\u%04x", 0xd800 + (cp >> 10), 0xdc00 + (cp & 0x3ff)); +} + +// repr() of a Python float: the shortest digits that read back the same, in exponent form below 1e-4 and from 1e16 +static std::string decision_py_float(double v) { + char buf[64]; + // the fewest digits that read back the same + for (int prec = 0; prec < 17; prec++) { + snprintf(buf, sizeof(buf), "%.*e", prec, v); + if (std::strtod(buf, nullptr) == v) { + break; + } + } + const std::string sci(buf); // for example "-1.25e+03" + const size_t pos_e = sci.find('e'); + const int exp = std::atoi(sci.c_str() + pos_e + 1); + std::string sign; + std::string digits; + for (size_t i = 0; i < pos_e; i++) { + if (sci[i] == '-') { + sign = "-"; + } else if (sci[i] != '.') { + digits += sci[i]; + } + } + const int n = (int) digits.size(); + if (exp < -4 || exp >= 16) { + return sign + digits.substr(0, 1) + (n > 1 ? "." + digits.substr(1) : "") + string_format("e%c%02d", exp < 0 ? '-' : '+', std::abs(exp)); + } + if (exp >= n - 1) { + return sign + digits + std::string(exp - (n - 1), '0') + ".0"; + } + if (exp >= 0) { + return sign + digits.substr(0, exp + 1) + "." + digits.substr(exp + 1); + } + return sign + "0." + std::string(-exp - 1, '0') + digits; +} + +static void decision_py_str(std::string & out, const std::string & s, bool ensure_ascii) { + out += '"'; + for (size_t i = 0; i < s.size();) { + const unsigned char c = s[i]; + switch (c) { + case '"': out += "\\\""; i++; continue; + case '\\': out += "\\\\"; i++; continue; + case '\b': out += "\\b"; i++; continue; + case '\f': out += "\\f"; i++; continue; + case '\n': out += "\\n"; i++; continue; + case '\r': out += "\\r"; i++; continue; + case '\t': out += "\\t"; i++; continue; + default: break; + } + if (c < 0x20 || (ensure_ascii && c == 0x7f)) { + out += string_format("\\u%04x", c); + i++; + } else if (c < 0x80 || !ensure_ascii) { + out += (char) c; + i++; + } else { + out += decision_py_escape_cp(decision_utf8_next(s, i)); + } + } + out += '"'; +} + +// json.dumps() of Python: indent < 0 is no indent, keys stay in their order +static void decision_py_dump(std::string & out, const json & val, int indent, int level, const std::string & item_sep, const std::string & key_sep, bool ensure_ascii) { + const auto newline = [&](int lvl) { + return indent < 0 ? std::string() : "\n" + std::string(indent * lvl, ' '); + }; + if (val.is_null()) { + out += "null"; + } else if (val.is_boolean()) { + out += val.get() ? "true" : "false"; + } else if (val.is_number_float()) { + out += decision_py_float(val.get()); + } else if (val.is_number()) { + out += val.dump(); + } else if (val.is_string()) { + decision_py_str(out, val.get(), ensure_ascii); + } else if (val.is_array()) { + if (val.empty()) { + out += "[]"; + return; + } + out += "[" + newline(level + 1); + for (size_t i = 0; i < val.size(); i++) { + if (i > 0) { + out += item_sep + newline(level + 1); + } + decision_py_dump(out, val.at(i), indent, level + 1, item_sep, key_sep, ensure_ascii); + } + out += newline(level) + "]"; + } else if (val.is_object()) { + if (val.empty()) { + out += "{}"; + return; + } + out += "{" + newline(level + 1); + bool first = true; + for (const auto & [key, item] : val.items()) { + if (!first) { + out += item_sep + newline(level + 1); + } + first = false; + decision_py_str(out, key, ensure_ascii); + out += key_sep; + decision_py_dump(out, item, indent, level + 1, item_sep, key_sep, ensure_ascii); + } + out += newline(level) + "}"; + } +} + +static std::string decision_py_dumps(const json & val, int indent, const std::string & item_sep, const std::string & key_sep, bool ensure_ascii) { + std::string out; + decision_py_dump(out, val, indent, 0, item_sep, key_sep, ensure_ascii); + return out; +} + +// a state longer than max_state_tokens keeps its head and its tail around a "..." line +std::string server_decision_context::h2o_cut_state(const std::string & text) const { + // a byte-level BPE never has more tokens than bytes + const size_t max_state_tokens = DECISION_H2O_MAX_STATE_TOKENS; + if (text.size() <= max_state_tokens) { + return text; + } + const llama_tokens tokens = common_tokenize(vocab, text, false, true); + if (tokens.size() <= max_state_tokens) { + return text; + } + // each half is the text of its token pieces, a cut inside a character gives U+FFFD as in a JSON response of the server + const size_t half = max_state_tokens / 2; + auto text_of = [&](llama_tokens::const_iterator begin, llama_tokens::const_iterator end) { + std::string out; + for (auto it = begin; it != end; ++it) { + out += common_token_to_piece(vocab, *it, true); + } + return json::parse(json::make(out).dump_safe()).get(); + }; + return text_of(tokens.begin(), tokens.begin() + half) + "\n...\n" + text_of(tokens.end() - half, tokens.end()); +} + +// the special token texts written in the input are quoted with JSON escapes, so they stay text +std::string server_decision_context::h2o_quote(const std::string & text) const { + std::string out; + for (size_t i = 0; i < text.size();) { + bool is_quoted = false; + if (quoted_first.find(text[i]) == std::string::npos) { + out += text[i++]; + continue; + } + for (const auto & spelling : quoted_spellings) { + if (text.compare(i, spelling.size(), spelling) == 0) { + std::string quoted = spelling; + string_replace_all(quoted, "<", "\\u003c"); + string_replace_all(quoted, ">", "\\u003e"); + out += quoted; + i += spelling.size(); + is_quoted = true; + break; + } + } + if (!is_quoted) { + out += text[i++]; + } + } + return out; +} + +json server_decision_context::prepare_state(const json & state) const { + if (type != COMMON_DECISION_TYPE_H2O_LIGHTNING) { + return state; + } + // done once per request: compact JSON, the cut, the quoting + return h2o_quote(h2o_cut_state(state.is_string() ? state.get() : decision_py_dumps(state, -1, ",", ":", false))); +} + +json server_decision_context::render_h2o(const json & state, const server_decision_question & question) const { + // the instructions are JSON with an indent of 1, the descriptions are default JSON, the state is from prepare_state() + const std::string instructions = decision_py_lstrip(question.instructions.is_string() ? + question.instructions.get() : decision_py_dumps(question.instructions, 1, ",", ": ", true)); + + json options = json::array(); + for (size_t i = 0; i < question.options.size(); i++) { + const auto & opt = question.options[i]; + json description; // null if not provided, the template has a default, except a score level that is shown as JSON null + if (question.type == SERVER_DECISION_QUESTION_SCORE && opt.description.is_null()) { + description = "null"; + } else if (!opt.description.is_null()) { + description = h2o_quote(opt.description.is_string() ? opt.description.get() : decision_py_dumps(opt.description, -1, ", ", ": ", false)); + } + const std::string key = h2o_quote(opt.key); + + // the user turn is stripped like Python's str.strip(), the template only trims ASCII whitespace + // so the text that ends the turn is stripped here, a space keeps a description that is only whitespace + if (i + 1 == question.options.size()) { + const bool is_key = question.type == SERVER_DECISION_QUESTION_CHOICE && description.is_null(); + if (is_key || description.is_string()) { + const std::string text = is_key ? key : description.get(); + const std::string stripped = decision_py_rstrip(text); + if (stripped != text) { + description = stripped.empty() ? " " : stripped; + } + } + } + + json option = json::object(); + option["label"] = label_texts[i]; + option["key"] = key; + option["description"] = description; + options.push_back(option); + } + + json inp = json::object(); + inp["type"] = decision_question_type_name(question.type); + inp["state"] = state; + inp["instructions"] = h2o_quote(instructions); + inp["options"] = options; + return inp; +} + // given to the lfm2-d1-omni template: text between the pieces of the prompt, and at the start of the pieces that are cut to a token budget static const std::string D1OMNI_MARKER = "<second; + } std::vector probs(n, 0.0); for (size_t v = 0; v < scores.size(); v++) { const auto & s = scores[v]; diff --git a/tools/server/server-decision.h b/tools/server/server-decision.h index ed9724429..b631099a8 100644 --- a/tools/server/server-decision.h +++ b/tools/server/server-decision.h @@ -46,6 +46,7 @@ struct server_decision_context { case COMMON_DECISION_TYPE_PPLX_DECIDER: case COMMON_DECISION_TYPE_VLLM_DECISION2: case COMMON_DECISION_TYPE_LFM2_D1: + case COMMON_DECISION_TYPE_H2O_LIGHTNING: return true; default: return false; @@ -66,6 +67,7 @@ struct server_decision_context { case COMMON_DECISION_TYPE_VLLM_D3: case COMMON_DECISION_TYPE_LFM2_D1: case COMMON_DECISION_TYPE_LFM2_D1_OMNI: + case COMMON_DECISION_TYPE_H2O_LIGHTNING: return true; default: return false; @@ -75,6 +77,9 @@ struct server_decision_context { // throw std::invalid_argument on bad input std::vector parse_questions(const json & body) const; + // the state as the model reads it, done once per request; returns the state for most models + json prepare_state(const json & state) const; + // returns the state without its images, they are appended to files in order // images come from "files" (alias "images") and from the image_url and input_audio parts of a state made of chat messages // an image can be an audio clip if the model supports it @@ -121,7 +126,7 @@ private: bool strict_payload = false; // state, instructions and descriptions must be text, an object or an array bool canonical_legend = false; // non-string score descriptions are returned as canonical JSON text - // OPENJEV, LEV, NIMBLE, PPLX_DECIDER, VLLM_D3 + // OPENJEV, LEV, NIMBLE, PPLX_DECIDER, VLLM_D3, H2O_LIGHTNING std::vector labels; std::vector label_texts; // only if the label of an option is given to the template @@ -132,6 +137,10 @@ private: size_t max_head_tokens = 0; // question + options size_t max_option_tokens = 48; + // H2O_LIGHTNING + std::vector quoted_spellings; // texts of the special tokens, quoted in the input, longest first + std::string quoted_first; // the first bytes of these texts + std::string render( const json & state, const std::vector & questions, @@ -143,6 +152,10 @@ private: size_t n_outputs(const server_decision_question & question) const; // LFM2_D1: label text and tokens of each option void d1_labels(const server_decision_question & question, std::vector & texts, std::vector & groups) const; + // H2O_LIGHTNING: the input of the template, its texts are rendered as the reference shim does + json render_h2o(const json & state, const server_decision_question & question) const; + std::string h2o_cut_state(const std::string & text) const; + std::string h2o_quote(const std::string & text) const; void fill_task_laya(llama_tokens & tokens, const server_decision_question & question, server_task & task) const; void fill_task_d1omni( const json & state,