vrr/koboldcpp
2
0
Fork 0
mirror of https://github.com/LostRuins/koboldcpp.git synced 2026-10-20 11:00:40 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

convert : support Qwen3.5 embedding models (#27920)

Browse source 
* Update convert_hf_to_gguf: support Qwen3.5 embedding models

* Behavior-preserving refactor for conventions.

* conversion: simplify pooling comment
This commit is contained in:
Sam Malayek 2026-10-08 01:40:01 -04:00 • committed by GitHub
parent 9b4ed0ca57
commit 75118a3a59
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 34 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

1
conversion/__init__.py
View file

@ -255,6 +255,7 @@ TEXT_MODEL_MAP: dict[str, str] = {
"Qwen3_5ForConditionalGeneration": "qwen",
"Qwen3_5MoeForCausalLM": "qwen",
"Qwen3_5MoeForConditionalGeneration": "qwen",
"Qwen3_5TextModel": "qwen",
"Qwen4ExpForCausalLM": "qwen4exp",
"Qwen4ExpForConditionalGeneration": "qwen4exp",
"RND1": "qwen",

10
conversion/base.py
View file

@ -2336,6 +2336,16 @@ class TextModel(ModelBase):
else:
raise NotImplementedError("Only MEAN, CLS, and LAST pooling types supported")
self.gguf_writer.add_pooling_type(pooling_type)
else:
embedding_config_path = self.dir_model / "embedding_config.json"
if embedding_config_path.is_file():
with open(embedding_config_path, encoding="utf-8") as f:
embedding_config = json.load(f)
pooling = embedding_config.get("pooling")
if pooling == "last_token":
self.gguf_writer.add_pooling_type(gguf.PoolingType.LAST)
elif pooling is not None:
raise NotImplementedError(f"unsupported embedding_config.json pooling {pooling!r}")
# pooling before a classification head (e.g. ModernBertForSequenceClassification)
if (classifier_pooling := self.hparams.get("classifier_pooling")) is not None:

15
conversion/qwen.py
View file

@ -650,11 +650,24 @@ class _Qwen35MRopeMixin:
self.gguf_writer.add_rope_dimension_sections(self._QWEN35_DEFAULT_MROPE_SECTION)
@ModelBase.register("Qwen3_5ForConditionalGeneration", "Qwen3_5ForCausalLM")
@ModelBase.register("Qwen3_5ForConditionalGeneration", "Qwen3_5ForCausalLM", "Qwen3_5TextModel")
@ModelBase.example("Qwen/Qwen3.5-9B")
class Qwen3_5TextModel(_Qwen35MRopeMixin, _LinearAttentionVReorderBase):
model_arch = gguf.MODEL_ARCH.QWEN35
def __init__(self, dir_model, *args, **kwargs):
# Inner TextModel does not own mtp.*. Set no_mtp before mixin bumps block_count.
hparams = kwargs.pop("hparams", None)
if hparams is None:
hparams = ModelBase.load_hparams(dir_model, self.is_mistral_format)
if get_model_architecture(hparams, ModelType.TEXT) == "Qwen3_5TextModel":
self.no_mtp = True
super().__init__(dir_model, *args, hparams=hparams, **kwargs)
def set_gguf_parameters(self):
super().set_gguf_parameters()
self._try_set_pooling_type()
def _is_openjev_checkpoint(dir_model: Path) -> bool:
return (dir_model / "helper" / "shim.py").is_file() and (dir_model / "config.json").is_file()

9
gguf-py/gguf/tensor_mapping.py
View file

@ -255,6 +255,7 @@ class TensorNameMap:
"model.layers.{bid}.self_attn.language_expert_query_key_value", # cogvlm
"model.layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5
"head.layers.{bid}.self_attn.in_proj", # laya
"layers.{bid}.linear_attn.in_proj_qkv", # qwen3.5 text
),
# Attention query
@ -397,6 +398,7 @@ class TensorNameMap:
MODEL_TENSOR.ATTN_GATE: (
"model.layers.{bid}.self_attn.gate_proj", # afmoe muse-glimmer
"model.layers.{bid}.linear_attn.in_proj_z", # qwen3.5
"layers.{bid}.linear_attn.in_proj_z", # qwen3.5 text
"model.layers.{bid}.self_attn.g_proj", # step3.5 head-wise attention gate
"model.layers.{bid}.self_attn.output_gate", # minimax-01
"model.layers.{bid}.self_attn.linear_gate", # hy-v4
@ -842,6 +844,7 @@ class TensorNameMap:
"model.layers.{bid}.mamba.conv1d", # jamba falcon-h1 granite-hybrid
"model.layers.layers.{bid}.mixer.conv1d", # plamo2
"model.layers.{bid}.linear_attn.conv1d", # qwen3next
"layers.{bid}.linear_attn.conv1d", # qwen3.5 text
),
MODEL_TENSOR.SSM_X: (
@ -857,6 +860,7 @@ class TensorNameMap:
"model.layers.{bid}.mamba.dt_proj", # jamba falcon-h1 granite-hybrid
"model.layers.layers.{bid}.mixer.dt_proj", # plamo2
"model.layers.{bid}.linear_attn.dt_proj", # qwen3next
"layers.{bid}.linear_attn.dt_proj", # qwen3.5 text
"backbone.layers.{bid}.mixer.dt", # nemotron-h-moe
"model.layers.{bid}.self_attn.dt_proj", # kimi
"model.layers.{bid}.attention.dt_proj", # bailingmoe3
@ -873,6 +877,7 @@ class TensorNameMap:
"model.layers.{bid}.mamba.A_log", # jamba falcon-h1 granite-hybrid
"model.layers.layers.{bid}.mixer.A_log", # plamo2
"model.layers.{bid}.linear_attn.A_log", # qwen3next
"layers.{bid}.linear_attn.A_log", # qwen3.5 text
"model.layers.{bid}.self_attn.A_log", # kimi
"model.layers.{bid}.attention.A_log", # bailingmoe3
),
@ -899,6 +904,7 @@ class TensorNameMap:
MODEL_TENSOR.SSM_NORM: (
"model.layers.{bid}.mamba.norm", # falcon-h1 granite-hybrid
"model.layers.{bid}.linear_attn.norm", # qwen3next
"layers.{bid}.linear_attn.norm", # qwen3.5 text
"backbone.layers.{bid}.mixer.norm", # mamba2
"model.layers.{bid}.self_attn.o_norm", # kimi
"model.layers.{bid}.attention.o_norm", # bailingmoe3
@ -909,11 +915,13 @@ class TensorNameMap:
"backbone.layers.{bid}.mixer.out_proj", # mamba
"model.layers.{bid}.mamba.out_proj", # jamba falcon-h1 granite-hybrid
"model.layers.{bid}.linear_attn.out_proj", # qwen3next
"layers.{bid}.linear_attn.out_proj", # qwen3.5 text
"model.layers.layers.{bid}.mixer.out_proj", # plamo2
),
MODEL_TENSOR.SSM_ALPHA: (
"model.layers.{bid}.linear_attn.in_proj_a", # qwen3.5
"layers.{bid}.linear_attn.in_proj_a", # qwen3.5 text
),
MODEL_TENSOR.SSM_BETA_ALPHA: (
@ -941,6 +949,7 @@ class TensorNameMap:
),
MODEL_TENSOR.SSM_BETA: (
"model.layers.{bid}.linear_attn.in_proj_b", # qwen3.5
"layers.{bid}.linear_attn.in_proj_b", # qwen3.5 text
"model.layers.{bid}.self_attn.b_proj", # Kimi Linear
"model.layers.{bid}.attention.b_proj", # bailingmoe3
),