[feature] add index share feature for DSA MTP (#44420)

Signed-off-by: JaredforReal <w13431838023@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Jared Wen
2026-06-06 22:04:14 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 9c7f7741d4
commit 32f34d3935
5 changed files with 114 additions and 25 deletions
@@ -50,7 +50,7 @@ class ModelArchConfigConvertorBase:
# special case for deepseek_v4
if hasattr(self.hf_text_config, "compress_ratios"):
return self.hf_text_config.head_dim
qk_rope_head_dim = getattr(self.hf_text_config, "qk_rope_head_dim", 0)
qk_rope_head_dim = self._get_qk_rope_head_dim()
if not envs.VLLM_MLA_DISABLE:
return self.hf_text_config.kv_lora_rank + qk_rope_head_dim
else:
@@ -71,6 +71,38 @@ class ModelArchConfigConvertorBase:
# FIXME(woosuk): This may not be true for all models.
return self.get_hidden_size() // total_num_attention_heads
def _get_qk_rope_head_dim(self) -> int:
"""Get qk_rope_head_dim, fixing the transformers v5.4+ attribute_map bug."""
cfg = self.hf_text_config
qk_rope_head_dim = getattr(cfg, "qk_rope_head_dim", 0)
qk_nope_head_dim = getattr(cfg, "qk_nope_head_dim", 0)
# In valid MLA configs, qk_rope_head_dim != qk_nope_head_dim.
if qk_rope_head_dim == 0 or qk_rope_head_dim != qk_nope_head_dim:
return qk_rope_head_dim # not corrupted
# Read the correct value from raw config.json.
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
model_path = self.hf_config.name_or_path
if not model_path:
return qk_rope_head_dim
raw = get_hf_file_to_dict("config.json", model_path)
if raw and "qk_rope_head_dim" in raw:
correct = raw["qk_rope_head_dim"]
if correct != qk_rope_head_dim:
logger.info(
"Fixing qk_rope_head_dim: %d -> %d "
"(transformers v5.4+ attribute_map bug)",
qk_rope_head_dim,
correct,
)
# Patch the config so downstream model layers also get
# the correct value.
cfg.qk_rope_head_dim = correct
return correct
return qk_rope_head_dim
def get_total_num_kv_heads(self) -> int:
attributes = [
# For Falcon: