[feature] add index share feature for DSA MTP (#44420)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9c7f7741d4
commit
32f34d3935
@@ -50,7 +50,7 @@ class ModelArchConfigConvertorBase:
|
||||
# special case for deepseek_v4
|
||||
if hasattr(self.hf_text_config, "compress_ratios"):
|
||||
return self.hf_text_config.head_dim
|
||||
qk_rope_head_dim = getattr(self.hf_text_config, "qk_rope_head_dim", 0)
|
||||
qk_rope_head_dim = self._get_qk_rope_head_dim()
|
||||
if not envs.VLLM_MLA_DISABLE:
|
||||
return self.hf_text_config.kv_lora_rank + qk_rope_head_dim
|
||||
else:
|
||||
@@ -71,6 +71,38 @@ class ModelArchConfigConvertorBase:
|
||||
# FIXME(woosuk): This may not be true for all models.
|
||||
return self.get_hidden_size() // total_num_attention_heads
|
||||
|
||||
def _get_qk_rope_head_dim(self) -> int:
|
||||
"""Get qk_rope_head_dim, fixing the transformers v5.4+ attribute_map bug."""
|
||||
cfg = self.hf_text_config
|
||||
qk_rope_head_dim = getattr(cfg, "qk_rope_head_dim", 0)
|
||||
qk_nope_head_dim = getattr(cfg, "qk_nope_head_dim", 0)
|
||||
|
||||
# In valid MLA configs, qk_rope_head_dim != qk_nope_head_dim.
|
||||
if qk_rope_head_dim == 0 or qk_rope_head_dim != qk_nope_head_dim:
|
||||
return qk_rope_head_dim # not corrupted
|
||||
|
||||
# Read the correct value from raw config.json.
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
||||
|
||||
model_path = self.hf_config.name_or_path
|
||||
if not model_path:
|
||||
return qk_rope_head_dim
|
||||
raw = get_hf_file_to_dict("config.json", model_path)
|
||||
if raw and "qk_rope_head_dim" in raw:
|
||||
correct = raw["qk_rope_head_dim"]
|
||||
if correct != qk_rope_head_dim:
|
||||
logger.info(
|
||||
"Fixing qk_rope_head_dim: %d -> %d "
|
||||
"(transformers v5.4+ attribute_map bug)",
|
||||
qk_rope_head_dim,
|
||||
correct,
|
||||
)
|
||||
# Patch the config so downstream model layers also get
|
||||
# the correct value.
|
||||
cfg.qk_rope_head_dim = correct
|
||||
return correct
|
||||
return qk_rope_head_dim
|
||||
|
||||
def get_total_num_kv_heads(self) -> int:
|
||||
attributes = [
|
||||
# For Falcon:
|
||||
|
||||
Reference in New Issue
Block a user