[Bugfix][MLA] Read sparse model settings from text config (#56160)

Signed-off-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
Andy Lo
2026-09-11 18:02:13 +00:00
committed by GitHub
parent 6fe67cbbf3
commit dc07f1638f
11 changed files with 32 additions and 22 deletions
@@ -168,6 +168,7 @@ def _run_sparse_backend_vs_sdpa(
)
model_config = vllm_config.model_config
model_config.hf_text_config = SimpleNamespace(
index_topk=topk_tokens,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
@@ -175,6 +176,7 @@ def _run_sparse_backend_vs_sdpa(
v_head_dim=v_head_dim,
model_type="deepseek_v2",
)
del model_config.hf_config.index_topk # Composite configs only nest this field.
model_config.dtype = dtype
model_config.get_num_attention_heads = MethodType(
lambda self, parallel_config: num_heads, model_config
@@ -287,7 +289,7 @@ def _run_sparse_backend_vs_sdpa(
causal_reference = torch.cat(causal_reference_outputs, dim=0)
vllm_config.cache_config.cache_dtype = kv_cache_dtype
vllm_config.model_config.hf_config.index_topk = topk_tokens
vllm_config.model_config.hf_text_config.index_topk = topk_tokens
common_attn_metadata = create_common_attn_metadata(
batch_spec, block_size, device, arange_block_indices=True
@@ -68,7 +68,9 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
model_config=SimpleNamespace(
hf_config=SimpleNamespace(compress_ratios=[0, 0, 4, 128, 0], index_kpool=32)
hf_text_config=SimpleNamespace(
compress_ratios=[0, 0, 4, 128, 0], index_kpool=32
)
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=1,
@@ -89,7 +91,7 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
def test_compressed_slot_mapping_warmup_includes_index_kpool():
config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=256),
model_config=SimpleNamespace(hf_config=SimpleNamespace(index_kpool=32)),
model_config=SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=32)),
)
keys = CompressedSlotMappingKernel().get_warmup_keys(config)
@@ -101,7 +103,7 @@ def test_index_conversion_warmup_uses_physical_block_stride():
cache_config=SimpleNamespace(block_size=64),
model_config=SimpleNamespace(
max_model_len=1024,
hf_config=SimpleNamespace(index_topk=2048),
hf_text_config=SimpleNamespace(index_topk=2048),
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=1,
@@ -423,6 +423,7 @@ def test_sparse_backend_decode_correctness(
)
model_config = vllm_config.model_config
model_config.hf_text_config = SimpleNamespace(
index_topk=topk_tokens,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
@@ -430,6 +431,7 @@ def test_sparse_backend_decode_correctness(
v_head_dim=v_head_dim,
model_type="deepseek_v2",
)
del model_config.hf_config.index_topk # Composite configs only nest this field.
model_config.dtype = dtype
model_config.get_num_attention_heads = MethodType(
lambda self, parallel_config: num_heads,
@@ -591,7 +593,7 @@ def test_sparse_backend_decode_correctness(
sdpa_reference = torch.cat(reference_outputs, dim=0)
vllm_config.cache_config.cache_dtype = kv_cache_dtype
vllm_config.model_config.hf_config.index_topk = topk_tokens
vllm_config.model_config.hf_text_config.index_topk = topk_tokens
common_attn_metadata = create_common_attn_metadata(
batch_spec,
@@ -1149,6 +1151,7 @@ def test_sparse_backend_prefill_correctness(
)
model_config = vllm_config.model_config
model_config.hf_text_config = SimpleNamespace(
index_topk=topk_tokens,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
@@ -1156,6 +1159,7 @@ def test_sparse_backend_prefill_correctness(
v_head_dim=v_head_dim,
model_type="deepseek_v2",
)
del model_config.hf_config.index_topk # Composite configs only nest this field.
model_config.dtype = dtype
model_config.model_arch_config.total_num_attention_heads = num_heads
model_config.get_num_attention_heads = MethodType(
@@ -1255,7 +1259,7 @@ def test_sparse_backend_prefill_correctness(
ref_output = torch.cat(reference_outputs, dim=0)
vllm_config.cache_config.cache_dtype = kv_cache_dtype
vllm_config.model_config.hf_config.index_topk = topk_tokens
vllm_config.model_config.hf_text_config.index_topk = topk_tokens
common_attn_metadata = create_common_attn_metadata(
batch_spec,
@@ -1747,6 +1751,7 @@ def _build_sparse_dcp_vllm_config(
model_config = vllm_config.model_config
model_config.dtype = torch.bfloat16
model_config.hf_text_config = SimpleNamespace(
index_topk=topk_tokens,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
@@ -1754,6 +1759,7 @@ def _build_sparse_dcp_vllm_config(
v_head_dim=v_head_dim,
model_type="deepseek_v2",
)
del model_config.hf_config.index_topk # Composite configs only nest this field.
model_config.get_num_attention_heads = MethodType(
lambda self, parallel_config: local_heads, model_config
)