[Bugfix][MLA] Read sparse model settings from text config (#56160)

Signed-off-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
Andy Lo
2026-09-11 18:02:13 +00:00
committed by GitHub
parent 6fe67cbbf3
commit dc07f1638f
11 changed files with 32 additions and 22 deletions
@@ -168,6 +168,7 @@ def _run_sparse_backend_vs_sdpa(
)
model_config = vllm_config.model_config
model_config.hf_text_config = SimpleNamespace(
index_topk=topk_tokens,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
@@ -175,6 +176,7 @@ def _run_sparse_backend_vs_sdpa(
v_head_dim=v_head_dim,
model_type="deepseek_v2",
)
del model_config.hf_config.index_topk # Composite configs only nest this field.
model_config.dtype = dtype
model_config.get_num_attention_heads = MethodType(
lambda self, parallel_config: num_heads, model_config
@@ -287,7 +289,7 @@ def _run_sparse_backend_vs_sdpa(
causal_reference = torch.cat(causal_reference_outputs, dim=0)
vllm_config.cache_config.cache_dtype = kv_cache_dtype
vllm_config.model_config.hf_config.index_topk = topk_tokens
vllm_config.model_config.hf_text_config.index_topk = topk_tokens
common_attn_metadata = create_common_attn_metadata(
batch_spec, block_size, device, arange_block_indices=True
@@ -68,7 +68,9 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
model_config=SimpleNamespace(
hf_config=SimpleNamespace(compress_ratios=[0, 0, 4, 128, 0], index_kpool=32)
hf_text_config=SimpleNamespace(
compress_ratios=[0, 0, 4, 128, 0], index_kpool=32
)
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=1,
@@ -89,7 +91,7 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
def test_compressed_slot_mapping_warmup_includes_index_kpool():
config = SimpleNamespace(
cache_config=SimpleNamespace(block_size=256),
model_config=SimpleNamespace(hf_config=SimpleNamespace(index_kpool=32)),
model_config=SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=32)),
)
keys = CompressedSlotMappingKernel().get_warmup_keys(config)
@@ -101,7 +103,7 @@ def test_index_conversion_warmup_uses_physical_block_stride():
cache_config=SimpleNamespace(block_size=64),
model_config=SimpleNamespace(
max_model_len=1024,
hf_config=SimpleNamespace(index_topk=2048),
hf_text_config=SimpleNamespace(index_topk=2048),
),
parallel_config=SimpleNamespace(
decode_context_parallel_size=1,
@@ -423,6 +423,7 @@ def test_sparse_backend_decode_correctness(
)
model_config = vllm_config.model_config
model_config.hf_text_config = SimpleNamespace(
index_topk=topk_tokens,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
@@ -430,6 +431,7 @@ def test_sparse_backend_decode_correctness(
v_head_dim=v_head_dim,
model_type="deepseek_v2",
)
del model_config.hf_config.index_topk # Composite configs only nest this field.
model_config.dtype = dtype
model_config.get_num_attention_heads = MethodType(
lambda self, parallel_config: num_heads,
@@ -591,7 +593,7 @@ def test_sparse_backend_decode_correctness(
sdpa_reference = torch.cat(reference_outputs, dim=0)
vllm_config.cache_config.cache_dtype = kv_cache_dtype
vllm_config.model_config.hf_config.index_topk = topk_tokens
vllm_config.model_config.hf_text_config.index_topk = topk_tokens
common_attn_metadata = create_common_attn_metadata(
batch_spec,
@@ -1149,6 +1151,7 @@ def test_sparse_backend_prefill_correctness(
)
model_config = vllm_config.model_config
model_config.hf_text_config = SimpleNamespace(
index_topk=topk_tokens,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
@@ -1156,6 +1159,7 @@ def test_sparse_backend_prefill_correctness(
v_head_dim=v_head_dim,
model_type="deepseek_v2",
)
del model_config.hf_config.index_topk # Composite configs only nest this field.
model_config.dtype = dtype
model_config.model_arch_config.total_num_attention_heads = num_heads
model_config.get_num_attention_heads = MethodType(
@@ -1255,7 +1259,7 @@ def test_sparse_backend_prefill_correctness(
ref_output = torch.cat(reference_outputs, dim=0)
vllm_config.cache_config.cache_dtype = kv_cache_dtype
vllm_config.model_config.hf_config.index_topk = topk_tokens
vllm_config.model_config.hf_text_config.index_topk = topk_tokens
common_attn_metadata = create_common_attn_metadata(
batch_spec,
@@ -1747,6 +1751,7 @@ def _build_sparse_dcp_vllm_config(
model_config = vllm_config.model_config
model_config.dtype = torch.bfloat16
model_config.hf_text_config = SimpleNamespace(
index_topk=topk_tokens,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
qk_nope_head_dim=qk_nope_head_dim,
@@ -1754,6 +1759,7 @@ def _build_sparse_dcp_vllm_config(
v_head_dim=v_head_dim,
model_type="deepseek_v2",
)
del model_config.hf_config.index_topk # Composite configs only nest this field.
model_config.get_num_attention_heads = MethodType(
lambda self, parallel_config: local_heads, model_config
)
@@ -173,7 +173,7 @@ class PackDCPTopkCandidatesKernel(
if dcp_world_size <= 1:
return []
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
topk = vllm_config.model_config.hf_config.index_topk
topk = vllm_config.model_config.hf_text_config.index_topk
if topk <= 0:
return []
@@ -546,7 +546,7 @@ class StableTopKFromGatheredCandidatesKernel(
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
if dcp_world_size <= 1:
return []
topk = vllm_config.model_config.hf_config.index_topk
topk = vllm_config.model_config.hf_text_config.index_topk
if topk <= 0:
return []
return self._trace_dispatch(self.dispatch)(
@@ -132,7 +132,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
self.device = device
self.model_config = vllm_config.model_config
self.mla_dims = get_mla_dims(self.model_config)
self.topk_tokens: int = vllm_config.model_config.hf_config.index_topk
self.topk_tokens: int = vllm_config.model_config.hf_text_config.index_topk
self.req_id_per_token_buffer = torch.empty(
(vllm_config.scheduler_config.max_num_batched_tokens,),
dtype=torch.int32,
@@ -202,7 +202,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
scheduler_config = vllm_config.scheduler_config
cache_config = vllm_config.cache_config
model_config = vllm_config.model_config
topk_tokens = model_config.hf_config.index_topk
topk_tokens = model_config.hf_text_config.index_topk
workspace_size = min(
max(
@@ -97,10 +97,10 @@ class CompressedSlotMappingKernel(
)
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
hf_config = vllm_config.model_config.hf_config
hf_text_config = vllm_config.model_config.hf_text_config
configured_ratios = (
*(getattr(hf_config, "compress_ratios", None) or ()),
getattr(hf_config, "index_kpool", 1) or 1,
*(getattr(hf_text_config, "compress_ratios", None) or ()),
getattr(hf_text_config, "index_kpool", 1) or 1,
)
compress_ratios = tuple(
dict.fromkeys(int(ratio) for ratio in configured_ratios if int(ratio) > 1)
@@ -99,8 +99,8 @@ class FlashAttnMLASparseBackend(AttentionBackend):
if vllm_config.parallel_config.decode_context_parallel_size > 1:
return "FlashAttention MLA Sparse does not support DCP for now"
hf_config = vllm_config.model_config.hf_config
if not hasattr(hf_config, "index_topk"):
hf_text_config = vllm_config.model_config.hf_text_config
if not hasattr(hf_text_config, "index_topk"):
return "FlashAttention MLA Sparse requires model with index_topk"
return None
+4 -4
View File
@@ -396,7 +396,7 @@ class BuildPrefillChunkMetadataKernel(
def get_warmup_keys(self, vllm_config: VllmConfig) -> list[CompileKey]:
max_tokens = max(1, min(vllm_config.scheduler_config.max_num_batched_tokens, 8))
hf_config = vllm_config.model_config.hf_config
hf_text_config = vllm_config.model_config.hf_text_config
parallel_config = vllm_config.parallel_config
dcp_world = parallel_config.decode_context_parallel_size
dcp_interleave = parallel_config.cp_kv_cache_interleave_size
@@ -405,12 +405,12 @@ class BuildPrefillChunkMetadataKernel(
dict.fromkeys(
max(1, int(ratio))
for ratio in (
*(getattr(hf_config, "compress_ratios", None) or (1,)),
getattr(hf_config, "index_kpool", 1) or 1,
*(getattr(hf_text_config, "compress_ratios", None) or (1,)),
getattr(hf_text_config, "index_kpool", 1) or 1,
)
)
)
index_kpool = getattr(hf_config, "index_kpool", None)
index_kpool = getattr(hf_text_config, "index_kpool", None)
if index_kpool and index_kpool > 1 and index_kpool not in compress_ratios:
compress_ratios = compress_ratios + (index_kpool,)
return self._trace_dispatch(self.dispatch)(
@@ -435,7 +435,7 @@ class ROCMAiterMLASparseMetadataBuilder(
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
self.mla_dims = get_mla_dims(self.model_config)
self.topk_tokens = vllm_config.model_config.hf_config.index_topk
self.topk_tokens = vllm_config.model_config.hf_text_config.index_topk
attention_context = vllm_config.compilation_config.static_forward_context
# Sink decode must use AITER's nonpersistent path. In particular,
# gfx942 has no persistent+LSE kernel, and its metadata heuristic
@@ -232,7 +232,7 @@ class ConvertReqIndexToGlobalIndexKernel(
dcp_size = vllm_config.parallel_config.decode_context_parallel_size
dcp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
dcp_rank = get_dcp_group().rank_in_group if dcp_size > 1 else 0
num_topk_tokens = vllm_config.model_config.hf_config.index_topk
num_topk_tokens = vllm_config.model_config.hf_text_config.index_topk
max_num_blocks = cdiv(
vllm_config.model_config.max_model_len,
block_size * dcp_size,
@@ -128,7 +128,7 @@ class XPUMLASparseMetadataBuilder(AttentionMetadataBuilder[XPUMLASparseMetadata]
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
self.mla_dims = get_mla_dims(self.model_config)
self.topk_tokens = vllm_config.model_config.hf_config.index_topk
self.topk_tokens = vllm_config.model_config.hf_text_config.index_topk
self.topk_tokens_tensor = torch.tensor(
[self.topk_tokens], device=device, dtype=torch.int32
)