[Bugfix][MLA] Read sparse model settings from text config (#56160)
Signed-off-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
@@ -168,6 +168,7 @@ def _run_sparse_backend_vs_sdpa(
|
||||
)
|
||||
model_config = vllm_config.model_config
|
||||
model_config.hf_text_config = SimpleNamespace(
|
||||
index_topk=topk_tokens,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
@@ -175,6 +176,7 @@ def _run_sparse_backend_vs_sdpa(
|
||||
v_head_dim=v_head_dim,
|
||||
model_type="deepseek_v2",
|
||||
)
|
||||
del model_config.hf_config.index_topk # Composite configs only nest this field.
|
||||
model_config.dtype = dtype
|
||||
model_config.get_num_attention_heads = MethodType(
|
||||
lambda self, parallel_config: num_heads, model_config
|
||||
@@ -287,7 +289,7 @@ def _run_sparse_backend_vs_sdpa(
|
||||
causal_reference = torch.cat(causal_reference_outputs, dim=0)
|
||||
|
||||
vllm_config.cache_config.cache_dtype = kv_cache_dtype
|
||||
vllm_config.model_config.hf_config.index_topk = topk_tokens
|
||||
vllm_config.model_config.hf_text_config.index_topk = topk_tokens
|
||||
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, block_size, device, arange_block_indices=True
|
||||
|
||||
@@ -68,7 +68,9 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=SimpleNamespace(compress_ratios=[0, 0, 4, 128, 0], index_kpool=32)
|
||||
hf_text_config=SimpleNamespace(
|
||||
compress_ratios=[0, 0, 4, 128, 0], index_kpool=32
|
||||
)
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=1,
|
||||
@@ -89,7 +91,7 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
|
||||
def test_compressed_slot_mapping_warmup_includes_index_kpool():
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=256),
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(index_kpool=32)),
|
||||
model_config=SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=32)),
|
||||
)
|
||||
|
||||
keys = CompressedSlotMappingKernel().get_warmup_keys(config)
|
||||
@@ -101,7 +103,7 @@ def test_index_conversion_warmup_uses_physical_block_stride():
|
||||
cache_config=SimpleNamespace(block_size=64),
|
||||
model_config=SimpleNamespace(
|
||||
max_model_len=1024,
|
||||
hf_config=SimpleNamespace(index_topk=2048),
|
||||
hf_text_config=SimpleNamespace(index_topk=2048),
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=1,
|
||||
|
||||
@@ -423,6 +423,7 @@ def test_sparse_backend_decode_correctness(
|
||||
)
|
||||
model_config = vllm_config.model_config
|
||||
model_config.hf_text_config = SimpleNamespace(
|
||||
index_topk=topk_tokens,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
@@ -430,6 +431,7 @@ def test_sparse_backend_decode_correctness(
|
||||
v_head_dim=v_head_dim,
|
||||
model_type="deepseek_v2",
|
||||
)
|
||||
del model_config.hf_config.index_topk # Composite configs only nest this field.
|
||||
model_config.dtype = dtype
|
||||
model_config.get_num_attention_heads = MethodType(
|
||||
lambda self, parallel_config: num_heads,
|
||||
@@ -591,7 +593,7 @@ def test_sparse_backend_decode_correctness(
|
||||
sdpa_reference = torch.cat(reference_outputs, dim=0)
|
||||
|
||||
vllm_config.cache_config.cache_dtype = kv_cache_dtype
|
||||
vllm_config.model_config.hf_config.index_topk = topk_tokens
|
||||
vllm_config.model_config.hf_text_config.index_topk = topk_tokens
|
||||
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec,
|
||||
@@ -1149,6 +1151,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
)
|
||||
model_config = vllm_config.model_config
|
||||
model_config.hf_text_config = SimpleNamespace(
|
||||
index_topk=topk_tokens,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
@@ -1156,6 +1159,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
v_head_dim=v_head_dim,
|
||||
model_type="deepseek_v2",
|
||||
)
|
||||
del model_config.hf_config.index_topk # Composite configs only nest this field.
|
||||
model_config.dtype = dtype
|
||||
model_config.model_arch_config.total_num_attention_heads = num_heads
|
||||
model_config.get_num_attention_heads = MethodType(
|
||||
@@ -1255,7 +1259,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
ref_output = torch.cat(reference_outputs, dim=0)
|
||||
|
||||
vllm_config.cache_config.cache_dtype = kv_cache_dtype
|
||||
vllm_config.model_config.hf_config.index_topk = topk_tokens
|
||||
vllm_config.model_config.hf_text_config.index_topk = topk_tokens
|
||||
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec,
|
||||
@@ -1747,6 +1751,7 @@ def _build_sparse_dcp_vllm_config(
|
||||
model_config = vllm_config.model_config
|
||||
model_config.dtype = torch.bfloat16
|
||||
model_config.hf_text_config = SimpleNamespace(
|
||||
index_topk=topk_tokens,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
@@ -1754,6 +1759,7 @@ def _build_sparse_dcp_vllm_config(
|
||||
v_head_dim=v_head_dim,
|
||||
model_type="deepseek_v2",
|
||||
)
|
||||
del model_config.hf_config.index_topk # Composite configs only nest this field.
|
||||
model_config.get_num_attention_heads = MethodType(
|
||||
lambda self, parallel_config: local_heads, model_config
|
||||
)
|
||||
|
||||
@@ -173,7 +173,7 @@ class PackDCPTopkCandidatesKernel(
|
||||
if dcp_world_size <= 1:
|
||||
return []
|
||||
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
topk = vllm_config.model_config.hf_config.index_topk
|
||||
topk = vllm_config.model_config.hf_text_config.index_topk
|
||||
if topk <= 0:
|
||||
return []
|
||||
|
||||
@@ -546,7 +546,7 @@ class StableTopKFromGatheredCandidatesKernel(
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if dcp_world_size <= 1:
|
||||
return []
|
||||
topk = vllm_config.model_config.hf_config.index_topk
|
||||
topk = vllm_config.model_config.hf_text_config.index_topk
|
||||
if topk <= 0:
|
||||
return []
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
|
||||
@@ -132,7 +132,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
self.device = device
|
||||
self.model_config = vllm_config.model_config
|
||||
self.mla_dims = get_mla_dims(self.model_config)
|
||||
self.topk_tokens: int = vllm_config.model_config.hf_config.index_topk
|
||||
self.topk_tokens: int = vllm_config.model_config.hf_text_config.index_topk
|
||||
self.req_id_per_token_buffer = torch.empty(
|
||||
(vllm_config.scheduler_config.max_num_batched_tokens,),
|
||||
dtype=torch.int32,
|
||||
@@ -202,7 +202,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
cache_config = vllm_config.cache_config
|
||||
model_config = vllm_config.model_config
|
||||
topk_tokens = model_config.hf_config.index_topk
|
||||
topk_tokens = model_config.hf_text_config.index_topk
|
||||
|
||||
workspace_size = min(
|
||||
max(
|
||||
|
||||
@@ -97,10 +97,10 @@ class CompressedSlotMappingKernel(
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, vllm_config: Any) -> list[CompileKey]:
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
configured_ratios = (
|
||||
*(getattr(hf_config, "compress_ratios", None) or ()),
|
||||
getattr(hf_config, "index_kpool", 1) or 1,
|
||||
*(getattr(hf_text_config, "compress_ratios", None) or ()),
|
||||
getattr(hf_text_config, "index_kpool", 1) or 1,
|
||||
)
|
||||
compress_ratios = tuple(
|
||||
dict.fromkeys(int(ratio) for ratio in configured_ratios if int(ratio) > 1)
|
||||
|
||||
@@ -99,8 +99,8 @@ class FlashAttnMLASparseBackend(AttentionBackend):
|
||||
if vllm_config.parallel_config.decode_context_parallel_size > 1:
|
||||
return "FlashAttention MLA Sparse does not support DCP for now"
|
||||
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
if not hasattr(hf_config, "index_topk"):
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
if not hasattr(hf_text_config, "index_topk"):
|
||||
return "FlashAttention MLA Sparse requires model with index_topk"
|
||||
return None
|
||||
|
||||
|
||||
@@ -396,7 +396,7 @@ class BuildPrefillChunkMetadataKernel(
|
||||
|
||||
def get_warmup_keys(self, vllm_config: VllmConfig) -> list[CompileKey]:
|
||||
max_tokens = max(1, min(vllm_config.scheduler_config.max_num_batched_tokens, 8))
|
||||
hf_config = vllm_config.model_config.hf_config
|
||||
hf_text_config = vllm_config.model_config.hf_text_config
|
||||
parallel_config = vllm_config.parallel_config
|
||||
dcp_world = parallel_config.decode_context_parallel_size
|
||||
dcp_interleave = parallel_config.cp_kv_cache_interleave_size
|
||||
@@ -405,12 +405,12 @@ class BuildPrefillChunkMetadataKernel(
|
||||
dict.fromkeys(
|
||||
max(1, int(ratio))
|
||||
for ratio in (
|
||||
*(getattr(hf_config, "compress_ratios", None) or (1,)),
|
||||
getattr(hf_config, "index_kpool", 1) or 1,
|
||||
*(getattr(hf_text_config, "compress_ratios", None) or (1,)),
|
||||
getattr(hf_text_config, "index_kpool", 1) or 1,
|
||||
)
|
||||
)
|
||||
)
|
||||
index_kpool = getattr(hf_config, "index_kpool", None)
|
||||
index_kpool = getattr(hf_text_config, "index_kpool", None)
|
||||
if index_kpool and index_kpool > 1 and index_kpool not in compress_ratios:
|
||||
compress_ratios = compress_ratios + (index_kpool,)
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
|
||||
@@ -435,7 +435,7 @@ class ROCMAiterMLASparseMetadataBuilder(
|
||||
|
||||
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
|
||||
self.mla_dims = get_mla_dims(self.model_config)
|
||||
self.topk_tokens = vllm_config.model_config.hf_config.index_topk
|
||||
self.topk_tokens = vllm_config.model_config.hf_text_config.index_topk
|
||||
attention_context = vllm_config.compilation_config.static_forward_context
|
||||
# Sink decode must use AITER's nonpersistent path. In particular,
|
||||
# gfx942 has no persistent+LSE kernel, and its metadata heuristic
|
||||
|
||||
@@ -232,7 +232,7 @@ class ConvertReqIndexToGlobalIndexKernel(
|
||||
dcp_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
dcp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
dcp_rank = get_dcp_group().rank_in_group if dcp_size > 1 else 0
|
||||
num_topk_tokens = vllm_config.model_config.hf_config.index_topk
|
||||
num_topk_tokens = vllm_config.model_config.hf_text_config.index_topk
|
||||
max_num_blocks = cdiv(
|
||||
vllm_config.model_config.max_model_len,
|
||||
block_size * dcp_size,
|
||||
|
||||
@@ -128,7 +128,7 @@ class XPUMLASparseMetadataBuilder(AttentionMetadataBuilder[XPUMLASparseMetadata]
|
||||
|
||||
self.num_heads = self.model_config.get_num_attention_heads(parallel_config)
|
||||
self.mla_dims = get_mla_dims(self.model_config)
|
||||
self.topk_tokens = vllm_config.model_config.hf_config.index_topk
|
||||
self.topk_tokens = vllm_config.model_config.hf_text_config.index_topk
|
||||
self.topk_tokens_tensor = torch.tensor(
|
||||
[self.topk_tokens], device=device, dtype=torch.int32
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user