[Bugfix][MLA] Read sparse model settings from text config (#56160)
Signed-off-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
@@ -168,6 +168,7 @@ def _run_sparse_backend_vs_sdpa(
|
||||
)
|
||||
model_config = vllm_config.model_config
|
||||
model_config.hf_text_config = SimpleNamespace(
|
||||
index_topk=topk_tokens,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
@@ -175,6 +176,7 @@ def _run_sparse_backend_vs_sdpa(
|
||||
v_head_dim=v_head_dim,
|
||||
model_type="deepseek_v2",
|
||||
)
|
||||
del model_config.hf_config.index_topk # Composite configs only nest this field.
|
||||
model_config.dtype = dtype
|
||||
model_config.get_num_attention_heads = MethodType(
|
||||
lambda self, parallel_config: num_heads, model_config
|
||||
@@ -287,7 +289,7 @@ def _run_sparse_backend_vs_sdpa(
|
||||
causal_reference = torch.cat(causal_reference_outputs, dim=0)
|
||||
|
||||
vllm_config.cache_config.cache_dtype = kv_cache_dtype
|
||||
vllm_config.model_config.hf_config.index_topk = topk_tokens
|
||||
vllm_config.model_config.hf_text_config.index_topk = topk_tokens
|
||||
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, block_size, device, arange_block_indices=True
|
||||
|
||||
@@ -68,7 +68,9 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
||||
model_config=SimpleNamespace(
|
||||
hf_config=SimpleNamespace(compress_ratios=[0, 0, 4, 128, 0], index_kpool=32)
|
||||
hf_text_config=SimpleNamespace(
|
||||
compress_ratios=[0, 0, 4, 128, 0], index_kpool=32
|
||||
)
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=1,
|
||||
@@ -89,7 +91,7 @@ def test_indexer_warmup_normalizes_zero_compress_ratios():
|
||||
def test_compressed_slot_mapping_warmup_includes_index_kpool():
|
||||
config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=256),
|
||||
model_config=SimpleNamespace(hf_config=SimpleNamespace(index_kpool=32)),
|
||||
model_config=SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=32)),
|
||||
)
|
||||
|
||||
keys = CompressedSlotMappingKernel().get_warmup_keys(config)
|
||||
@@ -101,7 +103,7 @@ def test_index_conversion_warmup_uses_physical_block_stride():
|
||||
cache_config=SimpleNamespace(block_size=64),
|
||||
model_config=SimpleNamespace(
|
||||
max_model_len=1024,
|
||||
hf_config=SimpleNamespace(index_topk=2048),
|
||||
hf_text_config=SimpleNamespace(index_topk=2048),
|
||||
),
|
||||
parallel_config=SimpleNamespace(
|
||||
decode_context_parallel_size=1,
|
||||
|
||||
@@ -423,6 +423,7 @@ def test_sparse_backend_decode_correctness(
|
||||
)
|
||||
model_config = vllm_config.model_config
|
||||
model_config.hf_text_config = SimpleNamespace(
|
||||
index_topk=topk_tokens,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
@@ -430,6 +431,7 @@ def test_sparse_backend_decode_correctness(
|
||||
v_head_dim=v_head_dim,
|
||||
model_type="deepseek_v2",
|
||||
)
|
||||
del model_config.hf_config.index_topk # Composite configs only nest this field.
|
||||
model_config.dtype = dtype
|
||||
model_config.get_num_attention_heads = MethodType(
|
||||
lambda self, parallel_config: num_heads,
|
||||
@@ -591,7 +593,7 @@ def test_sparse_backend_decode_correctness(
|
||||
sdpa_reference = torch.cat(reference_outputs, dim=0)
|
||||
|
||||
vllm_config.cache_config.cache_dtype = kv_cache_dtype
|
||||
vllm_config.model_config.hf_config.index_topk = topk_tokens
|
||||
vllm_config.model_config.hf_text_config.index_topk = topk_tokens
|
||||
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec,
|
||||
@@ -1149,6 +1151,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
)
|
||||
model_config = vllm_config.model_config
|
||||
model_config.hf_text_config = SimpleNamespace(
|
||||
index_topk=topk_tokens,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
@@ -1156,6 +1159,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
v_head_dim=v_head_dim,
|
||||
model_type="deepseek_v2",
|
||||
)
|
||||
del model_config.hf_config.index_topk # Composite configs only nest this field.
|
||||
model_config.dtype = dtype
|
||||
model_config.model_arch_config.total_num_attention_heads = num_heads
|
||||
model_config.get_num_attention_heads = MethodType(
|
||||
@@ -1255,7 +1259,7 @@ def test_sparse_backend_prefill_correctness(
|
||||
ref_output = torch.cat(reference_outputs, dim=0)
|
||||
|
||||
vllm_config.cache_config.cache_dtype = kv_cache_dtype
|
||||
vllm_config.model_config.hf_config.index_topk = topk_tokens
|
||||
vllm_config.model_config.hf_text_config.index_topk = topk_tokens
|
||||
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec,
|
||||
@@ -1747,6 +1751,7 @@ def _build_sparse_dcp_vllm_config(
|
||||
model_config = vllm_config.model_config
|
||||
model_config.dtype = torch.bfloat16
|
||||
model_config.hf_text_config = SimpleNamespace(
|
||||
index_topk=topk_tokens,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
@@ -1754,6 +1759,7 @@ def _build_sparse_dcp_vllm_config(
|
||||
v_head_dim=v_head_dim,
|
||||
model_type="deepseek_v2",
|
||||
)
|
||||
del model_config.hf_config.index_topk # Composite configs only nest this field.
|
||||
model_config.get_num_attention_heads = MethodType(
|
||||
lambda self, parallel_config: local_heads, model_config
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user