[Bugfix][MLA] Read sparse model settings from text config (#56160)

Signed-off-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
Andy Lo
2026-09-11 18:02:13 +00:00
committed by GitHub
parent 6fe67cbbf3
commit dc07f1638f
11 changed files with 32 additions and 22 deletions
@@ -132,7 +132,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
self.device = device
self.model_config = vllm_config.model_config
self.mla_dims = get_mla_dims(self.model_config)
self.topk_tokens: int = vllm_config.model_config.hf_config.index_topk
self.topk_tokens: int = vllm_config.model_config.hf_text_config.index_topk
self.req_id_per_token_buffer = torch.empty(
(vllm_config.scheduler_config.max_num_batched_tokens,),
dtype=torch.int32,
@@ -202,7 +202,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
scheduler_config = vllm_config.scheduler_config
cache_config = vllm_config.cache_config
model_config = vllm_config.model_config
topk_tokens = model_config.hf_config.index_topk
topk_tokens = model_config.hf_text_config.index_topk
workspace_size = min(
max(