[Bugfix][MLA] Read sparse model settings from text config (#56160)
Signed-off-by: Andy Lo <andy@mistral.ai>
This commit is contained in:
@@ -173,7 +173,7 @@ class PackDCPTopkCandidatesKernel(
|
||||
if dcp_world_size <= 1:
|
||||
return []
|
||||
cp_interleave = vllm_config.parallel_config.cp_kv_cache_interleave_size
|
||||
topk = vllm_config.model_config.hf_config.index_topk
|
||||
topk = vllm_config.model_config.hf_text_config.index_topk
|
||||
if topk <= 0:
|
||||
return []
|
||||
|
||||
@@ -546,7 +546,7 @@ class StableTopKFromGatheredCandidatesKernel(
|
||||
dcp_world_size = vllm_config.parallel_config.decode_context_parallel_size
|
||||
if dcp_world_size <= 1:
|
||||
return []
|
||||
topk = vllm_config.model_config.hf_config.index_topk
|
||||
topk = vllm_config.model_config.hf_text_config.index_topk
|
||||
if topk <= 0:
|
||||
return []
|
||||
return self._trace_dispatch(self.dispatch)(
|
||||
|
||||
@@ -132,7 +132,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
self.device = device
|
||||
self.model_config = vllm_config.model_config
|
||||
self.mla_dims = get_mla_dims(self.model_config)
|
||||
self.topk_tokens: int = vllm_config.model_config.hf_config.index_topk
|
||||
self.topk_tokens: int = vllm_config.model_config.hf_text_config.index_topk
|
||||
self.req_id_per_token_buffer = torch.empty(
|
||||
(vllm_config.scheduler_config.max_num_batched_tokens,),
|
||||
dtype=torch.int32,
|
||||
@@ -202,7 +202,7 @@ class SparseMLACommonMetadataBuilder(AttentionMetadataBuilder[T]):
|
||||
scheduler_config = vllm_config.scheduler_config
|
||||
cache_config = vllm_config.cache_config
|
||||
model_config = vllm_config.model_config
|
||||
topk_tokens = model_config.hf_config.index_topk
|
||||
topk_tokens = model_config.hf_text_config.index_topk
|
||||
|
||||
workspace_size = min(
|
||||
max(
|
||||
|
||||
Reference in New Issue
Block a user