[ROCm] [BugFix] Fix Quark GLM-5.2 Checkpoint inference: indexer wk per-channel FP8 dequant + missing sparse-MLA metadata fields (#48886)
Signed-off-by: Colin Zeng <Colin.Zeng@amd.com> Signed-off-by: ColinZ22 <Colin.Zeng@amd.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com> Co-authored-by: fanxingran <xingran.fan@amd.com>
This commit is contained in:
co-authored by
TJian
fanxingran
parent
53f6dd5c6f
commit
28158b2fc3
@@ -515,29 +515,36 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
compilation_config.static_forward_context[prefix] = self
|
||||
|
||||
self.prefill_backend: MLAPrefillBackend | None
|
||||
try:
|
||||
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
||||
except ValueError:
|
||||
if (
|
||||
not self.impl.is_sparse
|
||||
or vllm_config.attention_config.mla_prefill_backend is not None
|
||||
):
|
||||
raise
|
||||
if self.impl.is_sparse and not self.impl.supports_dense_mha_prefill:
|
||||
logger.warning_once(
|
||||
"No MLA prefill backend supports this model; sparse MLA will use the "
|
||||
"top-k MQA path only (no dense-MHA prefill)."
|
||||
"Sparse MLA impl has no dense-MHA prefill path; using the top-k "
|
||||
"MQA path only."
|
||||
)
|
||||
self.prefill_backend = None
|
||||
else:
|
||||
self.prefill_backend = prefill_backend_cls(
|
||||
num_heads=self.num_heads,
|
||||
scale=self.scale,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
try:
|
||||
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
|
||||
except ValueError:
|
||||
if (
|
||||
not self.impl.is_sparse
|
||||
or vllm_config.attention_config.mla_prefill_backend is not None
|
||||
):
|
||||
raise
|
||||
logger.warning_once(
|
||||
"No MLA prefill backend supports this model; sparse MLA will "
|
||||
"use the top-k MQA path only (no dense-MHA prefill)."
|
||||
)
|
||||
self.prefill_backend = None
|
||||
else:
|
||||
self.prefill_backend = prefill_backend_cls(
|
||||
num_heads=self.num_heads,
|
||||
scale=self.scale,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
vllm_config=vllm_config,
|
||||
)
|
||||
|
||||
self.kv_cache = torch.tensor([])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user