[ROCm] [BugFix] Fix Quark GLM-5.2 Checkpoint inference: indexer wk per-channel FP8 dequant + missing sparse-MLA metadata fields (#48886)

Signed-off-by: Colin Zeng <Colin.Zeng@amd.com>
Signed-off-by: ColinZ22 <Colin.Zeng@amd.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
Co-authored-by: fanxingran <xingran.fan@amd.com>
This commit is contained in:
Colin Z
2026-07-27 22:38:22 +00:00
committed by GitHub
co-authored by TJian fanxingran
parent 53f6dd5c6f
commit 28158b2fc3
5 changed files with 129 additions and 21 deletions
@@ -515,29 +515,36 @@ class MLAAttention(nn.Module, AttentionLayerBase):
compilation_config.static_forward_context[prefix] = self
self.prefill_backend: MLAPrefillBackend | None
try:
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
except ValueError:
if (
not self.impl.is_sparse
or vllm_config.attention_config.mla_prefill_backend is not None
):
raise
if self.impl.is_sparse and not self.impl.supports_dense_mha_prefill:
logger.warning_once(
"No MLA prefill backend supports this model; sparse MLA will use the "
"top-k MQA path only (no dense-MHA prefill)."
"Sparse MLA impl has no dense-MHA prefill path; using the top-k "
"MQA path only."
)
self.prefill_backend = None
else:
self.prefill_backend = prefill_backend_cls(
num_heads=self.num_heads,
scale=self.scale,
kv_lora_rank=self.kv_lora_rank,
qk_nope_head_dim=self.qk_nope_head_dim,
qk_rope_head_dim=self.qk_rope_head_dim,
v_head_dim=self.v_head_dim,
vllm_config=vllm_config,
)
try:
prefill_backend_cls = get_mla_prefill_backend(vllm_config)
except ValueError:
if (
not self.impl.is_sparse
or vllm_config.attention_config.mla_prefill_backend is not None
):
raise
logger.warning_once(
"No MLA prefill backend supports this model; sparse MLA will "
"use the top-k MQA path only (no dense-MHA prefill)."
)
self.prefill_backend = None
else:
self.prefill_backend = prefill_backend_cls(
num_heads=self.num_heads,
scale=self.scale,
kv_lora_rank=self.kv_lora_rank,
qk_nope_head_dim=self.qk_nope_head_dim,
qk_rope_head_dim=self.qk_rope_head_dim,
v_head_dim=self.v_head_dim,
vllm_config=vllm_config,
)
self.kv_cache = torch.tensor([])
+7 -2
View File
@@ -851,11 +851,16 @@ def _try_load_fp8_indexer_wk(
# We have both weight and scale: dequantize FP8 to BF16.
weight_fp8, scale_inv = entry["weight"], entry["scale"]
del buf[layer_prefix]
block_size = weight_fp8.shape[1] // scale_inv.shape[1]
if scale_inv.ndim == 1:
# Per-channel scale: one scale per row of [out, in]
group_shape = GroupShape(1, weight_fp8.shape[1])
else:
block_size = weight_fp8.shape[1] // scale_inv.shape[1]
group_shape = GroupShape(block_size, block_size)
weight_bf16 = scaled_dequantize(
weight_fp8,
scale_inv,
group_shape=GroupShape(block_size, block_size),
group_shape=group_shape,
out_dtype=torch.bfloat16,
)