[ROCm] [BugFix] Fix Quark GLM-5.2 Checkpoint inference: indexer wk per-channel FP8 dequant + missing sparse-MLA metadata fields (#48886)

Signed-off-by: Colin Zeng <Colin.Zeng@amd.com>
Signed-off-by: ColinZ22 <Colin.Zeng@amd.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
Co-authored-by: fanxingran <xingran.fan@amd.com>
This commit is contained in:
Colin Z
2026-07-27 22:38:22 +00:00
committed by GitHub
co-authored by TJian fanxingran
parent 53f6dd5c6f
commit 28158b2fc3
5 changed files with 129 additions and 21 deletions
@@ -88,6 +88,81 @@ def _make_common_metadata():
)
def _make_mixed_common_metadata():
# req0: query_len 1 (decode), req1: query_len 4 (prefill) -> decode-first
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32, device="cpu")
seq_lens = torch.tensor([16, 10], dtype=torch.int32, device="cpu")
return CommonAttentionMetadata(
query_start_loc=query_start_loc,
query_start_loc_cpu=query_start_loc,
seq_lens=seq_lens,
_seq_lens_cpu=seq_lens,
num_reqs=2,
num_actual_tokens=5,
max_query_len=4,
max_seq_len=16,
block_table_tensor=torch.arange(16, dtype=torch.int32, device="cpu").view(2, 8),
slot_mapping=torch.arange(5, dtype=torch.int64, device="cpu"),
)
def _patch_build_deps(monkeypatch, events=None):
"""Stub the aiter kernel, triton helper and CUDA sync so ``build()`` runs
on CPU."""
def fake_generate_sparse_seqlen_triton(
query_lens, seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
):
return torch.zeros(num_tokens, dtype=torch.int32, device="cpu")
fake_aiter = _FakeAiter("aiter")
fake_aiter.get_mla_metadata_v1 = Mock(side_effect=lambda *a, **k: None)
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
monkeypatch.setattr(
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen_triton
)
monkeypatch.setattr(
sparse_mod.torch.cuda,
"current_stream",
lambda device=None: SimpleNamespace(
synchronize=lambda: events.append("sync") if events is not None else None
),
)
def test_build_populates_decode_only_split_fields(monkeypatch):
"""Decode-only batch: all reqs count as decodes, prefill fields default."""
builder = _make_builder()
_patch_build_deps(monkeypatch)
md = builder.build(
common_prefix_len=0, common_attn_metadata=_make_common_metadata()
)
assert md.num_decodes == 2
assert md.num_prefills == 0
assert md.num_decode_tokens == 2
assert md.prefill_max_seq_len == 0
assert md.prefill is None
def test_build_populates_mixed_split_fields(monkeypatch):
"""Mixed decode+prefill batch: split is reported, prefill fields stay
default because this impl always runs the MQA path."""
builder = _make_builder()
_patch_build_deps(monkeypatch)
md = builder.build(
common_prefix_len=0, common_attn_metadata=_make_mixed_common_metadata()
)
assert md.num_decodes == 1
assert md.num_prefills == 1
assert md.num_decode_tokens == 1
assert md.prefill_max_seq_len == 0
assert md.prefill is None
def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
builder = _make_builder()
common_metadata = _make_common_metadata()