[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+65 -4
View File
@@ -140,7 +140,7 @@ def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
metadata = SimpleNamespace(
req_id_per_token=torch.zeros(1, dtype=torch.int32),
block_table=torch.zeros((1, 1), dtype=torch.int32),
block_size=1,
block_size=32,
)
recorded_scale = None
@@ -178,7 +178,7 @@ def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
)
impl.forward_mqa(
torch.zeros(1, 1, kv_lora_rank),
torch.zeros(1, 1, kv_lora_rank),
torch.zeros(1, 32, kv_lora_rank),
metadata,
SimpleNamespace(),
)
@@ -437,6 +437,7 @@ def test_sparse_backend_decode_correctness(
qk_rope_head_dim: int,
v_head_dim: int,
monkeypatch,
block_stride_rows: int | None = None,
):
if qk_rope_head_dim == 0 and (
backend_cls != FlashMLASparseBackend
@@ -478,8 +479,7 @@ def test_sparse_backend_decode_correctness(
if device_capability is None or device_capability.major != 10:
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
supported_block_sizes = backend_cls.get_supported_kernel_block_sizes()
if block_size not in supported_block_sizes:
if not backend_cls.supports_block_size(block_size):
pytest.skip(
f"{backend_cls.get_name()} does not support block_size={block_size}"
)
@@ -719,6 +719,11 @@ def test_sparse_backend_decode_correctness(
kv_cache_dtype=kv_cache_dtype,
scale=kv_cache_scale,
)
if block_stride_rows is not None:
pad = (0, 0, 0, block_stride_rows - block_size)
kv_cache = torch.nn.functional.pad(kv_cache, pad)[:, :, :block_size]
slot = int(common_attn_metadata.slot_mapping[-1])
query_row = kv_cache[slot // block_size, 0, slot % block_size].zero_()
# The sparse builder clones the layer's dense-MHA prefill backend from
# static_forward_context; register a mock layer carrying one.
@@ -814,6 +819,12 @@ def test_sparse_backend_decode_correctness(
out_buffer,
)
if block_stride_rows is not None and kv_cache_dtype == "auto":
expected_row = torch.cat((kv_c_vllm[-1], k_pe_vllm[-1, 0]))
torch.testing.assert_close(query_row, expected_row, rtol=0, atol=0)
elif block_stride_rows is not None:
assert query_row.any()
assert backend_output.shape == sdpa_reference.shape
assert backend_output.dtype == sdpa_reference.dtype
assert torch.isfinite(backend_output).all()
@@ -828,6 +839,56 @@ def test_sparse_backend_decode_correctness(
torch.testing.assert_close(backend_output, sdpa_reference, rtol=0.01, atol=0.01)
@pytest.mark.parametrize(
("backend_cls", "kv_cache_dtype", "block_size", "block_stride_rows", "head_dims"),
[
pytest.param(
FlashInferMLASparseTRTLLMBackend,
"auto",
256,
320,
(128, 128, 64, 128),
id="FlashInferTRTLLM",
),
# GLM-5.3 on SM90: the fp8 kernel's 64-token pages in larger blocks.
pytest.param(
FlashMLASparseBackend,
"fp8_ds_mla",
128,
192,
(64, 256, 0, 256),
id="FlashMLA-fp8_ds_mla",
),
],
)
def test_sparse_mla_packed_stride(
default_vllm_config,
dist_init,
workspace_init,
monkeypatch,
backend_cls,
kv_cache_dtype,
block_size,
block_stride_rows,
head_dims,
):
test_sparse_backend_decode_correctness(
default_vllm_config,
dist_init,
backend_cls,
"mixed_small",
kv_cache_dtype,
1,
block_size,
workspace_init,
1.0,
1.0,
*head_dims,
monkeypatch,
block_stride_rows=block_stride_rows,
)
def _triton_convert_reference_impl(
req_ids: torch.Tensor,
block_table: torch.Tensor,