[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -140,7 +140,7 @@ def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.zeros(1, dtype=torch.int32),
|
||||
block_table=torch.zeros((1, 1), dtype=torch.int32),
|
||||
block_size=1,
|
||||
block_size=32,
|
||||
)
|
||||
recorded_scale = None
|
||||
|
||||
@@ -178,7 +178,7 @@ def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
|
||||
)
|
||||
impl.forward_mqa(
|
||||
torch.zeros(1, 1, kv_lora_rank),
|
||||
torch.zeros(1, 1, kv_lora_rank),
|
||||
torch.zeros(1, 32, kv_lora_rank),
|
||||
metadata,
|
||||
SimpleNamespace(),
|
||||
)
|
||||
@@ -437,6 +437,7 @@ def test_sparse_backend_decode_correctness(
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
monkeypatch,
|
||||
block_stride_rows: int | None = None,
|
||||
):
|
||||
if qk_rope_head_dim == 0 and (
|
||||
backend_cls != FlashMLASparseBackend
|
||||
@@ -478,8 +479,7 @@ def test_sparse_backend_decode_correctness(
|
||||
if device_capability is None or device_capability.major != 10:
|
||||
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
|
||||
|
||||
supported_block_sizes = backend_cls.get_supported_kernel_block_sizes()
|
||||
if block_size not in supported_block_sizes:
|
||||
if not backend_cls.supports_block_size(block_size):
|
||||
pytest.skip(
|
||||
f"{backend_cls.get_name()} does not support block_size={block_size}"
|
||||
)
|
||||
@@ -719,6 +719,11 @@ def test_sparse_backend_decode_correctness(
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
scale=kv_cache_scale,
|
||||
)
|
||||
if block_stride_rows is not None:
|
||||
pad = (0, 0, 0, block_stride_rows - block_size)
|
||||
kv_cache = torch.nn.functional.pad(kv_cache, pad)[:, :, :block_size]
|
||||
slot = int(common_attn_metadata.slot_mapping[-1])
|
||||
query_row = kv_cache[slot // block_size, 0, slot % block_size].zero_()
|
||||
|
||||
# The sparse builder clones the layer's dense-MHA prefill backend from
|
||||
# static_forward_context; register a mock layer carrying one.
|
||||
@@ -814,6 +819,12 @@ def test_sparse_backend_decode_correctness(
|
||||
out_buffer,
|
||||
)
|
||||
|
||||
if block_stride_rows is not None and kv_cache_dtype == "auto":
|
||||
expected_row = torch.cat((kv_c_vllm[-1], k_pe_vllm[-1, 0]))
|
||||
torch.testing.assert_close(query_row, expected_row, rtol=0, atol=0)
|
||||
elif block_stride_rows is not None:
|
||||
assert query_row.any()
|
||||
|
||||
assert backend_output.shape == sdpa_reference.shape
|
||||
assert backend_output.dtype == sdpa_reference.dtype
|
||||
assert torch.isfinite(backend_output).all()
|
||||
@@ -828,6 +839,56 @@ def test_sparse_backend_decode_correctness(
|
||||
torch.testing.assert_close(backend_output, sdpa_reference, rtol=0.01, atol=0.01)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend_cls", "kv_cache_dtype", "block_size", "block_stride_rows", "head_dims"),
|
||||
[
|
||||
pytest.param(
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
"auto",
|
||||
256,
|
||||
320,
|
||||
(128, 128, 64, 128),
|
||||
id="FlashInferTRTLLM",
|
||||
),
|
||||
# GLM-5.3 on SM90: the fp8 kernel's 64-token pages in larger blocks.
|
||||
pytest.param(
|
||||
FlashMLASparseBackend,
|
||||
"fp8_ds_mla",
|
||||
128,
|
||||
192,
|
||||
(64, 256, 0, 256),
|
||||
id="FlashMLA-fp8_ds_mla",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_sparse_mla_packed_stride(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
workspace_init,
|
||||
monkeypatch,
|
||||
backend_cls,
|
||||
kv_cache_dtype,
|
||||
block_size,
|
||||
block_stride_rows,
|
||||
head_dims,
|
||||
):
|
||||
test_sparse_backend_decode_correctness(
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
backend_cls,
|
||||
"mixed_small",
|
||||
kv_cache_dtype,
|
||||
1,
|
||||
block_size,
|
||||
workspace_init,
|
||||
1.0,
|
||||
1.0,
|
||||
*head_dims,
|
||||
monkeypatch,
|
||||
block_stride_rows=block_stride_rows,
|
||||
)
|
||||
|
||||
|
||||
def _triton_convert_reference_impl(
|
||||
req_ids: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user