[Feat][Model] Enable KDA prefill checkpoints for GLM-5.3-Flash (#56960)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
This commit is contained in:
Chauncey
2026-09-29 09:14:30 +00:00
committed by GitHub
parent 3acddf01e7
commit 3e2a7e74a5
6 changed files with 307 additions and 11 deletions
+167
View File
@@ -193,3 +193,170 @@ def test_fused_recurrent_kda_rejects_unaddressable_layouts():
broken["cu_seqlens"] = None if q.shape[0] > 1 else inputs["cu_seqlens"]
with pytest.raises(AssertionError, match=r"torch.Size"):
run_kernel(broken, state)
@pytest.mark.parametrize("offset", [16, 144, 256])
@pytest.mark.parametrize("dim_first", [False, True])
@pytest.mark.parametrize("num_spec", [0, 3])
@torch.inference_mode()
def test_prefill_checkpoint_resumes_suffix(monkeypatch, dim_first, num_spec, offset):
"""Restoring both cached states must reproduce an uninterrupted prefill."""
from types import SimpleNamespace
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.kda_checkpoint import (
FlashKDAPrefillCheckpointExporter,
)
from vllm.models.glm5next.common import kda
from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata
from vllm.v1.attention.backends.utils import compute_causal_conv1d_metadata
from vllm.v1.worker.workspace import WorkspaceManager
if kda._resolve_kda_prefill_backend("auto", D, torch.bfloat16, -5) != "flashkda":
pytest.skip("requires FlashKDA")
import vllm._flashkda_C # noqa: F401
torch.manual_seed(42)
device = torch.device("cuda")
length, heads = 273, 2
width, state_len = 3 * heads * D, 3 + num_spec
layer = object.__new__(kda.Glm5NextLinearAttention)
torch.nn.Module.__init__(layer)
layer.prefix = "layer.0"
layer.local_num_heads = heads
layer.head_dim = D
layer.conv_size = 4
layer.local_projection_size = heads * D
layer.kda_safe_gate = True
layer.kda_lower_bound = -5.0
layer.kda_prefill_backend = "flashkda"
layer._conv_state_dim_first = dim_first
layer._merged_conv_weight = torch.randn(width, 4, device=device) * 0.1
layer.q_conv1d = SimpleNamespace(bias=None)
layer.A_log = torch.randn(1, 1, heads, 1, device=device) * 0.1
layer.dt_bias = torch.randn(heads * D, device=device) * 0.1
state_shape = (2, heads, D, D)
layer._flashkda_buffer_specs = (
(state_shape, torch.float32),
(state_shape, torch.float32),
(
(torch.ops._flashkda_C.get_workspace_size(2 * length, heads, 2),),
torch.uint8,
),
((1, 2 * length, heads, D), torch.bfloat16),
)
layer._checkpoint_exporter = FlashKDAPrefillCheckpointExporter(state_len=3)
workspace = WorkspaceManager(device)
monkeypatch.setattr(kda, "current_workspace_manager", lambda: workspace)
conv_shape = (width, state_len) if dim_first else (state_len, width)
num_slots = 5 + (num_spec + 1 if num_spec else 0)
conv_storage = torch.zeros(
num_slots, width * state_len + 8, device=device, dtype=torch.bfloat16
)
conv = conv_storage[:, : width * state_len].view(num_slots, *conv_shape)
recurrent_storage = torch.zeros(
num_slots, heads * D * D + 8, device=device, dtype=torch.float32
)
recurrent = recurrent_storage[:, : heads * D * D].view(num_slots, heads, D, D)
layer.kv_cache = (conv, recurrent)
projected = torch.randn(
2 * length, width + heads, device=device, dtype=torch.bfloat16
)
raw_qkv, beta = projected[:, :width], projected[:, width:].unsqueeze(0)
raw_before = raw_qkv.clone()
gate = torch.randn(1, 2 * length, heads, D, device=device, dtype=torch.bfloat16)
def run(qkv, g, b, slots, has_initial, checkpoint=None):
n = qkv.shape[0] // 2
cu_cpu = torch.tensor([0, n, 2 * n], dtype=torch.int32)
nums, batch_ptr, chunk_offsets = compute_causal_conv1d_metadata(
cu_cpu, device=device
)
metadata = GDNAttentionMetadata(
num_prefills=2,
num_prefill_tokens=2 * n,
num_decodes=0,
num_decode_tokens=0,
num_spec_decodes=0,
num_spec_decode_tokens=0,
num_actual_tokens=2 * n,
has_initial_state=torch.tensor(has_initial, device=device),
non_spec_query_start_loc=cu_cpu.to(device),
non_spec_state_indices_tensor=torch.tensor(
slots, dtype=torch.int32, device=device
),
checkpoint=checkpoint,
nums_dict=nums,
batch_ptr=batch_ptr,
token_chunk_offset_ptr=chunk_offsets,
)
monkeypatch.setattr(
kda,
"get_forward_context",
lambda: SimpleNamespace(attn_metadata={layer.prefix: metadata}),
)
spec_tokens = num_spec + 1 if num_spec else 0
if spec_tokens:
qkv = torch.cat([qkv[:spec_tokens], qkv])
g = torch.cat([g[:, :spec_tokens], g], dim=1)
b = torch.cat([b[:, :spec_tokens], b], dim=1)
metadata.num_spec_decodes = 1
metadata.num_spec_decode_tokens = spec_tokens
metadata.num_actual_tokens += spec_tokens
metadata.spec_sequence_masks = torch.tensor(
[True, False, False], device=device
)
metadata.spec_query_start_loc = torch.tensor(
[0, spec_tokens], dtype=torch.int32, device=device
)
metadata.spec_state_indices_tensor = torch.arange(
5, num_slots, dtype=torch.int32, device=device
).unsqueeze(0)
metadata.spec_token_indx = torch.arange(spec_tokens, device=device)
metadata.non_spec_token_indx = torch.arange(
spec_tokens, qkv.shape[0], device=device
)
metadata.num_accepted_tokens = torch.ones(
1, dtype=torch.int32, device=device
)
out = torch.empty(
1, qkv.shape[0], heads, D, device=device, dtype=torch.bfloat16
)
layer._forward(qkv, g, b, out)
return out[:, spec_tokens:]
checkpoint = MambaPrefillCheckpointMetadata(
torch.tensor([offset, 0], dtype=torch.int32, device=device),
torch.tensor([3, 0], dtype=torch.int32, device=device),
)
expected = run(raw_qkv, gate, beta, [1, 2], [False, False], checkpoint)
expected_state = recurrent[1].clone()
saved_conv, saved_recurrent = conv[3].clone(), recurrent[3].clone()
torch.testing.assert_close(raw_qkv, raw_before)
conv_view = conv if dim_first else conv.transpose(-1, -2)
torch.testing.assert_close(conv_view[3, :, :3], raw_qkv[offset - 3 : offset].T)
assert torch.count_nonzero(conv_view[3, :, 3:]) == 0
assert torch.count_nonzero(conv[0]) == torch.count_nonzero(recurrent[0]) == 0
run(
torch.cat([raw_qkv[:offset], raw_qkv[length : length + offset]]),
torch.cat([gate[:, :offset], gate[:, length : length + offset]], dim=1),
torch.cat([beta[:, :offset], beta[:, length : length + offset]], dim=1),
[1, 2],
[False, False],
)
torch.testing.assert_close(saved_recurrent, recurrent[1], atol=0.003, rtol=0.03)
suffix = slice(offset, length)
qkv = torch.cat([raw_qkv[suffix], raw_qkv[length + offset :]])
g = torch.cat([gate[:, suffix], gate[:, length + offset :]], dim=1)
b = torch.cat([beta[:, suffix], beta[:, length + offset :]], dim=1)
conv[4].copy_(saved_conv)
recurrent[4].copy_(saved_recurrent)
actual = run(qkv, g, b, [3, 4], [True, True])
torch.testing.assert_close(
actual[:, : length - offset], expected[:, suffix], atol=0.003, rtol=0.03
)
torch.testing.assert_close(recurrent[3], expected_state, atol=0.003, rtol=0.03)
@@ -367,3 +367,66 @@ def test_cudagraph_capture_batch_stays_decode_only():
assert staged is not None
assert staged.data_ptr() == builder.non_spec_state_indices_tensor.data_ptr()
torch.testing.assert_close(staged, common_attn_metadata.block_table_tensor[:, 0])
@pytest.mark.parametrize("num_spec", [0, 3])
@pytest.mark.parametrize("prefix_match_unit, expected_offset", [(16, 96), (8, 96)])
def test_checkpoint_metadata_preserves_non_spec_order(
num_spec, prefix_match_unit, expected_offset
):
"""Only eligible non-spec rows get a checkpoint in their reserved page."""
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
vllm_config = create_vllm_config(
model_name="Qwen/Qwen3.5-0.8B",
block_size=BLOCK_SIZE,
)
if num_spec > 0:
vllm_config.speculative_config = SpeculativeConfig(
method="ngram",
num_speculative_tokens=num_spec,
)
vllm_config.cache_config.mamba_cache_mode = "align"
vllm_config.cache_config.prefix_match_unit = prefix_match_unit
mamba_spec = MambaSpec(
block_size=64,
shapes=((16, 64),),
dtypes=(torch.float16,),
mamba_cache_mode="align",
num_speculative_blocks=num_spec,
num_prefill_checkpoint_blocks=1,
prefill_checkpoint_alignment=16,
)
builder = GDNAttentionMetadataBuilder(
kv_cache_spec=mamba_spec,
layer_names=["layer.0"],
vllm_config=vllm_config,
device=DEVICE,
)
batch = BatchSpec(
seq_lens=[65, 100, 100], query_lens=[3 if num_spec else 1, 100, 99]
)
common = create_common_attn_metadata(batch, 64, DEVICE, arange_block_indices=True)
table = common.block_table_tensor + 1
if num_spec:
table = torch.cat([table, torch.zeros(3, num_spec, dtype=table.dtype)], dim=1)
common = common.replace(block_table_tensor=table)
kwargs = {}
if num_spec:
kwargs = dict(
num_decode_draft_tokens_cpu=torch.tensor([2, -1, -1], dtype=torch.int32),
num_accepted_tokens=torch.ones(3, dtype=torch.int32),
)
actual = builder.build(0, common, **kwargs)
assert actual.checkpoint is not None
offsets = [expected_offset, 0] if num_spec else [0, expected_offset, 0]
slots = [common.block_table_tensor[1, 0].item(), NULL_BLOCK_ID]
if not num_spec:
slots.insert(0, NULL_BLOCK_ID)
torch.testing.assert_close(
actual.checkpoint.checkpoint_offsets, torch.tensor(offsets, dtype=torch.int32)
)
torch.testing.assert_close(
actual.checkpoint.state_indices, torch.tensor(slots, dtype=torch.int32)
)