diff --git a/tests/models/glm5next/test_kda_recurrent.py b/tests/models/glm5next/test_kda_recurrent.py index 234a63fb73..ddda0e0c34 100644 --- a/tests/models/glm5next/test_kda_recurrent.py +++ b/tests/models/glm5next/test_kda_recurrent.py @@ -193,3 +193,170 @@ def test_fused_recurrent_kda_rejects_unaddressable_layouts(): broken["cu_seqlens"] = None if q.shape[0] > 1 else inputs["cu_seqlens"] with pytest.raises(AssertionError, match=r"torch.Size"): run_kernel(broken, state) + + +@pytest.mark.parametrize("offset", [16, 144, 256]) +@pytest.mark.parametrize("dim_first", [False, True]) +@pytest.mark.parametrize("num_spec", [0, 3]) +@torch.inference_mode() +def test_prefill_checkpoint_resumes_suffix(monkeypatch, dim_first, num_spec, offset): + """Restoring both cached states must reproduce an uninterrupted prefill.""" + from types import SimpleNamespace + + from vllm.model_executor.layers.mamba.checkpoint import ( + MambaPrefillCheckpointMetadata, + ) + from vllm.model_executor.layers.mamba.kda_checkpoint import ( + FlashKDAPrefillCheckpointExporter, + ) + from vllm.models.glm5next.common import kda + from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata + from vllm.v1.attention.backends.utils import compute_causal_conv1d_metadata + from vllm.v1.worker.workspace import WorkspaceManager + + if kda._resolve_kda_prefill_backend("auto", D, torch.bfloat16, -5) != "flashkda": + pytest.skip("requires FlashKDA") + import vllm._flashkda_C # noqa: F401 + + torch.manual_seed(42) + device = torch.device("cuda") + length, heads = 273, 2 + width, state_len = 3 * heads * D, 3 + num_spec + layer = object.__new__(kda.Glm5NextLinearAttention) + torch.nn.Module.__init__(layer) + layer.prefix = "layer.0" + layer.local_num_heads = heads + layer.head_dim = D + layer.conv_size = 4 + layer.local_projection_size = heads * D + layer.kda_safe_gate = True + layer.kda_lower_bound = -5.0 + layer.kda_prefill_backend = "flashkda" + layer._conv_state_dim_first = dim_first + layer._merged_conv_weight = torch.randn(width, 4, device=device) * 0.1 + layer.q_conv1d = SimpleNamespace(bias=None) + layer.A_log = torch.randn(1, 1, heads, 1, device=device) * 0.1 + layer.dt_bias = torch.randn(heads * D, device=device) * 0.1 + state_shape = (2, heads, D, D) + layer._flashkda_buffer_specs = ( + (state_shape, torch.float32), + (state_shape, torch.float32), + ( + (torch.ops._flashkda_C.get_workspace_size(2 * length, heads, 2),), + torch.uint8, + ), + ((1, 2 * length, heads, D), torch.bfloat16), + ) + layer._checkpoint_exporter = FlashKDAPrefillCheckpointExporter(state_len=3) + workspace = WorkspaceManager(device) + monkeypatch.setattr(kda, "current_workspace_manager", lambda: workspace) + conv_shape = (width, state_len) if dim_first else (state_len, width) + num_slots = 5 + (num_spec + 1 if num_spec else 0) + conv_storage = torch.zeros( + num_slots, width * state_len + 8, device=device, dtype=torch.bfloat16 + ) + conv = conv_storage[:, : width * state_len].view(num_slots, *conv_shape) + recurrent_storage = torch.zeros( + num_slots, heads * D * D + 8, device=device, dtype=torch.float32 + ) + recurrent = recurrent_storage[:, : heads * D * D].view(num_slots, heads, D, D) + layer.kv_cache = (conv, recurrent) + projected = torch.randn( + 2 * length, width + heads, device=device, dtype=torch.bfloat16 + ) + raw_qkv, beta = projected[:, :width], projected[:, width:].unsqueeze(0) + raw_before = raw_qkv.clone() + gate = torch.randn(1, 2 * length, heads, D, device=device, dtype=torch.bfloat16) + + def run(qkv, g, b, slots, has_initial, checkpoint=None): + n = qkv.shape[0] // 2 + cu_cpu = torch.tensor([0, n, 2 * n], dtype=torch.int32) + nums, batch_ptr, chunk_offsets = compute_causal_conv1d_metadata( + cu_cpu, device=device + ) + metadata = GDNAttentionMetadata( + num_prefills=2, + num_prefill_tokens=2 * n, + num_decodes=0, + num_decode_tokens=0, + num_spec_decodes=0, + num_spec_decode_tokens=0, + num_actual_tokens=2 * n, + has_initial_state=torch.tensor(has_initial, device=device), + non_spec_query_start_loc=cu_cpu.to(device), + non_spec_state_indices_tensor=torch.tensor( + slots, dtype=torch.int32, device=device + ), + checkpoint=checkpoint, + nums_dict=nums, + batch_ptr=batch_ptr, + token_chunk_offset_ptr=chunk_offsets, + ) + monkeypatch.setattr( + kda, + "get_forward_context", + lambda: SimpleNamespace(attn_metadata={layer.prefix: metadata}), + ) + spec_tokens = num_spec + 1 if num_spec else 0 + if spec_tokens: + qkv = torch.cat([qkv[:spec_tokens], qkv]) + g = torch.cat([g[:, :spec_tokens], g], dim=1) + b = torch.cat([b[:, :spec_tokens], b], dim=1) + metadata.num_spec_decodes = 1 + metadata.num_spec_decode_tokens = spec_tokens + metadata.num_actual_tokens += spec_tokens + metadata.spec_sequence_masks = torch.tensor( + [True, False, False], device=device + ) + metadata.spec_query_start_loc = torch.tensor( + [0, spec_tokens], dtype=torch.int32, device=device + ) + metadata.spec_state_indices_tensor = torch.arange( + 5, num_slots, dtype=torch.int32, device=device + ).unsqueeze(0) + metadata.spec_token_indx = torch.arange(spec_tokens, device=device) + metadata.non_spec_token_indx = torch.arange( + spec_tokens, qkv.shape[0], device=device + ) + metadata.num_accepted_tokens = torch.ones( + 1, dtype=torch.int32, device=device + ) + out = torch.empty( + 1, qkv.shape[0], heads, D, device=device, dtype=torch.bfloat16 + ) + layer._forward(qkv, g, b, out) + return out[:, spec_tokens:] + + checkpoint = MambaPrefillCheckpointMetadata( + torch.tensor([offset, 0], dtype=torch.int32, device=device), + torch.tensor([3, 0], dtype=torch.int32, device=device), + ) + expected = run(raw_qkv, gate, beta, [1, 2], [False, False], checkpoint) + expected_state = recurrent[1].clone() + saved_conv, saved_recurrent = conv[3].clone(), recurrent[3].clone() + torch.testing.assert_close(raw_qkv, raw_before) + conv_view = conv if dim_first else conv.transpose(-1, -2) + torch.testing.assert_close(conv_view[3, :, :3], raw_qkv[offset - 3 : offset].T) + assert torch.count_nonzero(conv_view[3, :, 3:]) == 0 + assert torch.count_nonzero(conv[0]) == torch.count_nonzero(recurrent[0]) == 0 + + run( + torch.cat([raw_qkv[:offset], raw_qkv[length : length + offset]]), + torch.cat([gate[:, :offset], gate[:, length : length + offset]], dim=1), + torch.cat([beta[:, :offset], beta[:, length : length + offset]], dim=1), + [1, 2], + [False, False], + ) + torch.testing.assert_close(saved_recurrent, recurrent[1], atol=0.003, rtol=0.03) + + suffix = slice(offset, length) + qkv = torch.cat([raw_qkv[suffix], raw_qkv[length + offset :]]) + g = torch.cat([gate[:, suffix], gate[:, length + offset :]], dim=1) + b = torch.cat([beta[:, suffix], beta[:, length + offset :]], dim=1) + conv[4].copy_(saved_conv) + recurrent[4].copy_(saved_recurrent) + actual = run(qkv, g, b, [3, 4], [True, True]) + torch.testing.assert_close( + actual[:, : length - offset], expected[:, suffix], atol=0.003, rtol=0.03 + ) + torch.testing.assert_close(recurrent[3], expected_state, atol=0.003, rtol=0.03) diff --git a/tests/v1/attention/test_gdn_metadata_builder.py b/tests/v1/attention/test_gdn_metadata_builder.py index 44dee49a4e..0256a13141 100644 --- a/tests/v1/attention/test_gdn_metadata_builder.py +++ b/tests/v1/attention/test_gdn_metadata_builder.py @@ -367,3 +367,66 @@ def test_cudagraph_capture_batch_stays_decode_only(): assert staged is not None assert staged.data_ptr() == builder.non_spec_state_indices_tensor.data_ptr() torch.testing.assert_close(staged, common_attn_metadata.block_table_tensor[:, 0]) + + +@pytest.mark.parametrize("num_spec", [0, 3]) +@pytest.mark.parametrize("prefix_match_unit, expected_offset", [(16, 96), (8, 96)]) +def test_checkpoint_metadata_preserves_non_spec_order( + num_spec, prefix_match_unit, expected_offset +): + """Only eligible non-spec rows get a checkpoint in their reserved page.""" + from vllm.v1.attention.backends.utils import NULL_BLOCK_ID + + vllm_config = create_vllm_config( + model_name="Qwen/Qwen3.5-0.8B", + block_size=BLOCK_SIZE, + ) + if num_spec > 0: + vllm_config.speculative_config = SpeculativeConfig( + method="ngram", + num_speculative_tokens=num_spec, + ) + vllm_config.cache_config.mamba_cache_mode = "align" + vllm_config.cache_config.prefix_match_unit = prefix_match_unit + mamba_spec = MambaSpec( + block_size=64, + shapes=((16, 64),), + dtypes=(torch.float16,), + mamba_cache_mode="align", + num_speculative_blocks=num_spec, + num_prefill_checkpoint_blocks=1, + prefill_checkpoint_alignment=16, + ) + builder = GDNAttentionMetadataBuilder( + kv_cache_spec=mamba_spec, + layer_names=["layer.0"], + vllm_config=vllm_config, + device=DEVICE, + ) + batch = BatchSpec( + seq_lens=[65, 100, 100], query_lens=[3 if num_spec else 1, 100, 99] + ) + common = create_common_attn_metadata(batch, 64, DEVICE, arange_block_indices=True) + table = common.block_table_tensor + 1 + if num_spec: + table = torch.cat([table, torch.zeros(3, num_spec, dtype=table.dtype)], dim=1) + common = common.replace(block_table_tensor=table) + kwargs = {} + if num_spec: + kwargs = dict( + num_decode_draft_tokens_cpu=torch.tensor([2, -1, -1], dtype=torch.int32), + num_accepted_tokens=torch.ones(3, dtype=torch.int32), + ) + + actual = builder.build(0, common, **kwargs) + assert actual.checkpoint is not None + offsets = [expected_offset, 0] if num_spec else [0, expected_offset, 0] + slots = [common.block_table_tensor[1, 0].item(), NULL_BLOCK_ID] + if not num_spec: + slots.insert(0, NULL_BLOCK_ID) + torch.testing.assert_close( + actual.checkpoint.checkpoint_offsets, torch.tensor(offsets, dtype=torch.int32) + ) + torch.testing.assert_close( + actual.checkpoint.state_indices, torch.tensor(slots, dtype=torch.int32) + ) diff --git a/vllm/model_executor/layers/mamba/checkpoint.py b/vllm/model_executor/layers/mamba/checkpoint.py index 3ba4062079..1f4bf9dc37 100644 --- a/vllm/model_executor/layers/mamba/checkpoint.py +++ b/vllm/model_executor/layers/mamba/checkpoint.py @@ -76,6 +76,8 @@ class MambaPrefillCheckpointBuilder: m: CommonAttentionMetadata, request_rows: list[int], ) -> MambaPrefillCheckpointMetadata | None: + if self.vllm_config.cache_config.mamba_cache_mode != "align": + return None if self.kv_cache_spec.num_prefill_checkpoint_blocks == 0: return None assert m.seq_lens_cpu_upper_bound is not None diff --git a/vllm/models/glm5next/common/kda.py b/vllm/models/glm5next/common/kda.py index 436b9685cb..c67ac5b012 100644 --- a/vllm/models/glm5next/common/kda.py +++ b/vllm/models/glm5next/common/kda.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """GLM-5.3-Flash KDA layer with separate convolutions and a bounded safe gate.""" +from dataclasses import replace + import torch from torch import nn @@ -14,7 +16,14 @@ from vllm.model_executor.layers.linear import ( MergedColumnParallelLinear, RowParallelLinear, ) +from vllm.model_executor.layers.mamba.checkpoint import ( + MambaPrefillCheckpointMetadata, +) from vllm.model_executor.layers.mamba.gdn.base import GatedDeltaNetAttention +from vllm.model_executor.layers.mamba.kda_checkpoint import ( + FlashKDAPrefillCheckpointExporter, + kda_prefill_checkpoint_alignment, +) from vllm.model_executor.layers.mamba.mamba_utils import ( MambaStateDtypeCalculator, MambaStateShapeCalculator, @@ -37,6 +46,7 @@ from vllm.platforms import current_platform from vllm.third_party.flash_linear_attention.ops.kda import FusedRMSNormGated from vllm.transformers_utils.configs.glm5_next import Glm5NextConfig from vllm.v1.attention.backends.gdn_attn import GDNAttentionMetadata +from vllm.v1.kv_cache_interface import MambaSpec from vllm.v1.worker.workspace import current_workspace_manager if current_platform.is_rocm(): @@ -155,6 +165,16 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): num_heads: int conv_size: int + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> MambaSpec: + spec = super().get_kv_cache_spec(vllm_config) + assert isinstance(spec, MambaSpec) + alignment = kda_prefill_checkpoint_alignment(self.kda_prefill_backend) + return replace( + spec, + num_prefill_checkpoint_blocks=int(alignment is not None), + prefill_checkpoint_alignment=alignment, + ) + def get_state_dtype( self, ) -> tuple[torch.dtype, torch.dtype]: @@ -330,6 +350,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): self._flashkda_buffer_specs: ( tuple[tuple[tuple[int, ...], torch.dtype], ...] | None ) = None + self._checkpoint_exporter: FlashKDAPrefillCheckpointExporter | None = None if self.kda_prefill_backend == "flashkda": import vllm._flashkda_C # noqa: F401 @@ -339,6 +360,10 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): max_tokens, self.local_num_heads, max_seqs ) self._flashkda_buffer_specs = ( + ( + (max_seqs, self.local_num_heads, self.head_dim, self.head_dim), + self.get_state_dtype()[1], + ), ( (max_seqs, self.local_num_heads, self.head_dim, self.head_dim), self.get_state_dtype()[1], @@ -351,6 +376,9 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): vllm_config.model_config.dtype, ), ) + self._checkpoint_exporter = FlashKDAPrefillCheckpointExporter( + state_len=self.conv_size - 1 + ) def _flashkda_prefill( self, @@ -362,6 +390,10 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): initial_state: torch.Tensor, cu_seqlens: torch.Tensor, out: torch.Tensor | None, + checkpoint: MambaPrefillCheckpointMetadata | None = None, + raw_qkv: torch.Tensor | None = None, + conv_state: torch.Tensor | None = None, + recurrent_state: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: """Fused KDA chunked prefill (FlashKDA). Takes the raw gate logits ``g`` and raw ``beta`` logits, l2-normalizes q/k in-kernel and applies the @@ -370,10 +402,15 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): attention output into ``out`` (a workspace buffer when ``None``) and returns ``(out, final_state)``.""" assert self._flashkda_buffer_specs is not None - final_state, workspace, workspace_out = ( + final_state, checkpoint_state, workspace, workspace_out = ( current_workspace_manager().get_simultaneous(*self._flashkda_buffer_specs) ) final_state = final_state[: initial_state.shape[0]] + checkpoint_state = ( + checkpoint_state[: initial_state.shape[0]] + if checkpoint is not None + else None + ) if out is None: out = workspace_out[:, : q.shape[1]] # FlashKDA hardcodes dense q/k/v/g strides; beta may be row-strided. @@ -392,9 +429,23 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): initial_state.contiguous(), final_state, cu_seqlens.contiguous(), - None, - None, + checkpoint_state, + checkpoint.checkpoint_offsets.contiguous() + if checkpoint is not None + else None, ) + if checkpoint is not None: + assert self._checkpoint_exporter is not None + assert raw_qkv is not None and conv_state is not None + assert recurrent_state is not None + self._checkpoint_exporter.export( + checkpoint, + raw_qkv=raw_qkv, + conv_state=conv_state, + recurrent_checkpoint=checkpoint_state, + recurrent_state=recurrent_state, + cu_seqlens=cu_seqlens, + ) return out, final_state def forward( @@ -565,6 +616,7 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): q_spec, k_spec, v_spec = qkv_spec.split(self.local_projection_size, dim=-1) # --- causal conv1d: non-spec path (prefill or plain decode) --- + raw_qkv_ns = qkv_ns q_ns = k_ns = v_ns = None if attn_metadata_narrowed.num_prefills > 0: assert qkv_ns is not None @@ -662,6 +714,10 @@ class Glm5NextLinearAttention(GatedDeltaNetAttention): initial_state=initial_state, cu_seqlens=non_spec_query_start_loc, out=ns_out, + checkpoint=attn_metadata_narrowed.checkpoint, + raw_qkv=raw_qkv_ns, + conv_state=conv_state, + recurrent_state=recurrent_state, ) else: ( diff --git a/vllm/models/kimi_k3/nvidia/kda_metadata.py b/vllm/models/kimi_k3/nvidia/kda_metadata.py index 2562171109..fe50bd4f53 100644 --- a/vllm/models/kimi_k3/nvidia/kda_metadata.py +++ b/vllm/models/kimi_k3/nvidia/kda_metadata.py @@ -18,10 +18,6 @@ from typing import TYPE_CHECKING import torch from vllm.config import VllmConfig -from vllm.model_executor.layers.mamba.checkpoint import ( - MambaPrefillCheckpointBuilder, - MambaPrefillCheckpointMetadata, -) from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.torch_utils import async_tensor_h2d @@ -271,7 +267,6 @@ class KimiK3KDAMetadata(GDNAttentionMetadata, RecoverSSMMetadata): recoverssm_context: "KDARecoverSSMCommitContext | None" = field( default=None, repr=False, compare=False ) - checkpoint: MambaPrefillCheckpointMetadata | None = None def commit_recoverssm_state( self, num_accepted_tokens: torch.Tensor @@ -319,9 +314,6 @@ class KimiK3KDAMetadataBuilder(GDNAttentionMetadataBuilder): device: torch.device, ) -> None: super().__init__(kv_cache_spec, layer_names, vllm_config, device) - self.checkpoint_builder = MambaPrefillCheckpointBuilder( - vllm_config, kv_cache_spec - ) additional_config = vllm_config.additional_config self.use_flashinfer_prefill = ( isinstance(additional_config, dict) diff --git a/vllm/v1/attention/backends/gdn_attn.py b/vllm/v1/attention/backends/gdn_attn.py index ae8a9437d5..eb616644f3 100644 --- a/vllm/v1/attention/backends/gdn_attn.py +++ b/vllm/v1/attention/backends/gdn_attn.py @@ -8,6 +8,10 @@ from typing import Literal import torch from vllm.config import VllmConfig +from vllm.model_executor.layers.mamba.checkpoint import ( + MambaPrefillCheckpointBuilder, + MambaPrefillCheckpointMetadata, +) from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backend import ( AttentionBackend, @@ -48,6 +52,7 @@ class GDNAttentionMetadata: num_spec_decode_tokens: int num_actual_tokens: int + checkpoint: MambaPrefillCheckpointMetadata | None = None has_initial_state: torch.Tensor | None = None spec_query_start_loc: torch.Tensor | None = None # shape: [num_spec_decodes + 1,] @@ -97,6 +102,9 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] super().__init__(kv_cache_spec, layer_names, vllm_config, device) self.compilation_config = vllm_config.compilation_config self.speculative_config = vllm_config.speculative_config + self.checkpoint_builder = MambaPrefillCheckpointBuilder( + vllm_config, kv_cache_spec + ) from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import ( _resolve_gdn_prefill_backend, ) @@ -466,6 +474,13 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] else: has_initial_state = None + checkpoint: MambaPrefillCheckpointMetadata | None = None + if num_prefills > 0: + request_rows = list(range(m.num_reqs)) + if spec_sequence_masks_cpu is not None: + request_rows = (~spec_sequence_masks_cpu).nonzero().flatten().tolist() + checkpoint = self.checkpoint_builder.build(m, request_rows) + # Function code counted on either presency non-spec decode or spec decode, # but not both. assert not (num_decodes > 0 and num_spec_decodes > 0), ( @@ -543,6 +558,7 @@ class GDNAttentionMetadataBuilder(AttentionMetadataBuilder[GDNAttentionMetadata] num_spec_decodes=num_spec_decodes, num_spec_decode_tokens=num_spec_decode_tokens, num_actual_tokens=m.num_actual_tokens, + checkpoint=checkpoint, has_initial_state=has_initial_state, chunk_indices=chunk_indices, chunk_offsets=chunk_offsets,