diff --git a/tests/entrypoints/tool_parsers/test_granite4_tool_parser.py b/tests/entrypoints/tool_parsers/test_granite4_tool_parser.py index 71fe7637bf..67826d9564 100644 --- a/tests/entrypoints/tool_parsers/test_granite4_tool_parser.py +++ b/tests/entrypoints/tool_parsers/test_granite4_tool_parser.py @@ -24,6 +24,10 @@ def server(): "4096", "--max-num-seqs", "2", + # Streaming and non-streaming runs must match bitwise; a prefix-cache + # hit resumes the second from a mid-prefill Mamba checkpoint, whose + # numerics differ slightly from the cold single-pass prefill. + "--no-enable-prefix-caching", ] with RemoteOpenAIServer(model, args_for_model, max_wait_seconds=480) as server: yield server diff --git a/tests/kernels/mamba/test_mamba2_checkpoint_state.py b/tests/kernels/mamba/test_mamba2_checkpoint_state.py new file mode 100644 index 0000000000..54fcf80205 --- /dev/null +++ b/tests/kernels/mamba/test_mamba2_checkpoint_state.py @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""An SSD intermediate state equals an independent run over the same prefix. + +This is the central claim of internal prefill checkpoints: the state the SSD +scan already materializes at the checkpoint's chunk boundary is exactly the +state a fresh prefill over `[0, checkpoint)` would end with, so it can be +cached and later resumed from. The last test covers the other half: moving +that state, and the conv window, into the paged cache. +""" + +import pytest +import torch + +from tests.kernels.mamba.test_mamba_ssm_ssd import generate_random_inputs +from vllm.model_executor.layers.mamba.checkpoint import ( + MambaPrefillCheckpointMetadata, +) +from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import ( + store_prefill_checkpoint, +) +from vllm.model_executor.layers.mamba.ops.ssd_combined import ( + mamba_chunk_scan_combined_varlen, +) +from vllm.platforms import current_platform +from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadataBuilder +from vllm.v1.attention.backends.utils import NULL_BLOCK_ID + +DEVICE = current_platform.device_type +compute_chunk_metadata = BaseMambaAttentionMetadataBuilder._compute_chunk_metadata + + +def _run(inputs, seqlen, chunk_size, checkpoint_offset): + """Run one prefill of `seqlen` tokens, optionally exporting a checkpoint. + + Returns (states, checkpoint_chunk_index). `states` holds every chunk's + state when a checkpoint was requested, else the per-sequence final state. + """ + A, dt, X, B, C = inputs + offsets = [checkpoint_offset] if checkpoint_offset else None + cu, seq_idx, last, ckpt = compute_chunk_metadata( + chunk_size, + 1, + torch.tensor([0]), + torch.tensor([0, seqlen]), + checkpoint_offsets_p=offsets, + ) + + x = X[:seqlen] + out = torch.empty_like(x) + states = mamba_chunk_scan_combined_varlen( + x, + dt[:seqlen], + A, + B[:seqlen], + C[:seqlen], + chunk_size, + cu_seqlens=torch.tensor([0, seqlen], dtype=torch.int32, device=DEVICE), + cu_chunk_seqlens=torch.tensor(cu, dtype=torch.int32, device=DEVICE), + last_chunk_indices=torch.tensor(last, dtype=torch.int32, device=DEVICE), + seq_idx=torch.tensor(seq_idx, dtype=torch.int32, device=DEVICE), + out=out, + D=None, + return_intermediate_states=checkpoint_offset > 0, + state_dtype=torch.float32, + ) + return states, ckpt[0] + + +@pytest.mark.parametrize("checkpoint_offset", [32, 48, 96]) +def test_checkpoint_state_matches_an_independent_prefix_run(checkpoint_offset): + """states[checkpoint_chunk_idx] == the final state of a prefix-only run. + + 48 is deliberately not a multiple of chunk_size: align-mode block sizes + carry no chunk_size factor, so off-grid checkpoints are the common case + and are what the chunk split exists to support. + """ + seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32 + inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32) + inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs) + + states, ckpt_idx = _run(inputs, seqlen, chunk_size, checkpoint_offset) + assert ckpt_idx >= 0, "the checkpoint did not land on a chunk boundary" + checkpoint_state = states[ckpt_idx] + + prefix_final, _ = _run(inputs, checkpoint_offset, chunk_size, 0) + + torch.testing.assert_close( + checkpoint_state.to(torch.float32), + prefix_final[0].to(torch.float32), + atol=8e-3, + rtol=5e-3, + ) + + +def test_checkpoint_does_not_perturb_the_final_state(): + """Exporting a checkpoint must not change what the prefill produces. + + The chunk split is an extra boundary, not a different computation, so the + sequence's final state has to match a run that took no checkpoint. + """ + seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32 + inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32) + inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs) + + with_ckpt, ckpt_idx = _run(inputs, seqlen, chunk_size, 48) + without_ckpt, _ = _run(inputs, seqlen, chunk_size, 0) + + torch.testing.assert_close( + with_ckpt[-1].to(torch.float32), + without_ckpt[0].to(torch.float32), + atol=8e-3, + rtol=5e-3, + ) + + +@pytest.mark.parametrize("dim_first", [True, False]) +def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first): + """The store reads the checkpoint row out of every chunk state. + + Row 0 checkpoints 32 tokens in, its state being chunk 3 rather than the + program's own row, so the store only passes if ``chunk_idx`` is used. + Row 1 declines and must write nothing. The conv input is a column slice of + a wider projection, the SD conv layout is a transposed view, and SSM rows + are padded pages, matching the strides the mixer passes. + """ + dim, state_len, heads, head_dim, d_state = 16, 4, 2, 4, 8 + projection = torch.randn(64, dim + 8, device=DEVICE) + conv_input = projection[:, :dim] + if dim_first: + conv_state = torch.zeros(3, dim, state_len, device=DEVICE) + else: + conv_state = torch.zeros(3, state_len, dim, device=DEVICE).transpose(-1, -2) + varlen_states = torch.randn(5, heads, head_dim, d_state, device=DEVICE) + # Blocks are pages holding more than the SSM state, so rows are padded. + row = heads * head_dim * d_state + ssm_storage = torch.zeros(3, row + 8, device=DEVICE) + ssm_state = ssm_storage[:, :row].view(3, heads, head_dim, d_state) + checkpoint = MambaPrefillCheckpointMetadata( + checkpoint_offsets=torch.tensor([32, 0], dtype=torch.int32, device=DEVICE), + state_indices=torch.tensor( + [2, NULL_BLOCK_ID], dtype=torch.int32, device=DEVICE + ), + offsets=[32, 0], + ) + + store_prefill_checkpoint( + checkpoint, + torch.tensor([3, 0], device=DEVICE), + conv_input.T, + conv_state, + varlen_states, + ssm_state, + torch.tensor([0, 40, 64], dtype=torch.int32, device=DEVICE), + ) + + exact = dict(rtol=0, atol=0) + torch.testing.assert_close( + conv_state[2], conv_input[32 - state_len : 32].T, **exact + ) + torch.testing.assert_close(ssm_state[2], varlen_states[3], **exact) + assert not conv_state[:2].any() and not ssm_storage[:2].any() + assert not ssm_storage[:, row:].any(), "wrote into the page padding" diff --git a/tests/v1/attention/test_mamba2_prefill_checkpoints.py b/tests/v1/attention/test_mamba2_prefill_checkpoints.py new file mode 100644 index 0000000000..e5276786e2 --- /dev/null +++ b/tests/v1/attention/test_mamba2_prefill_checkpoints.py @@ -0,0 +1,163 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Internal prefill checkpoints for Mamba2 align-mode prefix caching.""" + +import torch + +from tests.v1.attention.utils import ( + BatchSpec, + create_common_attn_metadata, + create_vllm_config, +) +from vllm.v1.attention.backends.mamba2_attn import ( + Mamba2AttentionMetadataBuilder, +) +from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadataBuilder +from vllm.v1.attention.backends.utils import NULL_BLOCK_ID +from vllm.v1.kv_cache_interface import MambaSpec + +compute_chunk_metadata = BaseMambaAttentionMetadataBuilder._compute_chunk_metadata + + +def test_no_checkpoint_offsets_preserves_legacy_chunking(): + """Passing no offsets must reproduce the pre-feature chunk layout.""" + num_computed = torch.tensor([0, 300]) + qsl = torch.tensor([0, 700, 1000]) + + cu, seq_idx, last, ckpt = compute_chunk_metadata(256, 2, num_computed, qsl) + + # Row 0: 700 fresh tokens from sequence position 0 -> 256, 256, 188. + # Row 1: 300 new tokens resuming at 300. 300 % 256 == 44, so it finishes + # its partial chunk with 256 - 44 == 212 tokens, then 88. + assert cu == [0, 256, 512, 700, 912, 1000] + assert seq_idx == [0, 0, 0, 1, 1] + assert last == [2, 4] + assert ckpt == [-1, -1] + + +def test_chunking_realigns_to_the_grid_after_a_checkpoint(): + """After the forced break, chunks resume breaking on chunk_size. + + 100 -> then 156 to reach the 256 grid, then full chunks. Without the + realignment the second chunk would span 100..356 and cross a boundary. + """ + num_computed = torch.tensor([0]) + qsl = torch.tensor([0, 700]) + + cu, _, _, _ = compute_chunk_metadata( + 256, 1, num_computed, qsl, checkpoint_offsets_p=[100] + ) + + assert cu == [0, 100, 256, 512, 700] + + +def test_checkpoint_on_a_resumed_request_is_relative_to_the_query(): + """Offsets count from the query start, not the sequence start.""" + num_computed = torch.tensor([300]) + qsl = torch.tensor([0, 500]) + + cu, _, _, ckpt = compute_chunk_metadata( + 256, 1, num_computed, qsl, checkpoint_offsets_p=[212] + ) + + # Row resumes at sequence position 300. 300 % 256 == 44, so the partial + # chunk is 212 tokens and already ends exactly at the checkpoint (query + # offset 212 == sequence position 512) — no extra split needed. + assert cu == [0, 212, 468, 500] + assert ckpt == [0] + + +MAMBA_BLOCK_SIZE = 256 +CHUNK_SIZE = 256 +DEVICE = torch.device("cpu") + + +def _create_mamba2_builder( + mamba_cache_mode: str = "align", + num_prefill_checkpoint_blocks: int = 1, + prefill_checkpoint_alignment: int | None = 1, +) -> Mamba2AttentionMetadataBuilder: + vllm_config = create_vllm_config(block_size=MAMBA_BLOCK_SIZE) + vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode + # get_mamba_chunk_size() reads `mamba_chunk_size` then `chunk_size`. + vllm_config.model_config.hf_text_config.mamba_chunk_size = CHUNK_SIZE + spec = MambaSpec( + block_size=MAMBA_BLOCK_SIZE, + shapes=((16, 64),), + dtypes=(torch.float16,), + mamba_cache_mode=mamba_cache_mode, + num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks, + prefill_checkpoint_alignment=prefill_checkpoint_alignment, + ) + return Mamba2AttentionMetadataBuilder( + kv_cache_spec=spec, + layer_names=["layer.0"], + vllm_config=vllm_config, + device=DEVICE, + ) + + +def _build(builder, seq_lens, query_lens): + common = create_common_attn_metadata( + BatchSpec(seq_lens=seq_lens, query_lens=query_lens), + MAMBA_BLOCK_SIZE, + DEVICE, + arange_block_indices=True, + ) + # create_common_attn_metadata leaves is_prefilling unset; the builder + # asserts on it. Every row here is a full prefill. + common = common.replace(is_prefilling=torch.ones(len(seq_lens), dtype=torch.bool)) + return builder.build(common_prefix_len=0, common_attn_metadata=common) + + +def test_builder_emits_one_checkpoint_entry_per_prefill_row(): + """Declined rows stay in place, masked, and a chunk ends on the checkpoint.""" + builder = _create_mamba2_builder() + # Row 0 (seq_len 900) checkpoints at 768; row 1 (seq_len 100) is too short + # for a checkpoint at all. + meta = _build(builder, seq_lens=[900, 100], query_lens=[900, 100]) + + assert meta.checkpoint_chunk_idx is not None + assert meta.checkpoint_meta is not None + assert meta.checkpoint_chunk_idx.numel() == 2 + # The exporter masks row 1 off by its zero offset and null block, but + # still loads its chunk index first, so the placeholder must be in range. + assert meta.checkpoint_meta.checkpoint_offsets.tolist() == [768, 0] + assert meta.checkpoint_meta.state_indices[1].item() == NULL_BLOCK_ID + assert meta.checkpoint_chunk_idx[1].item() == 0 + # The checkpoint's token offset is the end of its chunk. Asserting it here + # also proves the chunk split actually landed on the checkpoint. + ckpt_chunk = meta.checkpoint_chunk_idx[0] + assert meta.cu_chunk_seqlen_p[ckpt_chunk + 1].item() == 768 + + +def test_update_block_table_regathers_checkpoint_blocks(): + """Groups sharing a spec reuse one group's metadata via update_block_table. + + Checkpoint destinations are block-table entries, so each group needs its + own: layers of different groups share physical tensors, and reusing the + first group's indices writes every group's checkpoint into its blocks. + """ + builder = _create_mamba2_builder() + meta = _build(builder, seq_lens=[900, 100], query_lens=[900, 100]) + assert meta.checkpoint_meta is not None + + # A second group's table: same shape as the first, disjoint block ids. + other_table = torch.arange(1000, 1016, dtype=torch.int32).reshape(2, 8) + updated = builder.update_block_table( + meta, other_table, torch.zeros(1000, dtype=torch.int64) + ) + + # Row 0 checkpoints in column cdiv(900, 256) - 2 == 2; row 1 declined. + assert updated.checkpoint_meta is not None + assert updated.checkpoint_meta.state_indices.tolist() == [ + other_table[0, 2].item(), + NULL_BLOCK_ID, + ] + + +def test_builder_emits_nothing_outside_align_mode(): + builder = _create_mamba2_builder(mamba_cache_mode="none") + meta = _build(builder, seq_lens=[900], query_lens=[900]) + + assert meta.checkpoint_chunk_idx is None diff --git a/vllm/model_executor/layers/mamba/checkpoint.py b/vllm/model_executor/layers/mamba/checkpoint.py index feef0afa05..45888113e1 100644 --- a/vllm/model_executor/layers/mamba/checkpoint.py +++ b/vllm/model_executor/layers/mamba/checkpoint.py @@ -70,6 +70,9 @@ def _gather_checkpoint_state_indices( class MambaPrefillCheckpointMetadata: checkpoint_offsets: torch.Tensor state_indices: torch.Tensor + # Host copy of ``checkpoint_offsets``, for backends that lay out the + # checkpoint before any tensor exists (Mamba2's SSD chunk split). + offsets: list[int] | None = None # Gather indices into the block table, kept so that another KV cache group # with the same spec can re-derive its own ``state_indices``. request_rows: torch.Tensor | None = None @@ -139,6 +142,7 @@ class MambaPrefillCheckpointBuilder: _gather_checkpoint_state_indices( m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor ), + offsets=checkpoint_offsets, request_rows=request_rows_tensor, block_cols=checkpoint_cols_tensor, ) diff --git a/vllm/model_executor/layers/mamba/mamba_mixer2.py b/vllm/model_executor/layers/mamba/mamba_mixer2.py index a67480ebc2..3753153fa3 100644 --- a/vllm/model_executor/layers/mamba/mamba_mixer2.py +++ b/vllm/model_executor/layers/mamba/mamba_mixer2.py @@ -3,11 +3,12 @@ from collections.abc import Sequence +from dataclasses import replace import torch from torch import nn -from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config +from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config from vllm.config.mamba import MambaBackendEnum from vllm.distributed import ( divide, @@ -39,6 +40,7 @@ from vllm.model_executor.layers.mamba.ops.layernorm_gated import rms_norm_gated from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501 selective_state_update_replayssm_output_only, ) +from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import store_prefill_checkpoint from vllm.model_executor.layers.mamba.ops.ssd_combined import ( mamba_chunk_scan_combined_varlen, ) @@ -66,7 +68,7 @@ from vllm.utils.torch_utils import ( from vllm.v1.attention.backend import AttentionMetadata from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum -from vllm.v1.kv_cache_interface import KVCacheGroupSpec +from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec, MambaSpec logger = init_logger(__name__) @@ -766,6 +768,8 @@ class MambaMixer2(MambaBase, PluggableLayer): query_start_loc_d = attn_metadata.query_start_loc_d num_decodes = attn_metadata.num_decodes num_decode_tokens = attn_metadata.num_decode_tokens + checkpoint_chunk_idx = attn_metadata.checkpoint_chunk_idx + checkpoint_meta = attn_metadata.checkpoint_meta if attn_metadata is None: # V1 profile run -- warm up SSD kernels so that autotuning @@ -842,6 +846,7 @@ class MambaMixer2(MambaBase, PluggableLayer): # NOTE: final output is an in-place update of out tensor assert preallocated_ssm_out_p is not None + has_checkpoints = checkpoint_chunk_idx is not None varlen_states = mamba_chunk_scan_combined_varlen( hidden_states_p.view( num_prefill_tokens, self.num_heads // self.tp_size, self.head_dim @@ -859,6 +864,7 @@ class MambaMixer2(MambaBase, PluggableLayer): cu_chunk_seqlens=cu_chunk_seqlen_p, last_chunk_indices=last_chunk_indices_p, initial_states=initial_states, + return_intermediate_states=has_checkpoints, dt_softplus=True, dt_limit=(0.0, float("inf")), out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim), @@ -869,7 +875,24 @@ class MambaMixer2(MambaBase, PluggableLayer): # - varlen state is a (num_prefills, nheads, headdim, dstate) # tensor assert state_indices_tensor_p is not None - ssm_state[state_indices_tensor_p] = varlen_states + final_states = ( + varlen_states[last_chunk_indices_p] + if has_checkpoints + else varlen_states + ) + if has_checkpoints: + assert checkpoint_meta is not None + assert query_start_loc_p is not None + store_prefill_checkpoint( + checkpoint_meta, + checkpoint_chunk_idx, + x, + conv_state, + varlen_states, + ssm_state, + query_start_loc_p, + ) + ssm_state[state_indices_tensor_p] = final_states if ring_start is not None and self._updates_replayssm_trackers: assert prev_num_accepted is not None reset_replayssm_ring_trackers( @@ -1070,6 +1093,18 @@ class MambaMixer2(MambaBase, PluggableLayer): is_blackwell=self.is_blackwell, ) + def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None: + spec = super().get_kv_cache_spec(vllm_config) + if not isinstance(spec, MambaSpec): + return spec + # The block-boundary state can be exported from inside one forward + # pass, so the scheduler need not split the prefill; every reader + # acts on this only in align mode. The metadata builder puts a chunk + # end exactly on the checkpoint, so any token position works. + return replace( + spec, num_prefill_checkpoint_blocks=1, prefill_checkpoint_alignment=1 + ) + def get_state_dtype(self) -> tuple[torch.dtype, ...]: assert self.model_config is not None assert self.cache_config is not None diff --git a/vllm/model_executor/layers/mamba/ops/ssd_checkpoint.py b/vllm/model_executor/layers/mamba/ops/ssd_checkpoint.py new file mode 100644 index 0000000000..25bfcedbdc --- /dev/null +++ b/vllm/model_executor/layers/mamba/ops/ssd_checkpoint.py @@ -0,0 +1,115 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import torch + +from vllm.model_executor.layers.mamba.checkpoint import ( + MambaPrefillCheckpointMetadata, +) +from vllm.triton_utils import tl, triton + + +@triton.jit +def _store_prefill_checkpoint_kernel( + x_ptr, + conv_state_ptr, + chunk_states_ptr, + ssm_state_ptr, + query_start_loc_ptr, + checkpoint_offsets_ptr, + state_indices_ptr, + chunk_idx_ptr, + stride_x_dim, + stride_x_token, + stride_conv_block, + stride_conv_dim, + stride_conv_token, + stride_chunk_states, + stride_ssm_block, + DIM: tl.constexpr, + STATE_LEN: tl.constexpr, + SSM_ROW_SIZE: tl.constexpr, + BLOCK_SIZE: tl.constexpr, +): + row = tl.program_id(0) + cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) + checkpoint_offset = tl.load(checkpoint_offsets_ptr + row) + valid = checkpoint_offset > 0 + dst = tl.load(state_indices_ptr + row).to(tl.int64) + + conv_mask = valid & (cols < DIM * STATE_LEN) + dim_idx = cols // STATE_LEN + history_idx = cols % STATE_LEN + token_idx = ( + tl.load(query_start_loc_ptr + row) + checkpoint_offset - STATE_LEN + history_idx + ).to(tl.int64) + conv = tl.load( + x_ptr + dim_idx * stride_x_dim + token_idx * stride_x_token, mask=conv_mask + ) + tl.store( + conv_state_ptr + + dst * stride_conv_block + + dim_idx * stride_conv_dim + + history_idx * stride_conv_token, + conv, + mask=conv_mask, + ) + + ssm_mask = valid & (cols < SSM_ROW_SIZE) + src = tl.load(chunk_idx_ptr + row).to(tl.int64) + ssm = tl.load(chunk_states_ptr + src * stride_chunk_states + cols, mask=ssm_mask) + tl.store(ssm_state_ptr + dst * stride_ssm_block + cols, ssm, mask=ssm_mask) + + +def store_prefill_checkpoint( + checkpoint: MambaPrefillCheckpointMetadata, + chunk_idx: torch.Tensor, + x: torch.Tensor, + conv_state: torch.Tensor, + chunk_states: torch.Tensor, + ssm_state: torch.Tensor, + query_start_loc: torch.Tensor, +) -> None: + """Write each prefill row's mid-prefill checkpoint into its paged block. + + Rows without a checkpoint (offset 0) write nothing. + + Args: + checkpoint: Per-row offsets and destination blocks. + chunk_idx: Per-row index into ``chunk_states`` of the checkpoint. + x: Pre-conv activations, ``(dim, num_prefill_tokens)``. + conv_state: Paged conv state, ``(num_blocks, dim, state_len)``. + chunk_states: SSD states at every chunk end. + ssm_state: Paged SSM state. + query_start_loc: Prefill query start locations. + + """ + dim = x.size(0) + state_len = conv_state.size(-1) + ssm_row_size = chunk_states[0].numel() + block_size = 1024 + grid = ( + chunk_idx.numel(), + triton.cdiv(max(dim * state_len, ssm_row_size), block_size), + ) + _store_prefill_checkpoint_kernel[grid]( + x, + conv_state, + chunk_states, + ssm_state, + query_start_loc, + checkpoint.checkpoint_offsets, + checkpoint.state_indices, + chunk_idx, + x.stride(0), + x.stride(1), + conv_state.stride(0), + conv_state.stride(1), + conv_state.stride(2), + chunk_states.stride(0), + ssm_state.stride(0), + DIM=dim, + STATE_LEN=state_len, + SSM_ROW_SIZE=ssm_row_size, + BLOCK_SIZE=block_size, + ) diff --git a/vllm/v1/attention/backends/mamba2_attn.py b/vllm/v1/attention/backends/mamba2_attn.py index e5e3827bcd..fc81cf680e 100644 --- a/vllm/v1/attention/backends/mamba2_attn.py +++ b/vllm/v1/attention/backends/mamba2_attn.py @@ -7,6 +7,10 @@ from typing import Any import torch from vllm.config import VllmConfig +from vllm.model_executor.layers.mamba.checkpoint import ( + MambaPrefillCheckpointBuilder, + MambaPrefillCheckpointMetadata, +) from vllm.utils.torch_utils import async_tensor_h2d from vllm.v1.attention.backend import ( AttentionBackend, @@ -109,6 +113,11 @@ class Mamba2AttentionMetadata(BaseMambaAttentionMetadata): # Chunk-related metadata (only for prefill) seq_idx_p: torch.Tensor | None = None + # Internal prefill checkpoints, one entry per prefill row. The chunk + # index selects the varlen_states row holding the checkpoint state. + checkpoint_chunk_idx: torch.Tensor | None = None + checkpoint_meta: MambaPrefillCheckpointMetadata | None = None + class Mamba2AttentionMetadataBuilder( BaseMambaAttentionMetadataBuilder[Mamba2AttentionMetadata] @@ -128,6 +137,9 @@ class Mamba2AttentionMetadataBuilder( "chunk_size needs to be set in the model config for Mamba2 models" ) self.chunk_size: int = chunk_size + self.checkpoint_builder = MambaPrefillCheckpointBuilder( + vllm_config, kv_cache_spec + ) def build( self, @@ -145,6 +157,8 @@ class Mamba2AttentionMetadataBuilder( seq_idx_p = None cu_chunk_seqlen_p = None last_chunk_indices_p = None + checkpoint_chunk_idx = None + checkpoint_meta = None prep_initial_states = False # Compute seq_idx for prefill only @@ -162,12 +176,27 @@ class Mamba2AttentionMetadataBuilder( ) prep_initial_states = bool((num_computed_tokens_p_cpu > 0).any()) - cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p = ( - self._build_chunk_metadata_tensors( - self.chunk_size, - common, - common_attn_metadata, - ) + checkpoint_offsets_p = None + first = common.num_reqs - common.num_prefills + checkpoint = self.checkpoint_builder.build( + common_attn_metadata, + list(range(first, common.num_reqs)), + ) + if checkpoint is not None: + # The host offsets place a chunk boundary on the checkpoint. + checkpoint_offsets_p = checkpoint.offsets + checkpoint_meta = checkpoint + + ( + cu_chunk_seqlen_p, + seq_idx_p, + last_chunk_indices_p, + checkpoint_chunk_idx, + ) = self._build_chunk_metadata_tensors( + self.chunk_size, + common, + common_attn_metadata, + checkpoint_offsets_p, ) return replace( @@ -177,4 +206,22 @@ class Mamba2AttentionMetadataBuilder( seq_idx_p=seq_idx_p, cu_chunk_seqlen_p=cu_chunk_seqlen_p, last_chunk_indices_p=last_chunk_indices_p, + checkpoint_chunk_idx=checkpoint_chunk_idx, + checkpoint_meta=checkpoint_meta, + ) + + def update_block_table( + self, + metadata: Mamba2AttentionMetadata, + blk_table: torch.Tensor, + slot_mapping: torch.Tensor, + ) -> Mamba2AttentionMetadata: + new_metadata = super().update_block_table(metadata, blk_table, slot_mapping) + if metadata.checkpoint_meta is None: + return new_metadata + # Checkpoint destinations are block-table entries, so each group + # re-gathers its own rather than writing into the source group's. + return replace( + new_metadata, + checkpoint_meta=metadata.checkpoint_meta.regather_state_indices(blk_table), ) diff --git a/vllm/v1/attention/backends/mamba_attn.py b/vllm/v1/attention/backends/mamba_attn.py index 9953461500..3e7ae905a3 100644 --- a/vllm/v1/attention/backends/mamba_attn.py +++ b/vllm/v1/attention/backends/mamba_attn.py @@ -233,13 +233,14 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu, ) + @staticmethod def _compute_chunk_metadata( - self, chunk_size: int, num_prefills: int, num_computed_tokens_p_cpu: torch.Tensor, query_start_loc_p_cpu: torch.Tensor, - ) -> tuple[list[int], list[int], list[int]]: + checkpoint_offsets_p: list[int] | None = None, + ) -> tuple[list[int], list[int], list[int], list[int]]: """Compute chunk-specific metadata for Mamba models. The code below carefully constructs the chunks such that: @@ -251,20 +252,34 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): of prefix caching for mamba (wip). We need to take care of the interaction with chunked prefill in order to satisfy constraint (2). + + `checkpoint_offsets_p[i]` forces an extra chunk end that many tokens + into row i's query (0 = none), because the SSD scan only materializes + states at chunk ends. Returns its chunk index per row, or -1. """ # TODO (tdoublep): This code could probably be optimized. cu_chunk_seqlen = [] seq_idx = [] last_chunk_indices = [] + checkpoint_chunk_indices = [] seqlen_pos = 0 + # A checkpoint splits its request into two segments so that a chunk + # ends exactly on it; each segment then chunks by the rules below. + segments = [] for req_idx in range(num_prefills): - this_num_computed = num_computed_tokens_p_cpu[req_idx].item() - this_new_tokens = ( + computed = num_computed_tokens_p_cpu[req_idx].item() + tokens = ( query_start_loc_p_cpu[req_idx + 1].item() - query_start_loc_p_cpu[req_idx].item() ) + ckpt = checkpoint_offsets_p[req_idx] if checkpoint_offsets_p else 0 + if ckpt: + segments.append((req_idx, computed, ckpt, True)) + segments.append((req_idx, computed + ckpt, tokens - ckpt, False)) + checkpoint_chunk_indices = [-1] * num_prefills + for req_idx, this_num_computed, this_new_tokens, is_ckpt_end in segments: # if computed tokens are not chunk-aligned, use the first # chunk to finish it off if this_num_computed % chunk_size != 0: @@ -288,11 +303,14 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): this_new_tokens -= chunk_len assert this_new_tokens == 0 - last_chunk_indices.append(len(cu_chunk_seqlen) - 1) + if is_ckpt_end: + checkpoint_chunk_indices[req_idx] = len(cu_chunk_seqlen) - 1 + else: + last_chunk_indices.append(len(cu_chunk_seqlen) - 1) cu_chunk_seqlen.append(seqlen_pos) - return cu_chunk_seqlen, seq_idx, last_chunk_indices + return cu_chunk_seqlen, seq_idx, last_chunk_indices, checkpoint_chunk_indices def _prefill_cpu_metadata( self, @@ -326,9 +344,13 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): chunk_size: int, common: M, common_attn_metadata: CommonAttentionMetadata, - ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + checkpoint_offsets_p: list[int] | None = None, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]: """Compute chunk metadata and return as device tensors. - Returns (cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p). + Returns (cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p, + checkpoint_chunk_idx_p). The last is None unless a row checkpoints; + it has one entry per prefill row, holding the logical chunk that ends + on the checkpoint, and 0 for rows that decline. """ num_prefills = common.num_prefills @@ -339,11 +361,14 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): common.num_decode_tokens, ) - cu_chunk_seqlen, seq_idx, last_chunk_indices = self._compute_chunk_metadata( - chunk_size, - num_prefills, - num_computed_tokens_p_cpu, - query_start_loc_p_cpu, + cu_chunk_seqlen, seq_idx, last_chunk_indices, ckpt_idxs = ( + self._compute_chunk_metadata( + chunk_size, + num_prefills, + num_computed_tokens_p_cpu, + query_start_loc_p_cpu, + checkpoint_offsets_p, + ) ) device = common_attn_metadata.query_start_loc.device @@ -356,7 +381,14 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC): last_chunk_indices_p = async_tensor_h2d( last_chunk_indices, dtype=torch.int32, device=device ) - return cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p + ckpt_idx_p = ( + async_tensor_h2d( + [max(idx, 0) for idx in ckpt_idxs], dtype=torch.int64, device=device + ) + if any(idx >= 0 for idx in ckpt_idxs) + else None + ) + return cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p, ckpt_idx_p def _compute_common_metadata( self,