[Feat][Mamba2] Enable internal prefill checkpoints (#57329)

Signed-off-by: Roi Koren <roik@nvidia.com>
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
roikoren755
2026-10-06 07:53:51 +00:00
committed by GitHub
co-authored by zjy0516
parent edfccc9469
commit e11962fc1c
8 changed files with 586 additions and 23 deletions
@@ -24,6 +24,10 @@ def server():
"4096",
"--max-num-seqs",
"2",
# Streaming and non-streaming runs must match bitwise; a prefix-cache
# hit resumes the second from a mid-prefill Mamba checkpoint, whose
# numerics differ slightly from the cold single-pass prefill.
"--no-enable-prefix-caching",
]
with RemoteOpenAIServer(model, args_for_model, max_wait_seconds=480) as server:
yield server
@@ -0,0 +1,163 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""An SSD intermediate state equals an independent run over the same prefix.
This is the central claim of internal prefill checkpoints: the state the SSD
scan already materializes at the checkpoint's chunk boundary is exactly the
state a fresh prefill over `[0, checkpoint)` would end with, so it can be
cached and later resumed from. The last test covers the other half: moving
that state, and the conv window, into the paged cache.
"""
import pytest
import torch
from tests.kernels.mamba.test_mamba_ssm_ssd import generate_random_inputs
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import (
store_prefill_checkpoint,
)
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadataBuilder
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
DEVICE = current_platform.device_type
compute_chunk_metadata = BaseMambaAttentionMetadataBuilder._compute_chunk_metadata
def _run(inputs, seqlen, chunk_size, checkpoint_offset):
"""Run one prefill of `seqlen` tokens, optionally exporting a checkpoint.
Returns (states, checkpoint_chunk_index). `states` holds every chunk's
state when a checkpoint was requested, else the per-sequence final state.
"""
A, dt, X, B, C = inputs
offsets = [checkpoint_offset] if checkpoint_offset else None
cu, seq_idx, last, ckpt = compute_chunk_metadata(
chunk_size,
1,
torch.tensor([0]),
torch.tensor([0, seqlen]),
checkpoint_offsets_p=offsets,
)
x = X[:seqlen]
out = torch.empty_like(x)
states = mamba_chunk_scan_combined_varlen(
x,
dt[:seqlen],
A,
B[:seqlen],
C[:seqlen],
chunk_size,
cu_seqlens=torch.tensor([0, seqlen], dtype=torch.int32, device=DEVICE),
cu_chunk_seqlens=torch.tensor(cu, dtype=torch.int32, device=DEVICE),
last_chunk_indices=torch.tensor(last, dtype=torch.int32, device=DEVICE),
seq_idx=torch.tensor(seq_idx, dtype=torch.int32, device=DEVICE),
out=out,
D=None,
return_intermediate_states=checkpoint_offset > 0,
state_dtype=torch.float32,
)
return states, ckpt[0]
@pytest.mark.parametrize("checkpoint_offset", [32, 48, 96])
def test_checkpoint_state_matches_an_independent_prefix_run(checkpoint_offset):
"""states[checkpoint_chunk_idx] == the final state of a prefix-only run.
48 is deliberately not a multiple of chunk_size: align-mode block sizes
carry no chunk_size factor, so off-grid checkpoints are the common case
and are what the chunk split exists to support.
"""
seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32
inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32)
inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs)
states, ckpt_idx = _run(inputs, seqlen, chunk_size, checkpoint_offset)
assert ckpt_idx >= 0, "the checkpoint did not land on a chunk boundary"
checkpoint_state = states[ckpt_idx]
prefix_final, _ = _run(inputs, checkpoint_offset, chunk_size, 0)
torch.testing.assert_close(
checkpoint_state.to(torch.float32),
prefix_final[0].to(torch.float32),
atol=8e-3,
rtol=5e-3,
)
def test_checkpoint_does_not_perturb_the_final_state():
"""Exporting a checkpoint must not change what the prefill produces.
The chunk split is an extra boundary, not a different computation, so the
sequence's final state has to match a run that took no checkpoint.
"""
seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32
inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32)
inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs)
with_ckpt, ckpt_idx = _run(inputs, seqlen, chunk_size, 48)
without_ckpt, _ = _run(inputs, seqlen, chunk_size, 0)
torch.testing.assert_close(
with_ckpt[-1].to(torch.float32),
without_ckpt[0].to(torch.float32),
atol=8e-3,
rtol=5e-3,
)
@pytest.mark.parametrize("dim_first", [True, False])
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first):
"""The store reads the checkpoint row out of every chunk state.
Row 0 checkpoints 32 tokens in, its state being chunk 3 rather than the
program's own row, so the store only passes if ``chunk_idx`` is used.
Row 1 declines and must write nothing. The conv input is a column slice of
a wider projection, the SD conv layout is a transposed view, and SSM rows
are padded pages, matching the strides the mixer passes.
"""
dim, state_len, heads, head_dim, d_state = 16, 4, 2, 4, 8
projection = torch.randn(64, dim + 8, device=DEVICE)
conv_input = projection[:, :dim]
if dim_first:
conv_state = torch.zeros(3, dim, state_len, device=DEVICE)
else:
conv_state = torch.zeros(3, state_len, dim, device=DEVICE).transpose(-1, -2)
varlen_states = torch.randn(5, heads, head_dim, d_state, device=DEVICE)
# Blocks are pages holding more than the SSM state, so rows are padded.
row = heads * head_dim * d_state
ssm_storage = torch.zeros(3, row + 8, device=DEVICE)
ssm_state = ssm_storage[:, :row].view(3, heads, head_dim, d_state)
checkpoint = MambaPrefillCheckpointMetadata(
checkpoint_offsets=torch.tensor([32, 0], dtype=torch.int32, device=DEVICE),
state_indices=torch.tensor(
[2, NULL_BLOCK_ID], dtype=torch.int32, device=DEVICE
),
offsets=[32, 0],
)
store_prefill_checkpoint(
checkpoint,
torch.tensor([3, 0], device=DEVICE),
conv_input.T,
conv_state,
varlen_states,
ssm_state,
torch.tensor([0, 40, 64], dtype=torch.int32, device=DEVICE),
)
exact = dict(rtol=0, atol=0)
torch.testing.assert_close(
conv_state[2], conv_input[32 - state_len : 32].T, **exact
)
torch.testing.assert_close(ssm_state[2], varlen_states[3], **exact)
assert not conv_state[:2].any() and not ssm_storage[:2].any()
assert not ssm_storage[:, row:].any(), "wrote into the page padding"
@@ -0,0 +1,163 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Internal prefill checkpoints for Mamba2 align-mode prefix caching."""
import torch
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
create_vllm_config,
)
from vllm.v1.attention.backends.mamba2_attn import (
Mamba2AttentionMetadataBuilder,
)
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadataBuilder
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.kv_cache_interface import MambaSpec
compute_chunk_metadata = BaseMambaAttentionMetadataBuilder._compute_chunk_metadata
def test_no_checkpoint_offsets_preserves_legacy_chunking():
"""Passing no offsets must reproduce the pre-feature chunk layout."""
num_computed = torch.tensor([0, 300])
qsl = torch.tensor([0, 700, 1000])
cu, seq_idx, last, ckpt = compute_chunk_metadata(256, 2, num_computed, qsl)
# Row 0: 700 fresh tokens from sequence position 0 -> 256, 256, 188.
# Row 1: 300 new tokens resuming at 300. 300 % 256 == 44, so it finishes
# its partial chunk with 256 - 44 == 212 tokens, then 88.
assert cu == [0, 256, 512, 700, 912, 1000]
assert seq_idx == [0, 0, 0, 1, 1]
assert last == [2, 4]
assert ckpt == [-1, -1]
def test_chunking_realigns_to_the_grid_after_a_checkpoint():
"""After the forced break, chunks resume breaking on chunk_size.
100 -> then 156 to reach the 256 grid, then full chunks. Without the
realignment the second chunk would span 100..356 and cross a boundary.
"""
num_computed = torch.tensor([0])
qsl = torch.tensor([0, 700])
cu, _, _, _ = compute_chunk_metadata(
256, 1, num_computed, qsl, checkpoint_offsets_p=[100]
)
assert cu == [0, 100, 256, 512, 700]
def test_checkpoint_on_a_resumed_request_is_relative_to_the_query():
"""Offsets count from the query start, not the sequence start."""
num_computed = torch.tensor([300])
qsl = torch.tensor([0, 500])
cu, _, _, ckpt = compute_chunk_metadata(
256, 1, num_computed, qsl, checkpoint_offsets_p=[212]
)
# Row resumes at sequence position 300. 300 % 256 == 44, so the partial
# chunk is 212 tokens and already ends exactly at the checkpoint (query
# offset 212 == sequence position 512) — no extra split needed.
assert cu == [0, 212, 468, 500]
assert ckpt == [0]
MAMBA_BLOCK_SIZE = 256
CHUNK_SIZE = 256
DEVICE = torch.device("cpu")
def _create_mamba2_builder(
mamba_cache_mode: str = "align",
num_prefill_checkpoint_blocks: int = 1,
prefill_checkpoint_alignment: int | None = 1,
) -> Mamba2AttentionMetadataBuilder:
vllm_config = create_vllm_config(block_size=MAMBA_BLOCK_SIZE)
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
# get_mamba_chunk_size() reads `mamba_chunk_size` then `chunk_size`.
vllm_config.model_config.hf_text_config.mamba_chunk_size = CHUNK_SIZE
spec = MambaSpec(
block_size=MAMBA_BLOCK_SIZE,
shapes=((16, 64),),
dtypes=(torch.float16,),
mamba_cache_mode=mamba_cache_mode,
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
prefill_checkpoint_alignment=prefill_checkpoint_alignment,
)
return Mamba2AttentionMetadataBuilder(
kv_cache_spec=spec,
layer_names=["layer.0"],
vllm_config=vllm_config,
device=DEVICE,
)
def _build(builder, seq_lens, query_lens):
common = create_common_attn_metadata(
BatchSpec(seq_lens=seq_lens, query_lens=query_lens),
MAMBA_BLOCK_SIZE,
DEVICE,
arange_block_indices=True,
)
# create_common_attn_metadata leaves is_prefilling unset; the builder
# asserts on it. Every row here is a full prefill.
common = common.replace(is_prefilling=torch.ones(len(seq_lens), dtype=torch.bool))
return builder.build(common_prefix_len=0, common_attn_metadata=common)
def test_builder_emits_one_checkpoint_entry_per_prefill_row():
"""Declined rows stay in place, masked, and a chunk ends on the checkpoint."""
builder = _create_mamba2_builder()
# Row 0 (seq_len 900) checkpoints at 768; row 1 (seq_len 100) is too short
# for a checkpoint at all.
meta = _build(builder, seq_lens=[900, 100], query_lens=[900, 100])
assert meta.checkpoint_chunk_idx is not None
assert meta.checkpoint_meta is not None
assert meta.checkpoint_chunk_idx.numel() == 2
# The exporter masks row 1 off by its zero offset and null block, but
# still loads its chunk index first, so the placeholder must be in range.
assert meta.checkpoint_meta.checkpoint_offsets.tolist() == [768, 0]
assert meta.checkpoint_meta.state_indices[1].item() == NULL_BLOCK_ID
assert meta.checkpoint_chunk_idx[1].item() == 0
# The checkpoint's token offset is the end of its chunk. Asserting it here
# also proves the chunk split actually landed on the checkpoint.
ckpt_chunk = meta.checkpoint_chunk_idx[0]
assert meta.cu_chunk_seqlen_p[ckpt_chunk + 1].item() == 768
def test_update_block_table_regathers_checkpoint_blocks():
"""Groups sharing a spec reuse one group's metadata via update_block_table.
Checkpoint destinations are block-table entries, so each group needs its
own: layers of different groups share physical tensors, and reusing the
first group's indices writes every group's checkpoint into its blocks.
"""
builder = _create_mamba2_builder()
meta = _build(builder, seq_lens=[900, 100], query_lens=[900, 100])
assert meta.checkpoint_meta is not None
# A second group's table: same shape as the first, disjoint block ids.
other_table = torch.arange(1000, 1016, dtype=torch.int32).reshape(2, 8)
updated = builder.update_block_table(
meta, other_table, torch.zeros(1000, dtype=torch.int64)
)
# Row 0 checkpoints in column cdiv(900, 256) - 2 == 2; row 1 declined.
assert updated.checkpoint_meta is not None
assert updated.checkpoint_meta.state_indices.tolist() == [
other_table[0, 2].item(),
NULL_BLOCK_ID,
]
def test_builder_emits_nothing_outside_align_mode():
builder = _create_mamba2_builder(mamba_cache_mode="none")
meta = _build(builder, seq_lens=[900], query_lens=[900])
assert meta.checkpoint_chunk_idx is None