[Feat][Mamba2] Enable internal prefill checkpoints (#57329)
Signed-off-by: Roi Koren <roik@nvidia.com> Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Co-authored-by: zjy0516 <riverclouds.zhu@qq.com>
This commit is contained in:
@@ -24,6 +24,10 @@ def server():
|
||||
"4096",
|
||||
"--max-num-seqs",
|
||||
"2",
|
||||
# Streaming and non-streaming runs must match bitwise; a prefix-cache
|
||||
# hit resumes the second from a mid-prefill Mamba checkpoint, whose
|
||||
# numerics differ slightly from the cold single-pass prefill.
|
||||
"--no-enable-prefix-caching",
|
||||
]
|
||||
with RemoteOpenAIServer(model, args_for_model, max_wait_seconds=480) as server:
|
||||
yield server
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""An SSD intermediate state equals an independent run over the same prefix.
|
||||
|
||||
This is the central claim of internal prefill checkpoints: the state the SSD
|
||||
scan already materializes at the checkpoint's chunk boundary is exactly the
|
||||
state a fresh prefill over `[0, checkpoint)` would end with, so it can be
|
||||
cached and later resumed from. The last test covers the other half: moving
|
||||
that state, and the conv window, into the paged cache.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.mamba.test_mamba_ssm_ssd import generate_random_inputs
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import (
|
||||
store_prefill_checkpoint,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
compute_chunk_metadata = BaseMambaAttentionMetadataBuilder._compute_chunk_metadata
|
||||
|
||||
|
||||
def _run(inputs, seqlen, chunk_size, checkpoint_offset):
|
||||
"""Run one prefill of `seqlen` tokens, optionally exporting a checkpoint.
|
||||
|
||||
Returns (states, checkpoint_chunk_index). `states` holds every chunk's
|
||||
state when a checkpoint was requested, else the per-sequence final state.
|
||||
"""
|
||||
A, dt, X, B, C = inputs
|
||||
offsets = [checkpoint_offset] if checkpoint_offset else None
|
||||
cu, seq_idx, last, ckpt = compute_chunk_metadata(
|
||||
chunk_size,
|
||||
1,
|
||||
torch.tensor([0]),
|
||||
torch.tensor([0, seqlen]),
|
||||
checkpoint_offsets_p=offsets,
|
||||
)
|
||||
|
||||
x = X[:seqlen]
|
||||
out = torch.empty_like(x)
|
||||
states = mamba_chunk_scan_combined_varlen(
|
||||
x,
|
||||
dt[:seqlen],
|
||||
A,
|
||||
B[:seqlen],
|
||||
C[:seqlen],
|
||||
chunk_size,
|
||||
cu_seqlens=torch.tensor([0, seqlen], dtype=torch.int32, device=DEVICE),
|
||||
cu_chunk_seqlens=torch.tensor(cu, dtype=torch.int32, device=DEVICE),
|
||||
last_chunk_indices=torch.tensor(last, dtype=torch.int32, device=DEVICE),
|
||||
seq_idx=torch.tensor(seq_idx, dtype=torch.int32, device=DEVICE),
|
||||
out=out,
|
||||
D=None,
|
||||
return_intermediate_states=checkpoint_offset > 0,
|
||||
state_dtype=torch.float32,
|
||||
)
|
||||
return states, ckpt[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("checkpoint_offset", [32, 48, 96])
|
||||
def test_checkpoint_state_matches_an_independent_prefix_run(checkpoint_offset):
|
||||
"""states[checkpoint_chunk_idx] == the final state of a prefix-only run.
|
||||
|
||||
48 is deliberately not a multiple of chunk_size: align-mode block sizes
|
||||
carry no chunk_size factor, so off-grid checkpoints are the common case
|
||||
and are what the chunk split exists to support.
|
||||
"""
|
||||
seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32
|
||||
inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32)
|
||||
inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs)
|
||||
|
||||
states, ckpt_idx = _run(inputs, seqlen, chunk_size, checkpoint_offset)
|
||||
assert ckpt_idx >= 0, "the checkpoint did not land on a chunk boundary"
|
||||
checkpoint_state = states[ckpt_idx]
|
||||
|
||||
prefix_final, _ = _run(inputs, checkpoint_offset, chunk_size, 0)
|
||||
|
||||
torch.testing.assert_close(
|
||||
checkpoint_state.to(torch.float32),
|
||||
prefix_final[0].to(torch.float32),
|
||||
atol=8e-3,
|
||||
rtol=5e-3,
|
||||
)
|
||||
|
||||
|
||||
def test_checkpoint_does_not_perturb_the_final_state():
|
||||
"""Exporting a checkpoint must not change what the prefill produces.
|
||||
|
||||
The chunk split is an extra boundary, not a different computation, so the
|
||||
sequence's final state has to match a run that took no checkpoint.
|
||||
"""
|
||||
seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32
|
||||
inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32)
|
||||
inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs)
|
||||
|
||||
with_ckpt, ckpt_idx = _run(inputs, seqlen, chunk_size, 48)
|
||||
without_ckpt, _ = _run(inputs, seqlen, chunk_size, 0)
|
||||
|
||||
torch.testing.assert_close(
|
||||
with_ckpt[-1].to(torch.float32),
|
||||
without_ckpt[0].to(torch.float32),
|
||||
atol=8e-3,
|
||||
rtol=5e-3,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dim_first", [True, False])
|
||||
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first):
|
||||
"""The store reads the checkpoint row out of every chunk state.
|
||||
|
||||
Row 0 checkpoints 32 tokens in, its state being chunk 3 rather than the
|
||||
program's own row, so the store only passes if ``chunk_idx`` is used.
|
||||
Row 1 declines and must write nothing. The conv input is a column slice of
|
||||
a wider projection, the SD conv layout is a transposed view, and SSM rows
|
||||
are padded pages, matching the strides the mixer passes.
|
||||
"""
|
||||
dim, state_len, heads, head_dim, d_state = 16, 4, 2, 4, 8
|
||||
projection = torch.randn(64, dim + 8, device=DEVICE)
|
||||
conv_input = projection[:, :dim]
|
||||
if dim_first:
|
||||
conv_state = torch.zeros(3, dim, state_len, device=DEVICE)
|
||||
else:
|
||||
conv_state = torch.zeros(3, state_len, dim, device=DEVICE).transpose(-1, -2)
|
||||
varlen_states = torch.randn(5, heads, head_dim, d_state, device=DEVICE)
|
||||
# Blocks are pages holding more than the SSM state, so rows are padded.
|
||||
row = heads * head_dim * d_state
|
||||
ssm_storage = torch.zeros(3, row + 8, device=DEVICE)
|
||||
ssm_state = ssm_storage[:, :row].view(3, heads, head_dim, d_state)
|
||||
checkpoint = MambaPrefillCheckpointMetadata(
|
||||
checkpoint_offsets=torch.tensor([32, 0], dtype=torch.int32, device=DEVICE),
|
||||
state_indices=torch.tensor(
|
||||
[2, NULL_BLOCK_ID], dtype=torch.int32, device=DEVICE
|
||||
),
|
||||
offsets=[32, 0],
|
||||
)
|
||||
|
||||
store_prefill_checkpoint(
|
||||
checkpoint,
|
||||
torch.tensor([3, 0], device=DEVICE),
|
||||
conv_input.T,
|
||||
conv_state,
|
||||
varlen_states,
|
||||
ssm_state,
|
||||
torch.tensor([0, 40, 64], dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
|
||||
exact = dict(rtol=0, atol=0)
|
||||
torch.testing.assert_close(
|
||||
conv_state[2], conv_input[32 - state_len : 32].T, **exact
|
||||
)
|
||||
torch.testing.assert_close(ssm_state[2], varlen_states[3], **exact)
|
||||
assert not conv_state[:2].any() and not ssm_storage[:2].any()
|
||||
assert not ssm_storage[:, row:].any(), "wrote into the page padding"
|
||||
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Internal prefill checkpoints for Mamba2 align-mode prefix caching."""
|
||||
|
||||
import torch
|
||||
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
create_vllm_config,
|
||||
)
|
||||
from vllm.v1.attention.backends.mamba2_attn import (
|
||||
Mamba2AttentionMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.kv_cache_interface import MambaSpec
|
||||
|
||||
compute_chunk_metadata = BaseMambaAttentionMetadataBuilder._compute_chunk_metadata
|
||||
|
||||
|
||||
def test_no_checkpoint_offsets_preserves_legacy_chunking():
|
||||
"""Passing no offsets must reproduce the pre-feature chunk layout."""
|
||||
num_computed = torch.tensor([0, 300])
|
||||
qsl = torch.tensor([0, 700, 1000])
|
||||
|
||||
cu, seq_idx, last, ckpt = compute_chunk_metadata(256, 2, num_computed, qsl)
|
||||
|
||||
# Row 0: 700 fresh tokens from sequence position 0 -> 256, 256, 188.
|
||||
# Row 1: 300 new tokens resuming at 300. 300 % 256 == 44, so it finishes
|
||||
# its partial chunk with 256 - 44 == 212 tokens, then 88.
|
||||
assert cu == [0, 256, 512, 700, 912, 1000]
|
||||
assert seq_idx == [0, 0, 0, 1, 1]
|
||||
assert last == [2, 4]
|
||||
assert ckpt == [-1, -1]
|
||||
|
||||
|
||||
def test_chunking_realigns_to_the_grid_after_a_checkpoint():
|
||||
"""After the forced break, chunks resume breaking on chunk_size.
|
||||
|
||||
100 -> then 156 to reach the 256 grid, then full chunks. Without the
|
||||
realignment the second chunk would span 100..356 and cross a boundary.
|
||||
"""
|
||||
num_computed = torch.tensor([0])
|
||||
qsl = torch.tensor([0, 700])
|
||||
|
||||
cu, _, _, _ = compute_chunk_metadata(
|
||||
256, 1, num_computed, qsl, checkpoint_offsets_p=[100]
|
||||
)
|
||||
|
||||
assert cu == [0, 100, 256, 512, 700]
|
||||
|
||||
|
||||
def test_checkpoint_on_a_resumed_request_is_relative_to_the_query():
|
||||
"""Offsets count from the query start, not the sequence start."""
|
||||
num_computed = torch.tensor([300])
|
||||
qsl = torch.tensor([0, 500])
|
||||
|
||||
cu, _, _, ckpt = compute_chunk_metadata(
|
||||
256, 1, num_computed, qsl, checkpoint_offsets_p=[212]
|
||||
)
|
||||
|
||||
# Row resumes at sequence position 300. 300 % 256 == 44, so the partial
|
||||
# chunk is 212 tokens and already ends exactly at the checkpoint (query
|
||||
# offset 212 == sequence position 512) — no extra split needed.
|
||||
assert cu == [0, 212, 468, 500]
|
||||
assert ckpt == [0]
|
||||
|
||||
|
||||
MAMBA_BLOCK_SIZE = 256
|
||||
CHUNK_SIZE = 256
|
||||
DEVICE = torch.device("cpu")
|
||||
|
||||
|
||||
def _create_mamba2_builder(
|
||||
mamba_cache_mode: str = "align",
|
||||
num_prefill_checkpoint_blocks: int = 1,
|
||||
prefill_checkpoint_alignment: int | None = 1,
|
||||
) -> Mamba2AttentionMetadataBuilder:
|
||||
vllm_config = create_vllm_config(block_size=MAMBA_BLOCK_SIZE)
|
||||
vllm_config.cache_config.mamba_cache_mode = mamba_cache_mode
|
||||
# get_mamba_chunk_size() reads `mamba_chunk_size` then `chunk_size`.
|
||||
vllm_config.model_config.hf_text_config.mamba_chunk_size = CHUNK_SIZE
|
||||
spec = MambaSpec(
|
||||
block_size=MAMBA_BLOCK_SIZE,
|
||||
shapes=((16, 64),),
|
||||
dtypes=(torch.float16,),
|
||||
mamba_cache_mode=mamba_cache_mode,
|
||||
num_prefill_checkpoint_blocks=num_prefill_checkpoint_blocks,
|
||||
prefill_checkpoint_alignment=prefill_checkpoint_alignment,
|
||||
)
|
||||
return Mamba2AttentionMetadataBuilder(
|
||||
kv_cache_spec=spec,
|
||||
layer_names=["layer.0"],
|
||||
vllm_config=vllm_config,
|
||||
device=DEVICE,
|
||||
)
|
||||
|
||||
|
||||
def _build(builder, seq_lens, query_lens):
|
||||
common = create_common_attn_metadata(
|
||||
BatchSpec(seq_lens=seq_lens, query_lens=query_lens),
|
||||
MAMBA_BLOCK_SIZE,
|
||||
DEVICE,
|
||||
arange_block_indices=True,
|
||||
)
|
||||
# create_common_attn_metadata leaves is_prefilling unset; the builder
|
||||
# asserts on it. Every row here is a full prefill.
|
||||
common = common.replace(is_prefilling=torch.ones(len(seq_lens), dtype=torch.bool))
|
||||
return builder.build(common_prefix_len=0, common_attn_metadata=common)
|
||||
|
||||
|
||||
def test_builder_emits_one_checkpoint_entry_per_prefill_row():
|
||||
"""Declined rows stay in place, masked, and a chunk ends on the checkpoint."""
|
||||
builder = _create_mamba2_builder()
|
||||
# Row 0 (seq_len 900) checkpoints at 768; row 1 (seq_len 100) is too short
|
||||
# for a checkpoint at all.
|
||||
meta = _build(builder, seq_lens=[900, 100], query_lens=[900, 100])
|
||||
|
||||
assert meta.checkpoint_chunk_idx is not None
|
||||
assert meta.checkpoint_meta is not None
|
||||
assert meta.checkpoint_chunk_idx.numel() == 2
|
||||
# The exporter masks row 1 off by its zero offset and null block, but
|
||||
# still loads its chunk index first, so the placeholder must be in range.
|
||||
assert meta.checkpoint_meta.checkpoint_offsets.tolist() == [768, 0]
|
||||
assert meta.checkpoint_meta.state_indices[1].item() == NULL_BLOCK_ID
|
||||
assert meta.checkpoint_chunk_idx[1].item() == 0
|
||||
# The checkpoint's token offset is the end of its chunk. Asserting it here
|
||||
# also proves the chunk split actually landed on the checkpoint.
|
||||
ckpt_chunk = meta.checkpoint_chunk_idx[0]
|
||||
assert meta.cu_chunk_seqlen_p[ckpt_chunk + 1].item() == 768
|
||||
|
||||
|
||||
def test_update_block_table_regathers_checkpoint_blocks():
|
||||
"""Groups sharing a spec reuse one group's metadata via update_block_table.
|
||||
|
||||
Checkpoint destinations are block-table entries, so each group needs its
|
||||
own: layers of different groups share physical tensors, and reusing the
|
||||
first group's indices writes every group's checkpoint into its blocks.
|
||||
"""
|
||||
builder = _create_mamba2_builder()
|
||||
meta = _build(builder, seq_lens=[900, 100], query_lens=[900, 100])
|
||||
assert meta.checkpoint_meta is not None
|
||||
|
||||
# A second group's table: same shape as the first, disjoint block ids.
|
||||
other_table = torch.arange(1000, 1016, dtype=torch.int32).reshape(2, 8)
|
||||
updated = builder.update_block_table(
|
||||
meta, other_table, torch.zeros(1000, dtype=torch.int64)
|
||||
)
|
||||
|
||||
# Row 0 checkpoints in column cdiv(900, 256) - 2 == 2; row 1 declined.
|
||||
assert updated.checkpoint_meta is not None
|
||||
assert updated.checkpoint_meta.state_indices.tolist() == [
|
||||
other_table[0, 2].item(),
|
||||
NULL_BLOCK_ID,
|
||||
]
|
||||
|
||||
|
||||
def test_builder_emits_nothing_outside_align_mode():
|
||||
builder = _create_mamba2_builder(mamba_cache_mode="none")
|
||||
meta = _build(builder, seq_lens=[900], query_lens=[900])
|
||||
|
||||
assert meta.checkpoint_chunk_idx is None
|
||||
@@ -70,6 +70,9 @@ def _gather_checkpoint_state_indices(
|
||||
class MambaPrefillCheckpointMetadata:
|
||||
checkpoint_offsets: torch.Tensor
|
||||
state_indices: torch.Tensor
|
||||
# Host copy of ``checkpoint_offsets``, for backends that lay out the
|
||||
# checkpoint before any tensor exists (Mamba2's SSD chunk split).
|
||||
offsets: list[int] | None = None
|
||||
# Gather indices into the block table, kept so that another KV cache group
|
||||
# with the same spec can re-derive its own ``state_indices``.
|
||||
request_rows: torch.Tensor | None = None
|
||||
@@ -139,6 +142,7 @@ class MambaPrefillCheckpointBuilder:
|
||||
_gather_checkpoint_state_indices(
|
||||
m.block_table_tensor, request_rows_tensor, checkpoint_cols_tensor
|
||||
),
|
||||
offsets=checkpoint_offsets,
|
||||
request_rows=request_rows_tensor,
|
||||
block_cols=checkpoint_cols_tensor,
|
||||
)
|
||||
|
||||
@@ -3,11 +3,12 @@
|
||||
|
||||
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import replace
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, get_current_vllm_config
|
||||
from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
|
||||
from vllm.config.mamba import MambaBackendEnum
|
||||
from vllm.distributed import (
|
||||
divide,
|
||||
@@ -39,6 +40,7 @@ from vllm.model_executor.layers.mamba.ops.layernorm_gated import rms_norm_gated
|
||||
from vllm.model_executor.layers.mamba.ops.selective_state_update_replayssm_output_only import ( # noqa: E501
|
||||
selective_state_update_replayssm_output_only,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import store_prefill_checkpoint
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
@@ -66,7 +68,7 @@ from vllm.utils.torch_utils import (
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec
|
||||
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, KVCacheSpec, MambaSpec
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -766,6 +768,8 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
query_start_loc_d = attn_metadata.query_start_loc_d
|
||||
num_decodes = attn_metadata.num_decodes
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
checkpoint_chunk_idx = attn_metadata.checkpoint_chunk_idx
|
||||
checkpoint_meta = attn_metadata.checkpoint_meta
|
||||
|
||||
if attn_metadata is None:
|
||||
# V1 profile run -- warm up SSD kernels so that autotuning
|
||||
@@ -842,6 +846,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
|
||||
# NOTE: final output is an in-place update of out tensor
|
||||
assert preallocated_ssm_out_p is not None
|
||||
has_checkpoints = checkpoint_chunk_idx is not None
|
||||
varlen_states = mamba_chunk_scan_combined_varlen(
|
||||
hidden_states_p.view(
|
||||
num_prefill_tokens, self.num_heads // self.tp_size, self.head_dim
|
||||
@@ -859,6 +864,7 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
cu_chunk_seqlens=cu_chunk_seqlen_p,
|
||||
last_chunk_indices=last_chunk_indices_p,
|
||||
initial_states=initial_states,
|
||||
return_intermediate_states=has_checkpoints,
|
||||
dt_softplus=True,
|
||||
dt_limit=(0.0, float("inf")),
|
||||
out=preallocated_ssm_out_p.view(num_prefill_tokens, -1, self.head_dim),
|
||||
@@ -869,7 +875,24 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
# - varlen state is a (num_prefills, nheads, headdim, dstate)
|
||||
# tensor
|
||||
assert state_indices_tensor_p is not None
|
||||
ssm_state[state_indices_tensor_p] = varlen_states
|
||||
final_states = (
|
||||
varlen_states[last_chunk_indices_p]
|
||||
if has_checkpoints
|
||||
else varlen_states
|
||||
)
|
||||
if has_checkpoints:
|
||||
assert checkpoint_meta is not None
|
||||
assert query_start_loc_p is not None
|
||||
store_prefill_checkpoint(
|
||||
checkpoint_meta,
|
||||
checkpoint_chunk_idx,
|
||||
x,
|
||||
conv_state,
|
||||
varlen_states,
|
||||
ssm_state,
|
||||
query_start_loc_p,
|
||||
)
|
||||
ssm_state[state_indices_tensor_p] = final_states
|
||||
if ring_start is not None and self._updates_replayssm_trackers:
|
||||
assert prev_num_accepted is not None
|
||||
reset_replayssm_ring_trackers(
|
||||
@@ -1070,6 +1093,18 @@ class MambaMixer2(MambaBase, PluggableLayer):
|
||||
is_blackwell=self.is_blackwell,
|
||||
)
|
||||
|
||||
def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None:
|
||||
spec = super().get_kv_cache_spec(vllm_config)
|
||||
if not isinstance(spec, MambaSpec):
|
||||
return spec
|
||||
# The block-boundary state can be exported from inside one forward
|
||||
# pass, so the scheduler need not split the prefill; every reader
|
||||
# acts on this only in align mode. The metadata builder puts a chunk
|
||||
# end exactly on the checkpoint, so any token position works.
|
||||
return replace(
|
||||
spec, num_prefill_checkpoint_blocks=1, prefill_checkpoint_alignment=1
|
||||
)
|
||||
|
||||
def get_state_dtype(self) -> tuple[torch.dtype, ...]:
|
||||
assert self.model_config is not None
|
||||
assert self.cache_config is not None
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_prefill_checkpoint_kernel(
|
||||
x_ptr,
|
||||
conv_state_ptr,
|
||||
chunk_states_ptr,
|
||||
ssm_state_ptr,
|
||||
query_start_loc_ptr,
|
||||
checkpoint_offsets_ptr,
|
||||
state_indices_ptr,
|
||||
chunk_idx_ptr,
|
||||
stride_x_dim,
|
||||
stride_x_token,
|
||||
stride_conv_block,
|
||||
stride_conv_dim,
|
||||
stride_conv_token,
|
||||
stride_chunk_states,
|
||||
stride_ssm_block,
|
||||
DIM: tl.constexpr,
|
||||
STATE_LEN: tl.constexpr,
|
||||
SSM_ROW_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
row = tl.program_id(0)
|
||||
cols = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
checkpoint_offset = tl.load(checkpoint_offsets_ptr + row)
|
||||
valid = checkpoint_offset > 0
|
||||
dst = tl.load(state_indices_ptr + row).to(tl.int64)
|
||||
|
||||
conv_mask = valid & (cols < DIM * STATE_LEN)
|
||||
dim_idx = cols // STATE_LEN
|
||||
history_idx = cols % STATE_LEN
|
||||
token_idx = (
|
||||
tl.load(query_start_loc_ptr + row) + checkpoint_offset - STATE_LEN + history_idx
|
||||
).to(tl.int64)
|
||||
conv = tl.load(
|
||||
x_ptr + dim_idx * stride_x_dim + token_idx * stride_x_token, mask=conv_mask
|
||||
)
|
||||
tl.store(
|
||||
conv_state_ptr
|
||||
+ dst * stride_conv_block
|
||||
+ dim_idx * stride_conv_dim
|
||||
+ history_idx * stride_conv_token,
|
||||
conv,
|
||||
mask=conv_mask,
|
||||
)
|
||||
|
||||
ssm_mask = valid & (cols < SSM_ROW_SIZE)
|
||||
src = tl.load(chunk_idx_ptr + row).to(tl.int64)
|
||||
ssm = tl.load(chunk_states_ptr + src * stride_chunk_states + cols, mask=ssm_mask)
|
||||
tl.store(ssm_state_ptr + dst * stride_ssm_block + cols, ssm, mask=ssm_mask)
|
||||
|
||||
|
||||
def store_prefill_checkpoint(
|
||||
checkpoint: MambaPrefillCheckpointMetadata,
|
||||
chunk_idx: torch.Tensor,
|
||||
x: torch.Tensor,
|
||||
conv_state: torch.Tensor,
|
||||
chunk_states: torch.Tensor,
|
||||
ssm_state: torch.Tensor,
|
||||
query_start_loc: torch.Tensor,
|
||||
) -> None:
|
||||
"""Write each prefill row's mid-prefill checkpoint into its paged block.
|
||||
|
||||
Rows without a checkpoint (offset 0) write nothing.
|
||||
|
||||
Args:
|
||||
checkpoint: Per-row offsets and destination blocks.
|
||||
chunk_idx: Per-row index into ``chunk_states`` of the checkpoint.
|
||||
x: Pre-conv activations, ``(dim, num_prefill_tokens)``.
|
||||
conv_state: Paged conv state, ``(num_blocks, dim, state_len)``.
|
||||
chunk_states: SSD states at every chunk end.
|
||||
ssm_state: Paged SSM state.
|
||||
query_start_loc: Prefill query start locations.
|
||||
|
||||
"""
|
||||
dim = x.size(0)
|
||||
state_len = conv_state.size(-1)
|
||||
ssm_row_size = chunk_states[0].numel()
|
||||
block_size = 1024
|
||||
grid = (
|
||||
chunk_idx.numel(),
|
||||
triton.cdiv(max(dim * state_len, ssm_row_size), block_size),
|
||||
)
|
||||
_store_prefill_checkpoint_kernel[grid](
|
||||
x,
|
||||
conv_state,
|
||||
chunk_states,
|
||||
ssm_state,
|
||||
query_start_loc,
|
||||
checkpoint.checkpoint_offsets,
|
||||
checkpoint.state_indices,
|
||||
chunk_idx,
|
||||
x.stride(0),
|
||||
x.stride(1),
|
||||
conv_state.stride(0),
|
||||
conv_state.stride(1),
|
||||
conv_state.stride(2),
|
||||
chunk_states.stride(0),
|
||||
ssm_state.stride(0),
|
||||
DIM=dim,
|
||||
STATE_LEN=state_len,
|
||||
SSM_ROW_SIZE=ssm_row_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
)
|
||||
@@ -7,6 +7,10 @@ from typing import Any
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointBuilder,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
@@ -109,6 +113,11 @@ class Mamba2AttentionMetadata(BaseMambaAttentionMetadata):
|
||||
# Chunk-related metadata (only for prefill)
|
||||
seq_idx_p: torch.Tensor | None = None
|
||||
|
||||
# Internal prefill checkpoints, one entry per prefill row. The chunk
|
||||
# index selects the varlen_states row holding the checkpoint state.
|
||||
checkpoint_chunk_idx: torch.Tensor | None = None
|
||||
checkpoint_meta: MambaPrefillCheckpointMetadata | None = None
|
||||
|
||||
|
||||
class Mamba2AttentionMetadataBuilder(
|
||||
BaseMambaAttentionMetadataBuilder[Mamba2AttentionMetadata]
|
||||
@@ -128,6 +137,9 @@ class Mamba2AttentionMetadataBuilder(
|
||||
"chunk_size needs to be set in the model config for Mamba2 models"
|
||||
)
|
||||
self.chunk_size: int = chunk_size
|
||||
self.checkpoint_builder = MambaPrefillCheckpointBuilder(
|
||||
vllm_config, kv_cache_spec
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
@@ -145,6 +157,8 @@ class Mamba2AttentionMetadataBuilder(
|
||||
seq_idx_p = None
|
||||
cu_chunk_seqlen_p = None
|
||||
last_chunk_indices_p = None
|
||||
checkpoint_chunk_idx = None
|
||||
checkpoint_meta = None
|
||||
prep_initial_states = False
|
||||
|
||||
# Compute seq_idx for prefill only
|
||||
@@ -162,12 +176,27 @@ class Mamba2AttentionMetadataBuilder(
|
||||
)
|
||||
prep_initial_states = bool((num_computed_tokens_p_cpu > 0).any())
|
||||
|
||||
cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p = (
|
||||
self._build_chunk_metadata_tensors(
|
||||
self.chunk_size,
|
||||
common,
|
||||
common_attn_metadata,
|
||||
)
|
||||
checkpoint_offsets_p = None
|
||||
first = common.num_reqs - common.num_prefills
|
||||
checkpoint = self.checkpoint_builder.build(
|
||||
common_attn_metadata,
|
||||
list(range(first, common.num_reqs)),
|
||||
)
|
||||
if checkpoint is not None:
|
||||
# The host offsets place a chunk boundary on the checkpoint.
|
||||
checkpoint_offsets_p = checkpoint.offsets
|
||||
checkpoint_meta = checkpoint
|
||||
|
||||
(
|
||||
cu_chunk_seqlen_p,
|
||||
seq_idx_p,
|
||||
last_chunk_indices_p,
|
||||
checkpoint_chunk_idx,
|
||||
) = self._build_chunk_metadata_tensors(
|
||||
self.chunk_size,
|
||||
common,
|
||||
common_attn_metadata,
|
||||
checkpoint_offsets_p,
|
||||
)
|
||||
|
||||
return replace(
|
||||
@@ -177,4 +206,22 @@ class Mamba2AttentionMetadataBuilder(
|
||||
seq_idx_p=seq_idx_p,
|
||||
cu_chunk_seqlen_p=cu_chunk_seqlen_p,
|
||||
last_chunk_indices_p=last_chunk_indices_p,
|
||||
checkpoint_chunk_idx=checkpoint_chunk_idx,
|
||||
checkpoint_meta=checkpoint_meta,
|
||||
)
|
||||
|
||||
def update_block_table(
|
||||
self,
|
||||
metadata: Mamba2AttentionMetadata,
|
||||
blk_table: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
) -> Mamba2AttentionMetadata:
|
||||
new_metadata = super().update_block_table(metadata, blk_table, slot_mapping)
|
||||
if metadata.checkpoint_meta is None:
|
||||
return new_metadata
|
||||
# Checkpoint destinations are block-table entries, so each group
|
||||
# re-gathers its own rather than writing into the source group's.
|
||||
return replace(
|
||||
new_metadata,
|
||||
checkpoint_meta=metadata.checkpoint_meta.regather_state_indices(blk_table),
|
||||
)
|
||||
|
||||
@@ -233,13 +233,14 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
num_decode_draft_tokens_cpu=num_decode_draft_tokens_cpu,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _compute_chunk_metadata(
|
||||
self,
|
||||
chunk_size: int,
|
||||
num_prefills: int,
|
||||
num_computed_tokens_p_cpu: torch.Tensor,
|
||||
query_start_loc_p_cpu: torch.Tensor,
|
||||
) -> tuple[list[int], list[int], list[int]]:
|
||||
checkpoint_offsets_p: list[int] | None = None,
|
||||
) -> tuple[list[int], list[int], list[int], list[int]]:
|
||||
"""Compute chunk-specific metadata for Mamba models.
|
||||
|
||||
The code below carefully constructs the chunks such that:
|
||||
@@ -251,20 +252,34 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
of prefix caching for mamba (wip). We need to take care
|
||||
of the interaction with chunked prefill in order to
|
||||
satisfy constraint (2).
|
||||
|
||||
`checkpoint_offsets_p[i]` forces an extra chunk end that many tokens
|
||||
into row i's query (0 = none), because the SSD scan only materializes
|
||||
states at chunk ends. Returns its chunk index per row, or -1.
|
||||
"""
|
||||
# TODO (tdoublep): This code could probably be optimized.
|
||||
cu_chunk_seqlen = []
|
||||
seq_idx = []
|
||||
last_chunk_indices = []
|
||||
checkpoint_chunk_indices = []
|
||||
seqlen_pos = 0
|
||||
|
||||
# A checkpoint splits its request into two segments so that a chunk
|
||||
# ends exactly on it; each segment then chunks by the rules below.
|
||||
segments = []
|
||||
for req_idx in range(num_prefills):
|
||||
this_num_computed = num_computed_tokens_p_cpu[req_idx].item()
|
||||
this_new_tokens = (
|
||||
computed = num_computed_tokens_p_cpu[req_idx].item()
|
||||
tokens = (
|
||||
query_start_loc_p_cpu[req_idx + 1].item()
|
||||
- query_start_loc_p_cpu[req_idx].item()
|
||||
)
|
||||
ckpt = checkpoint_offsets_p[req_idx] if checkpoint_offsets_p else 0
|
||||
if ckpt:
|
||||
segments.append((req_idx, computed, ckpt, True))
|
||||
segments.append((req_idx, computed + ckpt, tokens - ckpt, False))
|
||||
checkpoint_chunk_indices = [-1] * num_prefills
|
||||
|
||||
for req_idx, this_num_computed, this_new_tokens, is_ckpt_end in segments:
|
||||
# if computed tokens are not chunk-aligned, use the first
|
||||
# chunk to finish it off
|
||||
if this_num_computed % chunk_size != 0:
|
||||
@@ -288,11 +303,14 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
this_new_tokens -= chunk_len
|
||||
|
||||
assert this_new_tokens == 0
|
||||
last_chunk_indices.append(len(cu_chunk_seqlen) - 1)
|
||||
if is_ckpt_end:
|
||||
checkpoint_chunk_indices[req_idx] = len(cu_chunk_seqlen) - 1
|
||||
else:
|
||||
last_chunk_indices.append(len(cu_chunk_seqlen) - 1)
|
||||
|
||||
cu_chunk_seqlen.append(seqlen_pos)
|
||||
|
||||
return cu_chunk_seqlen, seq_idx, last_chunk_indices
|
||||
return cu_chunk_seqlen, seq_idx, last_chunk_indices, checkpoint_chunk_indices
|
||||
|
||||
def _prefill_cpu_metadata(
|
||||
self,
|
||||
@@ -326,9 +344,13 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
chunk_size: int,
|
||||
common: M,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
checkpoint_offsets_p: list[int] | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor | None]:
|
||||
"""Compute chunk metadata and return as device tensors.
|
||||
Returns (cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p).
|
||||
Returns (cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p,
|
||||
checkpoint_chunk_idx_p). The last is None unless a row checkpoints;
|
||||
it has one entry per prefill row, holding the logical chunk that ends
|
||||
on the checkpoint, and 0 for rows that decline.
|
||||
"""
|
||||
num_prefills = common.num_prefills
|
||||
|
||||
@@ -339,11 +361,14 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
common.num_decode_tokens,
|
||||
)
|
||||
|
||||
cu_chunk_seqlen, seq_idx, last_chunk_indices = self._compute_chunk_metadata(
|
||||
chunk_size,
|
||||
num_prefills,
|
||||
num_computed_tokens_p_cpu,
|
||||
query_start_loc_p_cpu,
|
||||
cu_chunk_seqlen, seq_idx, last_chunk_indices, ckpt_idxs = (
|
||||
self._compute_chunk_metadata(
|
||||
chunk_size,
|
||||
num_prefills,
|
||||
num_computed_tokens_p_cpu,
|
||||
query_start_loc_p_cpu,
|
||||
checkpoint_offsets_p,
|
||||
)
|
||||
)
|
||||
|
||||
device = common_attn_metadata.query_start_loc.device
|
||||
@@ -356,7 +381,14 @@ class BaseMambaAttentionMetadataBuilder(AttentionMetadataBuilder[M], abc.ABC):
|
||||
last_chunk_indices_p = async_tensor_h2d(
|
||||
last_chunk_indices, dtype=torch.int32, device=device
|
||||
)
|
||||
return cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p
|
||||
ckpt_idx_p = (
|
||||
async_tensor_h2d(
|
||||
[max(idx, 0) for idx in ckpt_idxs], dtype=torch.int64, device=device
|
||||
)
|
||||
if any(idx >= 0 for idx in ckpt_idxs)
|
||||
else None
|
||||
)
|
||||
return cu_chunk_seqlen_p, seq_idx_p, last_chunk_indices_p, ckpt_idx_p
|
||||
|
||||
def _compute_common_metadata(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user