Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 17:41:13 +00:00
1122 changed files with 60114 additions and 20956 deletions
@@ -0,0 +1,163 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""An SSD intermediate state equals an independent run over the same prefix.
This is the central claim of internal prefill checkpoints: the state the SSD
scan already materializes at the checkpoint's chunk boundary is exactly the
state a fresh prefill over `[0, checkpoint)` would end with, so it can be
cached and later resumed from. The last test covers the other half: moving
that state, and the conv window, into the paged cache.
"""
import pytest
import torch
from tests.kernels.mamba.test_mamba_ssm_ssd import generate_random_inputs
from vllm.model_executor.layers.mamba.checkpoint import (
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import (
store_prefill_checkpoint,
)
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadataBuilder
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
DEVICE = current_platform.device_type
compute_chunk_metadata = BaseMambaAttentionMetadataBuilder._compute_chunk_metadata
def _run(inputs, seqlen, chunk_size, checkpoint_offset):
"""Run one prefill of `seqlen` tokens, optionally exporting a checkpoint.
Returns (states, checkpoint_chunk_index). `states` holds every chunk's
state when a checkpoint was requested, else the per-sequence final state.
"""
A, dt, X, B, C = inputs
offsets = [checkpoint_offset] if checkpoint_offset else None
cu, seq_idx, last, ckpt = compute_chunk_metadata(
chunk_size,
1,
torch.tensor([0]),
torch.tensor([0, seqlen]),
checkpoint_offsets_p=offsets,
)
x = X[:seqlen]
out = torch.empty_like(x)
states = mamba_chunk_scan_combined_varlen(
x,
dt[:seqlen],
A,
B[:seqlen],
C[:seqlen],
chunk_size,
cu_seqlens=torch.tensor([0, seqlen], dtype=torch.int32, device=DEVICE),
cu_chunk_seqlens=torch.tensor(cu, dtype=torch.int32, device=DEVICE),
last_chunk_indices=torch.tensor(last, dtype=torch.int32, device=DEVICE),
seq_idx=torch.tensor(seq_idx, dtype=torch.int32, device=DEVICE),
out=out,
D=None,
return_intermediate_states=checkpoint_offset > 0,
state_dtype=torch.float32,
)
return states, ckpt[0]
@pytest.mark.parametrize("checkpoint_offset", [32, 48, 96])
def test_checkpoint_state_matches_an_independent_prefix_run(checkpoint_offset):
"""states[checkpoint_chunk_idx] == the final state of a prefix-only run.
48 is deliberately not a multiple of chunk_size: align-mode block sizes
carry no chunk_size factor, so off-grid checkpoints are the common case
and are what the chunk split exists to support.
"""
seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32
inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32)
inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs)
states, ckpt_idx = _run(inputs, seqlen, chunk_size, checkpoint_offset)
assert ckpt_idx >= 0, "the checkpoint did not land on a chunk boundary"
checkpoint_state = states[ckpt_idx]
prefix_final, _ = _run(inputs, checkpoint_offset, chunk_size, 0)
torch.testing.assert_close(
checkpoint_state.to(torch.float32),
prefix_final[0].to(torch.float32),
atol=8e-3,
rtol=5e-3,
)
def test_checkpoint_does_not_perturb_the_final_state():
"""Exporting a checkpoint must not change what the prefill produces.
The chunk split is an extra boundary, not a different computation, so the
sequence's final state has to match a run that took no checkpoint.
"""
seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32
inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32)
inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs)
with_ckpt, ckpt_idx = _run(inputs, seqlen, chunk_size, 48)
without_ckpt, _ = _run(inputs, seqlen, chunk_size, 0)
torch.testing.assert_close(
with_ckpt[-1].to(torch.float32),
without_ckpt[0].to(torch.float32),
atol=8e-3,
rtol=5e-3,
)
@pytest.mark.parametrize("dim_first", [True, False])
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first):
"""The store reads the checkpoint row out of every chunk state.
Row 0 checkpoints 32 tokens in, its state being chunk 3 rather than the
program's own row, so the store only passes if ``chunk_idx`` is used.
Row 1 declines and must write nothing. The conv input is a column slice of
a wider projection, the SD conv layout is a transposed view, and SSM rows
are padded pages, matching the strides the mixer passes.
"""
dim, state_len, heads, head_dim, d_state = 16, 4, 2, 4, 8
projection = torch.randn(64, dim + 8, device=DEVICE)
conv_input = projection[:, :dim]
if dim_first:
conv_state = torch.zeros(3, dim, state_len, device=DEVICE)
else:
conv_state = torch.zeros(3, state_len, dim, device=DEVICE).transpose(-1, -2)
varlen_states = torch.randn(5, heads, head_dim, d_state, device=DEVICE)
# Blocks are pages holding more than the SSM state, so rows are padded.
row = heads * head_dim * d_state
ssm_storage = torch.zeros(3, row + 8, device=DEVICE)
ssm_state = ssm_storage[:, :row].view(3, heads, head_dim, d_state)
checkpoint = MambaPrefillCheckpointMetadata(
checkpoint_offsets=torch.tensor([32, 0], dtype=torch.int32, device=DEVICE),
state_indices=torch.tensor(
[2, NULL_BLOCK_ID], dtype=torch.int32, device=DEVICE
),
offsets=[32, 0],
)
store_prefill_checkpoint(
checkpoint,
torch.tensor([3, 0], device=DEVICE),
conv_input.T,
conv_state,
varlen_states,
ssm_state,
torch.tensor([0, 40, 64], dtype=torch.int32, device=DEVICE),
)
exact = dict(rtol=0, atol=0)
torch.testing.assert_close(
conv_state[2], conv_input[32 - state_len : 32].T, **exact
)
torch.testing.assert_close(ssm_state[2], varlen_states[3], **exact)
assert not conv_state[:2].any() and not ssm_storage[:2].any()
assert not ssm_storage[:, row:].any(), "wrote into the page padding"
@@ -31,7 +31,11 @@ import torch
from vllm.model_executor.layers.mamba import mamba_utils as layer_mamba_utils
from vllm.platforms import current_platform
from vllm.v1.worker import mamba_utils as worker_mamba_utils
from vllm.v1.worker.mamba_utils import _TEMPORAL_TILES, precopy_mamba_align_fused_kernel
from vllm.v1.worker.mamba_utils import (
_TEMPORAL_TILES,
postprocess_mamba_fused_kernel,
precopy_mamba_align_fused_kernel,
)
_parametrize: Callable[..., Callable[[Any], Any]]
@@ -195,7 +199,8 @@ def test_precopy_matches_v1_copy_specs(
convs, ssms, device, conv_state_dim_first
)
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
# Non-identity mapping: block-table rows are request slots, not batch rows.
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device).flip(0)
grid = (num_reqs, NUM_LAYERS * 2, temporal_tiles)
precopy_mamba_align_fused_kernel[grid](
dst_col,
@@ -225,6 +230,77 @@ def test_precopy_matches_v1_copy_specs(
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
@_parametrize("conv_state_dim_first", [False, True])
@_cuda_required
def test_v2_postprocess_indexes_block_tables_by_req_slot(conv_state_dim_first):
"""Under PP the V2 postprocess runs after later steps have re-gathered the
batch-ordered tables, so it must index the request-slot tables by slot."""
device = torch.device("cuda")
torch.manual_seed(0)
num_reqs = 4
block_size = 8
num_blocks = num_reqs * MAX_COLS + 1
bt = torch.arange(1, num_blocks, dtype=torch.int32, device=device).view(
num_reqs, MAX_COLS
)
# slot 0: copy col 1 -> 0; slot 1: src == dst; slot 2: not aligned;
# slot 3: copy col 2 -> 0 with one accepted draft token.
num_accepted = torch.tensor([1, 1, 1, 2], dtype=torch.int32, device=device)
state_idx = torch.tensor([1, 0, 1, 2], dtype=torch.int32, device=device)
new_num_computed = torch.tensor([8, 8, 5, 8], dtype=torch.int32, device=device)
ref_src = torch.tensor([1, 0, -1, 2], dtype=torch.int32)
ref_dst = torch.zeros(num_reqs, dtype=torch.int32)
ref_bias = torch.tensor([0, 0, 0, 1], dtype=torch.int32)
convs, ssms = _build_state(num_blocks, device, conv_state_dim_first)
conv_ref, ssm_ref = _reference(
convs,
ssms,
bt.cpu(),
ref_src,
ref_dst,
ref_bias,
num_reqs,
conv_state_dim_first,
)
base, blk_stride, elem, inner, width, group, drc, drs = _build_meta(
convs, ssms, device, conv_state_dim_first
)
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
idx_mapping = torch.tensor([2, 0, 3, 1], dtype=torch.int32, device=device)
postprocess_mamba_fused_kernel[(num_reqs, NUM_LAYERS * 2, 1)](
num_accepted.clone(),
state_idx,
None,
new_num_computed,
None,
bt_ptrs,
bt.stride(0),
base,
blk_stride,
elem,
inner,
width,
group,
drc,
drs,
num_accepted,
idx_mapping,
num_reqs,
block_size=block_size,
COPY_BLOCK_SIZE=1024,
CONV_STATE_DIM_FIRST=conv_state_dim_first,
HAS_IDX_MAPPING=True,
PRECOMPUTED_NEW_COMPUTED=True,
TEMPORAL_TILES=1,
)
torch.accelerator.synchronize()
for layer in range(NUM_LAYERS):
torch.testing.assert_close(convs[layer], conv_ref[layer], rtol=0, atol=0)
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
def test_ds_conv_copy_spec_reproduces_multi_accept_assert(monkeypatch):
monkeypatch.setattr(
layer_mamba_utils,