Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-15 21:16:39 +07:00
1693 changed files with 99919 additions and 32440 deletions
@@ -13,10 +13,19 @@ if current_platform.is_rocm():
)
import flashinfer.gdn_prefill # noqa: E402
import torch.nn.functional as F # noqa: E402
from vllm.model_executor.layers.mamba.checkpoint import ( # noqa: E402
MambaPrefillCheckpointExporter,
MambaPrefillCheckpointMetadata,
)
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
fi_chunk_gated_delta_rule,
) # noqa: E402
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
plan_flashinfer_checkpoints,
)
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID # noqa: E402
def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
@@ -51,3 +60,130 @@ def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
assert captured_cu_seqlens.dtype == torch.int64
assert output.shape == q.shape
assert final_state is None
@pytest.mark.skipif(
not (
current_platform.is_device_capability(90)
or current_platform.is_device_capability_family(100)
or current_platform.is_device_capability_family(120)
),
reason="FlashInfer GDN prefill checkpoints need SM90, SM100 or SM120.",
)
@pytest.mark.parametrize("use_cp", [True, False])
@pytest.mark.parametrize("ssm_dtype", [torch.float32, torch.bfloat16])
def test_flashinfer_checkpoints_export_the_state_at_each_offset(use_cp, ssm_dtype):
"""Each exported checkpoint is the state of the prefill stopped there.
The leading decode row is outside the prefill kernel. The two checkpointed
prefills put FlashInfer's stride (their offsets' gcd) below both offsets,
so the exporter must pick each state out of the shared emitted buffer;
the third prefill takes no checkpoint. Both FlashInfer routes are covered
because ``use_cp="auto"`` may take either. The conv cache has MTP's spec
slots after the window, which the conv kernels leave alone.
"""
torch.manual_seed(0)
device = "cuda"
num_qk_heads, num_v_heads, head_dim, conv_dim, state_len = 2, 4, 128, 16, 3
num_spec = 2
offsets = [0, 128, 0, 192]
query_lens = [200, 50, 300]
state_indices = [NULL_BLOCK_ID, 3, NULL_BLOCK_ID, 5]
num_blocks = 8
num_tokens = sum(query_lens)
q = F.normalize(
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
).bfloat16()
k = F.normalize(
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
).bfloat16()
v = torch.randn(1, num_tokens, num_v_heads, head_dim, device=device).bfloat16()
g = F.logsigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
beta = torch.sigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
initial_state = 0.1 * torch.randn(
len(query_lens), num_v_heads, head_dim, head_dim, device=device
)
prefill_cu = [0]
for query_len in query_lens:
prefill_cu.append(prefill_cu[-1] + query_len)
every_n_tokens, cu_starts, rows = plan_flashinfer_checkpoints(offsets, query_lens)
state_checkpoints = torch.empty(
cu_starts[-1], num_v_heads, head_dim, head_dim, device=device
)
fi_chunk_gated_delta_rule(
q=q,
k=k,
v=v,
g=g,
beta=beta,
initial_state=initial_state,
output_final_state=True,
cu_seqlens=torch.tensor(prefill_cu, device=device),
use_qk_l2norm_in_kernel=False,
state_checkpoints=state_checkpoints,
checkpoint_cu_starts=torch.tensor(cu_starts, device=device),
checkpoint_every_n_tokens=every_n_tokens,
use_cp=use_cp,
)
# The exporter sees the decode row's token in front of the prefills.
raw_qkv = torch.randn(1 + num_tokens, conv_dim, device=device).bfloat16()
cu_seqlens = torch.tensor([0] + [1 + n for n in prefill_cu], device=device)
conv_state = torch.zeros(
num_blocks,
state_len + num_spec,
conv_dim,
device=device,
dtype=torch.bfloat16,
).transpose(-1, -2)
ssm_state = torch.zeros(
num_blocks, num_v_heads, head_dim, head_dim, device=device, dtype=ssm_dtype
)
MambaPrefillCheckpointExporter(state_len=state_len).export(
MambaPrefillCheckpointMetadata(
checkpoint_offsets=torch.tensor(offsets, device=device, dtype=torch.int32),
state_indices=torch.tensor(state_indices, device=device),
),
raw_qkv=raw_qkv,
conv_state=conv_state,
recurrent_checkpoint=state_checkpoints,
recurrent_state=ssm_state,
cu_seqlens=cu_seqlens,
recurrent_rows=torch.tensor(rows, device=device),
)
written = []
for row, (offset, block) in enumerate(zip(offsets, state_indices)):
if not offset:
continue
written.append(block)
seq = row - 1
start = prefill_cu[seq]
_, expected_state = fi_chunk_gated_delta_rule(
q=q[:, start : start + offset],
k=k[:, start : start + offset],
v=v[:, start : start + offset],
g=g[:, start : start + offset],
beta=beta[:, start : start + offset],
initial_state=initial_state[seq : seq + 1],
output_final_state=True,
cu_seqlens=torch.tensor([0, offset], device=device),
use_qk_l2norm_in_kernel=False,
)
torch.testing.assert_close(
ssm_state[block].float(), expected_state[0], atol=2e-2, rtol=2e-2
)
end = cu_seqlens[row].item() + offset
torch.testing.assert_close(
conv_state[block, :, :state_len],
raw_qkv[end - state_len : end].T,
atol=0,
rtol=0,
)
assert not conv_state[block, :, state_len:].any()
untouched = [i for i in range(num_blocks) if i not in written]
assert not ssm_state[untouched].any()
assert not conv_state[untouched].any()
@@ -178,7 +178,9 @@ def test_rocm_prefill_numerics_and_output_layout(state_dtype, correlated):
model="Qwen/Qwen3.5-0.8B",
revision="2fc06364715b967f1860aea9cf38778875588b17",
max_model_len=1024,
)
),
# auto picks AITER FlyDSL on gfx942/gfx950; this pins the Triton path.
additional_config={"gdn_prefill_backend": "triton"},
)
with set_current_vllm_config(config):
prefill = ChunkGatedDeltaRule()
@@ -114,23 +114,27 @@ def test_checkpoint_does_not_perturb_the_final_state():
)
@pytest.mark.parametrize("spec_slots", [0, 3])
@pytest.mark.parametrize("dim_first", [True, False])
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first):
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first, spec_slots):
"""The store reads the checkpoint row out of every chunk state.
Row 0 checkpoints 32 tokens in, its state being chunk 3 rather than the
program's own row, so the store only passes if ``chunk_idx`` is used.
Row 1 declines and must write nothing. The conv input is a column slice of
a wider projection, the SD conv layout is a transposed view, and SSM rows
are padded pages, matching the strides the mixer passes.
are padded pages, matching the strides the mixer passes. Spec decoding
widens the conv state, and the store must still fill only the first
``state_len`` slots with the newest inputs.
"""
dim, state_len, heads, head_dim, d_state = 16, 4, 2, 4, 8
projection = torch.randn(64, dim + 8, device=DEVICE)
conv_input = projection[:, :dim]
num_slots = state_len + spec_slots
if dim_first:
conv_state = torch.zeros(3, dim, state_len, device=DEVICE)
conv_state = torch.zeros(3, dim, num_slots, device=DEVICE)
else:
conv_state = torch.zeros(3, state_len, dim, device=DEVICE).transpose(-1, -2)
conv_state = torch.zeros(3, num_slots, dim, device=DEVICE).transpose(-1, -2)
varlen_states = torch.randn(5, heads, head_dim, d_state, device=DEVICE)
# Blocks are pages holding more than the SSM state, so rows are padded.
row = heads * head_dim * d_state
@@ -152,12 +156,14 @@ def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first):
varlen_states,
ssm_state,
torch.tensor([0, 40, 64], dtype=torch.int32, device=DEVICE),
state_len=state_len,
)
exact = dict(rtol=0, atol=0)
torch.testing.assert_close(
conv_state[2], conv_input[32 - state_len : 32].T, **exact
conv_state[2, :, :state_len], conv_input[32 - state_len : 32].T, **exact
)
assert not conv_state[2, :, state_len:].any(), "wrote into the spec slots"
torch.testing.assert_close(ssm_state[2], varlen_states[3], **exact)
assert not conv_state[:2].any() and not ssm_storage[:2].any()
assert not ssm_storage[:, row:].any(), "wrote into the page padding"
+6 -4
View File
@@ -96,15 +96,17 @@ def mixer2_gated_norm_tensor_parallel(
gate_states = torch.randn(batch_size, seq_len, hidden_size)
# create gated-norm with TP
mixer = Mixer2RMSNormGated(
full_hidden_size=hidden_size,
full_n_groups=n_groups,
)
with ensure_current_vllm_config():
mixer = Mixer2RMSNormGated(
full_hidden_size=hidden_size,
full_n_groups=n_groups,
)
mixer.weight.weight_loader(mixer.weight, weight) # load
# create gated-norm without TP to compute reference
# - utilize mock patching to disable TP when
with (
ensure_current_vllm_config(),
unittest.mock.patch(
"vllm.model_executor.layers.mamba.mamba_mixer2."
"get_tensor_model_parallel_world_size",
+24
View File
@@ -637,3 +637,27 @@ def test_replayssm_physical_ring_shape(
(8, expected_ring_len),
(2, expected_ring_len, 16),
)
def test_replayssm_ring_shape_extends_groups_for_head_shards():
"""A single group under TP is replicated to every head shard, as in
``mamba2_state_shape``, so each rank's B_cache holds one group."""
base_shapes = MambaStateShapeCalculator.mamba2_state_shape(
tp_world_size=2,
intermediate_size=256,
n_groups=1,
num_heads=4,
head_dim=64,
state_size=128,
conv_kernel=4,
)
shapes = MambaStateShapeCalculator.append_replayssm_ring(
base_shapes,
n_groups=1,
tp_world_size=2,
logical_window=16,
backend=MambaBackendEnum.TRITON,
)
assert shapes[4] == (1, 16, 128)