Merge branch 'main' into patches

This commit is contained in:
2026-10-17 08:06:56 +00:00
222 changed files with 14823 additions and 3606 deletions
+112
View File
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
import pytest
import torch
import torch.nn.functional as F
@@ -358,3 +360,113 @@ def test_causal_conv1d_varlen(
)
unpadded_out = out[:, : out_ref_tensor.shape[-1]]
assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol)
@pytest.mark.parametrize("itype", [torch.bfloat16])
@pytest.mark.parametrize("silu_activation", [True])
@pytest.mark.parametrize("has_bias", [True])
@pytest.mark.parametrize("width", [4])
@pytest.mark.parametrize("dim", [64, 2048])
@pytest.mark.parametrize("q_lens_cfg", [[3, 1, 0, 4, 2], [1, 5, 5, 1], [2, 0, 0, 3]])
@pytest.mark.skipif(
current_platform.is_cpu(),
reason="CPU kernel for causal_conv1d_update ignores num_accepted_tokens.",
)
def test_causal_conv1d_update_ragged_varlen(
q_lens_cfg, dim, width, has_bias, silu_activation, itype
):
mk_inputs = lambda: _create_ragged_varlen_inputs(
q_lens_cfg, dim, width, has_bias, silu_activation, itype
)
inputs, state = mk_inputs()
actual = causal_conv1d_update(conv_state=state, **inputs)
ref_inputs, ref_state = mk_inputs()
ref_outs = _create_ragged_varlen_ref_outs(conv_state=ref_state, **ref_inputs)
for i, ref_out in enumerate(ref_outs):
start = int(inputs["query_start_loc"][i])
end = int(inputs["query_start_loc"][i + 1])
entry = inputs["conv_state_indices"][i : i + 1]
assert torch.allclose(actual[start:end], ref_out, rtol=1e-2, atol=5e-2)
assert torch.equal(state[entry], ref_state[entry])
def _create_ragged_varlen_inputs(
q_lens_cfg, dim, width, has_bias, silu_activation, itype
):
device = DEVICE
set_random_seed(0)
batch_size = len(q_lens_cfg)
max_query_len = 5
assert max(q_lens_cfg) <= max_query_len
state_len = width - 1 + max_query_len
total_entries = 10 * batch_size
q_lens = torch.tensor(q_lens_cfg, dtype=torch.int32, device=device)
query_start_loc = torch.cat(
[
torch.zeros(1, dtype=torch.int32, device=device),
torch.cumsum(q_lens, dim=0, dtype=torch.int32),
]
)
total_tokens = int(q_lens.sum())
x = torch.randn(total_tokens, dim, device=device, dtype=itype)
conv_state = torch.randn(
total_entries, state_len, dim, device=device, dtype=itype
).transpose(1, 2)
weight = torch.randn(dim, width, device=device, dtype=itype)
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
activation = None if not silu_activation else "silu"
conv_state_indices = (
torch.randperm(total_entries - 1, device=device)[:batch_size] + 1
).to(torch.int32)
num_accepted_tokens = torch.randint(
1, max_query_len + 1, (batch_size,), dtype=torch.int32, device=device
)
inputs = dict(
x=x,
weight=weight,
bias=bias,
activation=activation,
conv_state_indices=conv_state_indices,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc,
max_query_len=max_query_len,
)
return inputs, conv_state
def _create_ragged_varlen_ref_outs(
x,
conv_state,
weight,
bias,
activation,
conv_state_indices,
num_accepted_tokens,
query_start_loc,
max_query_len,
) -> Iterator[torch.Tensor]:
for seq_idx in range(len(conv_state_indices)):
start = int(query_start_loc[seq_idx])
end = int(query_start_loc[seq_idx + 1])
entry = conv_state_indices[seq_idx : seq_idx + 1]
if start == end:
yield x[start:end]
continue
out_ref = causal_conv1d_update(
x[start:end],
conv_state,
weight,
bias,
activation=activation,
conv_state_indices=entry,
num_accepted_tokens=num_accepted_tokens[seq_idx : seq_idx + 1],
query_start_loc=query_start_loc.new_tensor([0, end - start]),
max_query_len=max_query_len,
)
yield out_ref
+47
View File
@@ -6,6 +6,10 @@ import torch
import torch.nn.functional as F
from einops import rearrange, repeat
from vllm.model_executor.layers.mamba.ops.ssd_chunk_scan import (
_chunk_scan_fwd,
_chunk_scan_fwd_kernel,
)
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
@@ -578,3 +582,46 @@ def test_mamba_chunk_scan_cont_batch_prefill_chunking(chunk_size, seqlens):
rtol=rtol,
msg=lambda x, i=i: f"seq{i} state " + x,
)
def test_chunk_scan_fwd_every_config_matches(monkeypatch):
"""The autotuner picks a config by speed only, so every config in the list
must give the same finite output. On sm90, configs that pipeline a wgmma
with `cb` in registers wrote NaN into whole rows (#61603)."""
from triton.runtime.errors import OutOfResources
set_random_seed(0)
nheads, headdim, ngroups, dstate, chunk_size = 64, 64, 8, 128, 128
cu_chunk_seqlens = torch.tensor([0, 128, 181], dtype=torch.int32, device=DEVICE)
seqlen, nchunks = 181, 2
x = torch.randn(seqlen, nheads, headdim, dtype=torch.bfloat16, device=DEVICE)
C = torch.randn(seqlen, ngroups, dstate, dtype=torch.bfloat16, device=DEVICE)
cb = torch.randn(nchunks, ngroups, chunk_size, chunk_size, device=DEVICE)
dt = torch.rand(nheads, nchunks, chunk_size, device=DEVICE) * 0.1
dA_cumsum = -torch.rand(nheads, nchunks, chunk_size, device=DEVICE).cumsum(-1)
states = torch.randn(nchunks, nheads, headdim, dstate, device=DEVICE) * 0.1
seq_idx = torch.zeros(nchunks, dtype=torch.int32, device=DEVICE)
D = torch.randn(nheads, headdim, device=DEVICE) * 0.1
ref = None
for config in list(_chunk_scan_fwd_kernel.configs):
monkeypatch.setattr(_chunk_scan_fwd_kernel, "configs", [config])
for _ in range(10):
out = torch.empty_like(x)
try:
_chunk_scan_fwd(
cb, x, dt, dA_cumsum, C, states, cu_chunk_seqlens, out, seq_idx, D
)
except OutOfResources:
# The autotuner skips configs that do not fit this device.
break
assert torch.isfinite(out).all(), f"non-finite output with {config}"
if ref is None:
ref = out
torch.testing.assert_close(
out,
ref,
atol=2e-2,
rtol=2e-2,
msg=lambda m, config=config: f"{config}: {m}",
)