Merge branch 'main' into patches

This commit is contained in:
2026-10-17 08:06:56 +00:00
222 changed files with 14823 additions and 3606 deletions
-40
View File
@@ -353,46 +353,6 @@ def test_silu_and_mul_with_clamp(
)
@pytest.mark.parametrize("linear_beta", [-1.0, 2.0])
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16])
@torch.inference_mode()
def test_masked_situ_and_mul(
default_vllm_config,
linear_beta: float,
dtype: torch.dtype,
) -> None:
"""Masked SITU computes valid expert rows and preserves padded zeros."""
device = CUDA_DEVICES[0]
num_experts, max_num_tokens, d = 4, 7, 512
beta = 1.5
input = torch.randn(num_experts, max_num_tokens, 2 * d, dtype=dtype, device=device)
expert_num_tokens = torch.tensor([0, 1, 4, 7], dtype=torch.int32, device=device)
output = torch.zeros(num_experts, max_num_tokens, d, dtype=dtype, device=device)
torch.ops._C.masked_situ_and_mul(
output, input, expert_num_tokens, beta, linear_beta
)
gate, up = input.float().chunk(2, dim=-1)
expected = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
if linear_beta > 0:
up = linear_beta * torch.tanh(up / linear_beta)
expected = (expected * up).to(dtype)
for expert, num_tokens in enumerate(expert_num_tokens.cpu().tolist()):
torch.testing.assert_close(
output[expert, :num_tokens],
expected[expert, :num_tokens],
atol=get_default_atol(output),
rtol=get_default_rtol(output),
)
assert torch.count_nonzero(output[expert, num_tokens:]) == 0
opcheck(
torch.ops._C.masked_situ_and_mul,
(output, input, expert_num_tokens, beta, linear_beta),
)
MOE_ACTIVATION_CASES = [
pytest.param(MoEActivation.SILU, ApplyMoEActivationConfig(), id="silu"),
pytest.param(
+112
View File
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterator
import pytest
import torch
import torch.nn.functional as F
@@ -358,3 +360,113 @@ def test_causal_conv1d_varlen(
)
unpadded_out = out[:, : out_ref_tensor.shape[-1]]
assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol)
@pytest.mark.parametrize("itype", [torch.bfloat16])
@pytest.mark.parametrize("silu_activation", [True])
@pytest.mark.parametrize("has_bias", [True])
@pytest.mark.parametrize("width", [4])
@pytest.mark.parametrize("dim", [64, 2048])
@pytest.mark.parametrize("q_lens_cfg", [[3, 1, 0, 4, 2], [1, 5, 5, 1], [2, 0, 0, 3]])
@pytest.mark.skipif(
current_platform.is_cpu(),
reason="CPU kernel for causal_conv1d_update ignores num_accepted_tokens.",
)
def test_causal_conv1d_update_ragged_varlen(
q_lens_cfg, dim, width, has_bias, silu_activation, itype
):
mk_inputs = lambda: _create_ragged_varlen_inputs(
q_lens_cfg, dim, width, has_bias, silu_activation, itype
)
inputs, state = mk_inputs()
actual = causal_conv1d_update(conv_state=state, **inputs)
ref_inputs, ref_state = mk_inputs()
ref_outs = _create_ragged_varlen_ref_outs(conv_state=ref_state, **ref_inputs)
for i, ref_out in enumerate(ref_outs):
start = int(inputs["query_start_loc"][i])
end = int(inputs["query_start_loc"][i + 1])
entry = inputs["conv_state_indices"][i : i + 1]
assert torch.allclose(actual[start:end], ref_out, rtol=1e-2, atol=5e-2)
assert torch.equal(state[entry], ref_state[entry])
def _create_ragged_varlen_inputs(
q_lens_cfg, dim, width, has_bias, silu_activation, itype
):
device = DEVICE
set_random_seed(0)
batch_size = len(q_lens_cfg)
max_query_len = 5
assert max(q_lens_cfg) <= max_query_len
state_len = width - 1 + max_query_len
total_entries = 10 * batch_size
q_lens = torch.tensor(q_lens_cfg, dtype=torch.int32, device=device)
query_start_loc = torch.cat(
[
torch.zeros(1, dtype=torch.int32, device=device),
torch.cumsum(q_lens, dim=0, dtype=torch.int32),
]
)
total_tokens = int(q_lens.sum())
x = torch.randn(total_tokens, dim, device=device, dtype=itype)
conv_state = torch.randn(
total_entries, state_len, dim, device=device, dtype=itype
).transpose(1, 2)
weight = torch.randn(dim, width, device=device, dtype=itype)
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
activation = None if not silu_activation else "silu"
conv_state_indices = (
torch.randperm(total_entries - 1, device=device)[:batch_size] + 1
).to(torch.int32)
num_accepted_tokens = torch.randint(
1, max_query_len + 1, (batch_size,), dtype=torch.int32, device=device
)
inputs = dict(
x=x,
weight=weight,
bias=bias,
activation=activation,
conv_state_indices=conv_state_indices,
num_accepted_tokens=num_accepted_tokens,
query_start_loc=query_start_loc,
max_query_len=max_query_len,
)
return inputs, conv_state
def _create_ragged_varlen_ref_outs(
x,
conv_state,
weight,
bias,
activation,
conv_state_indices,
num_accepted_tokens,
query_start_loc,
max_query_len,
) -> Iterator[torch.Tensor]:
for seq_idx in range(len(conv_state_indices)):
start = int(query_start_loc[seq_idx])
end = int(query_start_loc[seq_idx + 1])
entry = conv_state_indices[seq_idx : seq_idx + 1]
if start == end:
yield x[start:end]
continue
out_ref = causal_conv1d_update(
x[start:end],
conv_state,
weight,
bias,
activation=activation,
conv_state_indices=entry,
num_accepted_tokens=num_accepted_tokens[seq_idx : seq_idx + 1],
query_start_loc=query_start_loc.new_tensor([0, end - start]),
max_query_len=max_query_len,
)
yield out_ref
+47
View File
@@ -6,6 +6,10 @@ import torch
import torch.nn.functional as F
from einops import rearrange, repeat
from vllm.model_executor.layers.mamba.ops.ssd_chunk_scan import (
_chunk_scan_fwd,
_chunk_scan_fwd_kernel,
)
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
mamba_chunk_scan_combined_varlen,
)
@@ -578,3 +582,46 @@ def test_mamba_chunk_scan_cont_batch_prefill_chunking(chunk_size, seqlens):
rtol=rtol,
msg=lambda x, i=i: f"seq{i} state " + x,
)
def test_chunk_scan_fwd_every_config_matches(monkeypatch):
"""The autotuner picks a config by speed only, so every config in the list
must give the same finite output. On sm90, configs that pipeline a wgmma
with `cb` in registers wrote NaN into whole rows (#61603)."""
from triton.runtime.errors import OutOfResources
set_random_seed(0)
nheads, headdim, ngroups, dstate, chunk_size = 64, 64, 8, 128, 128
cu_chunk_seqlens = torch.tensor([0, 128, 181], dtype=torch.int32, device=DEVICE)
seqlen, nchunks = 181, 2
x = torch.randn(seqlen, nheads, headdim, dtype=torch.bfloat16, device=DEVICE)
C = torch.randn(seqlen, ngroups, dstate, dtype=torch.bfloat16, device=DEVICE)
cb = torch.randn(nchunks, ngroups, chunk_size, chunk_size, device=DEVICE)
dt = torch.rand(nheads, nchunks, chunk_size, device=DEVICE) * 0.1
dA_cumsum = -torch.rand(nheads, nchunks, chunk_size, device=DEVICE).cumsum(-1)
states = torch.randn(nchunks, nheads, headdim, dstate, device=DEVICE) * 0.1
seq_idx = torch.zeros(nchunks, dtype=torch.int32, device=DEVICE)
D = torch.randn(nheads, headdim, device=DEVICE) * 0.1
ref = None
for config in list(_chunk_scan_fwd_kernel.configs):
monkeypatch.setattr(_chunk_scan_fwd_kernel, "configs", [config])
for _ in range(10):
out = torch.empty_like(x)
try:
_chunk_scan_fwd(
cb, x, dt, dA_cumsum, C, states, cu_chunk_seqlens, out, seq_idx, D
)
except OutOfResources:
# The autotuner skips configs that do not fit this device.
break
assert torch.isfinite(out).all(), f"non-finite output with {config}"
if ref is None:
ref = out
torch.testing.assert_close(
out,
ref,
atol=2e-2,
rtol=2e-2,
msg=lambda m, config=config: f"{config}: {m}",
)
+13 -1
View File
@@ -10,7 +10,6 @@ import torch.nn.functional as F
import vllm.model_executor.kernels.mhc # noqa: F401
import vllm.model_executor.layers.mhc as mhc_layers
from vllm.model_executor.kernels.mhc.tilelang import (
_torch_hc_prenorm_gemm,
mhc_post_tilelang,
mhc_pre_delayed_tilelang,
)
@@ -1002,6 +1001,19 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult, fused_norm):
torch.testing.assert_close(actual, expected, atol=5e-2, rtol=1e-2)
def _torch_hc_prenorm_gemm(
x: torch.Tensor,
fn: torch.Tensor,
out: torch.Tensor,
sqrsum: torch.Tensor,
) -> None:
assert out.shape[0] == 1
assert sqrsum.shape[0] == 1
x_float = x.float()
out[0].copy_(x_float @ fn.t())
sqrsum[0].copy_(x_float.square().sum(dim=-1))
@pytest.mark.skipif(
not HAS_TILELANG_MHC,
reason="TileLang MHC support required",
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
import os
import time
import traceback
from collections.abc import Callable
from typing import Any, Concatenate
@@ -23,6 +24,8 @@ from vllm.utils.network_utils import get_open_port
P = ParamSpec("P")
LAUNCH_TIMEOUT_S = 900
@dataclasses.dataclass
class ProcessGroupInfo:
@@ -148,7 +151,7 @@ def parallel_launch_with_config(
*args: P.args,
**kwargs: P.kwargs,
) -> None:
spawn(
context = spawn(
_worker_parallel_launch,
args=(
world_size,
@@ -162,5 +165,16 @@ def parallel_launch_with_config(
)
+ args,
nprocs=world_size,
join=True,
join=False,
)
# A worker stuck in communicator setup never exits, so bound the wait.
deadline = time.monotonic() + LAUNCH_TIMEOUT_S
while not context.join(timeout=max(0.0, deadline - time.monotonic())):
if time.monotonic() >= deadline:
for process in context.processes:
process.kill()
for process in context.processes:
process.join()
raise TimeoutError(
f"Parallel workers did not finish within {LAUNCH_TIMEOUT_S}s"
)
+11 -1
View File
@@ -127,6 +127,14 @@ if has_nixl_ep():
BACKENDS += ["nixl_ep"]
DEEPEP_BACKENDS = {"deepep_high_throughput", "deepep_low_latency"}
# These tests run on one node, where NVSHMEM reaches peers over NVLink, so
# keep it off the NICs to avoid depending on IB setup on shared CI hosts.
DEEPEP_SINGLE_NODE_NVSHMEM_ENV = {
"NVSHMEM_REMOTE_TRANSPORT": "none",
# DeepEP forces IBGDA on, so also hide the NICs from it.
"NVSHMEM_HCA_LIST": "none",
}
MORI_BACKENDS = {"mori_high_throughput", "mori_low_latency"}
QUANT_METHODS = [
@@ -2187,6 +2195,8 @@ def test_moe_layer(
rocm_deepep = current_platform.is_rocm() and backend in DEEPEP_BACKENDS
parallel_worker = _parallel_worker_rocm_deepep if rocm_deepep else _parallel_worker
cuda_deepep = current_platform.is_cuda() and backend in DEEPEP_BACKENDS
env_dict = DEEPEP_SINGLE_NODE_NVSHMEM_ENV if cuda_deepep else None
launch_failures: list[str] = []
try:
@@ -2197,7 +2207,7 @@ def test_moe_layer(
world_size,
parallel_worker,
vllm_config,
None,
env_dict,
test_config_batch,
verbosity,
failure_report_path=failure_report_path,
+14
View File
@@ -582,6 +582,20 @@ def test_fused_topk_bias(
assert_routing_results_close(topk_weights, topk_ids, baseline_weights, baseline_ids)
@pytest.mark.parametrize("scoring_func", [0, 1])
def test_fused_grouped_topk_accepts_idle_rank(scoring_func):
"""Zero-token routing must leave CUDA ready for the next batch."""
if not current_platform.is_cuda():
pytest.skip("CUDA required")
from vllm import _custom_ops as ops
scores = torch.empty((0, 8), device="cuda", dtype=torch.float32)
bias = torch.zeros(8, device="cuda")
weights, ids = ops.grouped_topk(scores, 2, 1, 2, True, 1.0, bias, scoring_func)
torch.testing.assert_close(weights, scores.new_empty((0, 2)))
torch.testing.assert_close(ids, scores.new_empty((0, 2), dtype=torch.int32))
@pytest.mark.parametrize("m,k", MK_S)
@pytest.mark.parametrize("top_k", TOP_KS)
@pytest.mark.parametrize(
@@ -116,7 +116,7 @@ def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
@pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"])
@pytest.mark.parametrize("hidden_size", [256, 2048])
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA graph test")
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm graph test")
def test_silu_quant_skips_expert_padding_on_graph_replay(
counts: list[int],
group_size: int,
@@ -135,7 +135,10 @@ def test_silu_quant_skips_expert_padding_on_graph_replay(
x *= 16
clean_x = x.clone()
ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32)
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=FLOAT8_DTYPE)
output_dtype = (
FLOAT8_DTYPE if scale_format == "packed" else current_platform.fp8_dtype()
)
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=output_dtype)
def set_counts(values):
positions = []
@@ -187,6 +190,7 @@ def test_silu_quant_skips_expert_padding_on_graph_replay(
output.view(torch.uint8).fill_(91)
graph.replay()
reference_q, reference_s = quantize(x)
assert actual_q.dtype == reference_q.dtype
torch.testing.assert_close(
actual_q.view(torch.uint8)[live],
reference_q.view(torch.uint8)[live],
+57 -15
View File
@@ -3,11 +3,11 @@
"""Tests for fp32_router_gemm kernel: activation×weight→fp32.
Supported (hidden_size, num_experts) pairs:
(3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
(3072, 256), (6144, 128), (5120, 256), (4096, 352), (8192, 256)
Correctness baseline: F.linear in float32. Every M in [1, 32] is covered so
Correctness baseline: F.linear in float32. Every supported M is covered so
all tuned geometries (wide-block, experts-per-block, token-group; boundaries
at M=4/5, odd/even, M=15/16) are exercised on Blackwell, and the legacy
at M=4/5, M=9/10, M=12/13, odd/even, M=15/16) are exercised on Blackwell, and the legacy
128/1 geometry everywhere else.
"""
@@ -18,8 +18,12 @@ from vllm._custom_ops import fp32_router_gemm
from vllm.platforms import current_platform
# (hidden_size, num_experts)
SHAPES = [(3072, 256), (6144, 128)]
ALL_M = list(range(1, 33))
SHAPES = [(3072, 256), (6144, 128), (5120, 256), (4096, 352), (8192, 256)]
CASES = [
(m, hidden, experts)
for hidden, experts in SHAPES
for m in range(1, 49 if hidden == 5120 else 33)
]
# Absolute tolerance for fp32 kernel vs float64 reference
ATOL_FP32 = 2e-4
ATOL_BF16 = 2e-2 # bf16 activation has lower precision
@@ -40,8 +44,7 @@ def _ref(mat_a: torch.Tensor, mat_b: torch.Tensor) -> torch.Tensor:
return torch.nn.functional.linear(mat_a.float(), mat_b.float())
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
@pytest.mark.parametrize("num_tokens", ALL_M)
@pytest.mark.parametrize("num_tokens,hidden_dim,num_experts", CASES)
def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
"""fp32 activation → fp32 output should match reference closely."""
_requires_sm90()
@@ -58,8 +61,7 @@ def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
torch.testing.assert_close(out, ref, atol=ATOL_FP32, rtol=0)
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
@pytest.mark.parametrize("num_tokens", ALL_M)
@pytest.mark.parametrize("num_tokens,hidden_dim,num_experts", CASES)
def test_bf16_activation(num_tokens: int, hidden_dim: int, num_experts: int):
"""bf16 activation → fp32 output should match reference within bf16 error."""
_requires_sm90()
@@ -124,11 +126,11 @@ def test_topk_routing_consistency(num_tokens: int, hidden_dim: int, num_experts:
)
def test_zero_tokens_returns_empty():
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
def test_zero_tokens_returns_empty(hidden_dim, num_experts):
"""M=0 is a graceful no-op returning an empty [0, E] tensor."""
_requires_sm90()
device = torch.device("cuda")
hidden_dim, num_experts = SHAPES[0]
mat_a = torch.empty(0, hidden_dim, dtype=torch.float32, device=device)
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
out = fp32_router_gemm(mat_a, mat_b)
@@ -136,17 +138,19 @@ def test_zero_tokens_returns_empty():
assert out.dtype == torch.float32
def test_rejects_invalid_inputs():
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
def test_rejects_invalid_inputs(hidden_dim, num_experts):
"""The entry must fail loudly, never compute silently wrong results."""
_requires_sm90()
device = torch.device("cuda")
hidden_dim, num_experts = SHAPES[0]
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
# num_tokens > 32 (beyond the instantiated range)
# Beyond the instantiated range for this shape.
invalid_tokens = 49 if hidden_dim == 5120 else 33
with pytest.raises(Exception, match="num_tokens"):
fp32_router_gemm(
torch.randn(33, hidden_dim, dtype=torch.float32, device=device), mat_b
torch.randn(invalid_tokens, hidden_dim, dtype=torch.float32, device=device),
mat_b,
)
# unsupported (hidden_dim, num_experts) pair
@@ -173,3 +177,41 @@ def test_rejects_invalid_inputs():
fp32_router_gemm(
torch.randn(4, hidden_dim, dtype=torch.float16, device=device), mat_b
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
@pytest.mark.parametrize(
"tokens", [*range(1, 17), 17, 24, 32, 33, 40, 48, 56, 64, 65, 128]
)
@pytest.mark.parametrize(
"hidden_dim,num_experts", [(5120, 256), (4096, 352), (8192, 256)]
)
@torch.inference_mode()
def test_router_retains_fp32_weights_and_logits(
default_vllm_config, dist_init, tokens, hidden_dim, num_experts
):
"""Low-latency routing must retain FP32 weight precision and logits."""
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
from vllm.platforms import current_platform
if not current_platform.is_device_capability_family(100):
pytest.skip("SM100 required")
torch.manual_seed(42)
router = GateLinear(
hidden_dim, num_experts, params_dtype=torch.float32, out_dtype=torch.float32
).cuda()
router.weight.normal_(std=0.01)
x = torch.randn(tokens, hidden_dim, device="cuda", dtype=torch.bfloat16)
expected = (x.double() @ router.weight.double().T).float()
actual, bias = router(x)
assert bias is None and actual.dtype == torch.float32
torch.testing.assert_close(actual, expected, rtol=2e-5, atol=2e-5)
torch.testing.assert_close(actual.topk(8).indices, expected.topk(8).indices)
if tokens in (1, 16, 32, 33, 48, 56):
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
captured, _ = router(x)
x.normal_()
graph.replay()
expected = (x.double() @ router.weight.double().T).float()
torch.testing.assert_close(captured, expected, rtol=2e-5, atol=2e-5)
+25 -1
View File
@@ -566,7 +566,7 @@ def test_invalid_output_dtype():
with pytest.raises(ValueError, match="output_dtype=torch.float32"):
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
ll_bf16_gemm(a, b, output_dtype=torch.bfloat16)
ll_bf16_gemm(a, b, output_dtype=torch.float16)
def test_cache_miss_compiles_dotprod():
@@ -581,5 +581,29 @@ def test_cache_miss_compiles_dotprod():
_assert_close(out, _ref(a, b), context="cache miss dotprod")
@pytest.mark.parametrize("M", [1, 2, 8, 16])
@pytest.mark.parametrize("N", [1792, 3584])
@torch.inference_mode()
def test_bf16_output_and_fp32_cache_coexist(M, N):
"""BF16 epilogues retain FP32 accumulation and can replay with new inputs."""
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
torch.manual_seed(42)
x = torch.randn(M, 5120, device="cuda", dtype=torch.bfloat16)
w = torch.randn(N, 5120, device="cuda", dtype=torch.bfloat16) * 0.01
for dtype in (torch.float32, torch.bfloat16):
actual = ll_bf16_gemm(x, w, output_dtype=dtype)
assert actual.dtype == dtype
reference = (x.double() @ w.double().T).to(dtype)
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
actual = ll_bf16_gemm(x, w, output_dtype=torch.bfloat16)
x.normal_()
graph.replay()
reference = (x.double() @ w.double().T).bfloat16()
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
if __name__ == "__main__":
pytest.main([__file__, "-v"])