Merge branch 'main' into patches
This commit is contained in:
@@ -353,46 +353,6 @@ def test_silu_and_mul_with_clamp(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("linear_beta", [-1.0, 2.0])
|
||||
@pytest.mark.parametrize("dtype", [torch.half, torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_masked_situ_and_mul(
|
||||
default_vllm_config,
|
||||
linear_beta: float,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Masked SITU computes valid expert rows and preserves padded zeros."""
|
||||
device = CUDA_DEVICES[0]
|
||||
num_experts, max_num_tokens, d = 4, 7, 512
|
||||
beta = 1.5
|
||||
input = torch.randn(num_experts, max_num_tokens, 2 * d, dtype=dtype, device=device)
|
||||
expert_num_tokens = torch.tensor([0, 1, 4, 7], dtype=torch.int32, device=device)
|
||||
output = torch.zeros(num_experts, max_num_tokens, d, dtype=dtype, device=device)
|
||||
|
||||
torch.ops._C.masked_situ_and_mul(
|
||||
output, input, expert_num_tokens, beta, linear_beta
|
||||
)
|
||||
|
||||
gate, up = input.float().chunk(2, dim=-1)
|
||||
expected = beta * torch.tanh(gate / beta) * torch.sigmoid(gate)
|
||||
if linear_beta > 0:
|
||||
up = linear_beta * torch.tanh(up / linear_beta)
|
||||
expected = (expected * up).to(dtype)
|
||||
for expert, num_tokens in enumerate(expert_num_tokens.cpu().tolist()):
|
||||
torch.testing.assert_close(
|
||||
output[expert, :num_tokens],
|
||||
expected[expert, :num_tokens],
|
||||
atol=get_default_atol(output),
|
||||
rtol=get_default_rtol(output),
|
||||
)
|
||||
assert torch.count_nonzero(output[expert, num_tokens:]) == 0
|
||||
|
||||
opcheck(
|
||||
torch.ops._C.masked_situ_and_mul,
|
||||
(output, input, expert_num_tokens, beta, linear_beta),
|
||||
)
|
||||
|
||||
|
||||
MOE_ACTIVATION_CASES = [
|
||||
pytest.param(MoEActivation.SILU, ApplyMoEActivationConfig(), id="silu"),
|
||||
pytest.param(
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from collections.abc import Iterator
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
@@ -358,3 +360,113 @@ def test_causal_conv1d_varlen(
|
||||
)
|
||||
unpadded_out = out[:, : out_ref_tensor.shape[-1]]
|
||||
assert torch.allclose(unpadded_out, out_ref_tensor, rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("itype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("silu_activation", [True])
|
||||
@pytest.mark.parametrize("has_bias", [True])
|
||||
@pytest.mark.parametrize("width", [4])
|
||||
@pytest.mark.parametrize("dim", [64, 2048])
|
||||
@pytest.mark.parametrize("q_lens_cfg", [[3, 1, 0, 4, 2], [1, 5, 5, 1], [2, 0, 0, 3]])
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_cpu(),
|
||||
reason="CPU kernel for causal_conv1d_update ignores num_accepted_tokens.",
|
||||
)
|
||||
def test_causal_conv1d_update_ragged_varlen(
|
||||
q_lens_cfg, dim, width, has_bias, silu_activation, itype
|
||||
):
|
||||
mk_inputs = lambda: _create_ragged_varlen_inputs(
|
||||
q_lens_cfg, dim, width, has_bias, silu_activation, itype
|
||||
)
|
||||
inputs, state = mk_inputs()
|
||||
actual = causal_conv1d_update(conv_state=state, **inputs)
|
||||
|
||||
ref_inputs, ref_state = mk_inputs()
|
||||
ref_outs = _create_ragged_varlen_ref_outs(conv_state=ref_state, **ref_inputs)
|
||||
|
||||
for i, ref_out in enumerate(ref_outs):
|
||||
start = int(inputs["query_start_loc"][i])
|
||||
end = int(inputs["query_start_loc"][i + 1])
|
||||
entry = inputs["conv_state_indices"][i : i + 1]
|
||||
assert torch.allclose(actual[start:end], ref_out, rtol=1e-2, atol=5e-2)
|
||||
assert torch.equal(state[entry], ref_state[entry])
|
||||
|
||||
|
||||
def _create_ragged_varlen_inputs(
|
||||
q_lens_cfg, dim, width, has_bias, silu_activation, itype
|
||||
):
|
||||
device = DEVICE
|
||||
set_random_seed(0)
|
||||
|
||||
batch_size = len(q_lens_cfg)
|
||||
max_query_len = 5
|
||||
assert max(q_lens_cfg) <= max_query_len
|
||||
state_len = width - 1 + max_query_len
|
||||
total_entries = 10 * batch_size
|
||||
|
||||
q_lens = torch.tensor(q_lens_cfg, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.cat(
|
||||
[
|
||||
torch.zeros(1, dtype=torch.int32, device=device),
|
||||
torch.cumsum(q_lens, dim=0, dtype=torch.int32),
|
||||
]
|
||||
)
|
||||
total_tokens = int(q_lens.sum())
|
||||
|
||||
x = torch.randn(total_tokens, dim, device=device, dtype=itype)
|
||||
conv_state = torch.randn(
|
||||
total_entries, state_len, dim, device=device, dtype=itype
|
||||
).transpose(1, 2)
|
||||
weight = torch.randn(dim, width, device=device, dtype=itype)
|
||||
bias = torch.randn(dim, device=device, dtype=itype) if has_bias else None
|
||||
activation = None if not silu_activation else "silu"
|
||||
|
||||
conv_state_indices = (
|
||||
torch.randperm(total_entries - 1, device=device)[:batch_size] + 1
|
||||
).to(torch.int32)
|
||||
num_accepted_tokens = torch.randint(
|
||||
1, max_query_len + 1, (batch_size,), dtype=torch.int32, device=device
|
||||
)
|
||||
inputs = dict(
|
||||
x=x,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
activation=activation,
|
||||
conv_state_indices=conv_state_indices,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
return inputs, conv_state
|
||||
|
||||
|
||||
def _create_ragged_varlen_ref_outs(
|
||||
x,
|
||||
conv_state,
|
||||
weight,
|
||||
bias,
|
||||
activation,
|
||||
conv_state_indices,
|
||||
num_accepted_tokens,
|
||||
query_start_loc,
|
||||
max_query_len,
|
||||
) -> Iterator[torch.Tensor]:
|
||||
for seq_idx in range(len(conv_state_indices)):
|
||||
start = int(query_start_loc[seq_idx])
|
||||
end = int(query_start_loc[seq_idx + 1])
|
||||
entry = conv_state_indices[seq_idx : seq_idx + 1]
|
||||
if start == end:
|
||||
yield x[start:end]
|
||||
continue
|
||||
out_ref = causal_conv1d_update(
|
||||
x[start:end],
|
||||
conv_state,
|
||||
weight,
|
||||
bias,
|
||||
activation=activation,
|
||||
conv_state_indices=entry,
|
||||
num_accepted_tokens=num_accepted_tokens[seq_idx : seq_idx + 1],
|
||||
query_start_loc=query_start_loc.new_tensor([0, end - start]),
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
yield out_ref
|
||||
|
||||
@@ -6,6 +6,10 @@ import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange, repeat
|
||||
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_chunk_scan import (
|
||||
_chunk_scan_fwd,
|
||||
_chunk_scan_fwd_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
@@ -578,3 +582,46 @@ def test_mamba_chunk_scan_cont_batch_prefill_chunking(chunk_size, seqlens):
|
||||
rtol=rtol,
|
||||
msg=lambda x, i=i: f"seq{i} state " + x,
|
||||
)
|
||||
|
||||
|
||||
def test_chunk_scan_fwd_every_config_matches(monkeypatch):
|
||||
"""The autotuner picks a config by speed only, so every config in the list
|
||||
must give the same finite output. On sm90, configs that pipeline a wgmma
|
||||
with `cb` in registers wrote NaN into whole rows (#61603)."""
|
||||
from triton.runtime.errors import OutOfResources
|
||||
|
||||
set_random_seed(0)
|
||||
nheads, headdim, ngroups, dstate, chunk_size = 64, 64, 8, 128, 128
|
||||
cu_chunk_seqlens = torch.tensor([0, 128, 181], dtype=torch.int32, device=DEVICE)
|
||||
seqlen, nchunks = 181, 2
|
||||
x = torch.randn(seqlen, nheads, headdim, dtype=torch.bfloat16, device=DEVICE)
|
||||
C = torch.randn(seqlen, ngroups, dstate, dtype=torch.bfloat16, device=DEVICE)
|
||||
cb = torch.randn(nchunks, ngroups, chunk_size, chunk_size, device=DEVICE)
|
||||
dt = torch.rand(nheads, nchunks, chunk_size, device=DEVICE) * 0.1
|
||||
dA_cumsum = -torch.rand(nheads, nchunks, chunk_size, device=DEVICE).cumsum(-1)
|
||||
states = torch.randn(nchunks, nheads, headdim, dstate, device=DEVICE) * 0.1
|
||||
seq_idx = torch.zeros(nchunks, dtype=torch.int32, device=DEVICE)
|
||||
D = torch.randn(nheads, headdim, device=DEVICE) * 0.1
|
||||
|
||||
ref = None
|
||||
for config in list(_chunk_scan_fwd_kernel.configs):
|
||||
monkeypatch.setattr(_chunk_scan_fwd_kernel, "configs", [config])
|
||||
for _ in range(10):
|
||||
out = torch.empty_like(x)
|
||||
try:
|
||||
_chunk_scan_fwd(
|
||||
cb, x, dt, dA_cumsum, C, states, cu_chunk_seqlens, out, seq_idx, D
|
||||
)
|
||||
except OutOfResources:
|
||||
# The autotuner skips configs that do not fit this device.
|
||||
break
|
||||
assert torch.isfinite(out).all(), f"non-finite output with {config}"
|
||||
if ref is None:
|
||||
ref = out
|
||||
torch.testing.assert_close(
|
||||
out,
|
||||
ref,
|
||||
atol=2e-2,
|
||||
rtol=2e-2,
|
||||
msg=lambda m, config=config: f"{config}: {m}",
|
||||
)
|
||||
|
||||
@@ -10,7 +10,6 @@ import torch.nn.functional as F
|
||||
import vllm.model_executor.kernels.mhc # noqa: F401
|
||||
import vllm.model_executor.layers.mhc as mhc_layers
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
_torch_hc_prenorm_gemm,
|
||||
mhc_post_tilelang,
|
||||
mhc_pre_delayed_tilelang,
|
||||
)
|
||||
@@ -1002,6 +1001,19 @@ def test_mhc_pre_tilelang(num_tokens, hidden_size, hc_mult, fused_norm):
|
||||
torch.testing.assert_close(actual, expected, atol=5e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def _torch_hc_prenorm_gemm(
|
||||
x: torch.Tensor,
|
||||
fn: torch.Tensor,
|
||||
out: torch.Tensor,
|
||||
sqrsum: torch.Tensor,
|
||||
) -> None:
|
||||
assert out.shape[0] == 1
|
||||
assert sqrsum.shape[0] == 1
|
||||
x_float = x.float()
|
||||
out[0].copy_(x_float @ fn.t())
|
||||
sqrsum[0].copy_(x_float.square().sum(dim=-1))
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not HAS_TILELANG_MHC,
|
||||
reason="TileLang MHC support required",
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
import os
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Concatenate
|
||||
@@ -23,6 +24,8 @@ from vllm.utils.network_utils import get_open_port
|
||||
|
||||
P = ParamSpec("P")
|
||||
|
||||
LAUNCH_TIMEOUT_S = 900
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class ProcessGroupInfo:
|
||||
@@ -148,7 +151,7 @@ def parallel_launch_with_config(
|
||||
*args: P.args,
|
||||
**kwargs: P.kwargs,
|
||||
) -> None:
|
||||
spawn(
|
||||
context = spawn(
|
||||
_worker_parallel_launch,
|
||||
args=(
|
||||
world_size,
|
||||
@@ -162,5 +165,16 @@ def parallel_launch_with_config(
|
||||
)
|
||||
+ args,
|
||||
nprocs=world_size,
|
||||
join=True,
|
||||
join=False,
|
||||
)
|
||||
# A worker stuck in communicator setup never exits, so bound the wait.
|
||||
deadline = time.monotonic() + LAUNCH_TIMEOUT_S
|
||||
while not context.join(timeout=max(0.0, deadline - time.monotonic())):
|
||||
if time.monotonic() >= deadline:
|
||||
for process in context.processes:
|
||||
process.kill()
|
||||
for process in context.processes:
|
||||
process.join()
|
||||
raise TimeoutError(
|
||||
f"Parallel workers did not finish within {LAUNCH_TIMEOUT_S}s"
|
||||
)
|
||||
|
||||
@@ -127,6 +127,14 @@ if has_nixl_ep():
|
||||
BACKENDS += ["nixl_ep"]
|
||||
|
||||
DEEPEP_BACKENDS = {"deepep_high_throughput", "deepep_low_latency"}
|
||||
|
||||
# These tests run on one node, where NVSHMEM reaches peers over NVLink, so
|
||||
# keep it off the NICs to avoid depending on IB setup on shared CI hosts.
|
||||
DEEPEP_SINGLE_NODE_NVSHMEM_ENV = {
|
||||
"NVSHMEM_REMOTE_TRANSPORT": "none",
|
||||
# DeepEP forces IBGDA on, so also hide the NICs from it.
|
||||
"NVSHMEM_HCA_LIST": "none",
|
||||
}
|
||||
MORI_BACKENDS = {"mori_high_throughput", "mori_low_latency"}
|
||||
|
||||
QUANT_METHODS = [
|
||||
@@ -2187,6 +2195,8 @@ def test_moe_layer(
|
||||
|
||||
rocm_deepep = current_platform.is_rocm() and backend in DEEPEP_BACKENDS
|
||||
parallel_worker = _parallel_worker_rocm_deepep if rocm_deepep else _parallel_worker
|
||||
cuda_deepep = current_platform.is_cuda() and backend in DEEPEP_BACKENDS
|
||||
env_dict = DEEPEP_SINGLE_NODE_NVSHMEM_ENV if cuda_deepep else None
|
||||
launch_failures: list[str] = []
|
||||
|
||||
try:
|
||||
@@ -2197,7 +2207,7 @@ def test_moe_layer(
|
||||
world_size,
|
||||
parallel_worker,
|
||||
vllm_config,
|
||||
None,
|
||||
env_dict,
|
||||
test_config_batch,
|
||||
verbosity,
|
||||
failure_report_path=failure_report_path,
|
||||
|
||||
@@ -582,6 +582,20 @@ def test_fused_topk_bias(
|
||||
assert_routing_results_close(topk_weights, topk_ids, baseline_weights, baseline_ids)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scoring_func", [0, 1])
|
||||
def test_fused_grouped_topk_accepts_idle_rank(scoring_func):
|
||||
"""Zero-token routing must leave CUDA ready for the next batch."""
|
||||
if not current_platform.is_cuda():
|
||||
pytest.skip("CUDA required")
|
||||
from vllm import _custom_ops as ops
|
||||
|
||||
scores = torch.empty((0, 8), device="cuda", dtype=torch.float32)
|
||||
bias = torch.zeros(8, device="cuda")
|
||||
weights, ids = ops.grouped_topk(scores, 2, 1, 2, True, 1.0, bias, scoring_func)
|
||||
torch.testing.assert_close(weights, scores.new_empty((0, 2)))
|
||||
torch.testing.assert_close(ids, scores.new_empty((0, 2), dtype=torch.int32))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,k", MK_S)
|
||||
@pytest.mark.parametrize("top_k", TOP_KS)
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -116,7 +116,7 @@ def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
|
||||
@pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"])
|
||||
@pytest.mark.parametrize("hidden_size", [256, 2048])
|
||||
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA graph test")
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="CUDA/ROCm graph test")
|
||||
def test_silu_quant_skips_expert_padding_on_graph_replay(
|
||||
counts: list[int],
|
||||
group_size: int,
|
||||
@@ -135,7 +135,10 @@ def test_silu_quant_skips_expert_padding_on_graph_replay(
|
||||
x *= 16
|
||||
clean_x = x.clone()
|
||||
ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32)
|
||||
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=FLOAT8_DTYPE)
|
||||
output_dtype = (
|
||||
FLOAT8_DTYPE if scale_format == "packed" else current_platform.fp8_dtype()
|
||||
)
|
||||
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=output_dtype)
|
||||
|
||||
def set_counts(values):
|
||||
positions = []
|
||||
@@ -187,6 +190,7 @@ def test_silu_quant_skips_expert_padding_on_graph_replay(
|
||||
output.view(torch.uint8).fill_(91)
|
||||
graph.replay()
|
||||
reference_q, reference_s = quantize(x)
|
||||
assert actual_q.dtype == reference_q.dtype
|
||||
torch.testing.assert_close(
|
||||
actual_q.view(torch.uint8)[live],
|
||||
reference_q.view(torch.uint8)[live],
|
||||
|
||||
@@ -3,11 +3,11 @@
|
||||
"""Tests for fp32_router_gemm kernel: activation×weight→fp32.
|
||||
|
||||
Supported (hidden_size, num_experts) pairs:
|
||||
(3072, 256) -> MiniMax-M2/M2.5, (6144, 128) -> MiniMax-M3
|
||||
(3072, 256), (6144, 128), (5120, 256), (4096, 352), (8192, 256)
|
||||
|
||||
Correctness baseline: F.linear in float32. Every M in [1, 32] is covered so
|
||||
Correctness baseline: F.linear in float32. Every supported M is covered so
|
||||
all tuned geometries (wide-block, experts-per-block, token-group; boundaries
|
||||
at M=4/5, odd/even, M=15/16) are exercised on Blackwell, and the legacy
|
||||
at M=4/5, M=9/10, M=12/13, odd/even, M=15/16) are exercised on Blackwell, and the legacy
|
||||
128/1 geometry everywhere else.
|
||||
"""
|
||||
|
||||
@@ -18,8 +18,12 @@ from vllm._custom_ops import fp32_router_gemm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# (hidden_size, num_experts)
|
||||
SHAPES = [(3072, 256), (6144, 128)]
|
||||
ALL_M = list(range(1, 33))
|
||||
SHAPES = [(3072, 256), (6144, 128), (5120, 256), (4096, 352), (8192, 256)]
|
||||
CASES = [
|
||||
(m, hidden, experts)
|
||||
for hidden, experts in SHAPES
|
||||
for m in range(1, 49 if hidden == 5120 else 33)
|
||||
]
|
||||
# Absolute tolerance for fp32 kernel vs float64 reference
|
||||
ATOL_FP32 = 2e-4
|
||||
ATOL_BF16 = 2e-2 # bf16 activation has lower precision
|
||||
@@ -40,8 +44,7 @@ def _ref(mat_a: torch.Tensor, mat_b: torch.Tensor) -> torch.Tensor:
|
||||
return torch.nn.functional.linear(mat_a.float(), mat_b.float())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
|
||||
@pytest.mark.parametrize("num_tokens", ALL_M)
|
||||
@pytest.mark.parametrize("num_tokens,hidden_dim,num_experts", CASES)
|
||||
def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
|
||||
"""fp32 activation → fp32 output should match reference closely."""
|
||||
_requires_sm90()
|
||||
@@ -58,8 +61,7 @@ def test_fp32_activation(num_tokens: int, hidden_dim: int, num_experts: int):
|
||||
torch.testing.assert_close(out, ref, atol=ATOL_FP32, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
|
||||
@pytest.mark.parametrize("num_tokens", ALL_M)
|
||||
@pytest.mark.parametrize("num_tokens,hidden_dim,num_experts", CASES)
|
||||
def test_bf16_activation(num_tokens: int, hidden_dim: int, num_experts: int):
|
||||
"""bf16 activation → fp32 output should match reference within bf16 error."""
|
||||
_requires_sm90()
|
||||
@@ -124,11 +126,11 @@ def test_topk_routing_consistency(num_tokens: int, hidden_dim: int, num_experts:
|
||||
)
|
||||
|
||||
|
||||
def test_zero_tokens_returns_empty():
|
||||
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
|
||||
def test_zero_tokens_returns_empty(hidden_dim, num_experts):
|
||||
"""M=0 is a graceful no-op returning an empty [0, E] tensor."""
|
||||
_requires_sm90()
|
||||
device = torch.device("cuda")
|
||||
hidden_dim, num_experts = SHAPES[0]
|
||||
mat_a = torch.empty(0, hidden_dim, dtype=torch.float32, device=device)
|
||||
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
|
||||
out = fp32_router_gemm(mat_a, mat_b)
|
||||
@@ -136,17 +138,19 @@ def test_zero_tokens_returns_empty():
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_rejects_invalid_inputs():
|
||||
@pytest.mark.parametrize("hidden_dim,num_experts", SHAPES)
|
||||
def test_rejects_invalid_inputs(hidden_dim, num_experts):
|
||||
"""The entry must fail loudly, never compute silently wrong results."""
|
||||
_requires_sm90()
|
||||
device = torch.device("cuda")
|
||||
hidden_dim, num_experts = SHAPES[0]
|
||||
mat_b = torch.randn(num_experts, hidden_dim, dtype=torch.float32, device=device)
|
||||
|
||||
# num_tokens > 32 (beyond the instantiated range)
|
||||
# Beyond the instantiated range for this shape.
|
||||
invalid_tokens = 49 if hidden_dim == 5120 else 33
|
||||
with pytest.raises(Exception, match="num_tokens"):
|
||||
fp32_router_gemm(
|
||||
torch.randn(33, hidden_dim, dtype=torch.float32, device=device), mat_b
|
||||
torch.randn(invalid_tokens, hidden_dim, dtype=torch.float32, device=device),
|
||||
mat_b,
|
||||
)
|
||||
|
||||
# unsupported (hidden_dim, num_experts) pair
|
||||
@@ -173,3 +177,41 @@ def test_rejects_invalid_inputs():
|
||||
fp32_router_gemm(
|
||||
torch.randn(4, hidden_dim, dtype=torch.float16, device=device), mat_b
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
@pytest.mark.parametrize(
|
||||
"tokens", [*range(1, 17), 17, 24, 32, 33, 40, 48, 56, 64, 65, 128]
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"hidden_dim,num_experts", [(5120, 256), (4096, 352), (8192, 256)]
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_router_retains_fp32_weights_and_logits(
|
||||
default_vllm_config, dist_init, tokens, hidden_dim, num_experts
|
||||
):
|
||||
"""Low-latency routing must retain FP32 weight precision and logits."""
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_device_capability_family(100):
|
||||
pytest.skip("SM100 required")
|
||||
torch.manual_seed(42)
|
||||
router = GateLinear(
|
||||
hidden_dim, num_experts, params_dtype=torch.float32, out_dtype=torch.float32
|
||||
).cuda()
|
||||
router.weight.normal_(std=0.01)
|
||||
x = torch.randn(tokens, hidden_dim, device="cuda", dtype=torch.bfloat16)
|
||||
expected = (x.double() @ router.weight.double().T).float()
|
||||
actual, bias = router(x)
|
||||
assert bias is None and actual.dtype == torch.float32
|
||||
torch.testing.assert_close(actual, expected, rtol=2e-5, atol=2e-5)
|
||||
torch.testing.assert_close(actual.topk(8).indices, expected.topk(8).indices)
|
||||
if tokens in (1, 16, 32, 33, 48, 56):
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
captured, _ = router(x)
|
||||
x.normal_()
|
||||
graph.replay()
|
||||
expected = (x.double() @ router.weight.double().T).float()
|
||||
torch.testing.assert_close(captured, expected, rtol=2e-5, atol=2e-5)
|
||||
|
||||
@@ -566,7 +566,7 @@ def test_invalid_output_dtype():
|
||||
with pytest.raises(ValueError, match="output_dtype=torch.float32"):
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
|
||||
|
||||
ll_bf16_gemm(a, b, output_dtype=torch.bfloat16)
|
||||
ll_bf16_gemm(a, b, output_dtype=torch.float16)
|
||||
|
||||
|
||||
def test_cache_miss_compiles_dotprod():
|
||||
@@ -581,5 +581,29 @@ def test_cache_miss_compiles_dotprod():
|
||||
_assert_close(out, _ref(a, b), context="cache miss dotprod")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 2, 8, 16])
|
||||
@pytest.mark.parametrize("N", [1792, 3584])
|
||||
@torch.inference_mode()
|
||||
def test_bf16_output_and_fp32_cache_coexist(M, N):
|
||||
"""BF16 epilogues retain FP32 accumulation and can replay with new inputs."""
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_bf16 import ll_bf16_gemm
|
||||
|
||||
torch.manual_seed(42)
|
||||
x = torch.randn(M, 5120, device="cuda", dtype=torch.bfloat16)
|
||||
w = torch.randn(N, 5120, device="cuda", dtype=torch.bfloat16) * 0.01
|
||||
for dtype in (torch.float32, torch.bfloat16):
|
||||
actual = ll_bf16_gemm(x, w, output_dtype=dtype)
|
||||
assert actual.dtype == dtype
|
||||
reference = (x.double() @ w.double().T).to(dtype)
|
||||
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual = ll_bf16_gemm(x, w, output_dtype=torch.bfloat16)
|
||||
x.normal_()
|
||||
graph.replay()
|
||||
reference = (x.double() @ w.double().T).bfloat16()
|
||||
torch.testing.assert_close(actual, reference, rtol=0.008, atol=1e-5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
Reference in New Issue
Block a user