Merge branch 'main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-16 17:35:42 +07:00
347 changed files with 29531 additions and 3472 deletions
+25 -1
View File
@@ -122,6 +122,7 @@ def ref_single_query_cached_kv_attention(
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
@pytest.mark.parametrize("seed", SEEDS)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("use_interleaved_v_cache", [False, True])
def test_paged_attention(
kv_cache_factory,
num_seqs: int,
@@ -133,9 +134,14 @@ def test_paged_attention(
kv_cache_dtype: str,
seed: int,
device: str,
use_interleaved_v_cache: bool,
) -> None:
if current_platform.is_navi() and (
kv_cache_dtype == "fp8" or head_size != 128 or block_size != 16 or use_alibi
kv_cache_dtype == "fp8"
or head_size != 128
or block_size != 16
or use_alibi
or use_interleaved_v_cache
):
pytest.skip()
@@ -201,6 +207,20 @@ def test_paged_attention(
last_block_idx = block_tables_lst[seq_idx][seq_len // block_size]
value_cache[last_block_idx, :, :, padding_start:] = padding_nan
standard_value_cache = value_cache
if use_interleaved_v_cache:
# Hand the kernel V in the interleaved layout (same 4D shape); the
# reference below gets the standard layout back.
x_v = 16 // value_cache.element_size()
value_cache = (
value_cache.view(
NUM_BLOCKS, num_kv_heads, head_size, block_size // x_v, x_v
)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(NUM_BLOCKS, num_kv_heads, head_size, block_size)
)
# Call the paged attention kernel.
output = torch.empty_like(query)
num_partitions = (max_seq_len + PARTITION_SIZE_ROCM - 1) // PARTITION_SIZE_ROCM
@@ -234,6 +254,7 @@ def test_paged_attention(
kv_cache_dtype,
k_scale,
v_scale,
use_interleaved_v_cache=use_interleaved_v_cache,
)
opcheck(
@@ -259,10 +280,13 @@ def test_paged_attention(
v_scale,
None,
"f16",
use_interleaved_v_cache,
),
cond=(head_size == 64 and block_size == BLOCK_SIZES[0]),
)
value_cache = standard_value_cache
# Run the reference implementation.
if kv_cache_dtype == "fp8":
# Convert cache data back to dtype.
+90 -2
View File
@@ -109,6 +109,7 @@ def create_alibi_causal_mask(
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention(
num_heads: int,
@@ -119,6 +120,7 @@ def test_contexted_kv_attention(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
block_size: int = 32,
) -> None:
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
@@ -225,6 +227,23 @@ def test_contexted_kv_attention(
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
extra_op_kwargs = {}
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
# read it back interleaved.
x_v = 16 // v_cache.element_size()
v_cache = (
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(cache_size, num_kv_heads, head_size, block_size)
)
if op is context_attention_fwd:
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
else:
extra_op_kwargs["use_interleaved_v_cache"] = True
# Warm up the Triton kernel by calling it once before actually measuring
# generation time
op(
@@ -243,6 +262,7 @@ def test_contexted_kv_attention(
k_scale,
v_scale,
sliding_window=sliding_window,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
start_time = time.time()
@@ -262,6 +282,7 @@ def test_contexted_kv_attention(
k_scale,
v_scale,
sliding_window=sliding_window,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
end_time = time.time()
@@ -386,6 +407,7 @@ def test_rocm_backend_sliding_window_includes_boundary_token(
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_cached_kv(
num_heads: int,
@@ -394,6 +416,7 @@ def test_contexted_kv_attention_cached_kv(
dtype: torch.dtype,
kv_cache_dtype: str,
device: str,
interleave_v: bool,
block_size: int = 32,
) -> None:
# Exercises the KV_FROM_CACHE path of context_attention_fwd: the current
@@ -487,6 +510,18 @@ def test_contexted_kv_attention_cached_kv(
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
interleaved_v_pack_factor = 0
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape).
interleaved_v_pack_factor = 16 // v_cache.element_size()
x_v = interleaved_v_pack_factor
v_cache = (
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(cache_size, num_kv_heads, head_size, block_size)
)
# Cached-K/V path: current-chunk k and v are None.
context_attention_fwd(
query,
@@ -504,6 +539,7 @@ def test_contexted_kv_attention_cached_kv(
k_scale,
v_scale,
sliding_window=0,
interleaved_v_pack_factor=interleaved_v_pack_factor,
)
torch.accelerator.synchronize()
@@ -544,8 +580,11 @@ def test_contexted_kv_attention_cached_kv(
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> None:
def test_contexted_kv_attention_cached_kv_block_table_boundary(
device: str, interleave_v: bool
) -> None:
# Boundary guard for the KV_FROM_CACHE block-table load. With an
# exact-sized block table (row length == number of blocks for the
# sequence) and a query that ends the sequence, the last K/V tile has
@@ -611,6 +650,18 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
interleaved_v_pack_factor = 0
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape).
interleaved_v_pack_factor = 16 // v_cache.element_size()
x_v = interleaved_v_pack_factor
v_cache = (
v_cache.view(num_blocks, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(num_blocks, num_kv_heads, head_size, block_size)
)
# Cached-K/V path: current-chunk k and v are None.
context_attention_fwd(
query,
@@ -628,6 +679,7 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
k_scale,
v_scale,
sliding_window=0,
interleaved_v_pack_factor=interleaved_v_pack_factor,
)
torch.accelerator.synchronize()
@@ -723,6 +775,7 @@ def test_contexted_kv_attention_cached_kv_alibi_unsupported(device: str) -> None
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_alibi(
num_heads: int,
@@ -732,6 +785,7 @@ def test_contexted_kv_attention_alibi(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
block_size: int = 32,
) -> None:
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
@@ -860,6 +914,23 @@ def test_contexted_kv_attention_alibi(
)
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
extra_op_kwargs = {}
if interleave_v:
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
# read it back interleaved.
x_v = 16 // v_cache.element_size()
v_cache = (
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
.permute(0, 1, 3, 2, 4)
.contiguous()
.view(cache_size, num_kv_heads, head_size, block_size)
)
if op is context_attention_fwd:
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
else:
extra_op_kwargs["use_interleaved_v_cache"] = True
# Warm up the Triton kernel by calling it once before actually measuring
# generation time
op(
@@ -878,6 +949,7 @@ def test_contexted_kv_attention_alibi(
k_scale,
v_scale,
alibi_slopes=alibi_slopes,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
start_time = time.time()
@@ -897,6 +969,7 @@ def test_contexted_kv_attention_alibi(
k_scale,
v_scale,
alibi_slopes=alibi_slopes,
**extra_op_kwargs,
)
torch.accelerator.synchronize()
end_time = time.time()
@@ -992,6 +1065,7 @@ def test_contexted_kv_attention_alibi(
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_f32(
num_heads: int,
@@ -1002,6 +1076,7 @@ def test_contexted_kv_attention_f32(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
) -> None:
test_contexted_kv_attention(
num_heads,
@@ -1012,6 +1087,7 @@ def test_contexted_kv_attention_f32(
kv_cache_dtype,
device,
op,
interleave_v=interleave_v,
)
@@ -1023,6 +1099,7 @@ def test_contexted_kv_attention_f32(
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_contexted_kv_attention_alibi_f32(
num_heads: int,
@@ -1032,9 +1109,17 @@ def test_contexted_kv_attention_alibi_f32(
kv_cache_dtype: str,
device: str,
op: Callable,
interleave_v: bool,
) -> None:
test_contexted_kv_attention_alibi(
num_heads, num_queries_per_kv, head_size, dtype, kv_cache_dtype, device, op
num_heads,
num_queries_per_kv,
head_size,
dtype,
kv_cache_dtype,
device,
op,
interleave_v=interleave_v,
)
@@ -1052,6 +1137,7 @@ NONSTANDARD_BLOCK_SIZE_SHAPES = [
@pytest.mark.parametrize("dtype", DTYPES)
@pytest.mark.parametrize("device", CUDA_DEVICES)
@pytest.mark.parametrize("op", OPS)
@pytest.mark.parametrize("interleave_v", [False, True])
@torch.inference_mode()
def test_qwen3_nonstandard_block_size(
num_heads: int,
@@ -1061,6 +1147,7 @@ def test_qwen3_nonstandard_block_size(
dtype: torch.dtype,
device: str,
op: Callable,
interleave_v: bool,
) -> None:
"""Non-power-of-2 pages must match, even when a tile straddles a page."""
if not current_platform.is_rocm():
@@ -1076,4 +1163,5 @@ def test_qwen3_nonstandard_block_size(
kv_cache_dtype="auto",
device=device,
op=op,
interleave_v=interleave_v,
)
@@ -1061,3 +1061,25 @@ def test_aiter_mha_varlen_fp8_kv(dtype):
rtol=rtol,
)
torch.testing.assert_close(output, ref, atol=atol, rtol=rtol)
def test_asm_kv_scale_tables_shared_across_builders():
"""Builders share one AsmKvScaleTables; equal (shape, scale) requests must
return the same dense table, a different scale a different one."""
from vllm.v1.attention.backends.rocm_aiter_fa import (
AiterFlashAttentionMetadataBuilder,
AsmKvScaleTables,
)
tables = AsmKvScaleTables()
device = torch.device("cuda")
table = tables.get(device, 8, 2, 16, 0.5)
assert tables.get(device, 8, 2, 16, 0.5) is table
assert tables.get(device, 8, 2, 16, 1.0) is not table
assert table.shape == (8, 2, 16)
assert table.dtype == torch.float32
assert torch.all(table == 0.5)
# The builder class holds the single process-wide instance.
assert isinstance(
AiterFlashAttentionMetadataBuilder.asm_kv_scale_tables, AsmKvScaleTables
)
@@ -1989,8 +1989,8 @@ def test_dsv41_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
"""Replay V4.1 ROCm decode after device-only query reallocation."""
from tests.v1.attention.utils import create_vllm_config
from vllm.models.deepseek_v41.amd.rocm import (
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
DeepseekV41ROCMAiterSparseSWAMetadataBuilder,
compute_global_topk_ragged_indices_and_indptr,
)
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
@@ -2044,7 +2044,7 @@ def test_dsv41_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
mla_builder = DeepseekV41ROCMAiterMLASparseMetadataBuilder(
mla_spec, ["c2a"], vllm_config, device
)
swa_builder = DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
swa_builder = DeepseekV41ROCMAiterSparseSWAMetadataBuilder(
swa_spec, ["c2a"], vllm_config, device
)
assert (
File diff suppressed because it is too large Load Diff
+179 -15
View File
@@ -52,6 +52,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
)
from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
dequant_mxfp8_to_bf16,
mxfp8_e4m3_quantize,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
@@ -65,7 +66,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.model_executor.models.llama4 import Llama4MoE
from vllm.platforms import current_platform
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.math_utils import next_power_of_2, round_up
from vllm.utils.torch_utils import set_random_seed
try:
@@ -788,7 +789,7 @@ def test_trtllm_fp8_swiglu_clamp_support(
def _make_mxfp8_moe_weights(
e: int, n: int, k: int
e: int, n: int, k: int, is_gated: bool = True
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
q, s = zip(
@@ -796,8 +797,9 @@ def _make_mxfp8_moe_weights(
)
return torch.stack(q), torch.stack(s)
w1_rows = 2 * n if is_gated else n
w1, w1_scale = quantize(
torch.randn(e, 2 * n, k, device="cuda", dtype=torch.bfloat16) / 10
torch.randn(e, w1_rows, k, device="cuda", dtype=torch.bfloat16) / 10
)
w2, w2_scale = quantize(
torch.randn(e, k, n, device="cuda", dtype=torch.bfloat16) / 10
@@ -806,16 +808,20 @@ def _make_mxfp8_moe_weights(
@pytest.mark.parametrize("block_shape", [[128, 128], [1, 32]], ids=["block", "mxfp8"])
@pytest.mark.parametrize("transposed", [True, False], ids=["transposed", "row-major"])
@pytest.mark.parametrize("m", [1, 16])
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
)
def test_trtllm_fp8_block_moe_deferred_finalize(
m: int, block_shape: list[int], workspace_init
m: int, block_shape: list[int], transposed: bool, workspace_init
):
"""TRTLLM-Gen block-FP8 and MXFP8 modular experts can leave the top-k
finalize to the caller."""
finalize to the caller. For fp8-block, ``transposed`` toggles the oracle's
is_scale_transposed column-major activation-scale layout."""
if block_shape != [128, 128] and transposed:
pytest.skip("column-major scales apply to fp8 [128, 128] block only")
e, topk, n, k = 32, 4, 1024, 1024
set_random_seed(7)
with set_current_vllm_config(vllm_config):
@@ -849,6 +855,9 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
a2_scale=None,
block_shape=block_shape,
)
is_block = block_shape == [128, 128]
if is_block and not transposed:
quant_config = replace(quant_config, is_scale_transposed=False)
# One rank of a TP group, which deferral needs.
moe_config = FusedMoEConfig(
num_experts=e,
@@ -866,19 +875,27 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
routing_method=RoutingMethodType.TopK,
max_num_tokens=next_power_of_2(m),
)
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config, quant_config=quant_config, allow_new_interface=True
),
TrtLlmFp8ExpertsModular(moe_config=moe_config, quant_config=quant_config),
)
def make_kernel(quant_config: FusedMoEQuantConfig) -> mk.FusedMoEKernel:
return mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
),
TrtLlmFp8ExpertsModular(
moe_config=moe_config, quant_config=quant_config
),
)
kernel = make_kernel(quant_config)
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
score = torch.randn((m, e), device="cuda", dtype=torch.bfloat16)
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
check_deferred_moe_finalize(
moe_config,
lambda: kernel.apply(
def apply(k_: mk.FusedMoEKernel) -> torch.Tensor:
return k_.apply(
hidden_states=a,
w1=w1,
w2=w2,
@@ -888,7 +905,34 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
),
)
if is_block and transposed:
# The oracle gates column-major scales on TRTLLM fp8-block.
assert quant_config.is_scale_transposed
assert not make_fp8_moe_quant_config(
Fp8MoeBackend.DEEPGEMM,
w1_scale,
w2_scale,
a1_scale=None,
a2_scale=None,
block_shape=block_shape,
).is_scale_transposed
# no_dp_ep emits column-major [M, G] scales (physical [G, M]), so
# the experts' a1q_scale.t().contiguous() is a no-op.
_, a1q_scale, _, _, _ = kernel.prepare_finalize.prepare(
a, topk_weights, topk_ids, e, None, False, quant_config
)
assert a1q_scale.t().is_contiguous()
# Bitwise-equal to running with row-major scales.
row_major = replace(quant_config, is_scale_transposed=False)
torch.testing.assert_close(
apply(kernel), apply(make_kernel(row_major)), atol=0, rtol=0
)
check_deferred_moe_finalize(
moe_config,
lambda: apply(kernel),
router_weights=topk_weights,
)
@@ -986,3 +1030,123 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in
)
torch.testing.assert_close(fused, reference, atol=2e-2, rtol=2e-2)
@pytest.mark.parametrize(
("activation", "n", "k"),
[
(MoEActivation.SILU, 1024, 1024),
(MoEActivation.SILU, 928, 1024),
(MoEActivation.SILU, 1024, 2688),
(MoEActivation.RELU2_NO_MUL, 1024, 1024),
# Nemotron-3.5 Lightning's non-gated MoE: intermediate 1856, hidden 2688.
(MoEActivation.RELU2_NO_MUL, 1856, 2688),
],
)
@pytest.mark.parametrize("m", [1, 32])
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
)
def test_trtllm_mxfp8_moe_pads_intermediate(
m: int, activation: MoEActivation, n: int, k: int, workspace_init
):
"""TRTLLM-Gen MXFP8 experts need the intermediate size to be a multiple
of 128 and the hidden size a multiple of 256. Misaligned experts, gated or
not, are padded at load and, with the activations padded and the output
cropped as the MoE runner does, must still match the unquantized reference
on the dequantized weights."""
e, topk = 8, 2
is_gated = activation.is_gated
block_shape = [1, 32]
set_random_seed(7)
with set_current_vllm_config(vllm_config):
w1, w2, w1_scale, w2_scale = _make_mxfp8_moe_weights(e, n, k, is_gated)
w1_ref = dequant_mxfp8_to_bf16(w1, w1_scale)
w2_ref = dequant_mxfp8_to_bf16(w2, w2_scale)
layer = SimpleNamespace(
weight_block_size=block_shape,
moe_config=SimpleNamespace(
is_act_and_mul=is_gated, intermediate_size_per_partition=n
),
activation=activation,
)
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
Fp8MoeBackend.FLASHINFER_TRTLLM,
layer,
w1,
w2,
w1_scale,
w2_scale,
w13_input_scale=None,
w2_input_scale=None,
)
padded_n = layer.moe_config.intermediate_size_per_partition
padded_k = layer.moe_config.hidden_dim
assert padded_n == round_up(n, 128)
assert padded_k == round_up(k, 256)
quant_config = make_fp8_moe_quant_config(
Fp8MoeBackend.FLASHINFER_TRTLLM,
w1_scale,
w2_scale,
a1_scale=None,
a2_scale=None,
block_shape=block_shape,
)
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
hidden_dim=padded_k,
intermediate_size=padded_n,
num_local_experts=e,
num_logical_experts=e,
activation=activation,
device="cuda",
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=torch.bfloat16,
routing_method=RoutingMethodType.Renormalize,
max_num_tokens=next_power_of_2(m),
)
# The monolithic experts pass the activation type to the kernel; the
# modular ones always run it gated.
kernel = mk.FusedMoEKernel(
maybe_make_prepare_finalize(
moe=moe_config,
quant_config=quant_config,
allow_new_interface=True,
use_monolithic=True,
),
TrtLlmFp8ExpertsMonolithic(
moe_config=moe_config, quant_config=quant_config
),
)
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
logits = torch.randn((m, e), device="cuda", dtype=torch.float32)
output = kernel.apply_monolithic(
hidden_states=torch.nn.functional.pad(a, (0, padded_k - k)),
w1=w1,
w2=w2,
router_logits=logits,
activation=activation,
global_num_experts=e,
expert_map=None,
apply_router_weight_on_input=False,
routed_scaling_factor=1.0,
routing_sink=None,
)[:, :k]
topk_weights, topk_ids, _ = fused_topk(a, logits, topk, renormalize=True)
reference = fused_experts(
a,
w1_ref,
w2_ref,
topk_weights=topk_weights,
topk_ids=topk_ids,
activation=activation,
global_num_experts=e,
expert_map=None,
)
check_accuracy(ref_output=reference, actual_output=output)
+14 -1
View File
@@ -42,13 +42,15 @@ ROCM_AVAILABLE = current_platform.is_rocm()
ROCM_TRITON_KERNELS_AVAILABLE = False
ROCM_AITER_AVAILABLE = is_aiter_found()
ROCM_GFX950 = False
ROCM_GFX1250 = False
if ROCM_AVAILABLE:
from vllm.platforms.rocm import on_gfx950
from vllm.platforms.rocm import on_gfx950, on_gfx1250
from vllm.utils.import_utils import has_triton_kernels
ROCM_TRITON_KERNELS_AVAILABLE = has_triton_kernels()
ROCM_GFX950 = on_gfx950()
ROCM_GFX1250 = on_gfx1250()
if ROCM_AITER_AVAILABLE:
from aiter.ops.triton.quant import dynamic_mxfp4_quant
@@ -1568,6 +1570,15 @@ ROCM_BACKEND_CONFIGS = {
"requires_aiter": True,
"requires_gfx950": True,
},
"AITER_TRITON_MXFP4_MXFP4": {
"activation": "SILU",
"act_type": "mxfp4",
"rtol": 1.0,
"percent": 0.8,
"requires_aiter": True,
"requires_gfx950": False,
"requires_gfx1250": True,
},
}
@@ -1607,6 +1618,8 @@ def test_rocm_mxfp4_moe_oracle(
pytest.skip(f"Backend {backend_name} requires AITER")
if config["requires_gfx950"] and not ROCM_GFX950:
pytest.skip(f"Backend {backend_name} requires GFX950")
if config.get("requires_gfx1250") and not ROCM_GFX1250:
pytest.skip(f"Backend {backend_name} requires GFX1250")
import vllm.distributed.parallel_state as ps
from vllm.config import VllmConfig, set_current_vllm_config
+166
View File
@@ -219,3 +219,169 @@ def test_rocm_aiter_grouped_topk_torch_compile_compatibility():
topk_weights_original, topk_weights_compiled, rtol=1e-2, atol=1e-2
)
assert torch.allclose(topk_ids_original, topk_ids_compiled)
def test_rocm_aiter_topk_gating_custom_op_registration():
assert callable(torch.ops.vllm.rocm_aiter_topk_gating)
def _sort_routing(weights: torch.Tensor, ids: torch.Tensor):
ids_sorted, order = torch.sort(ids, dim=-1)
weights_sorted = torch.gather(weights, 1, order)
return weights_sorted, ids_sorted
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 64])
@pytest.mark.parametrize("num_experts", [256, 512])
@pytest.mark.parametrize("topk", [8, 10])
@pytest.mark.parametrize("renormalize", [True, False])
def test_rocm_aiter_topk_gating_matches_softmax_reference(
num_tokens: int,
num_experts: int,
topk: int,
renormalize: bool,
):
"""Numerical check vs softmax + topk, including Qwen3.8 (E=512, k=10)."""
torch.manual_seed(0)
gating_output = torch.randn(
(num_tokens, num_experts), dtype=torch.bfloat16, device="cuda"
)
scores = torch.softmax(gating_output.float(), dim=-1)
cutoff = torch.topk(scores, k=topk, dim=-1).values[:, -1:]
topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda")
topk_ids = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda")
token_expert_indices = torch.empty(
(num_tokens, topk), dtype=torch.int32, device="cuda"
)
torch.ops.vllm.rocm_aiter_topk_gating(
topk_weights,
topk_ids,
token_expert_indices,
gating_output,
renormalize,
0,
"",
)
# BF16 logits can tie at the top-k boundary, so a different tied expert is
# valid. Check selection against the cutoff and weights against the scores
# of the experts the kernel actually selected.
assert ((topk_ids >= 0) & (topk_ids < num_experts)).all()
sorted_ids = torch.sort(topk_ids, dim=-1).values
assert (sorted_ids[:, 1:] != sorted_ids[:, :-1]).all()
selected_scores = torch.gather(scores, 1, topk_ids.long())
assert (selected_scores >= cutoff).all()
expected_weights = selected_scores
if renormalize:
expected_weights = expected_weights / expected_weights.sum(dim=-1, keepdim=True)
torch.testing.assert_close(topk_weights, expected_weights, atol=2e-2, rtol=2e-2)
def test_rocm_aiter_topk_gating_shared_expert_suffix():
"""Fused shared expert: softmax top-k on the prefix, sigmoid appended.
Matches the Qwen3.8 FSE launch: 512 routed experts, top-10, one shared
column, weights wide enough for the suffix, and the shared id prefilled.
"""
torch.manual_seed(0)
num_tokens, num_experts, topk, num_shared = 4, 512, 10, 1
routed = torch.randn((num_tokens, num_experts), dtype=torch.bfloat16, device="cuda")
shared = torch.randn((num_tokens, num_shared), dtype=torch.bfloat16, device="cuda")
shared[0, 0] = 40
gating_output = torch.cat([routed, shared], dim=-1).contiguous()
width = topk + num_shared
shared_ids = torch.arange(
num_experts, num_experts + num_shared, device="cuda", dtype=torch.int32
)
topk_ids = torch.full((num_tokens, width), -1, dtype=torch.int32, device="cuda")
topk_ids[:, topk:] = shared_ids
topk_weights = torch.empty((num_tokens, width), dtype=torch.float32, device="cuda")
token_expert_indices = torch.empty(
(num_tokens, topk), dtype=torch.int32, device="cuda"
)
torch.ops.vllm.rocm_aiter_topk_gating(
topk_weights,
topk_ids[:, :topk],
token_expert_indices,
gating_output,
True,
num_shared,
"sigmoid",
)
scores = torch.softmax(routed.float(), dim=-1)
cutoff = torch.topk(scores, k=topk, dim=-1).values[:, -1:]
routed_ids = topk_ids[:, :topk]
assert ((routed_ids >= 0) & (routed_ids < num_experts)).all()
sorted_ids = torch.sort(routed_ids, dim=-1).values
assert (sorted_ids[:, 1:] != sorted_ids[:, :-1]).all()
selected = torch.gather(scores, 1, routed_ids.long())
assert (selected >= cutoff).all()
expected = selected / selected.sum(dim=-1, keepdim=True)
torch.testing.assert_close(topk_weights[:, :topk], expected, atol=2e-2, rtol=2e-2)
torch.testing.assert_close(
topk_weights[:, topk:],
torch.sigmoid(shared.float()),
atol=2e-2,
rtol=2e-2,
)
assert torch.equal(topk_ids[:, topk:], shared_ids.expand_as(topk_ids[:, topk:]))
def test_rocm_aiter_topk_gating_torch_compile_compatibility():
token, expert, topk, renormalize = 4, 512, 10, True
gating_output = torch.randn((token, expert), dtype=torch.bfloat16, device="cuda")
device = gating_output.device
def gating_fn(gating_output, topk_weights, topk_ids, token_expert_indices):
return torch.ops.vllm.rocm_aiter_topk_gating(
topk_weights,
topk_ids,
token_expert_indices,
gating_output,
renormalize,
0,
"",
)
torch.library.opcheck(
torch.ops.vllm.rocm_aiter_topk_gating,
(
torch.empty((token, topk), dtype=torch.float32, device=device),
torch.empty((token, topk), dtype=torch.int32, device=device),
torch.empty((token, topk), dtype=torch.int32, device=device),
gating_output,
),
kwargs={
"renormalize": renormalize,
"num_shared_experts": 0,
"shared_expert_scoring_func": "",
},
test_utils=("test_faketensor"),
)
compiled_fn = torch.compile(
gating_fn,
fullgraph=True,
backend="inductor",
mode="reduce-overhead",
dynamic=False,
)
def alloc():
return (
torch.empty((token, topk), dtype=torch.float32, device=device),
torch.empty((token, topk), dtype=torch.int32, device=device),
torch.empty((token, topk), dtype=torch.int32, device=device),
)
w0, i0, t0 = alloc()
w1, i1, t1 = alloc()
gating_fn(gating_output, w0, i0, t0)
compiled_fn(gating_output, w1, i1, t1)
w0, i0 = _sort_routing(w0, i0)
w1, i1 = _sort_routing(w1, i1)
assert torch.allclose(w0, w1, rtol=1e-2, atol=1e-2)
assert torch.equal(i0, i1)
+57 -29
View File
@@ -81,32 +81,11 @@ def _torch_topk_softplus_sqrt(
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
@pytest.mark.parametrize("has_hash_routing", [False, True])
@pytest.mark.parametrize(
"backend,expected_experts",
[
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
("aiter", "AiterMxfp4Experts"),
("triton_unfused", "UnfusedOAITritonExperts"),
],
)
def test_hash_routing_backend_selection(
dist_init,
default_vllm_config,
monkeypatch,
has_hash_routing,
backend,
expected_experts,
):
"""Hash tables must reach backend selection; modular AITER remains eligible."""
def _dsv4_moe_config(monkeypatch, default_vllm_config, has_hash_routing, backend):
from dataclasses import replace
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_deepseek_v4_mxfp4_moe_backend,
)
from vllm.platforms import rocm
# Exercise both native MX and monolithic AITER selection on any ROCm device.
@@ -130,14 +109,63 @@ def test_hash_routing_backend_selection(
),
prefix="hash_routing_selection",
)
config = replace(layer.routed_experts.moe_config, moe_backend=backend)
return layer, replace(layer.routed_experts.moe_config, moe_backend=backend)
if has_hash_routing and backend == "aiter_triton_mxfp4_bf16":
with pytest.raises(ValueError, match="hash routing"):
select_deepseek_v4_mxfp4_moe_backend(config)
else:
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
assert experts_cls.__name__ == expected_experts
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
@pytest.mark.parametrize("has_hash_routing", [False, True])
@pytest.mark.parametrize(
"backend,expected_experts",
[
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
("aiter", "AiterMxfp4Experts"),
("triton_unfused", "UnfusedOAITritonExperts"),
],
)
def test_hash_routing_backend_selection(
dist_init,
default_vllm_config,
monkeypatch,
has_hash_routing,
backend,
expected_experts,
):
"""Hash tables reach backend selection and hash-capable kernels."""
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_deepseek_v4_mxfp4_moe_backend,
)
layer, config = _dsv4_moe_config(
monkeypatch, default_vllm_config, has_hash_routing, backend
)
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
assert experts_cls.__name__ == expected_experts
# Monolithic kernels route internally, so they read the table from the layer.
assert (layer.routed_experts.hash_indices_table is not None) == has_hash_routing
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
def test_hash_routing_rejects_monolithic_without_support(
dist_init, default_vllm_config, monkeypatch
):
"""A monolithic kernel that does not implement hash routing is rejected."""
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
AiterW4A16ExpertsMonolithic,
)
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
select_deepseek_v4_mxfp4_moe_backend,
)
monkeypatch.setattr(
AiterW4A16ExpertsMonolithic,
"_supports_hash_routing",
staticmethod(lambda: False),
)
_, config = _dsv4_moe_config(
monkeypatch, default_vllm_config, True, "aiter_triton_mxfp4_bf16"
)
with pytest.raises(ValueError, match="hash routing"):
select_deepseek_v4_mxfp4_moe_backend(config)
def test_torch_topk_softplus_sqrt_breaks_ties_by_expert_id():
+18 -2
View File
@@ -50,7 +50,7 @@ SKINNY_GEMM_PASS_RATES = {
}
PRESHUFFLED_SHAPES = [
(64, 4096, 8192),
(32, 8192, 16384),
(32, 16384, 16384),
]
@@ -421,8 +421,12 @@ def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
M, K, N = shape
assert M <= 64
# The predicate takes K as packed bytes, matching `weight.shape[1]`.
# The predicate takes K as packed bytes, matching `weight.shape[1]`, and
# must agree with the config the kernel itself resolves.
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K // 2)
from aiter.ops.triton.utils.gemm_config_utils import get_gemm_config
assert get_gemm_config("GEMM-AFP4WFP4_PRESHUFFLED", M, N, K)[1]
A = torch.randn(M, K, dtype=torch.bfloat16)
B = torch.randn(N, K, dtype=torch.bfloat16)
@@ -464,6 +468,18 @@ def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
_assert_deterministic(run_preshuffled, n_runs=3)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fp4_preshuffled_tuned_gate_uses_logical_k():
"""AITER ships N=16384-K=16384 but not N=16384-K=8192, so a K=8192 layer
has no tuned preshuffled config even though twice its K does."""
_assert_aiter_supported()
from vllm._aiter_ops import rocm_aiter_ops
rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned.cache_clear()
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(16384, 16384 // 2)
assert not rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(16384, 8192 // 2)
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_fp4_gemm_a4w4_determinism():
"""AITER FP4 A4W4 GEMM is bitwise deterministic across repeated runs.
+528 -21
View File
@@ -14,6 +14,7 @@ These tests cover:
import math
from types import SimpleNamespace
import numpy as np
import pytest
import torch
@@ -157,6 +158,104 @@ def test_dspark_context_kv_matches_query_insert(
torch.testing.assert_close(kv, before, rtol=0, atol=0)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize("kv_mxfp8", [False, True])
def test_v41_pcp_swa_cache_matches_non_pcp_writer(kv_mxfp8, monkeypatch):
"""PCP SWA writes must preserve the established CUDA cache bytes."""
from vllm.models.deepseek_v41 import attention as attention_module
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
torch.manual_seed(43)
block_size = 64
num_local_tokens = 5
num_tokens = 2 * num_local_tokens
row_size = 528 if kv_mxfp8 else 584
num_blocks = 2
page_stride = math.ceil(block_size * row_size / 576) * 576 + 576
backing = torch.full(
(num_blocks, page_stride), 165, dtype=torch.uint8, device="cuda"
)
cache = backing.as_strided(
(num_blocks, block_size, row_size), (page_stride, row_size, 1)
)
reference_backing = backing.clone()
reference = reference_backing.as_strided(cache.shape, cache.stride())
local_kv = torch.randn(num_local_tokens, 512, dtype=torch.bfloat16, device="cuda")
remote_kv = torch.randn_like(local_kv)
gathered_kv = torch.cat((local_kv, remote_kv))
local_positions = torch.arange(num_local_tokens, device="cuda") + 3
cache_positions = torch.cat((local_positions, local_positions + 11))
cache_slots = torch.randperm(
num_blocks * block_size, dtype=torch.int64, device="cuda"
)[:num_tokens]
cache_slots[2] = -1
angles = torch.randn(32, 32, device="cuda")
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
num_heads = 8
q = torch.randn(
num_local_tokens,
num_heads,
512,
dtype=torch.bfloat16,
device="cuda",
)
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
torch.zeros(num_tokens, num_heads, 512, dtype=torch.bfloat16, device="cuda"),
gathered_kv,
reference.view(num_blocks, -1),
cache_slots,
cache_positions,
cos_sin,
num_heads,
1e-20,
block_size,
False,
kv_mxfp8,
)
monkeypatch.setattr(
attention_module,
"get_pcp_group",
lambda: SimpleNamespace(
all_gather=lambda tensor, dim: torch.cat((tensor, remote_kv), dim=dim)
),
)
metadata = SimpleNamespace(
slot_mapping=torch.full(
(num_local_tokens,), -1, dtype=torch.int64, device="cuda"
),
cache_slot_mapping=cache_slots,
cache_positions=cache_positions,
block_size=block_size,
)
attention = SimpleNamespace(
use_pcp=True,
swa_cache_layer=SimpleNamespace(
prefix="swa", kv_cache=cache, block_size=block_size
),
rotary_emb=SimpleNamespace(cos_sin_cache=cos_sin),
_flashinfer_fp8_kv_scale=None,
kv_mxfp8=kv_mxfp8,
accepts_unnormed_unroped_query=False,
n_local_heads=num_heads,
padded_heads=num_heads,
eps=1e-20,
head_dim=512,
)
DeepseekV4Attention._fused_qnorm_rope_kv_insert(
attention, q, local_kv, local_positions, {"swa": metadata}
)
torch.testing.assert_close(
backing.view(torch.uint8),
reference_backing.view(torch.uint8),
rtol=0,
atol=0,
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="graph capture coverage"
)
@@ -360,7 +459,7 @@ def _mxfp8_record_reference(
@pytest.mark.parametrize("compress_ratio", [1, 2])
def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
"""The 528-byte V4.1 record quantizes the RoPE dims instead of keeping them
in bf16, so a page is [block_size x 512 fp8][block_size x 16 UE8M0].
in bf16, so a token is [512 fp8][16 UE8M0].
Byte-exact against the reference quantizer, including the rows and page
padding the kernel must leave alone.
@@ -398,9 +497,8 @@ def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
values, scales = _mxfp8_record_reference(
latent[t], cos_sin[pos // compress_ratio * compress_ratio]
)
expected[page, row * 512 : (row + 1) * 512] = values
scale_offset = cache_block * 512 + row * 16
expected[page, scale_offset : scale_offset + 16] = scales
expected[page, row * 528 : row * 528 + 512] = values
expected[page, row * 528 + 512 : (row + 1) * 528] = scales
torch.testing.assert_close(cache_backing, expected, rtol=0, atol=0)
@@ -510,14 +608,13 @@ def test_v41_mxfp8_cache_round_trip():
values, scales = _ue8m0_reference(k[t].float(), 32, 448.0)
page, row = divmod(t, block_size)
torch.testing.assert_close(
cache[page, row * 512 : (row + 1) * 512],
cache[page, row * 528 : row * 528 + 512],
values.view(torch.uint8),
rtol=0,
atol=0,
)
scale_offset = block_size * 512 + row * 16
torch.testing.assert_close(
cache[page, scale_offset : scale_offset + 16],
cache[page, row * 528 + 512 : (row + 1) * 528],
(scales.log2() + 127).to(torch.uint8),
rtol=0,
atol=0,
@@ -564,7 +661,8 @@ def _decode_nvfp4_row(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tenso
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize("compress_ratio", [1, 2])
def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales.
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales
per token.
Scales are byte-exact against the reference (``amax / 6`` clamped to the
e4m3 range); values are checked as round-to-nearest on the e2m1 grid, whose
@@ -604,17 +702,13 @@ def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
amax = rotated.view(32, 16).abs().amax(-1)
scale = (amax / 6.0).clamp(2.0**-9, 448.0).to(torch.float8_e4m3fn)
got_scales = cache_backing[page, cache_block * 256 + row * 32 :][:32]
record = cache_backing[page, row * 288 : (row + 1) * 288]
got_scales = record[256:]
torch.testing.assert_close(got_scales, scale.view(torch.uint8), rtol=0, atol=0)
decoded = _decode_nvfp4_row(
cache_backing[page, row * 256 : (row + 1) * 256], got_scales
)
decoded = _decode_nvfp4_row(record[:256], got_scales)
step = scale.float().repeat_interleave(16)
assert ((decoded - rotated).abs() <= step).all()
untouched[page, row * 256 : (row + 1) * 256] = cache_backing[
page, row * 256 : (row + 1) * 256
]
untouched[page, cache_block * 256 + row * 32 :][:32] = got_scales
untouched[page, row * 288 : (row + 1) * 288] = record
# Rows the kernel must not have touched, page padding included.
torch.testing.assert_close(cache_backing, untouched, rtol=0, atol=0)
@@ -670,10 +764,8 @@ def test_v41_nvfp4_gather_matches_insert(num_reqs, num_tokens, partial):
offset=offset,
)
expected = _decode_nvfp4_row(
backing[:, : block_size * 256].reshape(-1, 256),
backing[:, block_size * 256 : block_size * 288].reshape(-1, 32),
)
records = backing[:, : block_size * 288].reshape(-1, 288)
expected = _decode_nvfp4_row(records[:, :256], records[:, 256:])
def check(shorter_by):
assert (storage[:, num_tokens + offset :] == -123).all()
@@ -702,6 +794,72 @@ def test_v41_nvfp4_gather_matches_insert(num_reqs, num_tokens, partial):
check(1)
@pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="The V4.1 records are SM100-only",
)
@pytest.mark.parametrize("record", ["swa_fp8", "compressed_fp8", "compressed_fp4"])
def test_v41_cache_writers_match_flashmla_decode(record: str):
"""FlashMLA and vLLM's gathers read back the token a V4.1 writer put mid
page in a padded block, so all agree on the interleaved record layout."""
import vllm.v1.attention.ops.flashmla as fm
from vllm.models.deepseek_v41.common.ops.cache_utils import (
dequantize_and_gather_k_cache,
dequantize_and_gather_k_cache_triton,
)
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
rope_quant_insert,
)
device, block_size, topk = "cuda", 64, 64
slot = block_size + 37
bytes_per_token = 288 if record == "compressed_fp4" else 528
torch.manual_seed(0)
kv = torch.randn(1, 512, dtype=torch.bfloat16, device=device)
positions = torch.tensor([slot], device=device)
# cos = 1, sin = 0: RoPE is the identity, so the cache should hold `kv`.
cos_sin = torch.zeros(slot + 1, 64, device=device)
cos_sin[:, :32] = 1
# Padded like a packed block: a multiple of the record and of 512 B.
cache = torch.zeros(2, block_size + 32, bytes_per_token, dtype=torch.uint8)
cache = cache.to(device)[:, :block_size]
if record == "swa_fp8":
torch.ops._C.fused_deepseek_v4_kv_rope_insert(
kv, cache, positions, positions, cos_sin, block_size, None, True
)
else:
rope_quant_insert(kv, positions, cos_sin, cache, positions, 1)
indices = torch.full((1, 1, topk), -1, dtype=torch.int32, device=device)
indices[..., 0] = slot
extra = None
if record == "compressed_fp4":
cache, extra = torch.zeros(2, block_size, 528).to(cache), cache
out, _ = fm.flash_mla_with_kvcache(
q=torch.randn(1, 1, 64, 512, dtype=torch.bfloat16, device=device),
k_cache=cache.unsqueeze(2),
block_table=None,
cache_seqlens=None,
head_dim_v=512,
tile_scheduler_metadata=fm.get_mla_metadata()[0],
is_fp8_kvcache=True,
indices=indices if extra is None else torch.full_like(indices, -1),
extra_k_cache=None if extra is None else extra.unsqueeze(2),
extra_indices_in_kvcache=None if extra is None else indices,
)
for gather in (dequantize_and_gather_k_cache, dequantize_and_gather_k_cache_triton):
rows = torch.zeros(1, slot + 1, 512, dtype=torch.bfloat16, device=device)
block_table = torch.arange(2, dtype=torch.int32, device=device)[None]
seq_lens = torch.tensor([slot + 1], dtype=torch.int32, device=device)
k_cache = cache if extra is None else extra
gather(rows, k_cache, seq_lens, None, block_table, block_size, offset=0)
torch.testing.assert_close(
out[0, 0], rows[0, slot].expand(64, -1), rtol=0, atol=0
)
cos = torch.nn.functional.cosine_similarity(rows[0, slot].float(), kv[0].float(), 0)
assert cos > (0.98 if extra is not None else 0.999)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.parametrize("compress_ratio", [1, 2])
@pytest.mark.parametrize("store_fp8", [False, True])
@@ -796,6 +954,7 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
capacity = 8
vllm_config = MagicMock()
vllm_config.scheduler_config.max_num_batched_tokens = 16
vllm_config.parallel_config.prefill_context_parallel_size = 1
spec = CircularBufferSpec(
block_size=capacity,
num_kv_heads=1,
@@ -832,6 +991,67 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
assert metadata.token_to_req_indices.tolist() == [0, 0, 0, 1, 1, 2, 2]
def test_v41_pcp_compressor_latent_is_gathered_for_cache_writes(monkeypatch):
"""PCP compresses locally and publishes the gathered latent to both caches."""
from vllm.forward_context import ForwardContext, override_forward_context
from vllm.models.deepseek_v41 import attention as attention_module
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
local_latent = torch.arange(8, dtype=torch.float32).view(2, 4)
cache_positions = torch.tensor([0, 1, 10, 11])
observed = {}
class Compressor:
def __call__(self, kv_score, positions):
return local_latent
def insert_cache(self, latent, positions, rotary_emb):
observed["insert"] = (latent, positions)
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale, **kwargs):
observed["indexer"] = (latent, kwargs)
return None, None, None
monkeypatch.setattr(
attention_module,
"get_pcp_group",
lambda: SimpleNamespace(
all_gather=lambda tensor, dim: torch.cat((tensor, tensor + 10), dim=dim)
),
)
metadata = {"swa": SimpleNamespace(cache_positions=cache_positions)}
attention = SimpleNamespace(
use_pcp=True,
compressor=Compressor(),
indexer=prepare_indexer,
aux_stream_list=None,
ln_events=[None, None],
swa_cache_layer=SimpleNamespace(prefix="swa"),
rotary_emb=SimpleNamespace(),
indexer_rotary_emb=SimpleNamespace(),
n_local_heads=1,
head_dim=4,
_wq_b_proj=lambda qr, scale: qr,
_fused_qnorm_rope_kv_insert=lambda q, kv, pos, meta: q,
_sparse_indexer_and_attn=lambda *args: None,
)
rows = torch.zeros(2, 4)
positions = torch.arange(2)
with override_forward_context(ForwardContext({}, metadata, {})):
DeepseekV4Attention._prepare_and_attn(
attention, rows, rows, rows, None, rows, rows, positions, rows
)
expected_global = torch.cat((local_latent, local_latent + 10))
insert_latent, insert_positions = observed["insert"]
torch.testing.assert_close(insert_latent, expected_global)
torch.testing.assert_close(insert_positions, cache_positions)
indexer_latent, indexer_kwargs = observed["indexer"]
torch.testing.assert_close(indexer_latent, local_latent)
torch.testing.assert_close(indexer_kwargs["cache_latent"], expected_global)
torch.testing.assert_close(indexer_kwargs["cache_positions"], cache_positions)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA stream coverage")
@pytest.mark.parametrize(
"use_aux,use_graph", [(False, False), (True, False), (True, True)]
@@ -871,6 +1091,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
context = ForwardContext({}, metadata, {})
compressor = DeepseekCompressor.__new__(DeepseekCompressor)
torch.nn.Module.__init__(compressor)
compressor.pcp_size = 1
compressor.head_dim, compressor.rope_head_dim, compressor.compress_ratio = (
512,
64,
@@ -896,7 +1117,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
use_fp4_kv=False,
)
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale):
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale, **kwargs):
DeepseekV4Indexer._produce_k(indexer, latent, positions, rotary)
return None, None, None
@@ -906,6 +1127,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
auxiliary = [torch.cuda.Stream()] if use_aux else None
attention = SimpleNamespace(
use_pcp=False,
compressor=compressor,
indexer=prepare_indexer,
aux_stream_list=auxiliary,
@@ -963,6 +1185,291 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
previous = expected
@pytest.mark.parametrize("pcp_size", [2, 4])
@pytest.mark.parametrize(
"lengths,prefilling,starts",
[
([9, 1], [True, False], [1, 4]),
([1, 1], [False, False], [16, 3]),
([3, 1], [True, False], [1, 0]),
],
)
def test_v41_pcp_ring_links_match_request_positions(
pcp_size, lengths, prefilling, starts, monkeypatch
):
"""Cross-chunk predecessors and open ring writes must belong to the request."""
from vllm.models.deepseek_v41 import compressor as compressor_module
from vllm.models.deepseek_v41.compressor import (
CompressorMetadata,
CompressorMetadataBuilder,
)
from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module
from vllm.v1.worker.gpu.pcp_manager import PCPManager
monkeypatch.setattr(
pcp_manager_module,
"async_tensor_h2d",
lambda array, **kwargs: torch.from_numpy(array),
)
manager = PCPManager(pcp_size, 0, torch.device("cpu"), shard_decode_requests=True)
query_start = np.array([0, *np.cumsum(lengths)], dtype=np.int32)
segments, counts = manager._build_batch_layout(
np.array(lengths), np.array(starts), np.array(prefilling), query_start
)
width = max(counts)
capacity = 8
blocks = [1, (1 << 20) + 2]
num_rows = min(width, 2 * len(lengths))
metadata_by_rank = []
positions_by_rank = []
gathered_keys = []
for local_segments in segments:
positions = torch.zeros(width, dtype=torch.int64)
slots = torch.full((width,), -1, dtype=torch.int64)
local_query_start = [0]
keys = [-1] * num_rows
for row, segment in enumerate(local_segments):
req = segment.global_batch_req_idx
begin = starts[req] + segment.global_batch_slice.start - query_start[req]
local = segment.rank_local_batch_slice
positions[local] = torch.arange(begin, begin + segment.num_tokens)
slots[local] = blocks[req] * capacity + positions[local] % capacity
local_query_start.append(local.stop)
keys[row] = (blocks[req] << 32) | int(positions[local.stop - 1])
if not local_segments:
local_query_start.append(0)
metadata_by_rank.append(
CompressorMetadata(
slot_mapping=slots,
query_start_loc=torch.tensor(local_query_start, dtype=torch.int32),
token_to_req_indices=torch.empty(0, dtype=torch.int32),
)
)
positions_by_rank.append(positions)
gathered_keys.extend(keys)
keys = torch.tensor(gathered_keys, dtype=torch.int64)
contributed_keys = []
def all_gather(tensor, dim):
contributed_keys.append(tensor.clone())
return keys
monkeypatch.setattr(
compressor_module,
"get_pcp_group",
lambda: SimpleNamespace(all_gather=all_gather),
)
builder = SimpleNamespace(
capacity=capacity,
device=torch.device("cpu"),
vllm_config=SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_seqs=len(lengths))
),
)
expected_rows = {
block: max(key for key in gathered_keys if key >> 32 == block)
for block in blocks
}
expected_slots = [
block * capacity + (key & 0xFFFFFFFF) % capacity
for block, key in expected_rows.items()
if key % 2 == 0
]
for metadata, positions in zip(metadata_by_rank, positions_by_rank):
num_reqs = metadata.query_start_loc.numel() - 1
CompressorMetadataBuilder._link_pcp_chunks(
builder, metadata, num_reqs, positions
)
assert metadata.prev_row_indices is not None
assert metadata.gather_tokens is not None
assert metadata.ring_write_slots is not None
for row, token in enumerate(metadata.gather_tokens.tolist()):
key = int(contributed_keys[-1][row])
if key >= 0:
assert int(metadata.slot_mapping[token]) // capacity == key >> 32
assert int(positions[token]) == key & 0xFFFFFFFF
for start, prev in zip(
metadata.query_start_loc[:-1].tolist(), metadata.prev_row_indices.tolist()
):
position = int(positions[start])
block = int(metadata.slot_mapping[start]) // capacity
wanted = (block << 32) | (position - 1)
if position % 2 == 1 and wanted in gathered_keys:
assert prev >= 0 and gathered_keys[prev] == wanted
else:
assert prev == -1
writes = metadata.ring_write_slots.tolist()
assert sorted(slot for slot in writes if slot >= 0) == sorted(expected_slots)
for row, slot in enumerate(writes):
if slot >= 0:
assert gathered_keys[row] == expected_rows[slot // capacity]
assert (gathered_keys[row] & 0xFFFFFFFF) % capacity == slot % capacity
torch.testing.assert_close(torch.cat(contributed_keys), keys)
def _build_pcp_ring_metadata(ranks, all_gather, spec, config, device, monkeypatch):
from vllm.models.deepseek_v41 import compressor as compressor_module
from vllm.models.deepseek_v41.compressor import CompressorMetadataBuilder
monkeypatch.setattr(
compressor_module,
"get_pcp_group",
lambda: SimpleNamespace(all_gather=all_gather),
)
for local in ranks:
builder = CompressorMetadataBuilder(spec, ["ring"], config, device)
local.metadata = builder.build(0, local.common)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Triton kernel")
@pytest.mark.parametrize("pcp_size", [2, 4])
def test_v41_pcp_compressor_preserves_pairs_and_decode_state(pcp_size, monkeypatch):
"""Pairs split across PCP chunks and decode owner changes keep exact latents."""
from vllm.forward_context import ForwardContext, override_forward_context
from vllm.models.deepseek_v41 import compressor as compressor_module
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
fused_save_compress_norm,
)
from vllm.models.deepseek_v41.compressor import DeepseekCompressor
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.kv_cache_interface import CircularBufferSpec
from vllm.v1.worker.gpu.pcp_manager import PCPManager
torch.manual_seed(42)
device = torch.device("cuda")
norm = torch.randn(512, dtype=torch.bfloat16, device=device)
# Block 0 is reserved as the null block; requests use physical blocks 1/2.
reference_state = torch.zeros(3, 8, 1024, device=device)
rank_states = [reference_state.clone() for _ in range(pcp_size)]
config = SimpleNamespace(
scheduler_config=SimpleNamespace(max_num_batched_tokens=32, max_num_seqs=4),
parallel_config=SimpleNamespace(prefill_context_parallel_size=pcp_size),
speculative_config=None,
)
spec = CircularBufferSpec(
block_size=8, num_kv_heads=1, head_size=1024, head_size_v=0, dtype=torch.float32
)
starts = np.zeros(2, dtype=np.int32)
for lengths, prefilling in [
([7, 3], [True, True]),
([9, 1], [True, False]),
([1, 1], [False, False]),
([1, 1], [False, False]),
]:
lengths = np.array(lengths, dtype=np.int32)
query_start = np.concatenate(([0], np.cumsum(lengths))).astype(np.int32)
qsl = torch.from_numpy(query_start).to(device)
positions = torch.cat(
[torch.arange(s, s + n, device=device) for s, n in zip(starts, lengths)]
)
req_ids = torch.repeat_interleave(
torch.arange(2, device=device, dtype=torch.int32),
torch.from_numpy(lengths).to(device),
)
slots = (req_ids.long() + 1) * 8 + positions % 8
raw = torch.randn(sum(lengths), 1024, device=device)
expected = torch.empty(sum(lengths), 512, dtype=torch.bfloat16, device=device)
fused_save_compress_norm(
raw,
positions,
reference_state,
slots,
qsl,
req_ids,
norm,
1e-20,
2,
expected,
)
ranks = []
for rank in range(pcp_size):
manager = PCPManager(pcp_size, rank, device, shard_decode_requests=True)
segments, counts = manager._build_batch_layout(
lengths, starts, np.array(prefilling), query_start
)
width = max(counts)
local_indices = manager._padded_gather_idx[
rank * width : (rank + 1) * width
]
local_lengths = [s.num_tokens for s in segments[rank]] or [0]
local_qsl = torch.tensor([0, *np.cumsum(local_lengths)], dtype=torch.int32)
local_reqs = [s.global_batch_req_idx for s in segments[rank]] or [0]
common = CommonAttentionMetadata(
query_start_loc=local_qsl.to(device),
query_start_loc_cpu=local_qsl,
seq_lens=torch.zeros(len(local_reqs), dtype=torch.int32, device=device),
num_reqs=len(local_reqs),
num_actual_tokens=counts[rank],
max_query_len=max(local_lengths),
max_seq_len=int(max(starts + lengths)),
block_table_tensor=torch.tensor(
np.array(local_reqs) + 1, dtype=torch.int32, device=device
).unsqueeze(1),
slot_mapping=torch.full((width * pcp_size,), -1, device=device),
positions=positions[local_indices],
)
ranks.append(
SimpleNamespace(
indices=local_indices[: counts[rank]],
raw=raw[local_indices],
positions=common.positions,
common=common,
)
)
# Each rank's builder contributes its last-row candidates to a gather.
candidates: list[torch.Tensor] = []
_build_pcp_ring_metadata(
ranks,
lambda t, dim, out=candidates: out.append(t.clone()) or t.repeat(pcp_size),
spec,
config,
device,
monkeypatch,
)
_build_pcp_ring_metadata(
ranks,
lambda t, dim, out=candidates: torch.cat(out),
spec,
config,
device,
monkeypatch,
)
gathered_rows = torch.cat([r.raw[r.metadata.gather_tokens] for r in ranks])
monkeypatch.setattr(
compressor_module,
"get_pcp_group",
lambda rows=gathered_rows: SimpleNamespace(all_gather=lambda t, dim: rows),
)
for rank, local in enumerate(ranks):
compressor = SimpleNamespace(
pcp_size=pcp_size,
head_dim=512,
compress_ratio=2,
rms_norm_eps=1e-20,
norm=SimpleNamespace(weight=norm),
state_cache=SimpleNamespace(prefix="ring", kv_cache=rank_states[rank]),
)
context = ForwardContext({}, {"ring": local.metadata}, {})
with override_forward_context(context):
actual = DeepseekCompressor.forward(
compressor, local.raw, local.positions
)
assert isinstance(actual, torch.Tensor)
valid = (positions[local.indices] + 1) % 2 == 0
torch.testing.assert_close(
actual[: local.indices.numel()][valid],
expected[local.indices][valid],
rtol=0,
atol=0,
)
for state in rank_states[1:]:
torch.testing.assert_close(state, rank_states[0], rtol=0, atol=0)
starts += lengths
def _on_gfx950() -> bool:
if not current_platform.is_rocm():
return False
+14 -5
View File
@@ -502,11 +502,12 @@ def test_lookback_window_reproduces_single_instance(runner, capture):
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
def test_v2_model_state_gathers_lookback_window():
@pytest.mark.parametrize("pcp", [False, True])
def test_v2_model_state_gathers_lookback_window(pcp):
"""The window is gathered on device from the runner's token history."""
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
depth, max_num_reqs, max_model_len = 3, 4, 16
depth, max_num_reqs, max_model_len = 3, 6, 16
state = DeepseekV41ModelState.__new__(DeepseekV41ModelState)
state.rope_state = None
state.supports_mm_inputs = False
@@ -527,15 +528,23 @@ def test_v2_model_state_gathers_lookback_window():
# Batch rows -> request state rows: a request two decode steps in, a fresh
# request, and one at the end of a 5-token prompt.
input_batch = SimpleNamespace(
idx_mapping=torch.tensor([2, 0, 3], dtype=torch.int64, device="cuda")
idx_mapping=torch.tensor(
[2, 0, 3, 2] if pcp else [2, 0, 3], dtype=torch.int64, device="cuda"
),
query_start_loc=torch.arange(5 if pcp else 4, dtype=torch.int32, device="cuda"),
positions=torch.tensor([8, 0, 6, 5] if pcp else [8, 0, 6], device="cuda"),
)
window = state.prepare_inputs(input_batch, req_states)["lookback_token_ids"]
assert window.cpu().tolist() == [
expected = [
[22, 21, 16],
[-1, -1, -1],
[0, 0, 0],
[-1, -1, -1],
]
if pcp:
# The second PCP chunk of request 2 starts at 5, not at its global 8.
expected.append([15, 14, 13])
expected.extend([[-1] * depth] * (max_num_reqs - len(expected)))
assert window.cpu().tolist() == expected
# Graph capture runs on the dummy inputs and replays read the same buffer.
dummy = state.prepare_dummy_inputs(num_reqs=3, num_tokens=3)["lookback_token_ids"]
assert dummy is window and torch.all(dummy == -1)
@@ -525,9 +525,8 @@ def test_kv_path_mxfp8_record_matches_reference(num_tokens: int, block_size: int
step = 32.0 * tile_scale.repeat_interleave(32, dim=-1)
assert ((rec_fused - rec_ref).abs() <= step).all()
# The NoPE dims see no rotation at all, so they must be byte-identical.
nope_bytes = slice(0, block_size * HEAD_DIM)
ref_rows = k_cache_ref[:, nope_bytes].view(num_blocks, block_size, HEAD_DIM)
fused_rows = k_cache_fused[:, nope_bytes].view(num_blocks, block_size, HEAD_DIM)
ref_rows = k_cache_ref.view(num_blocks, block_size, V41_HEAD_BYTES)
fused_rows = k_cache_fused.view(num_blocks, block_size, V41_HEAD_BYTES)
torch.testing.assert_close(
fused_rows[..., :NOPE_DIM], ref_rows[..., :NOPE_DIM], rtol=0, atol=0
)
@@ -109,7 +109,7 @@ def test_rocm_set_out_dtype_respects_bias_guard(monkeypatch):
@pytest.mark.parametrize(
("input_size", "output_size"),
[(3072, 256), (4096, 8), (4096, 192), (6144, 128), (6144, 256)],
[(3072, 256), (4096, 8), (6144, 128), (2048, 64), (2560, 512), (7168, 384)],
)
def test_rocm_gfx950_enables_fp32_router_gemm(
monkeypatch, input_size: int, output_size: int
@@ -147,7 +147,6 @@ def test_rocm_fp32_router_gemm_is_replicated_across_ep_sizes(
("input_size", "output_size", "params_dtype", "bias", "on_gfx950"),
[
pytest.param(6144, 128, torch.float32, False, False, id="gfx942"),
pytest.param(2048, 64, torch.float32, False, True, id="shape"),
pytest.param(6144, 128, torch.bfloat16, False, True, id="bf16-weight"),
pytest.param(6144, 128, torch.float32, True, True, id="bias"),
],
+171 -5
View File
@@ -5,6 +5,9 @@
import pytest
import torch
from vllm.model_executor.layers.fused_moe.router import (
rocm_fp32_router_gemm as rocm_router,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import (
fp32_router_gemm_dispatch_impl,
)
@@ -19,6 +22,11 @@ SHAPES = [
(4096, 192),
(6144, 128),
(6144, 256),
(2560, 512),
(2880, 128),
(5120, 16),
(7168, 384),
(1000, 100),
]
MAX_TOKENS = 32
ATOL = 5e-4
@@ -80,7 +88,7 @@ def test_rocm_fp32_router_gemm_preserves_topk(
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=device)
weight = torch.randn(num_experts, hidden_size, dtype=torch.float32, device=device)
top_k = 2 if (hidden_size, num_experts) == (4096, 8) else 8
top_k = 2 if num_experts <= 16 else 8
actual = rocm_fp32_router_gemm(x, weight).topk(top_k, dim=-1).indices
reference = _reference(x, weight)
reference_values, expected = reference.topk(top_k, dim=-1)
@@ -113,15 +121,22 @@ def test_rocm_fp32_router_gemm_rejects_invalid_inputs() -> None:
weight,
)
with pytest.raises(ValueError, match="shape"):
rocm_fp32_router_gemm(x[:, :2048].contiguous(), weight[:, :2048].contiguous())
with pytest.raises(ValueError, match="hidden sizes"):
rocm_fp32_router_gemm(x, weight[:, :2048].contiguous())
with pytest.raises(ValueError, match="num_experts"):
big_weight = torch.empty(1024, 7168, dtype=torch.float32, device=device)
rocm_fp32_router_gemm(
torch.empty(MAX_TOKENS, 7168, dtype=torch.bfloat16, device=device),
big_weight,
)
with pytest.raises(ValueError, match="contiguous"):
wide_x = torch.randn(4, hidden_size * 2, dtype=torch.bfloat16, device=device)
rocm_fp32_router_gemm(wide_x[:, ::2], weight)
with pytest.raises(ValueError, match="float32"):
rocm_fp32_router_gemm(x, weight.to(torch.bfloat16))
with pytest.raises(ValueError, match="float32 or bfloat16"):
rocm_fp32_router_gemm(x, weight.to(torch.float16))
with pytest.raises(ValueError, match="bfloat16 or float32"):
rocm_fp32_router_gemm(x.to(torch.float16), weight)
@@ -226,3 +241,154 @@ def test_rocm_fp32_router_gemm_dynamic_compile_dispatch() -> None:
)
output = compiled_dispatch(x)
torch.testing.assert_close(output, _reference(x, weight), atol=ATOL, rtol=RTOL)
@pytest.mark.parametrize("hidden_size", [64, 100, 513, 1000, 2049, 3000])
@pytest.mark.parametrize("num_tokens", [1, 5, 32])
@torch.inference_mode()
def test_rocm_fp32_router_gemm_any_hidden_size(num_tokens: int, hidden_size: int):
"""Hidden sizes that do not divide the K tile exercise the masked tail."""
torch.manual_seed(6100 + num_tokens + hidden_size)
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(64, hidden_size, dtype=torch.float32, device="cuda")
torch.testing.assert_close(
rocm_fp32_router_gemm(x, weight), _reference(x, weight), atol=ATOL, rtol=RTOL
)
@pytest.mark.parametrize("case", ["large", "near_zero", "zero_input", "ones"])
@torch.inference_mode()
def test_rocm_fp32_router_gemm_special_values(case: str) -> None:
torch.manual_seed(7100)
x = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(128, 5120, dtype=torch.float32, device="cuda")
if case in ("large", "near_zero"):
scale = 100.0 if case == "large" else 1e-4
x = (x * scale).to(torch.bfloat16)
weight *= scale
elif case == "zero_input":
x.zero_()
else:
x.fill_(1.0)
output = rocm_fp32_router_gemm(x, weight)
expected = _reference(x, weight)
assert torch.isfinite(output).all()
if case == "zero_input":
assert torch.equal(output, torch.zeros_like(output))
else:
assert ((output - expected).norm() / expected.norm()).item() < 1e-5
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 32])
@torch.inference_mode()
def test_rocm_fp32_router_gemm_deterministic(num_tokens: int) -> None:
torch.manual_seed(8100 + num_tokens)
x = torch.randn(num_tokens, 7168, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(384, 7168, dtype=torch.float32, device="cuda")
first = rocm_fp32_router_gemm(x, weight)
assert torch.equal(first, rocm_fp32_router_gemm(x, weight))
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 32])
@pytest.mark.parametrize(("hidden_size", "num_experts"), SHAPES)
@torch.inference_mode()
def test_rocm_fp32_router_gemm_bf16_weight(
num_tokens: int, hidden_size: int, num_experts: int
) -> None:
torch.manual_seed(9100 + num_tokens + hidden_size + num_experts)
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(num_experts, hidden_size, dtype=torch.bfloat16, device="cuda")
# Past this size hipBLASLt beats the kernel for BF16 weights on MI355.
eligible = num_tokens * num_experts <= 8192
assert rocm_router.can_use_rocm_fp32_router_gemm(x, weight) == eligible
if eligible:
torch.testing.assert_close(
rocm_fp32_router_gemm(x, weight),
_reference(x.float(), weight.float()),
atol=ATOL,
rtol=RTOL,
)
def _make_gate_linear(
monkeypatch: pytest.MonkeyPatch,
input_size,
output_size,
params_dtype=torch.float32,
):
for module in ("layers.linear", "parameter"):
monkeypatch.setattr(
f"vllm.model_executor.{module}.get_tensor_model_parallel_rank", lambda: 0
)
monkeypatch.setattr(
f"vllm.model_executor.{module}.get_tensor_model_parallel_world_size",
lambda: 1,
)
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
gate = GateLinear(
input_size=input_size,
output_size=output_size,
bias=False,
out_dtype=torch.float32,
params_dtype=params_dtype,
).cuda()
with torch.no_grad():
gate.weight.normal_()
gate.quant_method.process_weights_after_loading(gate)
return gate
@pytest.mark.parametrize("x_dtype", [torch.bfloat16, torch.float32])
@pytest.mark.parametrize("num_tokens", [1, 32, 33])
@torch.inference_mode()
def test_gate_linear_uses_rocm_fp32_router_gemm(
monkeypatch: pytest.MonkeyPatch, num_tokens: int, x_dtype: torch.dtype
) -> None:
"""A router shape outside the old tuned list takes the fast path for M<=32."""
gate = _make_gate_linear(monkeypatch, 2560, 512)
assert gate.allow_fp32_router_gemm
calls = []
kernel = rocm_router.rocm_fp32_router_gemm
def spy(hidden_states, router_weight):
calls.append(hidden_states.shape[0])
return kernel(hidden_states, router_weight)
monkeypatch.setattr(rocm_router, "rocm_fp32_router_gemm", spy)
x = torch.randn(num_tokens, 2560, dtype=x_dtype, device="cuda")
output, bias = gate(x)
assert bias is None
assert calls == ([num_tokens] if num_tokens <= MAX_TOKENS else [])
torch.testing.assert_close(output, _reference(x, gate.weight), atol=ATOL, rtol=RTOL)
@pytest.mark.parametrize(
("num_tokens", "num_experts", "uses_kernel"),
[(1, 256, True), (32, 256, True), (33, 256, False), (32, 512, False)],
)
@torch.inference_mode()
def test_gate_linear_bf16_weight_uses_rocm_router_gemm(
monkeypatch: pytest.MonkeyPatch, num_tokens, num_experts, uses_kernel
) -> None:
"""BF16-weight gates with FP32 logits use the kernel only where it beats
hipBLASLt; everything else keeps the torch.mm path."""
gate = _make_gate_linear(monkeypatch, 7168, num_experts, torch.bfloat16)
assert gate.allow_cublas_router_gemm
calls = []
kernel = rocm_router.rocm_fp32_router_gemm
def spy(hidden_states, router_weight):
calls.append(hidden_states.shape[0])
return kernel(hidden_states, router_weight)
monkeypatch.setattr(rocm_router, "rocm_fp32_router_gemm", spy)
x = torch.randn(num_tokens, 7168, dtype=torch.bfloat16, device="cuda")
output, _ = gate(x)
assert output.dtype == torch.float32
assert calls == ([num_tokens] if uses_kernel else [])
torch.testing.assert_close(
output, _reference(x.float(), gate.weight.float()), atol=ATOL, rtol=RTOL
)