Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -122,6 +122,7 @@ def ref_single_query_cached_kv_attention(
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("use_interleaved_v_cache", [False, True])
|
||||
def test_paged_attention(
|
||||
kv_cache_factory,
|
||||
num_seqs: int,
|
||||
@@ -133,9 +134,14 @@ def test_paged_attention(
|
||||
kv_cache_dtype: str,
|
||||
seed: int,
|
||||
device: str,
|
||||
use_interleaved_v_cache: bool,
|
||||
) -> None:
|
||||
if current_platform.is_navi() and (
|
||||
kv_cache_dtype == "fp8" or head_size != 128 or block_size != 16 or use_alibi
|
||||
kv_cache_dtype == "fp8"
|
||||
or head_size != 128
|
||||
or block_size != 16
|
||||
or use_alibi
|
||||
or use_interleaved_v_cache
|
||||
):
|
||||
pytest.skip()
|
||||
|
||||
@@ -201,6 +207,20 @@ def test_paged_attention(
|
||||
last_block_idx = block_tables_lst[seq_idx][seq_len // block_size]
|
||||
value_cache[last_block_idx, :, :, padding_start:] = padding_nan
|
||||
|
||||
standard_value_cache = value_cache
|
||||
if use_interleaved_v_cache:
|
||||
# Hand the kernel V in the interleaved layout (same 4D shape); the
|
||||
# reference below gets the standard layout back.
|
||||
x_v = 16 // value_cache.element_size()
|
||||
value_cache = (
|
||||
value_cache.view(
|
||||
NUM_BLOCKS, num_kv_heads, head_size, block_size // x_v, x_v
|
||||
)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(NUM_BLOCKS, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
|
||||
# Call the paged attention kernel.
|
||||
output = torch.empty_like(query)
|
||||
num_partitions = (max_seq_len + PARTITION_SIZE_ROCM - 1) // PARTITION_SIZE_ROCM
|
||||
@@ -234,6 +254,7 @@ def test_paged_attention(
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
use_interleaved_v_cache=use_interleaved_v_cache,
|
||||
)
|
||||
|
||||
opcheck(
|
||||
@@ -259,10 +280,13 @@ def test_paged_attention(
|
||||
v_scale,
|
||||
None,
|
||||
"f16",
|
||||
use_interleaved_v_cache,
|
||||
),
|
||||
cond=(head_size == 64 and block_size == BLOCK_SIZES[0]),
|
||||
)
|
||||
|
||||
value_cache = standard_value_cache
|
||||
|
||||
# Run the reference implementation.
|
||||
if kv_cache_dtype == "fp8":
|
||||
# Convert cache data back to dtype.
|
||||
|
||||
@@ -109,6 +109,7 @@ def create_alibi_causal_mask(
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention(
|
||||
num_heads: int,
|
||||
@@ -119,6 +120,7 @@ def test_contexted_kv_attention(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
block_size: int = 32,
|
||||
) -> None:
|
||||
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
|
||||
@@ -225,6 +227,23 @@ def test_contexted_kv_attention(
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
extra_op_kwargs = {}
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
|
||||
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
|
||||
# read it back interleaved.
|
||||
x_v = 16 // v_cache.element_size()
|
||||
v_cache = (
|
||||
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(cache_size, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
if op is context_attention_fwd:
|
||||
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
|
||||
else:
|
||||
extra_op_kwargs["use_interleaved_v_cache"] = True
|
||||
|
||||
# Warm up the Triton kernel by calling it once before actually measuring
|
||||
# generation time
|
||||
op(
|
||||
@@ -243,6 +262,7 @@ def test_contexted_kv_attention(
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=sliding_window,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
start_time = time.time()
|
||||
@@ -262,6 +282,7 @@ def test_contexted_kv_attention(
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=sliding_window,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
end_time = time.time()
|
||||
@@ -386,6 +407,7 @@ def test_rocm_backend_sliding_window_includes_boundary_token(
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_cached_kv(
|
||||
num_heads: int,
|
||||
@@ -394,6 +416,7 @@ def test_contexted_kv_attention_cached_kv(
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
interleave_v: bool,
|
||||
block_size: int = 32,
|
||||
) -> None:
|
||||
# Exercises the KV_FROM_CACHE path of context_attention_fwd: the current
|
||||
@@ -487,6 +510,18 @@ def test_contexted_kv_attention_cached_kv(
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
interleaved_v_pack_factor = 0
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape).
|
||||
interleaved_v_pack_factor = 16 // v_cache.element_size()
|
||||
x_v = interleaved_v_pack_factor
|
||||
v_cache = (
|
||||
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(cache_size, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
|
||||
# Cached-K/V path: current-chunk k and v are None.
|
||||
context_attention_fwd(
|
||||
query,
|
||||
@@ -504,6 +539,7 @@ def test_contexted_kv_attention_cached_kv(
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=0,
|
||||
interleaved_v_pack_factor=interleaved_v_pack_factor,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -544,8 +580,11 @@ def test_contexted_kv_attention_cached_kv(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> None:
|
||||
def test_contexted_kv_attention_cached_kv_block_table_boundary(
|
||||
device: str, interleave_v: bool
|
||||
) -> None:
|
||||
# Boundary guard for the KV_FROM_CACHE block-table load. With an
|
||||
# exact-sized block table (row length == number of blocks for the
|
||||
# sequence) and a query that ends the sequence, the last K/V tile has
|
||||
@@ -611,6 +650,18 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
interleaved_v_pack_factor = 0
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape).
|
||||
interleaved_v_pack_factor = 16 // v_cache.element_size()
|
||||
x_v = interleaved_v_pack_factor
|
||||
v_cache = (
|
||||
v_cache.view(num_blocks, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(num_blocks, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
|
||||
# Cached-K/V path: current-chunk k and v are None.
|
||||
context_attention_fwd(
|
||||
query,
|
||||
@@ -628,6 +679,7 @@ def test_contexted_kv_attention_cached_kv_block_table_boundary(device: str) -> N
|
||||
k_scale,
|
||||
v_scale,
|
||||
sliding_window=0,
|
||||
interleaved_v_pack_factor=interleaved_v_pack_factor,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -723,6 +775,7 @@ def test_contexted_kv_attention_cached_kv_alibi_unsupported(device: str) -> None
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_alibi(
|
||||
num_heads: int,
|
||||
@@ -732,6 +785,7 @@ def test_contexted_kv_attention_alibi(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
block_size: int = 32,
|
||||
) -> None:
|
||||
if "fp8" in kv_cache_dtype and not current_platform.has_device_capability(89):
|
||||
@@ -860,6 +914,23 @@ def test_contexted_kv_attention_alibi(
|
||||
)
|
||||
k_scale = v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
|
||||
extra_op_kwargs = {}
|
||||
if interleave_v:
|
||||
# Rewrite the V cache into the interleaved layout (same 4D shape). Both
|
||||
# context_attention_fwd (prefill) and kernel_paged_attention_2d (decode)
|
||||
# read it back interleaved.
|
||||
x_v = 16 // v_cache.element_size()
|
||||
v_cache = (
|
||||
v_cache.view(cache_size, num_kv_heads, head_size, block_size // x_v, x_v)
|
||||
.permute(0, 1, 3, 2, 4)
|
||||
.contiguous()
|
||||
.view(cache_size, num_kv_heads, head_size, block_size)
|
||||
)
|
||||
if op is context_attention_fwd:
|
||||
extra_op_kwargs["interleaved_v_pack_factor"] = x_v
|
||||
else:
|
||||
extra_op_kwargs["use_interleaved_v_cache"] = True
|
||||
|
||||
# Warm up the Triton kernel by calling it once before actually measuring
|
||||
# generation time
|
||||
op(
|
||||
@@ -878,6 +949,7 @@ def test_contexted_kv_attention_alibi(
|
||||
k_scale,
|
||||
v_scale,
|
||||
alibi_slopes=alibi_slopes,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
start_time = time.time()
|
||||
@@ -897,6 +969,7 @@ def test_contexted_kv_attention_alibi(
|
||||
k_scale,
|
||||
v_scale,
|
||||
alibi_slopes=alibi_slopes,
|
||||
**extra_op_kwargs,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
end_time = time.time()
|
||||
@@ -992,6 +1065,7 @@ def test_contexted_kv_attention_alibi(
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("sliding_window", SLIDING_WINDOW)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_f32(
|
||||
num_heads: int,
|
||||
@@ -1002,6 +1076,7 @@ def test_contexted_kv_attention_f32(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
) -> None:
|
||||
test_contexted_kv_attention(
|
||||
num_heads,
|
||||
@@ -1012,6 +1087,7 @@ def test_contexted_kv_attention_f32(
|
||||
kv_cache_dtype,
|
||||
device,
|
||||
op,
|
||||
interleave_v=interleave_v,
|
||||
)
|
||||
|
||||
|
||||
@@ -1023,6 +1099,7 @@ def test_contexted_kv_attention_f32(
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_contexted_kv_attention_alibi_f32(
|
||||
num_heads: int,
|
||||
@@ -1032,9 +1109,17 @@ def test_contexted_kv_attention_alibi_f32(
|
||||
kv_cache_dtype: str,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
) -> None:
|
||||
test_contexted_kv_attention_alibi(
|
||||
num_heads, num_queries_per_kv, head_size, dtype, kv_cache_dtype, device, op
|
||||
num_heads,
|
||||
num_queries_per_kv,
|
||||
head_size,
|
||||
dtype,
|
||||
kv_cache_dtype,
|
||||
device,
|
||||
op,
|
||||
interleave_v=interleave_v,
|
||||
)
|
||||
|
||||
|
||||
@@ -1052,6 +1137,7 @@ NONSTANDARD_BLOCK_SIZE_SHAPES = [
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("op", OPS)
|
||||
@pytest.mark.parametrize("interleave_v", [False, True])
|
||||
@torch.inference_mode()
|
||||
def test_qwen3_nonstandard_block_size(
|
||||
num_heads: int,
|
||||
@@ -1061,6 +1147,7 @@ def test_qwen3_nonstandard_block_size(
|
||||
dtype: torch.dtype,
|
||||
device: str,
|
||||
op: Callable,
|
||||
interleave_v: bool,
|
||||
) -> None:
|
||||
"""Non-power-of-2 pages must match, even when a tile straddles a page."""
|
||||
if not current_platform.is_rocm():
|
||||
@@ -1076,4 +1163,5 @@ def test_qwen3_nonstandard_block_size(
|
||||
kv_cache_dtype="auto",
|
||||
device=device,
|
||||
op=op,
|
||||
interleave_v=interleave_v,
|
||||
)
|
||||
|
||||
@@ -1061,3 +1061,25 @@ def test_aiter_mha_varlen_fp8_kv(dtype):
|
||||
rtol=rtol,
|
||||
)
|
||||
torch.testing.assert_close(output, ref, atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
def test_asm_kv_scale_tables_shared_across_builders():
|
||||
"""Builders share one AsmKvScaleTables; equal (shape, scale) requests must
|
||||
return the same dense table, a different scale a different one."""
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import (
|
||||
AiterFlashAttentionMetadataBuilder,
|
||||
AsmKvScaleTables,
|
||||
)
|
||||
|
||||
tables = AsmKvScaleTables()
|
||||
device = torch.device("cuda")
|
||||
table = tables.get(device, 8, 2, 16, 0.5)
|
||||
assert tables.get(device, 8, 2, 16, 0.5) is table
|
||||
assert tables.get(device, 8, 2, 16, 1.0) is not table
|
||||
assert table.shape == (8, 2, 16)
|
||||
assert table.dtype == torch.float32
|
||||
assert torch.all(table == 0.5)
|
||||
# The builder class holds the single process-wide instance.
|
||||
assert isinstance(
|
||||
AiterFlashAttentionMetadataBuilder.asm_kv_scale_tables, AsmKvScaleTables
|
||||
)
|
||||
|
||||
@@ -1989,8 +1989,8 @@ def test_dsv41_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
|
||||
"""Replay V4.1 ROCm decode after device-only query reallocation."""
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
|
||||
DeepseekV41ROCMAiterSparseSWAMetadataBuilder,
|
||||
compute_global_topk_ragged_indices_and_indptr,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
|
||||
@@ -2044,7 +2044,7 @@ def test_dsv41_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
|
||||
mla_builder = DeepseekV41ROCMAiterMLASparseMetadataBuilder(
|
||||
mla_spec, ["c2a"], vllm_config, device
|
||||
)
|
||||
swa_builder = DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
|
||||
swa_builder = DeepseekV41ROCMAiterSparseSWAMetadataBuilder(
|
||||
swa_spec, ["c2a"], vllm_config, device
|
||||
)
|
||||
assert (
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -52,6 +52,7 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import input_to_float8
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
dequant_mxfp8_to_bf16,
|
||||
mxfp8_e4m3_quantize,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
@@ -65,7 +66,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
)
|
||||
from vllm.model_executor.models.llama4 import Llama4MoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.math_utils import next_power_of_2, round_up
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
try:
|
||||
@@ -788,7 +789,7 @@ def test_trtllm_fp8_swiglu_clamp_support(
|
||||
|
||||
|
||||
def _make_mxfp8_moe_weights(
|
||||
e: int, n: int, k: int
|
||||
e: int, n: int, k: int, is_gated: bool = True
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
def quantize(w: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
q, s = zip(
|
||||
@@ -796,8 +797,9 @@ def _make_mxfp8_moe_weights(
|
||||
)
|
||||
return torch.stack(q), torch.stack(s)
|
||||
|
||||
w1_rows = 2 * n if is_gated else n
|
||||
w1, w1_scale = quantize(
|
||||
torch.randn(e, 2 * n, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
torch.randn(e, w1_rows, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
)
|
||||
w2, w2_scale = quantize(
|
||||
torch.randn(e, k, n, device="cuda", dtype=torch.bfloat16) / 10
|
||||
@@ -806,16 +808,20 @@ def _make_mxfp8_moe_weights(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_shape", [[128, 128], [1, 32]], ids=["block", "mxfp8"])
|
||||
@pytest.mark.parametrize("transposed", [True, False], ids=["transposed", "row-major"])
|
||||
@pytest.mark.parametrize("m", [1, 16])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
|
||||
)
|
||||
def test_trtllm_fp8_block_moe_deferred_finalize(
|
||||
m: int, block_shape: list[int], workspace_init
|
||||
m: int, block_shape: list[int], transposed: bool, workspace_init
|
||||
):
|
||||
"""TRTLLM-Gen block-FP8 and MXFP8 modular experts can leave the top-k
|
||||
finalize to the caller."""
|
||||
finalize to the caller. For fp8-block, ``transposed`` toggles the oracle's
|
||||
is_scale_transposed column-major activation-scale layout."""
|
||||
if block_shape != [128, 128] and transposed:
|
||||
pytest.skip("column-major scales apply to fp8 [128, 128] block only")
|
||||
e, topk, n, k = 32, 4, 1024, 1024
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
@@ -849,6 +855,9 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
|
||||
a2_scale=None,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
is_block = block_shape == [128, 128]
|
||||
if is_block and not transposed:
|
||||
quant_config = replace(quant_config, is_scale_transposed=False)
|
||||
# One rank of a TP group, which deferral needs.
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
@@ -866,19 +875,27 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
|
||||
routing_method=RoutingMethodType.TopK,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
kernel = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config, quant_config=quant_config, allow_new_interface=True
|
||||
),
|
||||
TrtLlmFp8ExpertsModular(moe_config=moe_config, quant_config=quant_config),
|
||||
)
|
||||
|
||||
def make_kernel(quant_config: FusedMoEQuantConfig) -> mk.FusedMoEKernel:
|
||||
return mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
),
|
||||
TrtLlmFp8ExpertsModular(
|
||||
moe_config=moe_config, quant_config=quant_config
|
||||
),
|
||||
)
|
||||
|
||||
kernel = make_kernel(quant_config)
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
|
||||
score = torch.randn((m, e), device="cuda", dtype=torch.bfloat16)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
check_deferred_moe_finalize(
|
||||
moe_config,
|
||||
lambda: kernel.apply(
|
||||
|
||||
def apply(k_: mk.FusedMoEKernel) -> torch.Tensor:
|
||||
return k_.apply(
|
||||
hidden_states=a,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
@@ -888,7 +905,34 @@ def test_trtllm_fp8_block_moe_deferred_finalize(
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
),
|
||||
)
|
||||
|
||||
if is_block and transposed:
|
||||
# The oracle gates column-major scales on TRTLLM fp8-block.
|
||||
assert quant_config.is_scale_transposed
|
||||
assert not make_fp8_moe_quant_config(
|
||||
Fp8MoeBackend.DEEPGEMM,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=block_shape,
|
||||
).is_scale_transposed
|
||||
# no_dp_ep emits column-major [M, G] scales (physical [G, M]), so
|
||||
# the experts' a1q_scale.t().contiguous() is a no-op.
|
||||
_, a1q_scale, _, _, _ = kernel.prepare_finalize.prepare(
|
||||
a, topk_weights, topk_ids, e, None, False, quant_config
|
||||
)
|
||||
assert a1q_scale.t().is_contiguous()
|
||||
# Bitwise-equal to running with row-major scales.
|
||||
row_major = replace(quant_config, is_scale_transposed=False)
|
||||
torch.testing.assert_close(
|
||||
apply(kernel), apply(make_kernel(row_major)), atol=0, rtol=0
|
||||
)
|
||||
|
||||
check_deferred_moe_finalize(
|
||||
moe_config,
|
||||
lambda: apply(kernel),
|
||||
router_weights=topk_weights,
|
||||
)
|
||||
|
||||
@@ -986,3 +1030,123 @@ def test_trtllm_mxfp8_minimax2_routing_applies_routed_scale(m: int, workspace_in
|
||||
)
|
||||
|
||||
torch.testing.assert_close(fused, reference, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("activation", "n", "k"),
|
||||
[
|
||||
(MoEActivation.SILU, 1024, 1024),
|
||||
(MoEActivation.SILU, 928, 1024),
|
||||
(MoEActivation.SILU, 1024, 2688),
|
||||
(MoEActivation.RELU2_NO_MUL, 1024, 1024),
|
||||
# Nemotron-3.5 Lightning's non-gated MoE: intermediate 1856, hidden 2688.
|
||||
(MoEActivation.RELU2_NO_MUL, 1856, 2688),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("m", [1, 32])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="Requires TRTLLM-Gen FP8 MoE (SM100)",
|
||||
)
|
||||
def test_trtllm_mxfp8_moe_pads_intermediate(
|
||||
m: int, activation: MoEActivation, n: int, k: int, workspace_init
|
||||
):
|
||||
"""TRTLLM-Gen MXFP8 experts need the intermediate size to be a multiple
|
||||
of 128 and the hidden size a multiple of 256. Misaligned experts, gated or
|
||||
not, are padded at load and, with the activations padded and the output
|
||||
cropped as the MoE runner does, must still match the unquantized reference
|
||||
on the dequantized weights."""
|
||||
e, topk = 8, 2
|
||||
is_gated = activation.is_gated
|
||||
block_shape = [1, 32]
|
||||
set_random_seed(7)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
w1, w2, w1_scale, w2_scale = _make_mxfp8_moe_weights(e, n, k, is_gated)
|
||||
w1_ref = dequant_mxfp8_to_bf16(w1, w1_scale)
|
||||
w2_ref = dequant_mxfp8_to_bf16(w2, w2_scale)
|
||||
|
||||
layer = SimpleNamespace(
|
||||
weight_block_size=block_shape,
|
||||
moe_config=SimpleNamespace(
|
||||
is_act_and_mul=is_gated, intermediate_size_per_partition=n
|
||||
),
|
||||
activation=activation,
|
||||
)
|
||||
w1, w2, w1_scale, w2_scale = convert_to_fp8_moe_kernel_format(
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
layer,
|
||||
w1,
|
||||
w2,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
w13_input_scale=None,
|
||||
w2_input_scale=None,
|
||||
)
|
||||
padded_n = layer.moe_config.intermediate_size_per_partition
|
||||
padded_k = layer.moe_config.hidden_dim
|
||||
assert padded_n == round_up(n, 128)
|
||||
assert padded_k == round_up(k, 256)
|
||||
|
||||
quant_config = make_fp8_moe_quant_config(
|
||||
Fp8MoeBackend.FLASHINFER_TRTLLM,
|
||||
w1_scale,
|
||||
w2_scale,
|
||||
a1_scale=None,
|
||||
a2_scale=None,
|
||||
block_shape=block_shape,
|
||||
)
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=padded_k,
|
||||
intermediate_size=padded_n,
|
||||
num_local_experts=e,
|
||||
num_logical_experts=e,
|
||||
activation=activation,
|
||||
device="cuda",
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
in_dtype=torch.bfloat16,
|
||||
routing_method=RoutingMethodType.Renormalize,
|
||||
max_num_tokens=next_power_of_2(m),
|
||||
)
|
||||
# The monolithic experts pass the activation type to the kernel; the
|
||||
# modular ones always run it gated.
|
||||
kernel = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=True,
|
||||
),
|
||||
TrtLlmFp8ExpertsMonolithic(
|
||||
moe_config=moe_config, quant_config=quant_config
|
||||
),
|
||||
)
|
||||
|
||||
a = torch.randn((m, k), device="cuda", dtype=torch.bfloat16) / 10
|
||||
logits = torch.randn((m, e), device="cuda", dtype=torch.float32)
|
||||
output = kernel.apply_monolithic(
|
||||
hidden_states=torch.nn.functional.pad(a, (0, padded_k - k)),
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
router_logits=logits,
|
||||
activation=activation,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
routed_scaling_factor=1.0,
|
||||
routing_sink=None,
|
||||
)[:, :k]
|
||||
topk_weights, topk_ids, _ = fused_topk(a, logits, topk, renormalize=True)
|
||||
reference = fused_experts(
|
||||
a,
|
||||
w1_ref,
|
||||
w2_ref,
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=activation,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
)
|
||||
|
||||
check_accuracy(ref_output=reference, actual_output=output)
|
||||
|
||||
@@ -42,13 +42,15 @@ ROCM_AVAILABLE = current_platform.is_rocm()
|
||||
ROCM_TRITON_KERNELS_AVAILABLE = False
|
||||
ROCM_AITER_AVAILABLE = is_aiter_found()
|
||||
ROCM_GFX950 = False
|
||||
ROCM_GFX1250 = False
|
||||
|
||||
if ROCM_AVAILABLE:
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
from vllm.platforms.rocm import on_gfx950, on_gfx1250
|
||||
from vllm.utils.import_utils import has_triton_kernels
|
||||
|
||||
ROCM_TRITON_KERNELS_AVAILABLE = has_triton_kernels()
|
||||
ROCM_GFX950 = on_gfx950()
|
||||
ROCM_GFX1250 = on_gfx1250()
|
||||
|
||||
if ROCM_AITER_AVAILABLE:
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
@@ -1568,6 +1570,15 @@ ROCM_BACKEND_CONFIGS = {
|
||||
"requires_aiter": True,
|
||||
"requires_gfx950": True,
|
||||
},
|
||||
"AITER_TRITON_MXFP4_MXFP4": {
|
||||
"activation": "SILU",
|
||||
"act_type": "mxfp4",
|
||||
"rtol": 1.0,
|
||||
"percent": 0.8,
|
||||
"requires_aiter": True,
|
||||
"requires_gfx950": False,
|
||||
"requires_gfx1250": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -1607,6 +1618,8 @@ def test_rocm_mxfp4_moe_oracle(
|
||||
pytest.skip(f"Backend {backend_name} requires AITER")
|
||||
if config["requires_gfx950"] and not ROCM_GFX950:
|
||||
pytest.skip(f"Backend {backend_name} requires GFX950")
|
||||
if config.get("requires_gfx1250") and not ROCM_GFX1250:
|
||||
pytest.skip(f"Backend {backend_name} requires GFX1250")
|
||||
|
||||
import vllm.distributed.parallel_state as ps
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
|
||||
@@ -219,3 +219,169 @@ def test_rocm_aiter_grouped_topk_torch_compile_compatibility():
|
||||
topk_weights_original, topk_weights_compiled, rtol=1e-2, atol=1e-2
|
||||
)
|
||||
assert torch.allclose(topk_ids_original, topk_ids_compiled)
|
||||
|
||||
|
||||
def test_rocm_aiter_topk_gating_custom_op_registration():
|
||||
assert callable(torch.ops.vllm.rocm_aiter_topk_gating)
|
||||
|
||||
|
||||
def _sort_routing(weights: torch.Tensor, ids: torch.Tensor):
|
||||
ids_sorted, order = torch.sort(ids, dim=-1)
|
||||
weights_sorted = torch.gather(weights, 1, order)
|
||||
return weights_sorted, ids_sorted
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 64])
|
||||
@pytest.mark.parametrize("num_experts", [256, 512])
|
||||
@pytest.mark.parametrize("topk", [8, 10])
|
||||
@pytest.mark.parametrize("renormalize", [True, False])
|
||||
def test_rocm_aiter_topk_gating_matches_softmax_reference(
|
||||
num_tokens: int,
|
||||
num_experts: int,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
):
|
||||
"""Numerical check vs softmax + topk, including Qwen3.8 (E=512, k=10)."""
|
||||
torch.manual_seed(0)
|
||||
gating_output = torch.randn(
|
||||
(num_tokens, num_experts), dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
scores = torch.softmax(gating_output.float(), dim=-1)
|
||||
cutoff = torch.topk(scores, k=topk, dim=-1).values[:, -1:]
|
||||
|
||||
topk_weights = torch.empty((num_tokens, topk), dtype=torch.float32, device="cuda")
|
||||
topk_ids = torch.empty((num_tokens, topk), dtype=torch.int32, device="cuda")
|
||||
token_expert_indices = torch.empty(
|
||||
(num_tokens, topk), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
torch.ops.vllm.rocm_aiter_topk_gating(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
0,
|
||||
"",
|
||||
)
|
||||
|
||||
# BF16 logits can tie at the top-k boundary, so a different tied expert is
|
||||
# valid. Check selection against the cutoff and weights against the scores
|
||||
# of the experts the kernel actually selected.
|
||||
assert ((topk_ids >= 0) & (topk_ids < num_experts)).all()
|
||||
sorted_ids = torch.sort(topk_ids, dim=-1).values
|
||||
assert (sorted_ids[:, 1:] != sorted_ids[:, :-1]).all()
|
||||
selected_scores = torch.gather(scores, 1, topk_ids.long())
|
||||
assert (selected_scores >= cutoff).all()
|
||||
expected_weights = selected_scores
|
||||
if renormalize:
|
||||
expected_weights = expected_weights / expected_weights.sum(dim=-1, keepdim=True)
|
||||
torch.testing.assert_close(topk_weights, expected_weights, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def test_rocm_aiter_topk_gating_shared_expert_suffix():
|
||||
"""Fused shared expert: softmax top-k on the prefix, sigmoid appended.
|
||||
|
||||
Matches the Qwen3.8 FSE launch: 512 routed experts, top-10, one shared
|
||||
column, weights wide enough for the suffix, and the shared id prefilled.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
num_tokens, num_experts, topk, num_shared = 4, 512, 10, 1
|
||||
routed = torch.randn((num_tokens, num_experts), dtype=torch.bfloat16, device="cuda")
|
||||
shared = torch.randn((num_tokens, num_shared), dtype=torch.bfloat16, device="cuda")
|
||||
shared[0, 0] = 40
|
||||
gating_output = torch.cat([routed, shared], dim=-1).contiguous()
|
||||
width = topk + num_shared
|
||||
shared_ids = torch.arange(
|
||||
num_experts, num_experts + num_shared, device="cuda", dtype=torch.int32
|
||||
)
|
||||
topk_ids = torch.full((num_tokens, width), -1, dtype=torch.int32, device="cuda")
|
||||
topk_ids[:, topk:] = shared_ids
|
||||
topk_weights = torch.empty((num_tokens, width), dtype=torch.float32, device="cuda")
|
||||
token_expert_indices = torch.empty(
|
||||
(num_tokens, topk), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
torch.ops.vllm.rocm_aiter_topk_gating(
|
||||
topk_weights,
|
||||
topk_ids[:, :topk],
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
True,
|
||||
num_shared,
|
||||
"sigmoid",
|
||||
)
|
||||
|
||||
scores = torch.softmax(routed.float(), dim=-1)
|
||||
cutoff = torch.topk(scores, k=topk, dim=-1).values[:, -1:]
|
||||
routed_ids = topk_ids[:, :topk]
|
||||
assert ((routed_ids >= 0) & (routed_ids < num_experts)).all()
|
||||
sorted_ids = torch.sort(routed_ids, dim=-1).values
|
||||
assert (sorted_ids[:, 1:] != sorted_ids[:, :-1]).all()
|
||||
selected = torch.gather(scores, 1, routed_ids.long())
|
||||
assert (selected >= cutoff).all()
|
||||
expected = selected / selected.sum(dim=-1, keepdim=True)
|
||||
torch.testing.assert_close(topk_weights[:, :topk], expected, atol=2e-2, rtol=2e-2)
|
||||
torch.testing.assert_close(
|
||||
topk_weights[:, topk:],
|
||||
torch.sigmoid(shared.float()),
|
||||
atol=2e-2,
|
||||
rtol=2e-2,
|
||||
)
|
||||
assert torch.equal(topk_ids[:, topk:], shared_ids.expand_as(topk_ids[:, topk:]))
|
||||
|
||||
|
||||
def test_rocm_aiter_topk_gating_torch_compile_compatibility():
|
||||
token, expert, topk, renormalize = 4, 512, 10, True
|
||||
gating_output = torch.randn((token, expert), dtype=torch.bfloat16, device="cuda")
|
||||
device = gating_output.device
|
||||
|
||||
def gating_fn(gating_output, topk_weights, topk_ids, token_expert_indices):
|
||||
return torch.ops.vllm.rocm_aiter_topk_gating(
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
token_expert_indices,
|
||||
gating_output,
|
||||
renormalize,
|
||||
0,
|
||||
"",
|
||||
)
|
||||
|
||||
torch.library.opcheck(
|
||||
torch.ops.vllm.rocm_aiter_topk_gating,
|
||||
(
|
||||
torch.empty((token, topk), dtype=torch.float32, device=device),
|
||||
torch.empty((token, topk), dtype=torch.int32, device=device),
|
||||
torch.empty((token, topk), dtype=torch.int32, device=device),
|
||||
gating_output,
|
||||
),
|
||||
kwargs={
|
||||
"renormalize": renormalize,
|
||||
"num_shared_experts": 0,
|
||||
"shared_expert_scoring_func": "",
|
||||
},
|
||||
test_utils=("test_faketensor"),
|
||||
)
|
||||
|
||||
compiled_fn = torch.compile(
|
||||
gating_fn,
|
||||
fullgraph=True,
|
||||
backend="inductor",
|
||||
mode="reduce-overhead",
|
||||
dynamic=False,
|
||||
)
|
||||
|
||||
def alloc():
|
||||
return (
|
||||
torch.empty((token, topk), dtype=torch.float32, device=device),
|
||||
torch.empty((token, topk), dtype=torch.int32, device=device),
|
||||
torch.empty((token, topk), dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
w0, i0, t0 = alloc()
|
||||
w1, i1, t1 = alloc()
|
||||
gating_fn(gating_output, w0, i0, t0)
|
||||
compiled_fn(gating_output, w1, i1, t1)
|
||||
w0, i0 = _sort_routing(w0, i0)
|
||||
w1, i1 = _sort_routing(w1, i1)
|
||||
assert torch.allclose(w0, w1, rtol=1e-2, atol=1e-2)
|
||||
assert torch.equal(i0, i1)
|
||||
|
||||
@@ -81,32 +81,11 @@ def _torch_topk_softplus_sqrt(
|
||||
return topk_weights.to(torch.float32), topk_ids.to(torch.int32)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
|
||||
@pytest.mark.parametrize("has_hash_routing", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"backend,expected_experts",
|
||||
[
|
||||
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
|
||||
("aiter", "AiterMxfp4Experts"),
|
||||
("triton_unfused", "UnfusedOAITritonExperts"),
|
||||
],
|
||||
)
|
||||
def test_hash_routing_backend_selection(
|
||||
dist_init,
|
||||
default_vllm_config,
|
||||
monkeypatch,
|
||||
has_hash_routing,
|
||||
backend,
|
||||
expected_experts,
|
||||
):
|
||||
"""Hash tables must reach backend selection; modular AITER remains eligible."""
|
||||
def _dsv4_moe_config(monkeypatch, default_vllm_config, has_hash_routing, backend):
|
||||
from dataclasses import replace
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_deepseek_v4_mxfp4_moe_backend,
|
||||
)
|
||||
from vllm.platforms import rocm
|
||||
|
||||
# Exercise both native MX and monolithic AITER selection on any ROCm device.
|
||||
@@ -130,14 +109,63 @@ def test_hash_routing_backend_selection(
|
||||
),
|
||||
prefix="hash_routing_selection",
|
||||
)
|
||||
config = replace(layer.routed_experts.moe_config, moe_backend=backend)
|
||||
return layer, replace(layer.routed_experts.moe_config, moe_backend=backend)
|
||||
|
||||
if has_hash_routing and backend == "aiter_triton_mxfp4_bf16":
|
||||
with pytest.raises(ValueError, match="hash routing"):
|
||||
select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
else:
|
||||
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
assert experts_cls.__name__ == expected_experts
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
|
||||
@pytest.mark.parametrize("has_hash_routing", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"backend,expected_experts",
|
||||
[
|
||||
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
|
||||
("aiter", "AiterMxfp4Experts"),
|
||||
("triton_unfused", "UnfusedOAITritonExperts"),
|
||||
],
|
||||
)
|
||||
def test_hash_routing_backend_selection(
|
||||
dist_init,
|
||||
default_vllm_config,
|
||||
monkeypatch,
|
||||
has_hash_routing,
|
||||
backend,
|
||||
expected_experts,
|
||||
):
|
||||
"""Hash tables reach backend selection and hash-capable kernels."""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_deepseek_v4_mxfp4_moe_backend,
|
||||
)
|
||||
|
||||
layer, config = _dsv4_moe_config(
|
||||
monkeypatch, default_vllm_config, has_hash_routing, backend
|
||||
)
|
||||
_, experts_cls = select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
assert experts_cls.__name__ == expected_experts
|
||||
# Monolithic kernels route internally, so they read the table from the layer.
|
||||
assert (layer.routed_experts.hash_indices_table is not None) == has_hash_routing
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm backend selection")
|
||||
def test_hash_routing_rejects_monolithic_without_support(
|
||||
dist_init, default_vllm_config, monkeypatch
|
||||
):
|
||||
"""A monolithic kernel that does not implement hash routing is rejected."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.aiter_mxfp4_w4a16_moe import (
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
select_deepseek_v4_mxfp4_moe_backend,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
AiterW4A16ExpertsMonolithic,
|
||||
"_supports_hash_routing",
|
||||
staticmethod(lambda: False),
|
||||
)
|
||||
_, config = _dsv4_moe_config(
|
||||
monkeypatch, default_vllm_config, True, "aiter_triton_mxfp4_bf16"
|
||||
)
|
||||
with pytest.raises(ValueError, match="hash routing"):
|
||||
select_deepseek_v4_mxfp4_moe_backend(config)
|
||||
|
||||
|
||||
def test_torch_topk_softplus_sqrt_breaks_ties_by_expert_id():
|
||||
|
||||
@@ -50,7 +50,7 @@ SKINNY_GEMM_PASS_RATES = {
|
||||
}
|
||||
PRESHUFFLED_SHAPES = [
|
||||
(64, 4096, 8192),
|
||||
(32, 8192, 16384),
|
||||
(32, 16384, 16384),
|
||||
]
|
||||
|
||||
|
||||
@@ -421,8 +421,12 @@ def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
|
||||
M, K, N = shape
|
||||
|
||||
assert M <= 64
|
||||
# The predicate takes K as packed bytes, matching `weight.shape[1]`.
|
||||
# The predicate takes K as packed bytes, matching `weight.shape[1]`, and
|
||||
# must agree with the config the kernel itself resolves.
|
||||
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(N, K // 2)
|
||||
from aiter.ops.triton.utils.gemm_config_utils import get_gemm_config
|
||||
|
||||
assert get_gemm_config("GEMM-AFP4WFP4_PRESHUFFLED", M, N, K)[1]
|
||||
|
||||
A = torch.randn(M, K, dtype=torch.bfloat16)
|
||||
B = torch.randn(N, K, dtype=torch.bfloat16)
|
||||
@@ -464,6 +468,18 @@ def test_aiter_fp4_gemm_preshuffled_tuned_shapes(shape):
|
||||
_assert_deterministic(run_preshuffled, n_runs=3)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fp4_preshuffled_tuned_gate_uses_logical_k():
|
||||
"""AITER ships N=16384-K=16384 but not N=16384-K=8192, so a K=8192 layer
|
||||
has no tuned preshuffled config even though twice its K does."""
|
||||
_assert_aiter_supported()
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned.cache_clear()
|
||||
assert rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(16384, 16384 // 2)
|
||||
assert not rocm_aiter_ops.is_triton_gemm_afp4wfp4_presh_ws_tuned(16384, 8192 // 2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_fp4_gemm_a4w4_determinism():
|
||||
"""AITER FP4 A4W4 GEMM is bitwise deterministic across repeated runs.
|
||||
|
||||
@@ -14,6 +14,7 @@ These tests cover:
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -157,6 +158,104 @@ def test_dspark_context_kv_matches_query_insert(
|
||||
torch.testing.assert_close(kv, before, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("kv_mxfp8", [False, True])
|
||||
def test_v41_pcp_swa_cache_matches_non_pcp_writer(kv_mxfp8, monkeypatch):
|
||||
"""PCP SWA writes must preserve the established CUDA cache bytes."""
|
||||
from vllm.models.deepseek_v41 import attention as attention_module
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
|
||||
torch.manual_seed(43)
|
||||
block_size = 64
|
||||
num_local_tokens = 5
|
||||
num_tokens = 2 * num_local_tokens
|
||||
row_size = 528 if kv_mxfp8 else 584
|
||||
num_blocks = 2
|
||||
page_stride = math.ceil(block_size * row_size / 576) * 576 + 576
|
||||
backing = torch.full(
|
||||
(num_blocks, page_stride), 165, dtype=torch.uint8, device="cuda"
|
||||
)
|
||||
cache = backing.as_strided(
|
||||
(num_blocks, block_size, row_size), (page_stride, row_size, 1)
|
||||
)
|
||||
reference_backing = backing.clone()
|
||||
reference = reference_backing.as_strided(cache.shape, cache.stride())
|
||||
|
||||
local_kv = torch.randn(num_local_tokens, 512, dtype=torch.bfloat16, device="cuda")
|
||||
remote_kv = torch.randn_like(local_kv)
|
||||
gathered_kv = torch.cat((local_kv, remote_kv))
|
||||
local_positions = torch.arange(num_local_tokens, device="cuda") + 3
|
||||
cache_positions = torch.cat((local_positions, local_positions + 11))
|
||||
cache_slots = torch.randperm(
|
||||
num_blocks * block_size, dtype=torch.int64, device="cuda"
|
||||
)[:num_tokens]
|
||||
cache_slots[2] = -1
|
||||
angles = torch.randn(32, 32, device="cuda")
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1)
|
||||
num_heads = 8
|
||||
q = torch.randn(
|
||||
num_local_tokens,
|
||||
num_heads,
|
||||
512,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
|
||||
torch.ops._C.fused_deepseek_v4_qnorm_rope_kv_rope_quant_insert(
|
||||
torch.zeros(num_tokens, num_heads, 512, dtype=torch.bfloat16, device="cuda"),
|
||||
gathered_kv,
|
||||
reference.view(num_blocks, -1),
|
||||
cache_slots,
|
||||
cache_positions,
|
||||
cos_sin,
|
||||
num_heads,
|
||||
1e-20,
|
||||
block_size,
|
||||
False,
|
||||
kv_mxfp8,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
attention_module,
|
||||
"get_pcp_group",
|
||||
lambda: SimpleNamespace(
|
||||
all_gather=lambda tensor, dim: torch.cat((tensor, remote_kv), dim=dim)
|
||||
),
|
||||
)
|
||||
metadata = SimpleNamespace(
|
||||
slot_mapping=torch.full(
|
||||
(num_local_tokens,), -1, dtype=torch.int64, device="cuda"
|
||||
),
|
||||
cache_slot_mapping=cache_slots,
|
||||
cache_positions=cache_positions,
|
||||
block_size=block_size,
|
||||
)
|
||||
attention = SimpleNamespace(
|
||||
use_pcp=True,
|
||||
swa_cache_layer=SimpleNamespace(
|
||||
prefix="swa", kv_cache=cache, block_size=block_size
|
||||
),
|
||||
rotary_emb=SimpleNamespace(cos_sin_cache=cos_sin),
|
||||
_flashinfer_fp8_kv_scale=None,
|
||||
kv_mxfp8=kv_mxfp8,
|
||||
accepts_unnormed_unroped_query=False,
|
||||
n_local_heads=num_heads,
|
||||
padded_heads=num_heads,
|
||||
eps=1e-20,
|
||||
head_dim=512,
|
||||
)
|
||||
DeepseekV4Attention._fused_qnorm_rope_kv_insert(
|
||||
attention, q, local_kv, local_positions, {"swa": metadata}
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
backing.view(torch.uint8),
|
||||
reference_backing.view(torch.uint8),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="graph capture coverage"
|
||||
)
|
||||
@@ -360,7 +459,7 @@ def _mxfp8_record_reference(
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
|
||||
"""The 528-byte V4.1 record quantizes the RoPE dims instead of keeping them
|
||||
in bf16, so a page is [block_size x 512 fp8][block_size x 16 UE8M0].
|
||||
in bf16, so a token is [512 fp8][16 UE8M0].
|
||||
|
||||
Byte-exact against the reference quantizer, including the rows and page
|
||||
padding the kernel must leave alone.
|
||||
@@ -398,9 +497,8 @@ def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
|
||||
values, scales = _mxfp8_record_reference(
|
||||
latent[t], cos_sin[pos // compress_ratio * compress_ratio]
|
||||
)
|
||||
expected[page, row * 512 : (row + 1) * 512] = values
|
||||
scale_offset = cache_block * 512 + row * 16
|
||||
expected[page, scale_offset : scale_offset + 16] = scales
|
||||
expected[page, row * 528 : row * 528 + 512] = values
|
||||
expected[page, row * 528 + 512 : (row + 1) * 528] = scales
|
||||
|
||||
torch.testing.assert_close(cache_backing, expected, rtol=0, atol=0)
|
||||
|
||||
@@ -510,14 +608,13 @@ def test_v41_mxfp8_cache_round_trip():
|
||||
values, scales = _ue8m0_reference(k[t].float(), 32, 448.0)
|
||||
page, row = divmod(t, block_size)
|
||||
torch.testing.assert_close(
|
||||
cache[page, row * 512 : (row + 1) * 512],
|
||||
cache[page, row * 528 : row * 528 + 512],
|
||||
values.view(torch.uint8),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
scale_offset = block_size * 512 + row * 16
|
||||
torch.testing.assert_close(
|
||||
cache[page, scale_offset : scale_offset + 16],
|
||||
cache[page, row * 528 + 512 : (row + 1) * 528],
|
||||
(scales.log2() + 127).to(torch.uint8),
|
||||
rtol=0,
|
||||
atol=0,
|
||||
@@ -564,7 +661,8 @@ def _decode_nvfp4_row(packed: torch.Tensor, scales: torch.Tensor) -> torch.Tenso
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
|
||||
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales.
|
||||
"""The 288-byte V4.1 NVFP4 record: 256 B of e2m1 pairs then 32 e4m3 scales
|
||||
per token.
|
||||
|
||||
Scales are byte-exact against the reference (``amax / 6`` clamped to the
|
||||
e4m3 range); values are checked as round-to-nearest on the e2m1 grid, whose
|
||||
@@ -604,17 +702,13 @@ def test_v41_rope_insert_nvfp4_record(compress_ratio: int):
|
||||
amax = rotated.view(32, 16).abs().amax(-1)
|
||||
scale = (amax / 6.0).clamp(2.0**-9, 448.0).to(torch.float8_e4m3fn)
|
||||
|
||||
got_scales = cache_backing[page, cache_block * 256 + row * 32 :][:32]
|
||||
record = cache_backing[page, row * 288 : (row + 1) * 288]
|
||||
got_scales = record[256:]
|
||||
torch.testing.assert_close(got_scales, scale.view(torch.uint8), rtol=0, atol=0)
|
||||
decoded = _decode_nvfp4_row(
|
||||
cache_backing[page, row * 256 : (row + 1) * 256], got_scales
|
||||
)
|
||||
decoded = _decode_nvfp4_row(record[:256], got_scales)
|
||||
step = scale.float().repeat_interleave(16)
|
||||
assert ((decoded - rotated).abs() <= step).all()
|
||||
untouched[page, row * 256 : (row + 1) * 256] = cache_backing[
|
||||
page, row * 256 : (row + 1) * 256
|
||||
]
|
||||
untouched[page, cache_block * 256 + row * 32 :][:32] = got_scales
|
||||
untouched[page, row * 288 : (row + 1) * 288] = record
|
||||
# Rows the kernel must not have touched, page padding included.
|
||||
torch.testing.assert_close(cache_backing, untouched, rtol=0, atol=0)
|
||||
|
||||
@@ -670,10 +764,8 @@ def test_v41_nvfp4_gather_matches_insert(num_reqs, num_tokens, partial):
|
||||
offset=offset,
|
||||
)
|
||||
|
||||
expected = _decode_nvfp4_row(
|
||||
backing[:, : block_size * 256].reshape(-1, 256),
|
||||
backing[:, block_size * 256 : block_size * 288].reshape(-1, 32),
|
||||
)
|
||||
records = backing[:, : block_size * 288].reshape(-1, 288)
|
||||
expected = _decode_nvfp4_row(records[:, :256], records[:, 256:])
|
||||
|
||||
def check(shorter_by):
|
||||
assert (storage[:, num_tokens + offset :] == -123).all()
|
||||
@@ -702,6 +794,72 @@ def test_v41_nvfp4_gather_matches_insert(num_reqs, num_tokens, partial):
|
||||
check(1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="The V4.1 records are SM100-only",
|
||||
)
|
||||
@pytest.mark.parametrize("record", ["swa_fp8", "compressed_fp8", "compressed_fp4"])
|
||||
def test_v41_cache_writers_match_flashmla_decode(record: str):
|
||||
"""FlashMLA and vLLM's gathers read back the token a V4.1 writer put mid
|
||||
page in a padded block, so all agree on the interleaved record layout."""
|
||||
import vllm.v1.attention.ops.flashmla as fm
|
||||
from vllm.models.deepseek_v41.common.ops.cache_utils import (
|
||||
dequantize_and_gather_k_cache,
|
||||
dequantize_and_gather_k_cache_triton,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
rope_quant_insert,
|
||||
)
|
||||
|
||||
device, block_size, topk = "cuda", 64, 64
|
||||
slot = block_size + 37
|
||||
bytes_per_token = 288 if record == "compressed_fp4" else 528
|
||||
torch.manual_seed(0)
|
||||
kv = torch.randn(1, 512, dtype=torch.bfloat16, device=device)
|
||||
positions = torch.tensor([slot], device=device)
|
||||
# cos = 1, sin = 0: RoPE is the identity, so the cache should hold `kv`.
|
||||
cos_sin = torch.zeros(slot + 1, 64, device=device)
|
||||
cos_sin[:, :32] = 1
|
||||
# Padded like a packed block: a multiple of the record and of 512 B.
|
||||
cache = torch.zeros(2, block_size + 32, bytes_per_token, dtype=torch.uint8)
|
||||
cache = cache.to(device)[:, :block_size]
|
||||
if record == "swa_fp8":
|
||||
torch.ops._C.fused_deepseek_v4_kv_rope_insert(
|
||||
kv, cache, positions, positions, cos_sin, block_size, None, True
|
||||
)
|
||||
else:
|
||||
rope_quant_insert(kv, positions, cos_sin, cache, positions, 1)
|
||||
|
||||
indices = torch.full((1, 1, topk), -1, dtype=torch.int32, device=device)
|
||||
indices[..., 0] = slot
|
||||
extra = None
|
||||
if record == "compressed_fp4":
|
||||
cache, extra = torch.zeros(2, block_size, 528).to(cache), cache
|
||||
out, _ = fm.flash_mla_with_kvcache(
|
||||
q=torch.randn(1, 1, 64, 512, dtype=torch.bfloat16, device=device),
|
||||
k_cache=cache.unsqueeze(2),
|
||||
block_table=None,
|
||||
cache_seqlens=None,
|
||||
head_dim_v=512,
|
||||
tile_scheduler_metadata=fm.get_mla_metadata()[0],
|
||||
is_fp8_kvcache=True,
|
||||
indices=indices if extra is None else torch.full_like(indices, -1),
|
||||
extra_k_cache=None if extra is None else extra.unsqueeze(2),
|
||||
extra_indices_in_kvcache=None if extra is None else indices,
|
||||
)
|
||||
for gather in (dequantize_and_gather_k_cache, dequantize_and_gather_k_cache_triton):
|
||||
rows = torch.zeros(1, slot + 1, 512, dtype=torch.bfloat16, device=device)
|
||||
block_table = torch.arange(2, dtype=torch.int32, device=device)[None]
|
||||
seq_lens = torch.tensor([slot + 1], dtype=torch.int32, device=device)
|
||||
k_cache = cache if extra is None else extra
|
||||
gather(rows, k_cache, seq_lens, None, block_table, block_size, offset=0)
|
||||
torch.testing.assert_close(
|
||||
out[0, 0], rows[0, slot].expand(64, -1), rtol=0, atol=0
|
||||
)
|
||||
cos = torch.nn.functional.cosine_similarity(rows[0, slot].float(), kv[0].float(), 0)
|
||||
assert cos > (0.98 if extra is not None else 0.999)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.parametrize("compress_ratio", [1, 2])
|
||||
@pytest.mark.parametrize("store_fp8", [False, True])
|
||||
@@ -796,6 +954,7 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
|
||||
capacity = 8
|
||||
vllm_config = MagicMock()
|
||||
vllm_config.scheduler_config.max_num_batched_tokens = 16
|
||||
vllm_config.parallel_config.prefill_context_parallel_size = 1
|
||||
spec = CircularBufferSpec(
|
||||
block_size=capacity,
|
||||
num_kv_heads=1,
|
||||
@@ -832,6 +991,67 @@ def test_v41_compressor_metadata_maps_tokens_to_their_ring():
|
||||
assert metadata.token_to_req_indices.tolist() == [0, 0, 0, 1, 1, 2, 2]
|
||||
|
||||
|
||||
def test_v41_pcp_compressor_latent_is_gathered_for_cache_writes(monkeypatch):
|
||||
"""PCP compresses locally and publishes the gathered latent to both caches."""
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v41 import attention as attention_module
|
||||
from vllm.models.deepseek_v41.attention import DeepseekV4Attention
|
||||
|
||||
local_latent = torch.arange(8, dtype=torch.float32).view(2, 4)
|
||||
cache_positions = torch.tensor([0, 1, 10, 11])
|
||||
observed = {}
|
||||
|
||||
class Compressor:
|
||||
def __call__(self, kv_score, positions):
|
||||
return local_latent
|
||||
|
||||
def insert_cache(self, latent, positions, rotary_emb):
|
||||
observed["insert"] = (latent, positions)
|
||||
|
||||
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale, **kwargs):
|
||||
observed["indexer"] = (latent, kwargs)
|
||||
return None, None, None
|
||||
|
||||
monkeypatch.setattr(
|
||||
attention_module,
|
||||
"get_pcp_group",
|
||||
lambda: SimpleNamespace(
|
||||
all_gather=lambda tensor, dim: torch.cat((tensor, tensor + 10), dim=dim)
|
||||
),
|
||||
)
|
||||
metadata = {"swa": SimpleNamespace(cache_positions=cache_positions)}
|
||||
attention = SimpleNamespace(
|
||||
use_pcp=True,
|
||||
compressor=Compressor(),
|
||||
indexer=prepare_indexer,
|
||||
aux_stream_list=None,
|
||||
ln_events=[None, None],
|
||||
swa_cache_layer=SimpleNamespace(prefix="swa"),
|
||||
rotary_emb=SimpleNamespace(),
|
||||
indexer_rotary_emb=SimpleNamespace(),
|
||||
n_local_heads=1,
|
||||
head_dim=4,
|
||||
_wq_b_proj=lambda qr, scale: qr,
|
||||
_fused_qnorm_rope_kv_insert=lambda q, kv, pos, meta: q,
|
||||
_sparse_indexer_and_attn=lambda *args: None,
|
||||
)
|
||||
rows = torch.zeros(2, 4)
|
||||
positions = torch.arange(2)
|
||||
with override_forward_context(ForwardContext({}, metadata, {})):
|
||||
DeepseekV4Attention._prepare_and_attn(
|
||||
attention, rows, rows, rows, None, rows, rows, positions, rows
|
||||
)
|
||||
|
||||
expected_global = torch.cat((local_latent, local_latent + 10))
|
||||
insert_latent, insert_positions = observed["insert"]
|
||||
torch.testing.assert_close(insert_latent, expected_global)
|
||||
torch.testing.assert_close(insert_positions, cache_positions)
|
||||
indexer_latent, indexer_kwargs = observed["indexer"]
|
||||
torch.testing.assert_close(indexer_latent, local_latent)
|
||||
torch.testing.assert_close(indexer_kwargs["cache_latent"], expected_global)
|
||||
torch.testing.assert_close(indexer_kwargs["cache_positions"], cache_positions)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA stream coverage")
|
||||
@pytest.mark.parametrize(
|
||||
"use_aux,use_graph", [(False, False), (True, False), (True, True)]
|
||||
@@ -871,6 +1091,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
|
||||
context = ForwardContext({}, metadata, {})
|
||||
compressor = DeepseekCompressor.__new__(DeepseekCompressor)
|
||||
torch.nn.Module.__init__(compressor)
|
||||
compressor.pcp_size = 1
|
||||
compressor.head_dim, compressor.rope_head_dim, compressor.compress_ratio = (
|
||||
512,
|
||||
64,
|
||||
@@ -896,7 +1117,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
|
||||
use_fp4_kv=False,
|
||||
)
|
||||
|
||||
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale):
|
||||
def prepare_indexer(qr, latent, weights, positions, rotary, qr_scale, **kwargs):
|
||||
DeepseekV4Indexer._produce_k(indexer, latent, positions, rotary)
|
||||
return None, None, None
|
||||
|
||||
@@ -906,6 +1127,7 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
|
||||
|
||||
auxiliary = [torch.cuda.Stream()] if use_aux else None
|
||||
attention = SimpleNamespace(
|
||||
use_pcp=False,
|
||||
compressor=compressor,
|
||||
indexer=prepare_indexer,
|
||||
aux_stream_list=auxiliary,
|
||||
@@ -963,6 +1185,291 @@ def test_v41_attention_joins_cache_writes_before_consumption(use_aux, use_graph)
|
||||
previous = expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pcp_size", [2, 4])
|
||||
@pytest.mark.parametrize(
|
||||
"lengths,prefilling,starts",
|
||||
[
|
||||
([9, 1], [True, False], [1, 4]),
|
||||
([1, 1], [False, False], [16, 3]),
|
||||
([3, 1], [True, False], [1, 0]),
|
||||
],
|
||||
)
|
||||
def test_v41_pcp_ring_links_match_request_positions(
|
||||
pcp_size, lengths, prefilling, starts, monkeypatch
|
||||
):
|
||||
"""Cross-chunk predecessors and open ring writes must belong to the request."""
|
||||
from vllm.models.deepseek_v41 import compressor as compressor_module
|
||||
from vllm.models.deepseek_v41.compressor import (
|
||||
CompressorMetadata,
|
||||
CompressorMetadataBuilder,
|
||||
)
|
||||
from vllm.v1.worker.gpu import pcp_manager as pcp_manager_module
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
|
||||
monkeypatch.setattr(
|
||||
pcp_manager_module,
|
||||
"async_tensor_h2d",
|
||||
lambda array, **kwargs: torch.from_numpy(array),
|
||||
)
|
||||
manager = PCPManager(pcp_size, 0, torch.device("cpu"), shard_decode_requests=True)
|
||||
query_start = np.array([0, *np.cumsum(lengths)], dtype=np.int32)
|
||||
segments, counts = manager._build_batch_layout(
|
||||
np.array(lengths), np.array(starts), np.array(prefilling), query_start
|
||||
)
|
||||
width = max(counts)
|
||||
capacity = 8
|
||||
blocks = [1, (1 << 20) + 2]
|
||||
num_rows = min(width, 2 * len(lengths))
|
||||
metadata_by_rank = []
|
||||
positions_by_rank = []
|
||||
gathered_keys = []
|
||||
for local_segments in segments:
|
||||
positions = torch.zeros(width, dtype=torch.int64)
|
||||
slots = torch.full((width,), -1, dtype=torch.int64)
|
||||
local_query_start = [0]
|
||||
keys = [-1] * num_rows
|
||||
for row, segment in enumerate(local_segments):
|
||||
req = segment.global_batch_req_idx
|
||||
begin = starts[req] + segment.global_batch_slice.start - query_start[req]
|
||||
local = segment.rank_local_batch_slice
|
||||
positions[local] = torch.arange(begin, begin + segment.num_tokens)
|
||||
slots[local] = blocks[req] * capacity + positions[local] % capacity
|
||||
local_query_start.append(local.stop)
|
||||
keys[row] = (blocks[req] << 32) | int(positions[local.stop - 1])
|
||||
if not local_segments:
|
||||
local_query_start.append(0)
|
||||
metadata_by_rank.append(
|
||||
CompressorMetadata(
|
||||
slot_mapping=slots,
|
||||
query_start_loc=torch.tensor(local_query_start, dtype=torch.int32),
|
||||
token_to_req_indices=torch.empty(0, dtype=torch.int32),
|
||||
)
|
||||
)
|
||||
positions_by_rank.append(positions)
|
||||
gathered_keys.extend(keys)
|
||||
|
||||
keys = torch.tensor(gathered_keys, dtype=torch.int64)
|
||||
contributed_keys = []
|
||||
|
||||
def all_gather(tensor, dim):
|
||||
contributed_keys.append(tensor.clone())
|
||||
return keys
|
||||
|
||||
monkeypatch.setattr(
|
||||
compressor_module,
|
||||
"get_pcp_group",
|
||||
lambda: SimpleNamespace(all_gather=all_gather),
|
||||
)
|
||||
builder = SimpleNamespace(
|
||||
capacity=capacity,
|
||||
device=torch.device("cpu"),
|
||||
vllm_config=SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_seqs=len(lengths))
|
||||
),
|
||||
)
|
||||
expected_rows = {
|
||||
block: max(key for key in gathered_keys if key >> 32 == block)
|
||||
for block in blocks
|
||||
}
|
||||
expected_slots = [
|
||||
block * capacity + (key & 0xFFFFFFFF) % capacity
|
||||
for block, key in expected_rows.items()
|
||||
if key % 2 == 0
|
||||
]
|
||||
for metadata, positions in zip(metadata_by_rank, positions_by_rank):
|
||||
num_reqs = metadata.query_start_loc.numel() - 1
|
||||
CompressorMetadataBuilder._link_pcp_chunks(
|
||||
builder, metadata, num_reqs, positions
|
||||
)
|
||||
assert metadata.prev_row_indices is not None
|
||||
assert metadata.gather_tokens is not None
|
||||
assert metadata.ring_write_slots is not None
|
||||
for row, token in enumerate(metadata.gather_tokens.tolist()):
|
||||
key = int(contributed_keys[-1][row])
|
||||
if key >= 0:
|
||||
assert int(metadata.slot_mapping[token]) // capacity == key >> 32
|
||||
assert int(positions[token]) == key & 0xFFFFFFFF
|
||||
for start, prev in zip(
|
||||
metadata.query_start_loc[:-1].tolist(), metadata.prev_row_indices.tolist()
|
||||
):
|
||||
position = int(positions[start])
|
||||
block = int(metadata.slot_mapping[start]) // capacity
|
||||
wanted = (block << 32) | (position - 1)
|
||||
if position % 2 == 1 and wanted in gathered_keys:
|
||||
assert prev >= 0 and gathered_keys[prev] == wanted
|
||||
else:
|
||||
assert prev == -1
|
||||
writes = metadata.ring_write_slots.tolist()
|
||||
assert sorted(slot for slot in writes if slot >= 0) == sorted(expected_slots)
|
||||
for row, slot in enumerate(writes):
|
||||
if slot >= 0:
|
||||
assert gathered_keys[row] == expected_rows[slot // capacity]
|
||||
assert (gathered_keys[row] & 0xFFFFFFFF) % capacity == slot % capacity
|
||||
torch.testing.assert_close(torch.cat(contributed_keys), keys)
|
||||
|
||||
|
||||
def _build_pcp_ring_metadata(ranks, all_gather, spec, config, device, monkeypatch):
|
||||
from vllm.models.deepseek_v41 import compressor as compressor_module
|
||||
from vllm.models.deepseek_v41.compressor import CompressorMetadataBuilder
|
||||
|
||||
monkeypatch.setattr(
|
||||
compressor_module,
|
||||
"get_pcp_group",
|
||||
lambda: SimpleNamespace(all_gather=all_gather),
|
||||
)
|
||||
for local in ranks:
|
||||
builder = CompressorMetadataBuilder(spec, ["ring"], config, device)
|
||||
local.metadata = builder.build(0, local.common)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Triton kernel")
|
||||
@pytest.mark.parametrize("pcp_size", [2, 4])
|
||||
def test_v41_pcp_compressor_preserves_pairs_and_decode_state(pcp_size, monkeypatch):
|
||||
"""Pairs split across PCP chunks and decode owner changes keep exact latents."""
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v41 import compressor as compressor_module
|
||||
from vllm.models.deepseek_v41.common.ops.fused_compress_quant_cache import (
|
||||
fused_save_compress_norm,
|
||||
)
|
||||
from vllm.models.deepseek_v41.compressor import DeepseekCompressor
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.kv_cache_interface import CircularBufferSpec
|
||||
from vllm.v1.worker.gpu.pcp_manager import PCPManager
|
||||
|
||||
torch.manual_seed(42)
|
||||
device = torch.device("cuda")
|
||||
norm = torch.randn(512, dtype=torch.bfloat16, device=device)
|
||||
# Block 0 is reserved as the null block; requests use physical blocks 1/2.
|
||||
reference_state = torch.zeros(3, 8, 1024, device=device)
|
||||
rank_states = [reference_state.clone() for _ in range(pcp_size)]
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=32, max_num_seqs=4),
|
||||
parallel_config=SimpleNamespace(prefill_context_parallel_size=pcp_size),
|
||||
speculative_config=None,
|
||||
)
|
||||
spec = CircularBufferSpec(
|
||||
block_size=8, num_kv_heads=1, head_size=1024, head_size_v=0, dtype=torch.float32
|
||||
)
|
||||
starts = np.zeros(2, dtype=np.int32)
|
||||
for lengths, prefilling in [
|
||||
([7, 3], [True, True]),
|
||||
([9, 1], [True, False]),
|
||||
([1, 1], [False, False]),
|
||||
([1, 1], [False, False]),
|
||||
]:
|
||||
lengths = np.array(lengths, dtype=np.int32)
|
||||
query_start = np.concatenate(([0], np.cumsum(lengths))).astype(np.int32)
|
||||
qsl = torch.from_numpy(query_start).to(device)
|
||||
positions = torch.cat(
|
||||
[torch.arange(s, s + n, device=device) for s, n in zip(starts, lengths)]
|
||||
)
|
||||
req_ids = torch.repeat_interleave(
|
||||
torch.arange(2, device=device, dtype=torch.int32),
|
||||
torch.from_numpy(lengths).to(device),
|
||||
)
|
||||
slots = (req_ids.long() + 1) * 8 + positions % 8
|
||||
raw = torch.randn(sum(lengths), 1024, device=device)
|
||||
expected = torch.empty(sum(lengths), 512, dtype=torch.bfloat16, device=device)
|
||||
fused_save_compress_norm(
|
||||
raw,
|
||||
positions,
|
||||
reference_state,
|
||||
slots,
|
||||
qsl,
|
||||
req_ids,
|
||||
norm,
|
||||
1e-20,
|
||||
2,
|
||||
expected,
|
||||
)
|
||||
|
||||
ranks = []
|
||||
for rank in range(pcp_size):
|
||||
manager = PCPManager(pcp_size, rank, device, shard_decode_requests=True)
|
||||
segments, counts = manager._build_batch_layout(
|
||||
lengths, starts, np.array(prefilling), query_start
|
||||
)
|
||||
width = max(counts)
|
||||
local_indices = manager._padded_gather_idx[
|
||||
rank * width : (rank + 1) * width
|
||||
]
|
||||
local_lengths = [s.num_tokens for s in segments[rank]] or [0]
|
||||
local_qsl = torch.tensor([0, *np.cumsum(local_lengths)], dtype=torch.int32)
|
||||
local_reqs = [s.global_batch_req_idx for s in segments[rank]] or [0]
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=local_qsl.to(device),
|
||||
query_start_loc_cpu=local_qsl,
|
||||
seq_lens=torch.zeros(len(local_reqs), dtype=torch.int32, device=device),
|
||||
num_reqs=len(local_reqs),
|
||||
num_actual_tokens=counts[rank],
|
||||
max_query_len=max(local_lengths),
|
||||
max_seq_len=int(max(starts + lengths)),
|
||||
block_table_tensor=torch.tensor(
|
||||
np.array(local_reqs) + 1, dtype=torch.int32, device=device
|
||||
).unsqueeze(1),
|
||||
slot_mapping=torch.full((width * pcp_size,), -1, device=device),
|
||||
positions=positions[local_indices],
|
||||
)
|
||||
ranks.append(
|
||||
SimpleNamespace(
|
||||
indices=local_indices[: counts[rank]],
|
||||
raw=raw[local_indices],
|
||||
positions=common.positions,
|
||||
common=common,
|
||||
)
|
||||
)
|
||||
|
||||
# Each rank's builder contributes its last-row candidates to a gather.
|
||||
candidates: list[torch.Tensor] = []
|
||||
_build_pcp_ring_metadata(
|
||||
ranks,
|
||||
lambda t, dim, out=candidates: out.append(t.clone()) or t.repeat(pcp_size),
|
||||
spec,
|
||||
config,
|
||||
device,
|
||||
monkeypatch,
|
||||
)
|
||||
_build_pcp_ring_metadata(
|
||||
ranks,
|
||||
lambda t, dim, out=candidates: torch.cat(out),
|
||||
spec,
|
||||
config,
|
||||
device,
|
||||
monkeypatch,
|
||||
)
|
||||
gathered_rows = torch.cat([r.raw[r.metadata.gather_tokens] for r in ranks])
|
||||
monkeypatch.setattr(
|
||||
compressor_module,
|
||||
"get_pcp_group",
|
||||
lambda rows=gathered_rows: SimpleNamespace(all_gather=lambda t, dim: rows),
|
||||
)
|
||||
for rank, local in enumerate(ranks):
|
||||
compressor = SimpleNamespace(
|
||||
pcp_size=pcp_size,
|
||||
head_dim=512,
|
||||
compress_ratio=2,
|
||||
rms_norm_eps=1e-20,
|
||||
norm=SimpleNamespace(weight=norm),
|
||||
state_cache=SimpleNamespace(prefix="ring", kv_cache=rank_states[rank]),
|
||||
)
|
||||
context = ForwardContext({}, {"ring": local.metadata}, {})
|
||||
with override_forward_context(context):
|
||||
actual = DeepseekCompressor.forward(
|
||||
compressor, local.raw, local.positions
|
||||
)
|
||||
assert isinstance(actual, torch.Tensor)
|
||||
valid = (positions[local.indices] + 1) % 2 == 0
|
||||
torch.testing.assert_close(
|
||||
actual[: local.indices.numel()][valid],
|
||||
expected[local.indices][valid],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
for state in rank_states[1:]:
|
||||
torch.testing.assert_close(state, rank_states[0], rtol=0, atol=0)
|
||||
starts += lengths
|
||||
|
||||
|
||||
def _on_gfx950() -> bool:
|
||||
if not current_platform.is_rocm():
|
||||
return False
|
||||
|
||||
@@ -502,11 +502,12 @@ def test_lookback_window_reproduces_single_instance(runner, capture):
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
|
||||
def test_v2_model_state_gathers_lookback_window():
|
||||
@pytest.mark.parametrize("pcp", [False, True])
|
||||
def test_v2_model_state_gathers_lookback_window(pcp):
|
||||
"""The window is gathered on device from the runner's token history."""
|
||||
from vllm.models.deepseek_v41.nvidia.model_state import DeepseekV41ModelState
|
||||
|
||||
depth, max_num_reqs, max_model_len = 3, 4, 16
|
||||
depth, max_num_reqs, max_model_len = 3, 6, 16
|
||||
state = DeepseekV41ModelState.__new__(DeepseekV41ModelState)
|
||||
state.rope_state = None
|
||||
state.supports_mm_inputs = False
|
||||
@@ -527,15 +528,23 @@ def test_v2_model_state_gathers_lookback_window():
|
||||
# Batch rows -> request state rows: a request two decode steps in, a fresh
|
||||
# request, and one at the end of a 5-token prompt.
|
||||
input_batch = SimpleNamespace(
|
||||
idx_mapping=torch.tensor([2, 0, 3], dtype=torch.int64, device="cuda")
|
||||
idx_mapping=torch.tensor(
|
||||
[2, 0, 3, 2] if pcp else [2, 0, 3], dtype=torch.int64, device="cuda"
|
||||
),
|
||||
query_start_loc=torch.arange(5 if pcp else 4, dtype=torch.int32, device="cuda"),
|
||||
positions=torch.tensor([8, 0, 6, 5] if pcp else [8, 0, 6], device="cuda"),
|
||||
)
|
||||
window = state.prepare_inputs(input_batch, req_states)["lookback_token_ids"]
|
||||
assert window.cpu().tolist() == [
|
||||
expected = [
|
||||
[22, 21, 16],
|
||||
[-1, -1, -1],
|
||||
[0, 0, 0],
|
||||
[-1, -1, -1],
|
||||
]
|
||||
if pcp:
|
||||
# The second PCP chunk of request 2 starts at 5, not at its global 8.
|
||||
expected.append([15, 14, 13])
|
||||
expected.extend([[-1] * depth] * (max_num_reqs - len(expected)))
|
||||
assert window.cpu().tolist() == expected
|
||||
# Graph capture runs on the dummy inputs and replays read the same buffer.
|
||||
dummy = state.prepare_dummy_inputs(num_reqs=3, num_tokens=3)["lookback_token_ids"]
|
||||
assert dummy is window and torch.all(dummy == -1)
|
||||
|
||||
@@ -525,9 +525,8 @@ def test_kv_path_mxfp8_record_matches_reference(num_tokens: int, block_size: int
|
||||
step = 32.0 * tile_scale.repeat_interleave(32, dim=-1)
|
||||
assert ((rec_fused - rec_ref).abs() <= step).all()
|
||||
# The NoPE dims see no rotation at all, so they must be byte-identical.
|
||||
nope_bytes = slice(0, block_size * HEAD_DIM)
|
||||
ref_rows = k_cache_ref[:, nope_bytes].view(num_blocks, block_size, HEAD_DIM)
|
||||
fused_rows = k_cache_fused[:, nope_bytes].view(num_blocks, block_size, HEAD_DIM)
|
||||
ref_rows = k_cache_ref.view(num_blocks, block_size, V41_HEAD_BYTES)
|
||||
fused_rows = k_cache_fused.view(num_blocks, block_size, V41_HEAD_BYTES)
|
||||
torch.testing.assert_close(
|
||||
fused_rows[..., :NOPE_DIM], ref_rows[..., :NOPE_DIM], rtol=0, atol=0
|
||||
)
|
||||
|
||||
@@ -109,7 +109,7 @@ def test_rocm_set_out_dtype_respects_bias_guard(monkeypatch):
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("input_size", "output_size"),
|
||||
[(3072, 256), (4096, 8), (4096, 192), (6144, 128), (6144, 256)],
|
||||
[(3072, 256), (4096, 8), (6144, 128), (2048, 64), (2560, 512), (7168, 384)],
|
||||
)
|
||||
def test_rocm_gfx950_enables_fp32_router_gemm(
|
||||
monkeypatch, input_size: int, output_size: int
|
||||
@@ -147,7 +147,6 @@ def test_rocm_fp32_router_gemm_is_replicated_across_ep_sizes(
|
||||
("input_size", "output_size", "params_dtype", "bias", "on_gfx950"),
|
||||
[
|
||||
pytest.param(6144, 128, torch.float32, False, False, id="gfx942"),
|
||||
pytest.param(2048, 64, torch.float32, False, True, id="shape"),
|
||||
pytest.param(6144, 128, torch.bfloat16, False, True, id="bf16-weight"),
|
||||
pytest.param(6144, 128, torch.float32, True, True, id="bias"),
|
||||
],
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
rocm_fp32_router_gemm as rocm_router,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import (
|
||||
fp32_router_gemm_dispatch_impl,
|
||||
)
|
||||
@@ -19,6 +22,11 @@ SHAPES = [
|
||||
(4096, 192),
|
||||
(6144, 128),
|
||||
(6144, 256),
|
||||
(2560, 512),
|
||||
(2880, 128),
|
||||
(5120, 16),
|
||||
(7168, 384),
|
||||
(1000, 100),
|
||||
]
|
||||
MAX_TOKENS = 32
|
||||
ATOL = 5e-4
|
||||
@@ -80,7 +88,7 @@ def test_rocm_fp32_router_gemm_preserves_topk(
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=device)
|
||||
weight = torch.randn(num_experts, hidden_size, dtype=torch.float32, device=device)
|
||||
|
||||
top_k = 2 if (hidden_size, num_experts) == (4096, 8) else 8
|
||||
top_k = 2 if num_experts <= 16 else 8
|
||||
actual = rocm_fp32_router_gemm(x, weight).topk(top_k, dim=-1).indices
|
||||
reference = _reference(x, weight)
|
||||
reference_values, expected = reference.topk(top_k, dim=-1)
|
||||
@@ -113,15 +121,22 @@ def test_rocm_fp32_router_gemm_rejects_invalid_inputs() -> None:
|
||||
weight,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="shape"):
|
||||
rocm_fp32_router_gemm(x[:, :2048].contiguous(), weight[:, :2048].contiguous())
|
||||
with pytest.raises(ValueError, match="hidden sizes"):
|
||||
rocm_fp32_router_gemm(x, weight[:, :2048].contiguous())
|
||||
|
||||
with pytest.raises(ValueError, match="num_experts"):
|
||||
big_weight = torch.empty(1024, 7168, dtype=torch.float32, device=device)
|
||||
rocm_fp32_router_gemm(
|
||||
torch.empty(MAX_TOKENS, 7168, dtype=torch.bfloat16, device=device),
|
||||
big_weight,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="contiguous"):
|
||||
wide_x = torch.randn(4, hidden_size * 2, dtype=torch.bfloat16, device=device)
|
||||
rocm_fp32_router_gemm(wide_x[:, ::2], weight)
|
||||
|
||||
with pytest.raises(ValueError, match="float32"):
|
||||
rocm_fp32_router_gemm(x, weight.to(torch.bfloat16))
|
||||
with pytest.raises(ValueError, match="float32 or bfloat16"):
|
||||
rocm_fp32_router_gemm(x, weight.to(torch.float16))
|
||||
|
||||
with pytest.raises(ValueError, match="bfloat16 or float32"):
|
||||
rocm_fp32_router_gemm(x.to(torch.float16), weight)
|
||||
@@ -226,3 +241,154 @@ def test_rocm_fp32_router_gemm_dynamic_compile_dispatch() -> None:
|
||||
)
|
||||
output = compiled_dispatch(x)
|
||||
torch.testing.assert_close(output, _reference(x, weight), atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hidden_size", [64, 100, 513, 1000, 2049, 3000])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 5, 32])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_fp32_router_gemm_any_hidden_size(num_tokens: int, hidden_size: int):
|
||||
"""Hidden sizes that do not divide the K tile exercise the masked tail."""
|
||||
torch.manual_seed(6100 + num_tokens + hidden_size)
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(64, hidden_size, dtype=torch.float32, device="cuda")
|
||||
torch.testing.assert_close(
|
||||
rocm_fp32_router_gemm(x, weight), _reference(x, weight), atol=ATOL, rtol=RTOL
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["large", "near_zero", "zero_input", "ones"])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_fp32_router_gemm_special_values(case: str) -> None:
|
||||
torch.manual_seed(7100)
|
||||
x = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(128, 5120, dtype=torch.float32, device="cuda")
|
||||
if case in ("large", "near_zero"):
|
||||
scale = 100.0 if case == "large" else 1e-4
|
||||
x = (x * scale).to(torch.bfloat16)
|
||||
weight *= scale
|
||||
elif case == "zero_input":
|
||||
x.zero_()
|
||||
else:
|
||||
x.fill_(1.0)
|
||||
output = rocm_fp32_router_gemm(x, weight)
|
||||
expected = _reference(x, weight)
|
||||
assert torch.isfinite(output).all()
|
||||
if case == "zero_input":
|
||||
assert torch.equal(output, torch.zeros_like(output))
|
||||
else:
|
||||
assert ((output - expected).norm() / expected.norm()).item() < 1e-5
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 32])
|
||||
@torch.inference_mode()
|
||||
def test_rocm_fp32_router_gemm_deterministic(num_tokens: int) -> None:
|
||||
torch.manual_seed(8100 + num_tokens)
|
||||
x = torch.randn(num_tokens, 7168, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(384, 7168, dtype=torch.float32, device="cuda")
|
||||
first = rocm_fp32_router_gemm(x, weight)
|
||||
assert torch.equal(first, rocm_fp32_router_gemm(x, weight))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 16, 32])
|
||||
@pytest.mark.parametrize(("hidden_size", "num_experts"), SHAPES)
|
||||
@torch.inference_mode()
|
||||
def test_rocm_fp32_router_gemm_bf16_weight(
|
||||
num_tokens: int, hidden_size: int, num_experts: int
|
||||
) -> None:
|
||||
torch.manual_seed(9100 + num_tokens + hidden_size + num_experts)
|
||||
x = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(num_experts, hidden_size, dtype=torch.bfloat16, device="cuda")
|
||||
# Past this size hipBLASLt beats the kernel for BF16 weights on MI355.
|
||||
eligible = num_tokens * num_experts <= 8192
|
||||
assert rocm_router.can_use_rocm_fp32_router_gemm(x, weight) == eligible
|
||||
if eligible:
|
||||
torch.testing.assert_close(
|
||||
rocm_fp32_router_gemm(x, weight),
|
||||
_reference(x.float(), weight.float()),
|
||||
atol=ATOL,
|
||||
rtol=RTOL,
|
||||
)
|
||||
|
||||
|
||||
def _make_gate_linear(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
input_size,
|
||||
output_size,
|
||||
params_dtype=torch.float32,
|
||||
):
|
||||
for module in ("layers.linear", "parameter"):
|
||||
monkeypatch.setattr(
|
||||
f"vllm.model_executor.{module}.get_tensor_model_parallel_rank", lambda: 0
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
f"vllm.model_executor.{module}.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
|
||||
gate = GateLinear(
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
bias=False,
|
||||
out_dtype=torch.float32,
|
||||
params_dtype=params_dtype,
|
||||
).cuda()
|
||||
with torch.no_grad():
|
||||
gate.weight.normal_()
|
||||
gate.quant_method.process_weights_after_loading(gate)
|
||||
return gate
|
||||
|
||||
|
||||
@pytest.mark.parametrize("x_dtype", [torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 32, 33])
|
||||
@torch.inference_mode()
|
||||
def test_gate_linear_uses_rocm_fp32_router_gemm(
|
||||
monkeypatch: pytest.MonkeyPatch, num_tokens: int, x_dtype: torch.dtype
|
||||
) -> None:
|
||||
"""A router shape outside the old tuned list takes the fast path for M<=32."""
|
||||
gate = _make_gate_linear(monkeypatch, 2560, 512)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
calls = []
|
||||
kernel = rocm_router.rocm_fp32_router_gemm
|
||||
|
||||
def spy(hidden_states, router_weight):
|
||||
calls.append(hidden_states.shape[0])
|
||||
return kernel(hidden_states, router_weight)
|
||||
|
||||
monkeypatch.setattr(rocm_router, "rocm_fp32_router_gemm", spy)
|
||||
x = torch.randn(num_tokens, 2560, dtype=x_dtype, device="cuda")
|
||||
output, bias = gate(x)
|
||||
|
||||
assert bias is None
|
||||
assert calls == ([num_tokens] if num_tokens <= MAX_TOKENS else [])
|
||||
torch.testing.assert_close(output, _reference(x, gate.weight), atol=ATOL, rtol=RTOL)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "num_experts", "uses_kernel"),
|
||||
[(1, 256, True), (32, 256, True), (33, 256, False), (32, 512, False)],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_gate_linear_bf16_weight_uses_rocm_router_gemm(
|
||||
monkeypatch: pytest.MonkeyPatch, num_tokens, num_experts, uses_kernel
|
||||
) -> None:
|
||||
"""BF16-weight gates with FP32 logits use the kernel only where it beats
|
||||
hipBLASLt; everything else keeps the torch.mm path."""
|
||||
gate = _make_gate_linear(monkeypatch, 7168, num_experts, torch.bfloat16)
|
||||
assert gate.allow_cublas_router_gemm
|
||||
calls = []
|
||||
kernel = rocm_router.rocm_fp32_router_gemm
|
||||
|
||||
def spy(hidden_states, router_weight):
|
||||
calls.append(hidden_states.shape[0])
|
||||
return kernel(hidden_states, router_weight)
|
||||
|
||||
monkeypatch.setattr(rocm_router, "rocm_fp32_router_gemm", spy)
|
||||
x = torch.randn(num_tokens, 7168, dtype=torch.bfloat16, device="cuda")
|
||||
output, _ = gate(x)
|
||||
|
||||
assert output.dtype == torch.float32
|
||||
assert calls == ([num_tokens] if uses_kernel else [])
|
||||
torch.testing.assert_close(
|
||||
output, _reference(x.float(), gate.weight.float()), atol=ATOL, rtol=RTOL
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user