Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -472,7 +472,7 @@ def varlen_with_paged_kv(
|
||||
scale = head_size**-0.5
|
||||
token_num = sum(query_lens)
|
||||
dynamic_causal_tensor = (
|
||||
torch.tensor(dynamic_causal, dtype=torch.bool)
|
||||
torch.tensor(dynamic_causal, dtype=torch.int32)
|
||||
if dynamic_causal is not None
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -6,6 +6,7 @@ import pytest
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
try:
|
||||
import flashinfer
|
||||
@@ -702,3 +703,35 @@ def test_flashinfer_decode_with_paged_fp8_kv(
|
||||
torch.testing.assert_close(output, ref_output, atol=2e-2, rtol=1e-2),
|
||||
f"{torch.max(torch.abs(output - ref_output))}",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not any(current_platform.is_device_capability_family(f) for f in (80, 90, 120)),
|
||||
reason="NVFP4 fa2 path",
|
||||
)
|
||||
@pytest.mark.parametrize("layout", ["LBNHC", "LBHNC", "BLNHC", "BLHNC"])
|
||||
def test_flashinfer_nvfp4_kv_cache_update(default_vllm_config, layout):
|
||||
from vllm.v1.attention.backends import flashinfer as fi
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
order = KVCacheLayout[layout].stride_order
|
||||
# [L, B, 2H, N, 64 + 8] NVFP4 pages for two layers; layer 1 is written.
|
||||
shape = (2, 6, 4, 16, 64 + 8)
|
||||
layers = torch.zeros([shape[i] for i in order], dtype=torch.uint8)
|
||||
layers = layers.permute(*(order.index(i) for i in range(5)))
|
||||
one = torch.ones(())
|
||||
layer = torch.nn.Module()
|
||||
layer._k_scale = layer._v_scale = one
|
||||
kv = torch.randn(2, 96, 2, 128, dtype=torch.bfloat16)
|
||||
impl = fi.FlashInferImpl(4, 128, 1.0, 2, None, None, "nvfp4")
|
||||
impl.do_kv_cache_update(layer, kv[0], kv[1], layers[1], torch.arange(96))
|
||||
|
||||
data, scales, kv_layout = fi._nvfp4_kv_views(layers[1], 2)
|
||||
pages = torch.arange(6, dtype=torch.int32)[None]
|
||||
seq_lens = torch.tensor([96], dtype=torch.int32)
|
||||
deq = torch.empty_like(kv)[:, None]
|
||||
flashinfer.nvfp4_kv_dequantize_paged(
|
||||
data, scales, pages, seq_lens, one, one, *deq, kv_layout
|
||||
)
|
||||
assert ((deq[:, 0] - kv).norm() / kv.norm()) < 0.2
|
||||
assert not layers[0].any()
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for MiniMax M3 sparse prefill attention kernels."""
|
||||
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -1444,19 +1446,21 @@ def test_decode_index_topk_fp8(num_idx_heads: int):
|
||||
reason="CuteDSL index decode score requires Blackwell.",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("dtype", "decode_query_len", "max_decode_query_len"),
|
||||
("dtype", "decode_query_len", "max_decode_query_len", "concurrent_writes"),
|
||||
[
|
||||
(torch.bfloat16, 1, 1),
|
||||
(torch.bfloat16, 3, 8),
|
||||
(torch.float8_e4m3fn, 1, 1),
|
||||
(torch.float8_e4m3fn, 3, 8),
|
||||
(torch.float8_e4m3fn, 8, 8),
|
||||
(torch.bfloat16, 1, 1, False),
|
||||
(torch.bfloat16, 3, 8, False),
|
||||
(torch.float8_e4m3fn, 1, 1, False),
|
||||
(torch.float8_e4m3fn, 3, 8, False),
|
||||
(torch.float8_e4m3fn, 8, 8, False),
|
||||
pytest.param(torch.float8_e4m3fn, 1, 1, True, id="concurrent-writes"),
|
||||
],
|
||||
)
|
||||
def test_decode_index_score_cutedsl_correctness(
|
||||
dtype: torch.dtype,
|
||||
decode_query_len: int,
|
||||
max_decode_query_len: int,
|
||||
concurrent_writes: bool,
|
||||
):
|
||||
pytest.importorskip("cutlass")
|
||||
from vllm.models.minimax_m3.nvidia.ops import (
|
||||
@@ -1466,7 +1470,8 @@ def test_decode_index_score_cutedsl_correctness(
|
||||
torch.manual_seed(0)
|
||||
init_blocks, local_blocks = 0, 0
|
||||
num_idx_heads, head_dim = 4, 128
|
||||
active_seq_lens = torch.tensor((1025, 4097), device="cuda", dtype=torch.int32)
|
||||
lengths = (131326, 131289, 131252, 131215) if concurrent_writes else (1025, 4097)
|
||||
active_seq_lens = torch.tensor(lengths, device="cuda", dtype=torch.int32)
|
||||
batch = active_seq_lens.numel()
|
||||
total_q = batch * decode_query_len
|
||||
max_seq_len = int(active_seq_lens.max())
|
||||
@@ -1488,7 +1493,8 @@ def test_decode_index_score_cutedsl_correctness(
|
||||
)
|
||||
score = unified_score.transpose(0, 1)
|
||||
|
||||
minimax_m3_index_decode_score_cutedsl(
|
||||
launch = partial(
|
||||
minimax_m3_index_decode_score_cutedsl,
|
||||
idx_q,
|
||||
index_kv_cache,
|
||||
block_table,
|
||||
@@ -1501,6 +1507,7 @@ def test_decode_index_score_cutedsl_correctness(
|
||||
max_decode_query_len=max_decode_query_len,
|
||||
score_out=score,
|
||||
)
|
||||
launch()
|
||||
expected = _reference_decode_index_score(
|
||||
idx_q,
|
||||
index_kv_cache,
|
||||
@@ -1511,6 +1518,23 @@ def test_decode_index_score_cutedsl_correctness(
|
||||
)
|
||||
torch.testing.assert_close(score, expected)
|
||||
|
||||
if concurrent_writes:
|
||||
# Concurrent stores expose shared-memory reuse before ldmatrix reads finish.
|
||||
quiet = score.clone()
|
||||
noise = torch.zeros(1 << 26, device="cuda")
|
||||
side = torch.cuda.Stream()
|
||||
side.wait_stream(torch.cuda.current_stream())
|
||||
try:
|
||||
for _ in range(20):
|
||||
with torch.cuda.stream(side):
|
||||
for _ in range(40):
|
||||
noise.mul_(1.0)
|
||||
launch()
|
||||
torch.testing.assert_close(score, quiet, atol=0, rtol=0)
|
||||
side.synchronize()
|
||||
finally:
|
||||
side.synchronize()
|
||||
|
||||
|
||||
# Sparse attention kernels.
|
||||
def _reference_sparse_attn(
|
||||
@@ -2588,6 +2612,103 @@ def test_decode_sparse_attention_correctness(
|
||||
assert error.max().item() < 1.7e-2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_layout", ["NHD", "HND"], indirect=True)
|
||||
@pytest.mark.parametrize("capture_graph", [False, True])
|
||||
@pytest.mark.parametrize("num_reqs", [2, 17], ids=["split-k", "single-chunk"])
|
||||
@pytest.mark.parametrize(
|
||||
"invalid_block", [-1, 1_000_000], ids=["sentinel", "out-of-range"]
|
||||
)
|
||||
def test_decode_sparse_attention_ignores_invalid_topk(
|
||||
kv_layout: KVCacheLayout, capture_graph: bool, num_reqs: int, invalid_block: int
|
||||
):
|
||||
"""Unused top-k entries must not be dereferenced as block-table indices."""
|
||||
torch.manual_seed(0)
|
||||
decode_query_len = 4
|
||||
seq_lens = torch.full(
|
||||
(num_reqs,), decode_query_len, device="cuda", dtype=torch.int32
|
||||
)
|
||||
seq_lens[1] = 4096
|
||||
q = torch.randn(
|
||||
seq_lens.numel() * decode_query_len,
|
||||
NUM_Q_HEADS,
|
||||
HEAD_DIM,
|
||||
device="cuda",
|
||||
dtype=DTYPE,
|
||||
)
|
||||
kv_cache = _allocate_main_kv_via_contract(1, kv_layout)
|
||||
|
||||
# All logical blocks alias the single valid page. With a contiguous multi-row
|
||||
# table, row 1's raw pointer offset -1 resolves to the final element of row
|
||||
# 0. Poison that exact location so an unsafe load deterministically forms an
|
||||
# out-of-range KV-cache pointer.
|
||||
clean_block_table = torch.zeros(num_reqs, 32, device="cuda", dtype=torch.int32)
|
||||
block_table = clean_block_table.clone()
|
||||
block_table[0, -1] = 1_000_000
|
||||
topk_idx = torch.full(
|
||||
(NUM_KV_HEADS, q.shape[0], TOPK),
|
||||
-1,
|
||||
device="cuda",
|
||||
dtype=torch.int32,
|
||||
)
|
||||
topk_idx[..., 0] = 0
|
||||
# Request 1 has invalid entries both before and after its valid block.
|
||||
# Two requests produce empty split-K chunks; 17 put the full top-k in one
|
||||
# chunk, exercising neutral updates before and after a nonempty softmax.
|
||||
request_topk = topk_idx[:, decode_query_len : 2 * decode_query_len]
|
||||
request_topk.fill_(invalid_block)
|
||||
request_topk[..., TOPK // 2] = 0
|
||||
|
||||
# A valid-only run isolates sentinel handling from BF16-vs-FP32 rounding.
|
||||
# Include block 0 exactly once; replacing sentinels with 0 would duplicate
|
||||
# its contribution. The general correctness test covers reference accuracy.
|
||||
valid_topk = torch.zeros(
|
||||
(NUM_KV_HEADS, q.shape[0], 1), device="cuda", dtype=torch.int32
|
||||
)
|
||||
expected = torch.empty_like(q)
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q,
|
||||
kv_cache,
|
||||
valid_topk,
|
||||
clean_block_table,
|
||||
seq_lens,
|
||||
NUM_KV_HEADS,
|
||||
SM_SCALE,
|
||||
expected,
|
||||
decode_query_len,
|
||||
)
|
||||
actual = torch.empty_like(q)
|
||||
|
||||
def run_decode() -> None:
|
||||
minimax_m3_sparse_attn_decode(
|
||||
q,
|
||||
kv_cache,
|
||||
topk_idx,
|
||||
block_table,
|
||||
seq_lens,
|
||||
NUM_KV_HEADS,
|
||||
SM_SCALE,
|
||||
actual,
|
||||
decode_query_len,
|
||||
)
|
||||
|
||||
if capture_graph:
|
||||
# An unfixed kernel faults during capture and leaves the CUDA context
|
||||
# unusable. Run negative-control reproductions in a dedicated process.
|
||||
run_decode()
|
||||
current_platform.synchronize()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run_decode()
|
||||
for _ in range(3):
|
||||
graph.replay()
|
||||
else:
|
||||
run_decode()
|
||||
current_platform.synchronize()
|
||||
|
||||
assert torch.isfinite(actual).all()
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_decode_wrong_layout_breaks_parity():
|
||||
"""Negative (AC-3/AC-5): consuming the physical HND buffer as if it were
|
||||
already contiguous-NHD (i.e. skipping the allocator's inverse permute)
|
||||
|
||||
@@ -156,6 +156,8 @@ def test_sparse_mla_sink_matches_ragged_reference(
|
||||
num_decodes=batch_size,
|
||||
num_decode_tokens=batch_size,
|
||||
max_query_len=1,
|
||||
max_seq_len=max(seq_lens),
|
||||
topk_tokens=max(seq_lens),
|
||||
)
|
||||
sinks = torch.linspace(-2.0, 6.0, real_heads, device=device)
|
||||
|
||||
@@ -367,12 +369,15 @@ def test_sparse_mla_sink_matches_dense_attention_with_empty_rows_and_paged_cache
|
||||
for page, offset in rows
|
||||
]
|
||||
lengths = torch.tensor([0, *(len(rows) for rows in selected_rows)])
|
||||
longest_row = max(len(rows) for rows in selected_rows)
|
||||
metadata = SimpleNamespace(
|
||||
block_size=block_size,
|
||||
num_prefills=0,
|
||||
num_decodes=len(selected_rows),
|
||||
num_decode_tokens=len(selected_rows),
|
||||
max_query_len=1,
|
||||
max_seq_len=longest_row,
|
||||
topk_tokens=longest_row,
|
||||
qo_indptr=torch.arange(
|
||||
len(selected_rows) + 1, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
@@ -479,6 +484,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
|
||||
num_decodes=num_tokens,
|
||||
num_decode_tokens=num_tokens,
|
||||
max_query_len=1,
|
||||
max_seq_len=kv.shape[0],
|
||||
block_size=block_size,
|
||||
topk_tokens=impl.topk_indices_buffer.shape[1],
|
||||
req_id_per_token=torch.zeros(num_tokens, dtype=torch.int32, device="cuda"),
|
||||
|
||||
@@ -41,7 +41,7 @@ def _on_gfx950() -> bool:
|
||||
# architectures take the fallback decode kernel, so its tests are skipped there.
|
||||
requires_split_decode_arch = pytest.mark.skipif(
|
||||
not _on_split_decode_arch(),
|
||||
reason="split-K decode kernel is only tuned for AMD gfx942/gfx950",
|
||||
reason="split-K decode and the V4.1 combine tests need an AMD gfx942/gfx950",
|
||||
)
|
||||
requires_gfx950 = pytest.mark.skipif(
|
||||
not _on_gfx950(),
|
||||
@@ -596,6 +596,87 @@ def test_sparse_attn_prefill_ragged_kernel() -> None:
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
NOPE_ONLY_HEAD_DIM = 512
|
||||
|
||||
|
||||
def _nope_only_ragged_inputs(
|
||||
kv_lens: list[int], num_heads: int, device: torch.device
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Build a GLM-like 512+0 NoPE decode batch with ragged KV segments."""
|
||||
num_kv = 4096
|
||||
kv = (
|
||||
torch.randn(num_kv, NOPE_ONLY_HEAD_DIM, dtype=torch.bfloat16, device=device)
|
||||
* 0.125
|
||||
)
|
||||
q = (
|
||||
torch.randn(
|
||||
len(kv_lens),
|
||||
num_heads,
|
||||
NOPE_ONLY_HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125
|
||||
)
|
||||
rows = [torch.randperm(num_kv)[:length].tolist() for length in kv_lens]
|
||||
indices, indptr = _ragged_from_rows(rows, device)
|
||||
return q, kv, indices, indptr
|
||||
|
||||
|
||||
# Lengths below the split count leave splits empty; non-multiples force a tail.
|
||||
_NOPE_ONLY_KV_LENS = [0, 1, 3, 37, 129, 2048]
|
||||
|
||||
|
||||
@requires_split_decode_arch
|
||||
@pytest.mark.parametrize("num_splits", [2, 32])
|
||||
@pytest.mark.parametrize("with_sink", [True, False])
|
||||
@torch.inference_mode()
|
||||
def test_sparse_attn_decode_bf16_split_k_matches_ragged(
|
||||
num_splits: int, with_sink: bool
|
||||
) -> None:
|
||||
"""Split-K decode must reproduce the single-workgroup ragged kernel."""
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_rocm_sparse_attn_decode_ragged_bf16_triton,
|
||||
_rocm_sparse_attn_prefill_ragged_triton,
|
||||
)
|
||||
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(0)
|
||||
num_heads = 16
|
||||
q, kv, indices, indptr = _nope_only_ragged_inputs(
|
||||
_NOPE_ONLY_KV_LENS, num_heads, device
|
||||
)
|
||||
attn_sink = (
|
||||
torch.randn(num_heads, dtype=torch.float32, device=device)
|
||||
if with_sink
|
||||
else None
|
||||
)
|
||||
scale = NOPE_ONLY_HEAD_DIM**-0.5
|
||||
|
||||
expected = _rocm_sparse_attn_prefill_ragged_triton(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
indptr=indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=NOPE_ONLY_HEAD_DIM,
|
||||
rope_head_dim=0,
|
||||
)
|
||||
actual = _rocm_sparse_attn_decode_ragged_bf16_triton(
|
||||
q=q,
|
||||
kv=kv,
|
||||
indices=indices,
|
||||
indptr=indptr,
|
||||
scale=scale,
|
||||
attn_sink=attn_sink,
|
||||
nope_head_dim=NOPE_ONLY_HEAD_DIM,
|
||||
rope_head_dim=0,
|
||||
num_splits=num_splits,
|
||||
)
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
def _sparse_prefill_ragged_inputs(nope_dim: int, rope_dim: int) -> dict:
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(7)
|
||||
@@ -1105,6 +1186,21 @@ def test_decode_num_splits_gfx950(monkeypatch) -> None:
|
||||
assert mod._decode_gfx950_num_splits(1, 1, 128, 8192) == 32
|
||||
assert mod._decode_gfx950_num_splits(17, 1, 128, 32) == 4
|
||||
assert mod._decode_gfx950_num_splits(512, 1, 128, 7812) == 1
|
||||
assert mod._decode_gfx950_num_splits(4, 1, 2048.0, 0.0, 32) == 32
|
||||
|
||||
|
||||
def test_sparse_decode_bf16_num_splits_floor_and_tile_clamp(monkeypatch) -> None:
|
||||
"""Short rows skip splitting, and splits never exceed the tiles they walk."""
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
monkeypatch.setattr(mod, "_decode_num_splits", lambda *args: 32)
|
||||
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 32)
|
||||
|
||||
floor = mod._SPARSE_DECODE_BF16_MIN_SPLIT_LEN
|
||||
block_k = mod._SPARSE_DECODE_BF16_BLOCK_K
|
||||
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, floor - 1) == 1
|
||||
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, floor) == floor // block_k
|
||||
assert mod.rocm_sparse_decode_bf16_num_splits(1, 1, 2048) == 32
|
||||
|
||||
|
||||
@requires_split_decode_arch
|
||||
@@ -2311,3 +2407,159 @@ def test_paged_mqa_logits_gate_v41_shuffle_vs_c4a_flat() -> None:
|
||||
torch.testing.assert_close(
|
||||
gated_c4a[:, :seq_len], ref[:, :seq_len], atol=2e-3, rtol=2e-3
|
||||
)
|
||||
|
||||
|
||||
def _combine_topk_swa_indices_torch(
|
||||
topk_indices,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
gather_lens,
|
||||
window_size,
|
||||
compress_ratio,
|
||||
topk,
|
||||
M,
|
||||
N,
|
||||
):
|
||||
"""The Torch path this kernel replaces, copied from before the switch.
|
||||
|
||||
Kept verbatim so the reference cannot drift toward the kernel.
|
||||
"""
|
||||
from vllm.models.deepseek_v41.common.ops.cache_utils import (
|
||||
_SPARSE_PREFILL_TOPK_ALIGNMENT,
|
||||
)
|
||||
|
||||
topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous()
|
||||
num_tokens = topk_indices.shape[0]
|
||||
combined_topk = (
|
||||
(topk + window_size + _SPARSE_PREFILL_TOPK_ALIGNMENT - 1)
|
||||
// _SPARSE_PREFILL_TOPK_ALIGNMENT
|
||||
* _SPARSE_PREFILL_TOPK_ALIGNMENT
|
||||
)
|
||||
combined_indices = torch.full(
|
||||
(num_tokens, combined_topk),
|
||||
fill_value=-1,
|
||||
dtype=torch.int32,
|
||||
device=topk_indices.device,
|
||||
)
|
||||
combined_lens = torch.empty(
|
||||
num_tokens, dtype=torch.int32, device=topk_indices.device
|
||||
)
|
||||
|
||||
# query_start_loc may have a non-zero base for a narrowed mixed batch.
|
||||
query_lens = query_start_loc[1:] - query_start_loc[:-1]
|
||||
req_ids = torch.repeat_interleave(
|
||||
torch.arange(seq_lens.shape[0], device=seq_lens.device), query_lens
|
||||
)
|
||||
query_starts = query_start_loc[:-1] - query_start_loc[0]
|
||||
token_offsets = torch.arange(num_tokens, device=seq_lens.device) - (
|
||||
torch.repeat_interleave(query_starts, query_lens)
|
||||
)
|
||||
positions = seq_lens[req_ids] - query_lens[req_ids] + token_offsets
|
||||
|
||||
logical_topk_width = min(topk, topk_indices.shape[1])
|
||||
topk_lens = torch.minimum(
|
||||
(positions + 1) // compress_ratio,
|
||||
torch.full_like(positions, logical_topk_width),
|
||||
).clamp_min(0)
|
||||
topk_offsets = torch.arange(logical_topk_width, device=seq_lens.device)
|
||||
topk_mask = topk_offsets[None, :] < topk_lens[:, None]
|
||||
topk_values = topk_indices[:, :logical_topk_width].to(torch.int32)
|
||||
topk_valid = topk_mask & (topk_values >= 0) & (topk_values < N)
|
||||
combined_indices[:, :logical_topk_width] = torch.where(
|
||||
topk_valid,
|
||||
topk_values + (M * req_ids).to(torch.int32)[:, None],
|
||||
-1,
|
||||
)
|
||||
|
||||
swa_lens = torch.minimum(
|
||||
positions + 1, torch.full_like(positions, window_size)
|
||||
).clamp_min(0)
|
||||
swa_offsets = torch.arange(window_size, device=seq_lens.device)
|
||||
swa_mask = swa_offsets[None, :] < swa_lens[:, None]
|
||||
swa_columns = topk_lens[:, None] + swa_offsets[None, :]
|
||||
gather_starts = seq_lens - gather_lens
|
||||
swa_values = (
|
||||
M * req_ids[:, None]
|
||||
+ N
|
||||
+ swa_offsets[None, :]
|
||||
+ positions[:, None]
|
||||
- swa_lens[:, None]
|
||||
+ 1
|
||||
- gather_starts[req_ids, None]
|
||||
).to(torch.int32)
|
||||
rows = torch.arange(num_tokens, device=seq_lens.device)[:, None].expand_as(
|
||||
swa_columns
|
||||
)
|
||||
flat_dst = rows[swa_mask] * combined_topk + swa_columns[swa_mask]
|
||||
combined_indices.view(-1).index_copy_(0, flat_dst, swa_values[swa_mask])
|
||||
combined_lens.copy_((topk_lens + swa_lens).to(torch.int32))
|
||||
return combined_indices, combined_lens
|
||||
|
||||
|
||||
def _v41_combine_case(case, window):
|
||||
dev = "cuda"
|
||||
ratio, topk = 2, 512
|
||||
if case == "seq_len_below_query_len":
|
||||
qsl = torch.tensor([0, 64], dtype=torch.int32, device=dev)
|
||||
sl = torch.tensor([32], dtype=torch.int32, device=dev)
|
||||
rows = 64
|
||||
elif case == "seq_len_zero":
|
||||
# The warmup batch: every pos is negative, so both lengths clamp to 0.
|
||||
qsl = torch.tensor([0, 32, 64], dtype=torch.int32, device=dev)
|
||||
sl = torch.zeros(2, dtype=torch.int32, device=dev)
|
||||
rows = 64
|
||||
elif case == "compress_ratio_zero":
|
||||
qsl = torch.tensor([0, 4, 8], dtype=torch.int32, device=dev)
|
||||
sl = torch.tensor([4, 4], dtype=torch.int32, device=dev)
|
||||
rows, ratio, topk = 8, 0, 0
|
||||
elif case == "nonzero_base":
|
||||
# Prefill rows of a mixed batch: query_start_loc starts past the decodes.
|
||||
qsl = torch.tensor([32, 64, 128], dtype=torch.int32, device=dev)
|
||||
sl = torch.tensor([48, 80], dtype=torch.int32, device=dev)
|
||||
rows = 96
|
||||
else:
|
||||
qsl = torch.tensor([0, 32, 96], dtype=torch.int32, device=dev)
|
||||
sl = torch.tensor([48, 80], dtype=torch.int32, device=dev)
|
||||
rows = 96
|
||||
# What the SWA builder gathers: the query plus up to window - 1 rows before it.
|
||||
gl = torch.minimum(sl, qsl[1:] - qsl[:-1] + window - 1)
|
||||
set_random_seed(0)
|
||||
ti = torch.randint(0, 4096, (rows, max(topk, 1)), dtype=torch.int32)
|
||||
if case == "invalid_topk":
|
||||
# Failed candidates stay -1 instead of pointing into another request.
|
||||
ti[:, ::7] = -1
|
||||
return ti.to(dev), qsl, sl, gl, window, ratio, topk, 100000, 4096
|
||||
|
||||
|
||||
@requires_split_decode_arch
|
||||
@pytest.mark.parametrize(
|
||||
"case,window",
|
||||
[
|
||||
("seq_len_below_query_len", 128),
|
||||
("seq_len_zero", 128),
|
||||
("compress_ratio_zero", 128),
|
||||
("nonzero_base", 128),
|
||||
("invalid_topk", 128),
|
||||
("plain", 64),
|
||||
("plain", 128),
|
||||
("plain", 256),
|
||||
],
|
||||
)
|
||||
def test_v41_combine_topk_swa_indices_matches_torch(case, window) -> None:
|
||||
from vllm.models.deepseek_v41.amd.rocm import combine_topk_swa_indices
|
||||
|
||||
args = _v41_combine_case(case, window)
|
||||
got_indices, got_lens = combine_topk_swa_indices(*args)
|
||||
ref_indices, ref_lens = _combine_topk_swa_indices_torch(*args)
|
||||
assert torch.equal(got_lens, ref_lens), (
|
||||
f"lens differ: ref={ref_lens.tolist()} got={got_lens.tolist()}"
|
||||
)
|
||||
assert torch.equal(got_indices, ref_indices)
|
||||
|
||||
|
||||
def test_v41_rocm_combine_uses_shared_kernel() -> None:
|
||||
"""ROCm prefill takes the shared combine, so it inherits the replay bound."""
|
||||
from vllm.models.deepseek_v41.amd import rocm
|
||||
from vllm.models.deepseek_v41.common.ops import cache_utils
|
||||
|
||||
assert rocm.combine_topk_swa_indices is cache_utils.combine_topk_swa_indices
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -181,6 +181,53 @@ def ref_paged_attn(
|
||||
return torch.cat(outputs, dim=0)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_fp8_softmax_preserves_small_probabilities() -> None:
|
||||
"""Keep exp(-8) contributions that underflow when cast directly to E4M3."""
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
num_tokens = block_size = 32
|
||||
head_size = 128
|
||||
|
||||
query = torch.zeros(1, 1, head_size, dtype=FP8_DTYPE, device=device)
|
||||
query[..., 0] = 1
|
||||
key_cache = torch.zeros(1, block_size, 1, head_size, dtype=FP8_DTYPE, device=device)
|
||||
key_cache[:, 1:, :, 0] = -8
|
||||
value_cache = torch.ones_like(key_cache)
|
||||
value_cache[:, 0] = 0
|
||||
output = torch.empty(1, 1, head_size, dtype=torch.bfloat16, device=device)
|
||||
|
||||
cu_seqlens_q = torch.tensor([0, 1], dtype=torch.int32, device=device)
|
||||
seqused_k = torch.tensor([num_tokens], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor([[0]], dtype=torch.int32, device=device)
|
||||
scale = torch.ones(1, dtype=torch.float32, device=device)
|
||||
|
||||
unified_attention(
|
||||
q=query,
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
out=output,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
max_seqlen_q=1,
|
||||
seqused_k=seqused_k,
|
||||
max_seqlen_k=num_tokens,
|
||||
softmax_scale=1.0,
|
||||
causal=True,
|
||||
window_size=(-1, -1),
|
||||
block_table=block_table,
|
||||
softcap=0,
|
||||
q_descale=scale,
|
||||
k_descale=scale,
|
||||
v_descale=scale,
|
||||
kv_quant_mode=KVQuantMode.FP8_PER_TENSOR,
|
||||
)
|
||||
|
||||
scores = torch.einsum("qhd,thd->qht", query.float(), key_cache[0].float())
|
||||
probabilities = torch.softmax(scores, dim=-1)
|
||||
expected = torch.einsum("qht,thd->qhd", probabilities, value_cache[0].float())
|
||||
|
||||
torch.testing.assert_close(output.float(), expected, atol=5e-5, rtol=1e-2)
|
||||
|
||||
|
||||
def ref_paged_clamped_mm_attn(
|
||||
query: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
|
||||
@@ -35,6 +35,8 @@ NUM_HEADS = [(4, 4), (8, 2), (5, 1)]
|
||||
HEAD_SIZES = [(128, 128), (192, 128)]
|
||||
BLOCK_SIZES = [16]
|
||||
DTYPES = [torch.bfloat16]
|
||||
# None keeps the KV cache in the query dtype.
|
||||
KV_CACHE_DTYPES = [None, current_platform.fp8_dtype()]
|
||||
|
||||
NUM_BLOCKS = 2048
|
||||
|
||||
@@ -80,6 +82,7 @@ def _alloc_segm_buffers(seq_threshold_3D: int, num_query_heads: int, head_size_v
|
||||
@pytest.mark.parametrize("sliding_window", [None, 128])
|
||||
@pytest.mark.parametrize("soft_cap", [None, 50.0])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPES)
|
||||
@pytest.mark.parametrize("seq_threshold_3D", SEQ_THRESHOLD_3D_VALUES)
|
||||
@torch.inference_mode()
|
||||
def test_triton_unified_attn_diffkv_vs_reference(
|
||||
@@ -89,13 +92,17 @@ def test_triton_unified_attn_diffkv_vs_reference(
|
||||
sliding_window: int | None,
|
||||
soft_cap: float | None,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: torch.dtype | None,
|
||||
block_size: int,
|
||||
seq_threshold_3D: int,
|
||||
quant_query: bool = False,
|
||||
) -> None:
|
||||
head_size_qk, head_size_v = head_sizes
|
||||
|
||||
# Keep the FA3/FA4 comparison on NVIDIA; ROCm uses the PyTorch oracle.
|
||||
if not current_platform.is_rocm():
|
||||
# FA3/FA4 is the reference on NVIDIA; ROCm and the FP8 KV cache (FA would
|
||||
# also quantize Q) use the PyTorch oracle.
|
||||
use_torch_ref = current_platform.is_rocm() or kv_cache_dtype is not None
|
||||
if not use_torch_ref:
|
||||
fa_version = get_flash_attn_version(
|
||||
head_size=head_size_qk, head_size_v=head_size_v
|
||||
)
|
||||
@@ -126,6 +133,20 @@ def test_triton_unified_attn_diffkv_vs_reference(
|
||||
)
|
||||
key_cache = kv_cache[..., :head_size_qk]
|
||||
value_cache = kv_cache[..., head_size_qk:]
|
||||
q_descale = k_descale = v_descale = None
|
||||
if kv_cache_dtype is not None:
|
||||
# Non-power-of-two scales exercise descale rounding.
|
||||
k_descale = torch.tensor(0.3, dtype=torch.float32)
|
||||
v_descale = torch.tensor(0.7, dtype=torch.float32)
|
||||
kv_cache = torch.cat(
|
||||
[key_cache / k_descale, value_cache / v_descale], dim=-1
|
||||
).to(kv_cache_dtype)
|
||||
key_cache = kv_cache[..., :head_size_qk]
|
||||
value_cache = kv_cache[..., head_size_qk:]
|
||||
|
||||
if quant_query:
|
||||
q_descale = torch.tensor(0.2, dtype=torch.float32)
|
||||
query = (query / q_descale).to(kv_cache_dtype)
|
||||
|
||||
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
|
||||
dim=0, dtype=torch.int32
|
||||
@@ -137,12 +158,20 @@ def test_triton_unified_attn_diffkv_vs_reference(
|
||||
0, NUM_BLOCKS, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
if current_platform.is_rocm():
|
||||
if use_torch_ref:
|
||||
ref_key_cache = key_cache.float()
|
||||
ref_value_cache = value_cache.float()
|
||||
if kv_cache_dtype is not None:
|
||||
ref_key_cache *= k_descale
|
||||
ref_value_cache *= v_descale
|
||||
# FP32 also keeps the helper's in-place scaling off the kernel input.
|
||||
ref_query = query.float()
|
||||
if quant_query:
|
||||
ref_query *= q_descale
|
||||
ref_out = ref_paged_attn(
|
||||
query.float(),
|
||||
key_cache.float(),
|
||||
value_cache.float(),
|
||||
ref_query,
|
||||
ref_key_cache,
|
||||
ref_value_cache,
|
||||
query_lens,
|
||||
kv_lens,
|
||||
block_tables,
|
||||
@@ -200,6 +229,32 @@ def test_triton_unified_attn_diffkv_vs_reference(
|
||||
softmax_segm_output=segm_output,
|
||||
softmax_segm_max=segm_max,
|
||||
softmax_segm_expsum=segm_expsum,
|
||||
q_descale=q_descale,
|
||||
k_descale=k_descale,
|
||||
v_descale=v_descale,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(triton_out, ref_out, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not current_platform.has_device_capability(89),
|
||||
reason="FP8 query requires CUDA SM89+",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,seq_threshold_3D",
|
||||
[([(129, 463)], 0), ([(1, 2011)], 0), ([(1, 2011)], 8)],
|
||||
)
|
||||
def test_triton_unified_attn_diffkv_fp8_query(seq_lens, seq_threshold_3D):
|
||||
test_triton_unified_attn_diffkv_vs_reference(
|
||||
seq_lens=seq_lens,
|
||||
num_heads=(8, 2),
|
||||
head_sizes=(192, 128),
|
||||
sliding_window=None,
|
||||
soft_cap=None,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype=current_platform.fp8_dtype(),
|
||||
block_size=16,
|
||||
seq_threshold_3D=seq_threshold_3D,
|
||||
quant_query=True,
|
||||
)
|
||||
|
||||
@@ -34,6 +34,7 @@ def test_cpu_write(device):
|
||||
cpu_tensor[4, 5] = -1
|
||||
|
||||
gpu_view.mul_(2)
|
||||
torch.accelerator.synchronize(gpu_view.device)
|
||||
assert gpu_view[0, 0] == 2
|
||||
assert gpu_view[2, 3] == 4
|
||||
assert gpu_view[4, 5] == -2
|
||||
@@ -55,6 +56,7 @@ def test_gpu_write(device):
|
||||
gpu_view[2, 3] = 2
|
||||
gpu_view[4, 5] = -1
|
||||
gpu_view.mul_(2)
|
||||
torch.accelerator.synchronize(gpu_view.device)
|
||||
|
||||
assert cpu_tensor[0, 0] == 2
|
||||
assert cpu_tensor[2, 3] == 4
|
||||
@@ -232,6 +234,25 @@ def test_non_pinned_cpu_tensor(device):
|
||||
assert gpu_view[9, 9] == 198
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only sync check.")
|
||||
def test_non_uva_buffer_copies_without_stream_sync():
|
||||
buffer = buffer_utils.NonUvaBuffer(8, torch.int32)
|
||||
buffer.np[:] = np.arange(8)
|
||||
|
||||
torch.cuda.set_sync_debug_mode("error")
|
||||
try:
|
||||
prefix = buffer.uva(4)
|
||||
full = buffer.uva()
|
||||
finally:
|
||||
torch.cuda.set_sync_debug_mode(0)
|
||||
# The next step overwrites the host buffer right after the copy is issued.
|
||||
buffer.np[:] = -1
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert prefix.tolist() == list(range(4))
|
||||
assert full.tolist() == list(range(8))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_uva_available(), reason="UVA is not available.")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_xpu(), reason="XPU non-contiguous UVA test."
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""An SSD intermediate state equals an independent run over the same prefix.
|
||||
|
||||
This is the central claim of internal prefill checkpoints: the state the SSD
|
||||
scan already materializes at the checkpoint's chunk boundary is exactly the
|
||||
state a fresh prefill over `[0, checkpoint)` would end with, so it can be
|
||||
cached and later resumed from. The last test covers the other half: moving
|
||||
that state, and the conv window, into the paged cache.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.mamba.test_mamba_ssm_ssd import generate_random_inputs
|
||||
from vllm.model_executor.layers.mamba.checkpoint import (
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_checkpoint import (
|
||||
store_prefill_checkpoint,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.ops.ssd_combined import (
|
||||
mamba_chunk_scan_combined_varlen,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.mamba_attn import BaseMambaAttentionMetadataBuilder
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
|
||||
DEVICE = current_platform.device_type
|
||||
compute_chunk_metadata = BaseMambaAttentionMetadataBuilder._compute_chunk_metadata
|
||||
|
||||
|
||||
def _run(inputs, seqlen, chunk_size, checkpoint_offset):
|
||||
"""Run one prefill of `seqlen` tokens, optionally exporting a checkpoint.
|
||||
|
||||
Returns (states, checkpoint_chunk_index). `states` holds every chunk's
|
||||
state when a checkpoint was requested, else the per-sequence final state.
|
||||
"""
|
||||
A, dt, X, B, C = inputs
|
||||
offsets = [checkpoint_offset] if checkpoint_offset else None
|
||||
cu, seq_idx, last, ckpt = compute_chunk_metadata(
|
||||
chunk_size,
|
||||
1,
|
||||
torch.tensor([0]),
|
||||
torch.tensor([0, seqlen]),
|
||||
checkpoint_offsets_p=offsets,
|
||||
)
|
||||
|
||||
x = X[:seqlen]
|
||||
out = torch.empty_like(x)
|
||||
states = mamba_chunk_scan_combined_varlen(
|
||||
x,
|
||||
dt[:seqlen],
|
||||
A,
|
||||
B[:seqlen],
|
||||
C[:seqlen],
|
||||
chunk_size,
|
||||
cu_seqlens=torch.tensor([0, seqlen], dtype=torch.int32, device=DEVICE),
|
||||
cu_chunk_seqlens=torch.tensor(cu, dtype=torch.int32, device=DEVICE),
|
||||
last_chunk_indices=torch.tensor(last, dtype=torch.int32, device=DEVICE),
|
||||
seq_idx=torch.tensor(seq_idx, dtype=torch.int32, device=DEVICE),
|
||||
out=out,
|
||||
D=None,
|
||||
return_intermediate_states=checkpoint_offset > 0,
|
||||
state_dtype=torch.float32,
|
||||
)
|
||||
return states, ckpt[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("checkpoint_offset", [32, 48, 96])
|
||||
def test_checkpoint_state_matches_an_independent_prefix_run(checkpoint_offset):
|
||||
"""states[checkpoint_chunk_idx] == the final state of a prefix-only run.
|
||||
|
||||
48 is deliberately not a multiple of chunk_size: align-mode block sizes
|
||||
carry no chunk_size factor, so off-grid checkpoints are the common case
|
||||
and are what the chunk split exists to support.
|
||||
"""
|
||||
seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32
|
||||
inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32)
|
||||
inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs)
|
||||
|
||||
states, ckpt_idx = _run(inputs, seqlen, chunk_size, checkpoint_offset)
|
||||
assert ckpt_idx >= 0, "the checkpoint did not land on a chunk boundary"
|
||||
checkpoint_state = states[ckpt_idx]
|
||||
|
||||
prefix_final, _ = _run(inputs, checkpoint_offset, chunk_size, 0)
|
||||
|
||||
torch.testing.assert_close(
|
||||
checkpoint_state.to(torch.float32),
|
||||
prefix_final[0].to(torch.float32),
|
||||
atol=8e-3,
|
||||
rtol=5e-3,
|
||||
)
|
||||
|
||||
|
||||
def test_checkpoint_does_not_perturb_the_final_state():
|
||||
"""Exporting a checkpoint must not change what the prefill produces.
|
||||
|
||||
The chunk split is an extra boundary, not a different computation, so the
|
||||
sequence's final state has to match a run that took no checkpoint.
|
||||
"""
|
||||
seqlen, chunk_size, n_heads, d_head = 128, 32, 4, 32
|
||||
inputs = generate_random_inputs(1, seqlen, n_heads, d_head, torch.float32)
|
||||
inputs = tuple(t.squeeze(0) if t.dim() > 1 else t for t in inputs)
|
||||
|
||||
with_ckpt, ckpt_idx = _run(inputs, seqlen, chunk_size, 48)
|
||||
without_ckpt, _ = _run(inputs, seqlen, chunk_size, 0)
|
||||
|
||||
torch.testing.assert_close(
|
||||
with_ckpt[-1].to(torch.float32),
|
||||
without_ckpt[0].to(torch.float32),
|
||||
atol=8e-3,
|
||||
rtol=5e-3,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dim_first", [True, False])
|
||||
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first):
|
||||
"""The store reads the checkpoint row out of every chunk state.
|
||||
|
||||
Row 0 checkpoints 32 tokens in, its state being chunk 3 rather than the
|
||||
program's own row, so the store only passes if ``chunk_idx`` is used.
|
||||
Row 1 declines and must write nothing. The conv input is a column slice of
|
||||
a wider projection, the SD conv layout is a transposed view, and SSM rows
|
||||
are padded pages, matching the strides the mixer passes.
|
||||
"""
|
||||
dim, state_len, heads, head_dim, d_state = 16, 4, 2, 4, 8
|
||||
projection = torch.randn(64, dim + 8, device=DEVICE)
|
||||
conv_input = projection[:, :dim]
|
||||
if dim_first:
|
||||
conv_state = torch.zeros(3, dim, state_len, device=DEVICE)
|
||||
else:
|
||||
conv_state = torch.zeros(3, state_len, dim, device=DEVICE).transpose(-1, -2)
|
||||
varlen_states = torch.randn(5, heads, head_dim, d_state, device=DEVICE)
|
||||
# Blocks are pages holding more than the SSM state, so rows are padded.
|
||||
row = heads * head_dim * d_state
|
||||
ssm_storage = torch.zeros(3, row + 8, device=DEVICE)
|
||||
ssm_state = ssm_storage[:, :row].view(3, heads, head_dim, d_state)
|
||||
checkpoint = MambaPrefillCheckpointMetadata(
|
||||
checkpoint_offsets=torch.tensor([32, 0], dtype=torch.int32, device=DEVICE),
|
||||
state_indices=torch.tensor(
|
||||
[2, NULL_BLOCK_ID], dtype=torch.int32, device=DEVICE
|
||||
),
|
||||
offsets=[32, 0],
|
||||
)
|
||||
|
||||
store_prefill_checkpoint(
|
||||
checkpoint,
|
||||
torch.tensor([3, 0], device=DEVICE),
|
||||
conv_input.T,
|
||||
conv_state,
|
||||
varlen_states,
|
||||
ssm_state,
|
||||
torch.tensor([0, 40, 64], dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
|
||||
exact = dict(rtol=0, atol=0)
|
||||
torch.testing.assert_close(
|
||||
conv_state[2], conv_input[32 - state_len : 32].T, **exact
|
||||
)
|
||||
torch.testing.assert_close(ssm_state[2], varlen_states[3], **exact)
|
||||
assert not conv_state[:2].any() and not ssm_storage[:2].any()
|
||||
assert not ssm_storage[:, row:].any(), "wrote into the page padding"
|
||||
@@ -31,7 +31,11 @@ import torch
|
||||
from vllm.model_executor.layers.mamba import mamba_utils as layer_mamba_utils
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker import mamba_utils as worker_mamba_utils
|
||||
from vllm.v1.worker.mamba_utils import _TEMPORAL_TILES, precopy_mamba_align_fused_kernel
|
||||
from vllm.v1.worker.mamba_utils import (
|
||||
_TEMPORAL_TILES,
|
||||
postprocess_mamba_fused_kernel,
|
||||
precopy_mamba_align_fused_kernel,
|
||||
)
|
||||
|
||||
_parametrize: Callable[..., Callable[[Any], Any]]
|
||||
|
||||
@@ -195,7 +199,8 @@ def test_precopy_matches_v1_copy_specs(
|
||||
convs, ssms, device, conv_state_dim_first
|
||||
)
|
||||
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device)
|
||||
# Non-identity mapping: block-table rows are request slots, not batch rows.
|
||||
idx_mapping = torch.arange(num_reqs, dtype=torch.int32, device=device).flip(0)
|
||||
grid = (num_reqs, NUM_LAYERS * 2, temporal_tiles)
|
||||
precopy_mamba_align_fused_kernel[grid](
|
||||
dst_col,
|
||||
@@ -225,6 +230,77 @@ def test_precopy_matches_v1_copy_specs(
|
||||
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
|
||||
|
||||
|
||||
@_parametrize("conv_state_dim_first", [False, True])
|
||||
@_cuda_required
|
||||
def test_v2_postprocess_indexes_block_tables_by_req_slot(conv_state_dim_first):
|
||||
"""Under PP the V2 postprocess runs after later steps have re-gathered the
|
||||
batch-ordered tables, so it must index the request-slot tables by slot."""
|
||||
device = torch.device("cuda")
|
||||
torch.manual_seed(0)
|
||||
num_reqs = 4
|
||||
block_size = 8
|
||||
num_blocks = num_reqs * MAX_COLS + 1
|
||||
bt = torch.arange(1, num_blocks, dtype=torch.int32, device=device).view(
|
||||
num_reqs, MAX_COLS
|
||||
)
|
||||
# slot 0: copy col 1 -> 0; slot 1: src == dst; slot 2: not aligned;
|
||||
# slot 3: copy col 2 -> 0 with one accepted draft token.
|
||||
num_accepted = torch.tensor([1, 1, 1, 2], dtype=torch.int32, device=device)
|
||||
state_idx = torch.tensor([1, 0, 1, 2], dtype=torch.int32, device=device)
|
||||
new_num_computed = torch.tensor([8, 8, 5, 8], dtype=torch.int32, device=device)
|
||||
ref_src = torch.tensor([1, 0, -1, 2], dtype=torch.int32)
|
||||
ref_dst = torch.zeros(num_reqs, dtype=torch.int32)
|
||||
ref_bias = torch.tensor([0, 0, 0, 1], dtype=torch.int32)
|
||||
|
||||
convs, ssms = _build_state(num_blocks, device, conv_state_dim_first)
|
||||
conv_ref, ssm_ref = _reference(
|
||||
convs,
|
||||
ssms,
|
||||
bt.cpu(),
|
||||
ref_src,
|
||||
ref_dst,
|
||||
ref_bias,
|
||||
num_reqs,
|
||||
conv_state_dim_first,
|
||||
)
|
||||
base, blk_stride, elem, inner, width, group, drc, drs = _build_meta(
|
||||
convs, ssms, device, conv_state_dim_first
|
||||
)
|
||||
bt_ptrs = torch.tensor([bt.data_ptr()], dtype=torch.int64, device=device)
|
||||
idx_mapping = torch.tensor([2, 0, 3, 1], dtype=torch.int32, device=device)
|
||||
postprocess_mamba_fused_kernel[(num_reqs, NUM_LAYERS * 2, 1)](
|
||||
num_accepted.clone(),
|
||||
state_idx,
|
||||
None,
|
||||
new_num_computed,
|
||||
None,
|
||||
bt_ptrs,
|
||||
bt.stride(0),
|
||||
base,
|
||||
blk_stride,
|
||||
elem,
|
||||
inner,
|
||||
width,
|
||||
group,
|
||||
drc,
|
||||
drs,
|
||||
num_accepted,
|
||||
idx_mapping,
|
||||
num_reqs,
|
||||
block_size=block_size,
|
||||
COPY_BLOCK_SIZE=1024,
|
||||
CONV_STATE_DIM_FIRST=conv_state_dim_first,
|
||||
HAS_IDX_MAPPING=True,
|
||||
PRECOMPUTED_NEW_COMPUTED=True,
|
||||
TEMPORAL_TILES=1,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
for layer in range(NUM_LAYERS):
|
||||
torch.testing.assert_close(convs[layer], conv_ref[layer], rtol=0, atol=0)
|
||||
torch.testing.assert_close(ssms[layer], ssm_ref[layer], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_ds_conv_copy_spec_reproduces_multi_accept_assert(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
layer_mamba_utils,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, replace
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
@@ -702,6 +702,21 @@ def make_modular_kernel(
|
||||
return modular_kernel
|
||||
|
||||
|
||||
def _shuffle_weights_for_aiter(rank_weights: WeightTensors) -> WeightTensors:
|
||||
"""Pre-shuffle weights so AITER selects its prebuilt `preshuffle_on` module.
|
||||
|
||||
Production shuffles in `process_weights_after_loading`; this harness builds its
|
||||
tensors directly, so without this every call asks for a `preshuffle_off` module
|
||||
that is not prebuilt, and JIT-compiles a kernel the image already ships.
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
w1, w2 = rocm_aiter_ops.shuffle_weights(rank_weights.w1, rank_weights.w2)
|
||||
w1.is_shuffled = True
|
||||
w2.is_shuffled = True
|
||||
return replace(rank_weights, w1=w1, w2=w2)
|
||||
|
||||
|
||||
def _maybe_convert_weights_for_experts(
|
||||
config: Config,
|
||||
rank_weights: WeightTensors,
|
||||
@@ -715,6 +730,15 @@ def _maybe_convert_weights_for_experts(
|
||||
fe_type = config.fused_experts_type
|
||||
fe_name = getattr(fe_type, "__name__", "")
|
||||
|
||||
# AITER's prebuilt modules carry a gfx950 instance table even on gfx942 and reject
|
||||
# intermediate sizes that are not a multiple of 256, but only for the fp8 per-tensor
|
||||
# and per-token schemes. Those keep JIT-compiling until
|
||||
# https://github.com/ROCm/aiter/issues/5766 is fixed.
|
||||
if fe_name == "AiterExperts" and (
|
||||
config.quant_dtype is None or config.quant_block_shape is not None
|
||||
):
|
||||
return _shuffle_weights_for_aiter(rank_weights)
|
||||
|
||||
backend: Fp8MoeBackend | None = None
|
||||
if fe_name == "TrtLlmFp8ExpertsModular":
|
||||
backend = Fp8MoeBackend.FLASHINFER_TRTLLM
|
||||
|
||||
@@ -37,9 +37,12 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
run_cutlass_moe_fp4,
|
||||
run_cutlass_moe_fp8,
|
||||
run_cutlass_moe_mxfp4,
|
||||
swizzle_mxfp4_scales,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle
|
||||
from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -294,6 +297,87 @@ def test_cutlass_fp4_moe_padded_routes_do_not_change_valid_output(quantization:
|
||||
torch.testing.assert_close(actual[1:], torch.zeros_like(actual[1:]))
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_cutlass_mxfp4_oracle_matches_direct_kernel(monkeypatch, workspace_init):
|
||||
"""The MXFP4 oracle selects CUTLASS for W4A4 and swizzles checkpoint
|
||||
scales into the layout the kernel expects."""
|
||||
if not CutlassExpertsMxfp4._supports_current_device():
|
||||
pytest.skip("CUTLASS mxfp4 MoE is not supported on this GPU")
|
||||
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
|
||||
|
||||
set_random_seed(7)
|
||||
e, m, n, k, topk = 8, 33, 256, 512, 2
|
||||
device = torch.device("cuda")
|
||||
w1 = torch.randint(0, 256, (e, 2 * n, k // 2), device=device, dtype=torch.uint8)
|
||||
w2 = torch.randint(0, 256, (e, k, n // 2), device=device, dtype=torch.uint8)
|
||||
w1_scale = torch.randint(118, 124, (e, 2 * n, k // 32), device=device).byte()
|
||||
w2_scale = torch.randint(118, 124, (e, k, n // 32), device=device).byte()
|
||||
a = torch.randn((m, k), device=device, dtype=torch.bfloat16)
|
||||
score = torch.randn((m, e), device=device, dtype=torch.bfloat16)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
|
||||
def swizzle(scale: torch.Tensor) -> torch.Tensor:
|
||||
rows, cols = scale.shape[1:]
|
||||
return torch.stack(
|
||||
[swizzle_mxfp4_scales(s, rows, cols * 32).view(rows, cols) for s in scale]
|
||||
)
|
||||
|
||||
expected = torch.empty_like(a)
|
||||
run_cutlass_moe_mxfp4(
|
||||
output=expected,
|
||||
a=a,
|
||||
w1_fp4=w1,
|
||||
w1_blockscale=swizzle(w1_scale),
|
||||
w2_fp4=w2,
|
||||
w2_blockscale=swizzle(w2_scale),
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
workspace13=torch.empty((m * topk, max(2 * n, k)), device=device).to(a),
|
||||
workspace2=torch.empty((m * topk, n), device=device).to(a),
|
||||
m=m,
|
||||
n=n,
|
||||
k=k,
|
||||
e=e,
|
||||
device=device,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=e, experts_per_token=topk, hidden_dim=k, intermediate_size=n
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
backend, experts_cls = mxfp4_oracle.select_mxfp4_moe_backend(
|
||||
moe_config, activation_key=kMxfp4Dynamic
|
||||
)
|
||||
assert backend == mxfp4_oracle.Mxfp4MoeBackend.CUTLASS_MXFP4_MXFP4
|
||||
|
||||
w13, w2, w13_scale, w2_scale, _, _ = (
|
||||
mxfp4_oracle.convert_weight_to_mxfp4_moe_kernel_format(
|
||||
backend, torch.nn.Module(), w1, w2, w1_scale, w2_scale
|
||||
)
|
||||
)
|
||||
kernel = mxfp4_oracle.make_mxfp4_moe_kernel(
|
||||
mxfp4_oracle.make_mxfp4_moe_quant_config(backend, w13_scale, w2_scale),
|
||||
moe_config,
|
||||
experts_cls,
|
||||
backend,
|
||||
)
|
||||
actual = kernel.apply(
|
||||
a,
|
||||
w13,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_ids,
|
||||
activation=MoEActivation.SILU,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class MOETensors:
|
||||
a: torch.Tensor
|
||||
|
||||
@@ -70,6 +70,28 @@ def test_ep_gather_uses_64_bit_row_offsets():
|
||||
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_ep_gather_uses_64_bit_output_row_offsets():
|
||||
hidden_size = 6144
|
||||
output_row = (1 << 31) // hidden_size + 1
|
||||
num_rows = output_row + 1
|
||||
output = torch.empty(
|
||||
(num_rows, hidden_size),
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
ep_gather(
|
||||
input_tensor=torch.ones((1, hidden_size), device="cuda", dtype=torch.bfloat16),
|
||||
recv_topk_ids=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int64),
|
||||
recv_topk_weight=torch.ones((num_rows, 1), device="cuda", dtype=torch.float32),
|
||||
input_index=torch.zeros((num_rows, 1), device="cuda", dtype=torch.int32),
|
||||
expert_map=None,
|
||||
output_tensor=output,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(output, torch.ones_like(output), rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("workspace_dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("num_columns", [2048, 6144, 6145])
|
||||
def test_fp8_workspace_shape(workspace_dtype, num_columns):
|
||||
@@ -158,6 +180,153 @@ def make_block_quant_fp8_weights(
|
||||
return w1, w2, w1_s, w2_s
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [16, 129, 512])
|
||||
@pytest.mark.parametrize("with_metadata", [False, True])
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_block_fp8_padding_on_graph_replay(num_tokens, with_metadata, monkeypatch):
|
||||
"""Both FP8 GEMMs consume only live rows as routing changes on replay."""
|
||||
from unittest.mock import Mock
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.experts.deep_gemm_moe as impl
|
||||
from vllm.utils.deep_gemm import DeepGemmQuantScaleFMT
|
||||
|
||||
if not (
|
||||
impl.current_platform.is_device_capability_family(90)
|
||||
or (
|
||||
impl.current_platform.is_device_capability_family(100)
|
||||
and DeepGemmQuantScaleFMT.from_oracle() == DeepGemmQuantScaleFMT.UE8M0
|
||||
)
|
||||
):
|
||||
pytest.skip("Prefix-sum FP8 dispatch requires SM90 or SM10.x")
|
||||
|
||||
torch.manual_seed(0)
|
||||
e, n, k, topk = 8, 1024, 512, 2
|
||||
x = torch.randn(num_tokens, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
|
||||
w1, w2, s1, s2 = make_block_quant_fp8_weights(e, n, k, BLOCK_SIZE)
|
||||
quant = fp8_w8a8_moe_quant_config(w1_scale=s1, w2_scale=s2, block_shape=BLOCK_SIZE)
|
||||
experts = impl.DeepGemmExperts(make_dummy_moe_config(), quant)
|
||||
ids = torch.zeros(num_tokens, topk, device="cuda", dtype=torch.int64)
|
||||
weights = torch.full((num_tokens, topk), 0.5, device="cuda")
|
||||
counts = torch.zeros(e, device="cuda", dtype=torch.int32)
|
||||
metadata = mk.ExpertTokensMetadata(counts, None) if with_metadata else None
|
||||
shapes = experts.workspace_shapes(
|
||||
num_tokens, 2 * n, k, topk, e, e, metadata, MoEActivation.SILU
|
||||
)
|
||||
|
||||
def route(mode):
|
||||
ids[:, 0] = torch.arange(num_tokens, device="cuda") % e
|
||||
ids[:, 1] = (ids[:, 0] + 1) % e
|
||||
if mode == 1:
|
||||
ids.fill_(-1)
|
||||
ids[:1, 0] = e - 1
|
||||
elif mode == 2:
|
||||
ids.fill_(-1)
|
||||
counts.copy_(torch.bincount(ids[ids >= 0], minlength=e).int())
|
||||
|
||||
route(0)
|
||||
variants = []
|
||||
for dense in (True, False):
|
||||
ws1, ws2, output = [
|
||||
torch.empty(shape, device="cuda", dtype=torch.bfloat16) for shape in shapes
|
||||
]
|
||||
|
||||
def invoke(output=output, ws1=ws1, ws2=ws2):
|
||||
experts.apply(
|
||||
output,
|
||||
aq,
|
||||
w1,
|
||||
w2,
|
||||
weights,
|
||||
ids,
|
||||
MoEActivation.SILU,
|
||||
e,
|
||||
None,
|
||||
scales,
|
||||
None,
|
||||
ws1,
|
||||
ws2,
|
||||
metadata,
|
||||
False,
|
||||
)
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
if dense:
|
||||
platform = Mock(wraps=impl.current_platform)
|
||||
platform.is_device_capability_family.return_value = False
|
||||
mp.setattr(impl, "current_platform", platform)
|
||||
gemm = impl.m_grouped_fp8_gemm_nt_contiguous
|
||||
calls: list[int] = []
|
||||
|
||||
def check_layout(*args, dense=dense, calls=calls, gemm=gemm, **kwargs):
|
||||
assert kwargs.get("use_psum_layout", False) == (not dense)
|
||||
calls.append(args[3].numel())
|
||||
return gemm(*args, **kwargs)
|
||||
|
||||
mp.setattr(impl, "m_grouped_fp8_gemm_nt_contiguous", check_layout)
|
||||
invoke()
|
||||
assert len(calls) == 2
|
||||
if not dense:
|
||||
assert calls == [e, e]
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
invoke()
|
||||
variants.append((graph, output, ws1, ws2))
|
||||
|
||||
for mode in (0, 1, 2, 0):
|
||||
route(mode)
|
||||
for graph, _, ws1, ws2 in variants:
|
||||
ws1.fill_(float("nan"))
|
||||
ws2.fill_(float("nan"))
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(variants[0][1], variants[1][1], rtol=0, atol=0)
|
||||
assert torch.isfinite(variants[1][1]).all()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("n", "k"), [(2048, 512), (512, 1024)])
|
||||
@pytest.mark.skipif(not is_deep_gemm_supported(), reason="Requires deep_gemm kernels")
|
||||
def test_deepgemm_prefix_sum_leaves_unused_capacity_untouched(n, k):
|
||||
"""Both projection shapes skip unused tiles as counts change on replay."""
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
m_grouped_fp8_gemm_nt_contiguous,
|
||||
mk_alignment_scope,
|
||||
)
|
||||
|
||||
if not (
|
||||
current_platform.is_device_capability_family(90)
|
||||
or current_platform.is_device_capability_family(100)
|
||||
):
|
||||
pytest.skip("Requires SM90 or SM10.x")
|
||||
|
||||
capacity, experts, alignment = 512, 4, 128
|
||||
x = torch.randn(capacity, k, device="cuda", dtype=torch.bfloat16) / 10
|
||||
aq, scales = per_token_group_quant_fp8(x, 128, use_ue8m0=True)
|
||||
w, _, ws, _ = make_block_quant_fp8_weights(experts, n // 2, k, BLOCK_SIZE)
|
||||
ends = torch.tensor([0, 1, 128, 129], device="cuda", dtype=torch.int32)
|
||||
output = torch.empty(capacity, n, device="cuda", dtype=torch.bfloat16)
|
||||
|
||||
def invoke():
|
||||
with mk_alignment_scope(alignment):
|
||||
m_grouped_fp8_gemm_nt_contiguous(
|
||||
(aq, scales), (w, ws), output, ends, use_psum_layout=True
|
||||
)
|
||||
|
||||
invoke()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
invoke()
|
||||
for bounds, allocated_end in (([0, 1, 128, 129], 256), ([0] * 4, 0)):
|
||||
ends.copy_(torch.tensor(bounds, device="cuda", dtype=torch.int32))
|
||||
output.fill_(91)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.all(output[allocated_end:] == 91)
|
||||
if allocated_end:
|
||||
assert torch.all(output[[0, 128]] != 91)
|
||||
|
||||
|
||||
def run_single_case(m, n, k, topk, num_experts, block_size):
|
||||
"""Run one (M,N,K) configuration on a single GPU and assert DeepGEMM ==
|
||||
Triton baseline within tolerance.
|
||||
|
||||
@@ -32,8 +32,11 @@ from triton_kernels.testing import assert_close
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.config import mxfp4_w4a16_moe_quant_config
|
||||
from vllm.model_executor.layers.fused_moe.experts.gpt_oss_triton_kernels_moe import (
|
||||
_sm12x_block_m,
|
||||
triton_kernel_moe_forward,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import Mxfp4MoeBackend
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import mx_scale_kwargs
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
@@ -314,9 +317,13 @@ def test_equiv(num_token, a_dtype, w_dtype, tp, workspace_init):
|
||||
pc2,
|
||||
) = init_compute_data(M, K, N, E, a_dtype, w_dtype, num_warps=8)
|
||||
|
||||
if current_platform.is_device_capability_family(100):
|
||||
if current_platform.is_device_capability_family(
|
||||
100
|
||||
) or current_platform.is_device_capability_family(120):
|
||||
# Mirrors `_swizzle_mxfp4`; SM12x also needs epilogue_subtile=1.
|
||||
constraints = {
|
||||
"is_persistent": True,
|
||||
"epilogue_subtile": 1,
|
||||
}
|
||||
opt_flags.update_opt_flags_constraints(constraints)
|
||||
|
||||
@@ -356,6 +363,38 @@ def test_equiv(num_token, a_dtype, w_dtype, tp, workspace_init):
|
||||
assert_close(ref=out_ref, tri=out_triton_monolithic, maxtol=0.025, rmstol=0.005)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_rows", "num_experts", "expected"),
|
||||
[
|
||||
(4, 32, 16), # bs=1 decode: below the 16-row floor
|
||||
(32 * 20, 32, 32), # 20 tokens/expert rounds up to 32
|
||||
(32 * 33, 32, 32), # heuristic would pick 64; capped for 99KB smem
|
||||
(12000, 32, 32), # prefill: heuristic would pick 128
|
||||
],
|
||||
)
|
||||
def test_sm12x_block_m(num_rows, num_experts, expected):
|
||||
assert _sm12x_block_m(num_rows, num_experts) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sm12x", [True, False])
|
||||
def test_gpt_oss_priority_prefers_marlin_on_sm12x(
|
||||
monkeypatch: pytest.MonkeyPatch, sm12x: bool
|
||||
):
|
||||
"""SM12x runs the OAI Triton kernels at small tiles only, so MARLIN must
|
||||
stay the default there while TRITON remains selectable explicitly."""
|
||||
monkeypatch.setattr(mxfp4_oracle.current_platform, "is_cuda", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
mxfp4_oracle.current_platform,
|
||||
"is_device_capability_family",
|
||||
lambda family: sm12x and family == 120,
|
||||
)
|
||||
order = mxfp4_oracle._get_priority_backends_for_gpt_oss()
|
||||
triton_first = order.index(Mxfp4MoeBackend.TRITON) < order.index(
|
||||
Mxfp4MoeBackend.MARLIN
|
||||
)
|
||||
assert triton_first is not sm12x
|
||||
|
||||
|
||||
def test_unit_shuffle():
|
||||
N = ModelConfig.intermediate_size
|
||||
K = ModelConfig.hidden_size
|
||||
|
||||
@@ -17,6 +17,10 @@ from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_cutlass_fused_moe
|
||||
from vllm.utils.import_utils import has_aiter, has_deep_ep, has_deep_gemm
|
||||
@@ -988,6 +992,173 @@ def test_aiter_moe_padding_matrix_mxfp4():
|
||||
padding matrix once MI350/gfx950 hardware is available in CI."""
|
||||
|
||||
|
||||
# --- 4c: HIP-graph token padding -- inf/nan garbage rows (topk_ids=-1) ------
|
||||
|
||||
# AiterExperts isolates garbage rows from real rows under both AITER sorting
|
||||
# backends, and reference_moe_impl's dispatch (`mask = topk_ids == i`) is
|
||||
# likewise row-local, so comparing real rows against it is valid. Exception:
|
||||
# AiterExperts's a1 quant scale is normally fixed before garbage injection
|
||||
# (computed in RankTensors.make()), but "fp8_tensor_token" computes it live
|
||||
# from the (already-injected) hidden_states -- see the skip below.
|
||||
|
||||
# {mode name -> {row index: fill value}}. Covers multiple garbage-row counts
|
||||
# and positions, and both inf and nan.
|
||||
_TOKEN_PADDING_GARBAGE_MODES: dict[str, dict[int, float]] = {
|
||||
"single_inf": {3: float("inf")},
|
||||
"single_nan": {5: float("nan")},
|
||||
"multiple_mixed": {
|
||||
1: float("inf"),
|
||||
4: float("-inf"),
|
||||
7: float("nan"),
|
||||
12: float("inf"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _apply_token_padding_garbage_rows(
|
||||
rank_tensors: RankTensors, garbage_rows: dict[int, float]
|
||||
) -> None:
|
||||
"""Simulate the HIP-graph padding-token contract in-place: garbage rows
|
||||
get inf/nan hidden_states and their routing masked to topk_ids=-1 /
|
||||
topk_weights=0, mirroring grouped_topk_router.py's `is_padding` masking.
|
||||
"""
|
||||
for row, fill in garbage_rows.items():
|
||||
rank_tensors.hidden_states[row].fill_(fill)
|
||||
rank_tensors.topk_ids[row].fill_(-1)
|
||||
rank_tensors.topk_weights[row].fill_(0.0)
|
||||
|
||||
|
||||
def _aiter_token_padding_worker(
|
||||
pgi: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
cpu_group,
|
||||
config: Config,
|
||||
weights: WeightTensors,
|
||||
verbose: bool,
|
||||
garbage_rows: dict[int, float],
|
||||
):
|
||||
device = torch.device(f"cuda:{pgi.local_rank}")
|
||||
init_workspace_manager(device)
|
||||
set_random_seed(pgi.rank)
|
||||
|
||||
weights = copy.deepcopy(weights)
|
||||
weights.to_current_device()
|
||||
|
||||
rank_tensors = RankTensors.make(config, pgi)
|
||||
_apply_token_padding_garbage_rows(rank_tensors, garbage_rows)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
mk_out = run_modular_kernel(pgi, vllm_config, config, weights, rank_tensors)
|
||||
ref_out = reference_moe_impl(config, weights, rank_tensors)
|
||||
|
||||
real_rows = [i for i in range(config.Ms) if i not in garbage_rows]
|
||||
|
||||
assert torch.isfinite(mk_out[real_rows]).all(), (
|
||||
"Garbage (padding) token rows leaked NaN/Inf into real rows' output "
|
||||
"through AiterExperts."
|
||||
)
|
||||
|
||||
# ref_out's magnitude here (~1e-3-1e-2) is below atol=3e-2, so a zeroed-out
|
||||
# mk_out would still pass the checks below -- guard against that.
|
||||
ref_scale = ref_out[real_rows].abs().mean()
|
||||
mk_scale = mk_out[real_rows].abs().mean()
|
||||
assert mk_scale > 0.5 * ref_scale, (
|
||||
f"AiterExperts output magnitude (mean |mk_out|={mk_scale:.6f}) looks "
|
||||
f"degenerate/zeroed vs. reference (mean |ref_out|={ref_scale:.6f})."
|
||||
)
|
||||
|
||||
if config.quant_config is not None:
|
||||
check_accuracy(
|
||||
ref_out[real_rows], mk_out[real_rows], atol=3e-2, rtol=3e-2, percent=0.9
|
||||
)
|
||||
else:
|
||||
torch.testing.assert_close(
|
||||
ref_out[real_rows], mk_out[real_rows], atol=3e-2, rtol=3e-2
|
||||
)
|
||||
|
||||
|
||||
@require_aiter_moe
|
||||
@pytest.mark.parametrize("quant_config", _PADDING_QUANT_CONFIGS, ids=_PADDING_QUANT_IDS)
|
||||
@pytest.mark.parametrize("garbage_mode", list(_TOKEN_PADDING_GARBAGE_MODES))
|
||||
def test_aiter_moe_token_padding_garbage_rows(
|
||||
garbage_mode: str, quant_config: TestMoEQuantConfig | None
|
||||
):
|
||||
"""See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").
|
||||
|
||||
Simulates HIP-graph token padding: a subset of rows get inf/nan
|
||||
hidden_states with topk_ids=-1 / topk_weights=0, mirroring the router's
|
||||
padding-token masking contract (grouped_topk_router.py). Verifies the
|
||||
real (non-garbage) rows' output through AiterExperts remains finite and
|
||||
matches reference_moe_impl, across multiple garbage-row counts/positions
|
||||
and both inf and nan fill values, for unquantized + the 4 fp8 quant
|
||||
schemes AiterExperts supports.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import (
|
||||
MoEPrepareAndFinalizeNoDPEPModular,
|
||||
)
|
||||
|
||||
garbage_rows = _TOKEN_PADDING_GARBAGE_MODES[garbage_mode]
|
||||
|
||||
# Reuses 4b's already-confirmed 128-aligned ("padded") K/N -- these are
|
||||
# known to work across all 5 quant schemes here, including AITER's
|
||||
# block-quant kernel (which 4b's _PADDING_K_UNPADDED/_PADDING_N_UNPADDED
|
||||
# sizes do not support). Token-padding safety doesn't depend on
|
||||
# hidden_pad/intermediate_pad, so there's no need for a separate shape.
|
||||
config = Config(
|
||||
Ms=_PADDING_M,
|
||||
K=_PADDING_K_PADDED,
|
||||
N=_PADDING_N_PADDED,
|
||||
E=_PADDING_E,
|
||||
topks=_PADDING_TOPK,
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
world_size=1,
|
||||
)
|
||||
assert config.is_valid()[0]
|
||||
assert config.fe_supports_quant_scheme(), (
|
||||
f"AiterExperts does not support quant scheme {quant_config}."
|
||||
)
|
||||
|
||||
# This (weight, activation) pair is AiterExperts's real per-tensor
|
||||
# dynamic-activation-quant scheme -- its batch-wide max-abs scale is
|
||||
# poisoned by a real inf garbage row. Upstream bug:
|
||||
# https://github.com/ROCm/aiter/issues/6275.
|
||||
if (
|
||||
quant_config is not None
|
||||
and config.fp8_quant_key_pair() == (kFp8StaticTensorSym, kFp8DynamicTensorSym)
|
||||
and any(v in (float("inf"), float("-inf")) for v in garbage_rows.values())
|
||||
):
|
||||
pytest.skip(
|
||||
"Batch-wide dynamic per-tensor quant scale is poisoned by a "
|
||||
"real inf garbage row. See "
|
||||
"https://github.com/ROCm/aiter/issues/6275."
|
||||
)
|
||||
|
||||
weights = WeightTensors.make(config)
|
||||
vllm_config, env_dict = config.make_env_data()
|
||||
|
||||
parallel_launch_with_config(
|
||||
config.world_size,
|
||||
_aiter_token_padding_worker,
|
||||
vllm_config,
|
||||
env_dict,
|
||||
config,
|
||||
weights,
|
||||
False,
|
||||
garbage_rows,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason="MXFP4 AiterExperts token padding requires MI350/gfx950 hardware. "
|
||||
'See https://github.com/vllm-project/vllm/issues/54966 ("Test padding").'
|
||||
)
|
||||
def test_aiter_moe_token_padding_garbage_rows_mxfp4():
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Ability to test individual PrepareAndFinalize and FusedExperts combination
|
||||
from .modular_kernel_tools.cli_args import make_config, make_config_arg_parser
|
||||
|
||||
@@ -1437,6 +1437,7 @@ def test_humming_global_valid_shape_m(
|
||||
def test_humming_permute_scratch_is_shared_by_config(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Layers share one scratch whose key is independent of batch shape."""
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
@@ -1447,34 +1448,19 @@ def test_humming_permute_scratch_is_shared_by_config(
|
||||
manager = workspace.WorkspaceManager(torch.device("cpu"))
|
||||
monkeypatch.setattr(workspace, "_manager", manager)
|
||||
|
||||
scratch_topk6 = Mock()
|
||||
scratch_topk1 = Mock()
|
||||
scratch_type = Mock(side_effect=[scratch_topk6, scratch_topk1])
|
||||
scratch = Mock()
|
||||
scratch_type = Mock(side_effect=[scratch])
|
||||
monkeypatch.setattr(humming, "moe_permute_unpermute_supported", lambda: True)
|
||||
monkeypatch.setattr(permute, "MoEPermuteScratch", scratch_type)
|
||||
moe_config = make_dummy_moe_config(max_num_tokens=512, experts_per_token=6)
|
||||
moe_config.moe_parallel_config.dp_size = 2
|
||||
experts = SimpleNamespace(moe_config=moe_config)
|
||||
other_layer = SimpleNamespace(moe_config=moe_config)
|
||||
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
|
||||
assert (
|
||||
humming.HummingExpertsBase._get_permute_scratch(other_layer, 6) is scratch_topk6
|
||||
)
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 1) is scratch_topk1
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
|
||||
|
||||
assert scratch_type.call_count == 2
|
||||
first_call, second_call = scratch_type.call_args_list
|
||||
assert first_call.kwargs["max_num_tokens"] == 1024
|
||||
assert first_call.kwargs["topk"] == 6
|
||||
assert second_call.kwargs["max_num_tokens"] == 6144
|
||||
assert second_call.kwargs["topk"] == 1
|
||||
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts) is scratch
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(other_layer) is scratch
|
||||
manager.lock()
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts, 6) is scratch_topk6
|
||||
with pytest.raises(AssertionError, match="was not allocated during warmup"):
|
||||
humming.HummingExpertsBase._get_permute_scratch(experts, 2)
|
||||
assert humming.HummingExpertsBase._get_permute_scratch(experts) is scratch
|
||||
assert scratch_type.call_count == 1
|
||||
|
||||
|
||||
def test_humming_delegates_to_instance_activation():
|
||||
@@ -1531,7 +1517,7 @@ def test_humming_grouped_apply_forwards_valid_prefix(
|
||||
num_experts=2,
|
||||
estimate_local_valid_shape_m=lambda _: 6,
|
||||
prepare_buffers=lambda *_: buffers,
|
||||
_get_permute_scratch=lambda _, *, indices_only=False: object(),
|
||||
_get_permute_scratch=lambda: object(),
|
||||
process_input=Mock(
|
||||
side_effect=lambda _, **kwargs: (
|
||||
kwargs["inputs"],
|
||||
|
||||
@@ -169,13 +169,9 @@ def test_moe_permute_unpermute(
|
||||
scratch = None
|
||||
if use_scratch:
|
||||
scratch = get_moe_permute_scratch(
|
||||
max_num_tokens=n_token,
|
||||
topk=topk,
|
||||
num_experts=n_expert,
|
||||
num_local_experts=n_local_expert,
|
||||
device=hidden_states.device,
|
||||
hidden_size=n_hidden,
|
||||
hidden_dtype=dtype,
|
||||
)
|
||||
|
||||
(
|
||||
@@ -247,13 +243,9 @@ def test_moe_permute_reuses_scratch_buffers(
|
||||
_, topk_ids, _ = fused_topk(hidden_states, gating_output, topk, False)
|
||||
|
||||
scratch_config = dict(
|
||||
max_num_tokens=n_token,
|
||||
topk=topk,
|
||||
num_experts=n_expert,
|
||||
num_local_experts=n_expert,
|
||||
device=hidden_states.device,
|
||||
hidden_size=n_hidden,
|
||||
hidden_dtype=hidden_states.dtype,
|
||||
)
|
||||
scratch = get_moe_permute_scratch(**scratch_config)
|
||||
|
||||
@@ -300,15 +292,10 @@ def test_moe_permute_reuses_scratch_buffers(
|
||||
permuted_idx_2,
|
||||
) = second
|
||||
|
||||
torch.testing.assert_close(permuted_hidden_states_1, permuted_hidden_states_2)
|
||||
torch.testing.assert_close(expert_first_token_offset_1, expert_first_token_offset_2)
|
||||
torch.testing.assert_close(inv_permuted_idx_1, inv_permuted_idx_2)
|
||||
torch.testing.assert_close(permuted_idx_1, permuted_idx_2)
|
||||
|
||||
assert (
|
||||
permuted_hidden_states_1.untyped_storage().data_ptr()
|
||||
== permuted_hidden_states_2.untyped_storage().data_ptr()
|
||||
)
|
||||
assert (
|
||||
expert_first_token_offset_1.untyped_storage().data_ptr()
|
||||
== expert_first_token_offset_2.untyped_storage().data_ptr()
|
||||
@@ -330,14 +317,11 @@ def test_moe_permute_scratch_reused_across_graph_sizes(workspace_init) -> None:
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
scratch = get_moe_permute_scratch(
|
||||
max_num_tokens=128,
|
||||
topk=6,
|
||||
num_experts=16,
|
||||
num_local_experts=16,
|
||||
device=device,
|
||||
hidden_size=2048,
|
||||
hidden_dtype=torch.bfloat16,
|
||||
)
|
||||
scratch.reserve(128 * 6)
|
||||
current_workspace_manager().lock()
|
||||
runs = []
|
||||
for n_token in (128, 1, 33):
|
||||
@@ -372,13 +356,9 @@ def test_moe_permute_scratch_isolated_across_execution_slots(monkeypatch) -> Non
|
||||
ubatch = 0
|
||||
monkeypatch.setattr(workspace, "dbo_current_ubatch_id", lambda: ubatch)
|
||||
scratch_config = dict(
|
||||
max_num_tokens=64,
|
||||
topk=6,
|
||||
num_experts=16,
|
||||
num_local_experts=16,
|
||||
device=device,
|
||||
hidden_size=2048,
|
||||
hidden_dtype=torch.bfloat16,
|
||||
)
|
||||
runs = []
|
||||
for i, n_token in enumerate((1, 7, 33, 64)):
|
||||
@@ -396,7 +376,7 @@ def test_moe_permute_scratch_isolated_across_execution_slots(monkeypatch) -> Non
|
||||
runs.append((stream, graph, hidden, topk_ids, output, scratch))
|
||||
torch.cuda.current_stream().wait_stream(stream)
|
||||
|
||||
assert len({run[-1].permuted_hidden_states.data_ptr() for run in runs}) == 4
|
||||
assert len({run[-1].permuted_idx.data_ptr() for run in runs}) == 4
|
||||
manager.lock()
|
||||
for _ in range(3):
|
||||
for i, (stream, graph, hidden, topk_ids, _, scratch) in enumerate(runs):
|
||||
@@ -423,8 +403,6 @@ def test_moe_permute_scratch_without_manager(monkeypatch) -> None:
|
||||
|
||||
monkeypatch.setattr(workspace, "_manager", None)
|
||||
config = dict(
|
||||
max_num_tokens=4,
|
||||
topk=2,
|
||||
num_experts=4,
|
||||
num_local_experts=4,
|
||||
device=torch.device("cuda"),
|
||||
@@ -432,6 +410,8 @@ def test_moe_permute_scratch_without_manager(monkeypatch) -> None:
|
||||
first = get_moe_permute_scratch(**config)
|
||||
second = get_moe_permute_scratch(**config)
|
||||
|
||||
first.reserve(8)
|
||||
second.reserve(8)
|
||||
assert (
|
||||
first.token_expert_indices.data_ptr() != second.token_expert_indices.data_ptr()
|
||||
)
|
||||
@@ -450,13 +430,9 @@ def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
|
||||
topk_ids = torch.tensor([[0], [-1], [1], [4], [2]], device="cuda")
|
||||
expert_map = torch.tensor([0, 1, -1, -1], dtype=torch.int32, device="cuda")
|
||||
scratch = MoEPermuteScratch(
|
||||
max_num_tokens=5,
|
||||
topk=1,
|
||||
num_experts=4,
|
||||
num_local_experts=2,
|
||||
device=hidden_states.device,
|
||||
hidden_size=16,
|
||||
hidden_dtype=hidden_states.dtype,
|
||||
)
|
||||
|
||||
permuted, _, expert_offsets, inverse, _ = moe_permute(
|
||||
@@ -498,3 +474,39 @@ def test_moe_permute_ignores_invalid_expert_ids_with_scratch() -> None:
|
||||
expected_offsets = torch.tensor([0, 5, 5], dtype=torch.int64, device="cuda")
|
||||
torch.testing.assert_close(indices.flatten(), expected_inverse)
|
||||
torch.testing.assert_close(expert_offsets, expected_offsets)
|
||||
|
||||
|
||||
def test_moe_permute_scratch_grows_until_workspace_lock(workspace_init) -> None:
|
||||
"""Capacity follows the inputs seen in warmup, whatever their topk or size."""
|
||||
if not moe_permute_unpermute_supported():
|
||||
pytest.skip("moe_permute_unpermute is not supported on this platform.")
|
||||
|
||||
device = torch.device("cuda:0")
|
||||
scratch = get_moe_permute_scratch(
|
||||
num_experts=16, num_local_experts=16, device=device
|
||||
)
|
||||
|
||||
def run(n_token: int, topk: int, dtype: torch.dtype) -> None:
|
||||
hidden = torch.randn((n_token, 64), device=device).to(dtype)
|
||||
topk_ids = torch.randint(16, (n_token, topk), device=device)
|
||||
actual = moe_permute(hidden, None, topk_ids, 16, scratch=scratch)
|
||||
expected = moe_permute(hidden, None, topk_ids, 16)
|
||||
for a, e in zip(actual, expected):
|
||||
if a is not None:
|
||||
torch.testing.assert_close(a.float(), e.float())
|
||||
|
||||
run(8, 6, torch.bfloat16)
|
||||
run(64, 6, torch.bfloat16)
|
||||
run(384, 1, torch.float8_e4m3fn)
|
||||
assert scratch.max_expanded_rows == 384
|
||||
|
||||
current_workspace_manager().lock()
|
||||
run(64, 6, torch.bfloat16)
|
||||
with pytest.raises(AssertionError, match="Scratch growth is not allowed"):
|
||||
run(65, 6, torch.bfloat16)
|
||||
|
||||
# Experts rebuilt after the lock (e.g. a weight reload) keep the capacity.
|
||||
rebuilt = get_moe_permute_scratch(
|
||||
num_experts=16, num_local_experts=16, device=device
|
||||
)
|
||||
assert rebuilt.max_expanded_rows == 384
|
||||
|
||||
@@ -9,6 +9,8 @@ have the original unpadded size. These tests verify that weight loading
|
||||
correctly handles this mismatch.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -564,9 +566,11 @@ class TestLoadWeightsExpertMapping:
|
||||
transposed=False,
|
||||
quant_method="tensor",
|
||||
layer_name="model.layers.0.mlp.experts",
|
||||
is_gated=True,
|
||||
):
|
||||
experts = torch.nn.Module()
|
||||
experts.layer_name = layer_name
|
||||
experts.moe_config = SimpleNamespace(is_act_and_mul=is_gated)
|
||||
experts.get_expert_mapping = lambda **_: mapping
|
||||
experts.is_fused_checkpoint_transposed = transposed
|
||||
experts._orient_fused_weight = RoutedExperts._orient_fused_weight
|
||||
@@ -593,7 +597,7 @@ class TestLoadWeightsExpertMapping:
|
||||
(("gate_proj", "down_proj", "up_proj"), ""),
|
||||
(("w1", "w2", "w3"), ""),
|
||||
(("up_proj", "down_proj", "up_proj"), ""),
|
||||
(("up_proj", "down_proj", ""), ""),
|
||||
(("up_proj", "down_proj", None), ""),
|
||||
(("gate_proj", "down_proj", "up_proj"), "base_layer."),
|
||||
],
|
||||
)
|
||||
@@ -609,7 +613,10 @@ class TestLoadWeightsExpertMapping:
|
||||
self, projs, lora_prefix, suffix, shape, dtype
|
||||
):
|
||||
mapping = self._mapping(
|
||||
projs, num_redundant_experts=4, lora_base_layer_prefix=lora_prefix
|
||||
projs,
|
||||
num_redundant_experts=4,
|
||||
lora_base_layer_prefix=lora_prefix,
|
||||
is_gated=projs[2] is not None,
|
||||
)
|
||||
for logical_id, physical_ids in [(1, (1, 17)), (10, (10,))]:
|
||||
for proj in dict.fromkeys(projs):
|
||||
@@ -651,6 +658,47 @@ class TestLoadWeightsExpertMapping:
|
||||
for call, expected in zip(calls, weight.chunk(2) * 2):
|
||||
torch.testing.assert_close(call[3], expected, rtol=0, atol=0)
|
||||
|
||||
def test_non_gated_mapping_uses_stacked_up_proj_without_warning(self, caplog_vllm):
|
||||
mapping = self._mapping(("up_proj", "down_proj", None), is_gated=False)
|
||||
assert "Unexpected gate/up projection names" not in caplog_vllm.text
|
||||
assert mapping[:2] == [
|
||||
("experts.w13_weight", "experts.up_proj", 0, "w1"),
|
||||
("experts.w2_weight", "experts.down_proj", 0, "w2"),
|
||||
]
|
||||
assert mapping[2:] == [
|
||||
(param, f"experts.{expert}.{proj}.", expert, shard)
|
||||
for expert in range(16)
|
||||
for param, proj, shard in (
|
||||
("experts.w13_", "up_proj", "w1"),
|
||||
("experts.w2_", "down_proj", "w2"),
|
||||
)
|
||||
]
|
||||
|
||||
def test_gated_mapping_with_unknown_names_warns_and_skips_fused(self, caplog_vllm):
|
||||
mapping = self._mapping(("fc1", "fc2", "fc3"))
|
||||
assert "Unexpected gate/up projection names: fc1, fc3" in caplog_vllm.text
|
||||
assert [name for _, name, _, _ in mapping[:3]] == [
|
||||
"experts.0.fc1.",
|
||||
"experts.0.fc2.",
|
||||
"experts.0.fc3.",
|
||||
]
|
||||
|
||||
def test_stacked_non_gated_tensors_load_unsplit(self):
|
||||
mapping = self._mapping(("up_proj", "down_proj", None), is_gated=False)
|
||||
up_proj = torch.arange(16 * 4 * 3).reshape(16, 4, 3)
|
||||
down_proj = torch.arange(16 * 3 * 4).reshape(16, 3, 4)
|
||||
calls = self._load(
|
||||
mapping,
|
||||
[("up_proj", up_proj), ("down_proj", down_proj)],
|
||||
is_gated=False,
|
||||
)
|
||||
assert [call[:3] for call in calls] == [
|
||||
*(("w13_weight", "w1", expert) for expert in range(16)),
|
||||
*(("w2_weight", "w2", expert) for expert in range(16)),
|
||||
]
|
||||
for call, expected in zip(calls, [*up_proj, *down_proj]):
|
||||
torch.testing.assert_close(call[3], expected, rtol=0, atol=0)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"suffix,quant_method,transposed",
|
||||
[
|
||||
|
||||
@@ -7,7 +7,11 @@ from dataclasses import dataclass, replace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.utils import check_accuracy, check_deferred_moe_finalize
|
||||
from tests.kernels.moe.utils import (
|
||||
check_accuracy,
|
||||
check_deferred_moe_finalize,
|
||||
make_dummy_moe_config,
|
||||
)
|
||||
from vllm._aiter_ops import (
|
||||
is_aiter_found,
|
||||
is_aiter_found_and_supported,
|
||||
@@ -2009,6 +2013,52 @@ def test_select_mxfp4_moe_backend_raises_with_unsupported_reasons(
|
||||
mxfp4_oracle.select_mxfp4_moe_backend(moe_config)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"moe_backend,w4a4,supported,expected",
|
||||
[
|
||||
# The checkpoint's activation wins over BF16 backends listed first.
|
||||
("auto", True, {"CUTLASS_MXFP4_MXFP4", "MARLIN"}, "CUTLASS_MXFP4_MXFP4"),
|
||||
# No W4A4 kernel: run at BF16 rather than another quantized activation.
|
||||
("auto", True, {"FLASHINFER_TRTLLM_MXFP4_MXFP8", "MARLIN"}, "MARLIN"),
|
||||
("marlin", True, {"MARLIN"}, "MARLIN"),
|
||||
# A BF16 checkpoint never gets its activations quantized.
|
||||
("auto", False, {"CUTLASS_MXFP4_MXFP4", "MARLIN"}, "MARLIN"),
|
||||
],
|
||||
)
|
||||
def test_select_mxfp4_moe_backend_respects_checkpoint_activation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
moe_backend: str,
|
||||
w4a4: bool,
|
||||
supported: set[str],
|
||||
expected: str,
|
||||
):
|
||||
"""Activations run at the checkpoint's precision when a backend supports
|
||||
it and fall back to BF16 otherwise, never to a lower precision."""
|
||||
import vllm.model_executor.layers.fused_moe.oracle.mxfp4 as mxfp4_oracle
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Dynamic,
|
||||
)
|
||||
|
||||
def backend_to_kernel_cls(backend):
|
||||
class Experts:
|
||||
@staticmethod
|
||||
def is_supported_config(*args):
|
||||
return backend.name in supported, None
|
||||
|
||||
return [Experts]
|
||||
|
||||
monkeypatch.setattr(mxfp4_oracle, "backend_to_kernel_cls", backend_to_kernel_cls)
|
||||
monkeypatch.setattr(mxfp4_oracle, "_user_moe_activation_override", lambda: None)
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.moe_backend = moe_backend
|
||||
|
||||
backend, _ = mxfp4_oracle.select_mxfp4_moe_backend(
|
||||
moe_config, activation_key=kMxfp4Dynamic if w4a4 else None
|
||||
)
|
||||
|
||||
assert backend.name == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend_name", ["AITER_TRITON_MXFP4_BF16", "AITER_MXFP4_FP8"])
|
||||
@pytest.mark.parametrize("use_ep", [False, True])
|
||||
def test_aiter_mxfp4_monolithic_rejects_expert_parallel(
|
||||
|
||||
@@ -7,6 +7,7 @@ import torch
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
_per_token_group_quant_fp8_colmajor,
|
||||
silu_mul_per_token_group_quant_fp8_colmajor,
|
||||
silu_mul_quant_fp8_packed_triton,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import triton
|
||||
@@ -108,6 +109,94 @@ def test_silu_mul_fp8_quant_deep_gemm(T: int, N: int):
|
||||
torch.testing.assert_close(output_scales, ref_output_scales)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"counts", [[0, 1, 127, 128, 129, 0, 257, 13], [128] * 8, [0] * 8]
|
||||
)
|
||||
@pytest.mark.parametrize("group_size", [32, 128])
|
||||
@pytest.mark.parametrize("scale_format", ["packed", "float32", "ceil_ue8m0"])
|
||||
@pytest.mark.parametrize("hidden_size", [256, 2048])
|
||||
@pytest.mark.parametrize("clamp_limit", [None, 10.0])
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA graph test")
|
||||
def test_silu_quant_skips_expert_padding_on_graph_replay(
|
||||
counts: list[int],
|
||||
group_size: int,
|
||||
scale_format: str,
|
||||
hidden_size: int,
|
||||
clamp_limit: float | None,
|
||||
):
|
||||
"""Live rows match dense quantization; gaps stay untouched as counts change."""
|
||||
alignment = 128
|
||||
capacity = sum(triton.cdiv(n, alignment) * alignment for n in counts)
|
||||
# Exercise unused capacity as well as the gaps between experts.
|
||||
capacity = max(capacity, alignment)
|
||||
if counts != [128] * 8:
|
||||
capacity += alignment
|
||||
x = torch.randn((capacity, 2 * hidden_size), device=DEVICE, dtype=torch.bfloat16)
|
||||
x *= 16
|
||||
clean_x = x.clone()
|
||||
ends = torch.empty(len(counts), device=DEVICE, dtype=torch.int32)
|
||||
output = torch.empty((capacity, hidden_size), device=DEVICE, dtype=FLOAT8_DTYPE)
|
||||
|
||||
def set_counts(values):
|
||||
positions = []
|
||||
mask = torch.zeros(capacity, device=DEVICE, dtype=torch.bool)
|
||||
start = 0
|
||||
for count in values:
|
||||
mask[start : start + count] = True
|
||||
positions.append(start + count)
|
||||
start += triton.cdiv(count, alignment) * alignment
|
||||
ends.copy_(torch.tensor(positions, device=DEVICE, dtype=torch.int32))
|
||||
return mask
|
||||
|
||||
def quantize(x, output=None, **kwargs):
|
||||
if scale_format == "packed":
|
||||
return silu_mul_quant_fp8_packed_triton(
|
||||
x,
|
||||
group_size=group_size,
|
||||
output_q=output,
|
||||
clamp_limit=clamp_limit,
|
||||
**kwargs,
|
||||
)
|
||||
return silu_mul_per_token_group_quant_fp8_colmajor(
|
||||
x,
|
||||
group_size=group_size,
|
||||
output=output,
|
||||
clamp_limit=clamp_limit,
|
||||
use_ue8m0=scale_format == "ceil_ue8m0",
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def invoke():
|
||||
return quantize(
|
||||
x,
|
||||
output=output,
|
||||
expert_ends=ends,
|
||||
expert_alignment=alignment,
|
||||
)
|
||||
|
||||
set_counts(counts)
|
||||
invoke()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
actual_q, actual_s = invoke()
|
||||
|
||||
for values in (counts, counts[::-1]):
|
||||
live = set_counts(values)
|
||||
x.copy_(clean_x)
|
||||
x[~live] = float("nan")
|
||||
output.view(torch.uint8).fill_(91)
|
||||
graph.replay()
|
||||
reference_q, reference_s = quantize(x)
|
||||
torch.testing.assert_close(
|
||||
actual_q.view(torch.uint8)[live],
|
||||
reference_q.view(torch.uint8)[live],
|
||||
rtol=0,
|
||||
atol=0,
|
||||
)
|
||||
torch.testing.assert_close(actual_s[live], reference_s[live], rtol=0, atol=0)
|
||||
assert torch.all(actual_q.view(torch.uint8)[~live] == 91)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("T", [128, 256, 512])
|
||||
@pytest.mark.parametrize("N", [128 * 2, 256 * 2, 768 * 2, 2048 * 2, 7168 * 2])
|
||||
@pytest.mark.parametrize("clamp_limit", [7.0, 10.0])
|
||||
|
||||
@@ -16,8 +16,7 @@ import torch
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import check_deferred_moe_finalize, make_test_quant_config
|
||||
from tests.kernels.quantization.nvfp4_utils import (
|
||||
FLOAT4_E2M1_MAX,
|
||||
FLOAT8_E4M3_MAX,
|
||||
convert_swizzled_to_linear,
|
||||
dequantize_nvfp4_to_dtype,
|
||||
)
|
||||
from tests.kernels.utils import torch_moe
|
||||
@@ -39,6 +38,13 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_nvfp4_moe import (
|
||||
TrtLlmNvFp4ExpertsModular,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_static_weights_for_trtllm_fp4_moe,
|
||||
reorder_w1w3_to_w3w1,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
@@ -96,10 +102,14 @@ SITU = op_registry[_SITU_OP_NAME]
|
||||
|
||||
|
||||
def _make_trtllm_fp4_moe_kernel(
|
||||
moe_config: FusedMoEConfig, quant_config: FusedMoEQuantConfig
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
per_token_activation: bool = False,
|
||||
) -> mk.FusedMoEKernel:
|
||||
experts = TrtLlmNvFp4ExpertsModular(
|
||||
moe_config=moe_config, quant_config=quant_config
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
per_token_activation=per_token_activation,
|
||||
)
|
||||
# Mimic the production weight-loader path so per-expert tensors that
|
||||
# are normally precomputed in process_weights_after_loading (g1_scale_c
|
||||
@@ -126,16 +136,37 @@ def _make_trtllm_fp4_moe_kernel(
|
||||
|
||||
|
||||
ACTIVATION_CASES = [
|
||||
pytest.param(MoEActivation.SILU, MoEActivation.SILU, None, id="silu"),
|
||||
pytest.param(MoEActivation.SILU, SILU_WITH_CLAMP, _SWIGLU_LIMIT, id="silu_clamp"),
|
||||
pytest.param(MoEActivation.SITU, SITU, None, id="situ"),
|
||||
pytest.param(MoEActivation.SILU, MoEActivation.SILU, None, False, id="silu"),
|
||||
pytest.param(
|
||||
MoEActivation.SILU,
|
||||
MoEActivation.SILU,
|
||||
None,
|
||||
True,
|
||||
id="silu_per_token",
|
||||
),
|
||||
pytest.param(
|
||||
MoEActivation.SILU,
|
||||
SILU_WITH_CLAMP,
|
||||
_SWIGLU_LIMIT,
|
||||
False,
|
||||
id="silu_clamp",
|
||||
),
|
||||
pytest.param(MoEActivation.SITU, SITU, None, False, id="situ"),
|
||||
pytest.param(
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
None,
|
||||
False,
|
||||
id="relu2_no_mul",
|
||||
),
|
||||
pytest.param(MoEActivation.GELU, MoEActivation.GELU, None, id="gelu"),
|
||||
pytest.param(
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
MoEActivation.RELU2_NO_MUL,
|
||||
None,
|
||||
True,
|
||||
id="relu2_no_mul_per_token",
|
||||
),
|
||||
pytest.param(MoEActivation.GELU, MoEActivation.GELU, None, False, id="gelu"),
|
||||
]
|
||||
|
||||
|
||||
@@ -143,7 +174,10 @@ ACTIVATION_CASES = [
|
||||
@pytest.mark.parametrize("e", [128])
|
||||
@pytest.mark.parametrize("topk", [8])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("activation,torch_activation,swiglu_limit", ACTIVATION_CASES)
|
||||
@pytest.mark.parametrize(
|
||||
"activation,torch_activation,swiglu_limit,per_token_activation",
|
||||
ACTIVATION_CASES,
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_trtllm_fp4_moe_no_graph(
|
||||
m: int,
|
||||
@@ -155,6 +189,7 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
activation: MoEActivation,
|
||||
torch_activation: MoEActivation | type[SiluAndMulWithClamp] | type[SituAndMul],
|
||||
swiglu_limit: float | None,
|
||||
per_token_activation: bool,
|
||||
workspace_init,
|
||||
):
|
||||
# FlashInfer's trtllm_batched_gemm_runner has no precompiled tile
|
||||
@@ -202,6 +237,42 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
# clamp/output-scale coupling in the FlashInfer kernel wrapper.
|
||||
quant_config.g1_alphas.fill_(_LARGE_OUTPUT1_SCALE)
|
||||
|
||||
w1_q_linear = w1_q
|
||||
w2_q_linear = w2_q
|
||||
w1_scale_reference = quant_config.w1_scale
|
||||
w2_scale_reference = quant_config.w2_scale
|
||||
assert w1_scale_reference is not None
|
||||
assert w2_scale_reference is not None
|
||||
w1_scale_linear = torch.stack(
|
||||
[
|
||||
convert_swizzled_to_linear(scale, w1_q.size(1), k, quant_blocksize)
|
||||
for scale in w1_scale_reference
|
||||
]
|
||||
)
|
||||
w2_scale_linear = torch.stack(
|
||||
[
|
||||
convert_swizzled_to_linear(scale, w2_q.size(1), n, quant_blocksize)
|
||||
for scale in w2_scale_reference
|
||||
]
|
||||
)
|
||||
w1_q_for_kernel = w1_q
|
||||
if is_gated_act:
|
||||
w1_q_for_kernel, w1_scale_linear = reorder_w1w3_to_w3w1(
|
||||
w1_q.clone(), w1_scale_linear
|
||||
)
|
||||
w1_q, w1_scale, w2_q, w2_scale = prepare_static_weights_for_trtllm_fp4_moe(
|
||||
w1_q_for_kernel,
|
||||
w2_q,
|
||||
w1_scale_linear,
|
||||
w2_scale_linear,
|
||||
hidden_size=k,
|
||||
intermediate_size=n,
|
||||
num_experts=e,
|
||||
is_gated_activation=is_gated_act,
|
||||
)
|
||||
quant_config._w1.scale = w1_scale
|
||||
quant_config._w2.scale = w2_scale
|
||||
|
||||
score = torch.randn((m, e), device="cuda", dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(a, score, topk, renormalize=False)
|
||||
|
||||
@@ -224,7 +295,11 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
),
|
||||
)
|
||||
|
||||
trtllm_experts = _make_trtllm_fp4_moe_kernel(moe_config, quant_config)
|
||||
trtllm_experts = _make_trtllm_fp4_moe_kernel(
|
||||
moe_config,
|
||||
quant_config,
|
||||
per_token_activation=per_token_activation,
|
||||
)
|
||||
if activation == MoEActivation.SITU:
|
||||
torch.testing.assert_close(
|
||||
trtllm_experts.fused_experts.g1_scale_c, quant_config.a2_gscale
|
||||
@@ -245,10 +320,20 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
# Reference: round-trip activations and weights through FP4
|
||||
# quant/dequant so the comparison isolates kernel/activation behavior
|
||||
# from quantization error.
|
||||
a_global_scale = ((FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / a.abs().max()).to(
|
||||
torch.float32
|
||||
)
|
||||
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(a, a_global_scale)
|
||||
if per_token_activation:
|
||||
from flashinfer import SfLayout, nvfp4_quantize
|
||||
|
||||
a_fp4, a_scale_interleaved, per_token_scale = nvfp4_quantize(
|
||||
a,
|
||||
NVFP4_PER_TOKEN_BASE_GLOBAL_SCALE,
|
||||
sfLayout=SfLayout.layout_linear,
|
||||
per_token_activation=True,
|
||||
)
|
||||
a_global_scale = 1.0 / per_token_scale.unsqueeze(1)
|
||||
else:
|
||||
assert quant_config.a1_gscale is not None
|
||||
a_global_scale = quant_config.a1_gscale[0]
|
||||
a_fp4, a_scale_interleaved = ops.scaled_fp4_quant(a, a_global_scale)
|
||||
a_in_dtype = dequantize_nvfp4_to_dtype(
|
||||
a_fp4,
|
||||
a_scale_interleaved,
|
||||
@@ -256,6 +341,7 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
dtype=a.dtype,
|
||||
device=a.device,
|
||||
block_size=quant_blocksize,
|
||||
is_sf_linear_layout=per_token_activation,
|
||||
)
|
||||
|
||||
w1_d = torch.empty(
|
||||
@@ -264,16 +350,16 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
|
||||
for idx in range(e):
|
||||
w1_d[idx] = dequantize_nvfp4_to_dtype(
|
||||
w1_q[idx],
|
||||
quant_config.w1_scale[idx],
|
||||
w1_q_linear[idx],
|
||||
w1_scale_reference[idx],
|
||||
(1 / quant_config.g1_alphas[idx]),
|
||||
dtype=dtype,
|
||||
device=w1_q.device,
|
||||
block_size=quant_blocksize,
|
||||
)
|
||||
w2_d[idx] = dequantize_nvfp4_to_dtype(
|
||||
w2_q[idx],
|
||||
quant_config.w2_scale[idx],
|
||||
w2_q_linear[idx],
|
||||
w2_scale_reference[idx],
|
||||
(1 / quant_config.g2_alphas[idx]),
|
||||
dtype=dtype,
|
||||
device=w2_q.device,
|
||||
@@ -284,6 +370,13 @@ def test_trtllm_fp4_moe_no_graph(
|
||||
a_in_dtype, w1_d, w2_d, score, topk, activation=torch_activation
|
||||
)
|
||||
|
||||
if swiglu_limit is None:
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
torch_output.float().flatten(),
|
||||
trtllm_output.float().flatten(),
|
||||
dim=0,
|
||||
)
|
||||
assert cosine > 0.99
|
||||
torch.testing.assert_close(torch_output, trtllm_output, atol=2e-1, rtol=2e-1)
|
||||
|
||||
|
||||
@@ -416,5 +509,6 @@ if __name__ == "__main__":
|
||||
MoEActivation.GELU,
|
||||
MoEActivation.GELU,
|
||||
None,
|
||||
False,
|
||||
None,
|
||||
)
|
||||
|
||||
@@ -446,8 +446,15 @@ def test_select_cuda_deepep_ht_falls_back_from_trtllm(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
|
||||
return_value=True,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"routing_method",
|
||||
[RoutingMethodType.Renormalize, RoutingMethodType.MiniMax2],
|
||||
)
|
||||
@pytest.mark.parametrize("moe_backend", ["auto", "flashinfer_trtllm"])
|
||||
def test_select_cuda_flashinfer_trtllm_ag_rs_uses_monolithic(
|
||||
mock_supports_current_device,
|
||||
routing_method,
|
||||
moe_backend,
|
||||
):
|
||||
"""Test AG/RS stays on BF16 TRTLLM monolithic when TRTLLM is supported."""
|
||||
with (
|
||||
@@ -462,8 +469,8 @@ def test_select_cuda_flashinfer_trtllm_ag_rs_uses_monolithic(
|
||||
),
|
||||
):
|
||||
moe_config = make_dummy_moe_config(num_experts=4, num_local_experts=2)
|
||||
moe_config.moe_backend = "flashinfer_trtllm"
|
||||
moe_config.routing_method = RoutingMethodType.Renormalize
|
||||
moe_config.moe_backend = moe_backend
|
||||
moe_config.routing_method = routing_method
|
||||
moe_config.moe_parallel_config.use_ep = True
|
||||
moe_config.moe_parallel_config.dp_size = 2
|
||||
moe_config.moe_parallel_config.ep_size = 2
|
||||
|
||||
@@ -29,8 +29,11 @@ if not current_platform.is_cuda():
|
||||
# `is_quant_method_supported` conflates kernels with quantization methods
|
||||
# an assumption which is breaking down as quantizations methods can have
|
||||
# have kernels and some kernels support multiple quantization methods.
|
||||
capability = current_platform.get_device_capability()
|
||||
IS_SUPPORTED_BY_GPU = capability is not None and capability[0] >= 9
|
||||
# CUTLASS W4A8 kernels are only built for SM90 (9.0a); the kernel selector
|
||||
# requires exactly SM90 too, so match it here rather than accepting >= 90.
|
||||
IS_SUPPORTED_BY_GPU = (
|
||||
current_platform.is_cuda() and current_platform.is_device_capability(90)
|
||||
)
|
||||
|
||||
MNK_SHAPES = [
|
||||
(1, 128, 128),
|
||||
@@ -106,10 +109,6 @@ TEST_TYPES = [
|
||||
|
||||
# TODO: in future PR refactor this and `is_quant_method_supported` in the kernel
|
||||
# unit tests to a common utility function. Currently the use of
|
||||
# `is_quant_method_supported` conflates kernels with quantization methods
|
||||
# an assumption which is breaking down as quantizations methods can have
|
||||
# have kernels and some kernels support multiple quantization methods.
|
||||
IS_SUPPORTED_BY_GPU = current_platform.has_device_capability(90)
|
||||
|
||||
|
||||
# For testing quantized linear kernels
|
||||
|
||||
@@ -17,9 +17,8 @@ from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
IS_SUPPORTED_BY_GPU = (
|
||||
current_platform.is_cuda() and capability is not None and capability[0] >= 9
|
||||
current_platform.is_cuda() and current_platform.is_device_capability(90)
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -13,11 +13,15 @@ from nvfp4_utils import (
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.kernels.linear.nvfp4 import NvFp4LinearLayerConfig
|
||||
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferCuteDslNvFp4LinearKernel,
|
||||
FlashInferCuteDslNvFp4W4A16LinearKernel,
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.quant_activation import QuantizedActivation
|
||||
from vllm.model_executor.layers.quantization.utils.nvfp4_emulation_utils import (
|
||||
dequantize_to_dtype,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import (
|
||||
flashinfer_scaled_fp4_mm,
|
||||
@@ -47,6 +51,54 @@ SEEDS = [42]
|
||||
CUDA_DEVICES = ["cuda:0"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("token_shape", [(17,), (1, 17), (2, 17)])
|
||||
@pytest.mark.parametrize(
|
||||
"kernel_cls",
|
||||
[FlashInferCutlassNvFp4LinearKernel, FlashInferCuteDslNvFp4LinearKernel],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_nvfp4_preserves_quantized_input_shape(
|
||||
token_shape: tuple[int, ...],
|
||||
kernel_cls: type[FlashInferCutlassNvFp4LinearKernel]
|
||||
| type[FlashInferCuteDslNvFp4LinearKernel],
|
||||
) -> None:
|
||||
"""Pre-quantized inputs must match the tensor path and retain token dimensions."""
|
||||
supported, reason = kernel_cls.is_supported()
|
||||
if not supported:
|
||||
pytest.skip(reason)
|
||||
|
||||
set_random_seed(42)
|
||||
hidden_size, output_size = 128, 128
|
||||
x = torch.randn(*token_shape, hidden_size, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(output_size, hidden_size, dtype=x.dtype, device=x.device)
|
||||
input_scale = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / x.abs().max()).float()
|
||||
weight_scale = (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX / weight.abs().max()).float()
|
||||
x_fp4, x_blockscale = ops.scaled_fp4_quant(x.reshape(-1, hidden_size), input_scale)
|
||||
weight_fp4, weight_blockscale = ops.scaled_fp4_quant(weight, weight_scale)
|
||||
quantized_input = QuantizedActivation(
|
||||
data=x_fp4.reshape(*token_shape, hidden_size // 2),
|
||||
scale=x_blockscale,
|
||||
orig_dtype=x.dtype,
|
||||
orig_shape=x.shape,
|
||||
quant_key=kNvfp4Dynamic,
|
||||
)
|
||||
layer = torch.nn.Module()
|
||||
layer.input_size_per_partition = hidden_size
|
||||
layer.output_size_per_partition = output_size
|
||||
layer.weight = weight_fp4
|
||||
layer.weight_scale = weight_blockscale
|
||||
layer.input_global_scale_inv = input_scale
|
||||
layer.alpha = (input_scale * weight_scale).reciprocal()
|
||||
kernel = kernel_cls(NvFp4LinearLayerConfig())
|
||||
|
||||
expected = kernel.apply_weights(layer, x.reshape(-1, hidden_size))
|
||||
actual = kernel.apply_weights(layer, quantized_input)
|
||||
|
||||
assert actual.shape == (*token_shape, output_size)
|
||||
assert actual.dtype == x.dtype
|
||||
torch.testing.assert_close(actual.reshape(expected.shape), expected)
|
||||
|
||||
|
||||
def get_ref_results(
|
||||
a_fp4,
|
||||
b_fp4,
|
||||
|
||||
@@ -18,6 +18,7 @@ Run `pytest tests/kernels/quantization/test_rdna3_moe_w4a16.py`.
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -32,6 +33,9 @@ from vllm.model_executor.layers.fused_moe.activation import ( # noqa: E402
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import ( # noqa: E402
|
||||
moe_align_block_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import ( # noqa: E402
|
||||
_process_weights_rdna3,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import ( # noqa: E402
|
||||
pack_quantized_values_into_int32,
|
||||
)
|
||||
@@ -95,6 +99,16 @@ def _make_qzeros(E, groups, N):
|
||||
return qz.unsqueeze(0).expand(E, -1, -1).contiguous()
|
||||
|
||||
|
||||
def _make_ct_weights(E, N, K, group_size, dtype):
|
||||
"""Compressed-tensors canonical N-first layout: packed ``[E, N, K // 8]``
|
||||
and scales ``[E, N, K // group_size]``, plus the dequantized weights."""
|
||||
q = torch.randint(0, 16, (E, N, K), dtype=torch.int32, device=device)
|
||||
s = torch.rand(E, N, K // group_size, dtype=dtype, device=device) * 0.1 + 0.01
|
||||
packed = pack_quantized_values_into_int32(q, scalar_types.uint4b8, packed_dim=2)
|
||||
w = (q - 8).float() * s.float().repeat_interleave(group_size, dim=2)
|
||||
return packed.contiguous(), s, w
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("E, K, N_inter, top_k, group_size", MODEL_CONFIGS)
|
||||
@pytest.mark.parametrize("M", NUM_TOKENS)
|
||||
@@ -359,3 +373,70 @@ def test_expert_id_minus_one():
|
||||
|
||||
# Output should remain zero (expert skipped)
|
||||
assert torch.equal(out, torch.zeros_like(out))
|
||||
|
||||
|
||||
@gfx1100_only
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_process_weights_from_compressed_tensors_layout(dtype):
|
||||
"""Weights in the layout compressed-tensors hands to the oracle go through
|
||||
the RDNA3 post-processing and a full MoE forward, against a dequantized
|
||||
reference. The tests above build the kernel layout directly, so they do
|
||||
not notice when the layout upstream of the kernel changes (#52798)."""
|
||||
E, K, N_inter, top_k, group_size, M = 8, 512, 256, 2, 32, 16
|
||||
torch.manual_seed(5)
|
||||
w13, w13_s, w13_ref = _make_ct_weights(E, 2 * N_inter, K, group_size, dtype)
|
||||
w2, w2_s, w2_ref = _make_ct_weights(E, K, N_inter, group_size, dtype)
|
||||
w13, w2, w13_s, w2_s, w13_z, w2_z, *_ = _process_weights_rdna3(
|
||||
w13, w2, w13_s, w2_s, group_size
|
||||
)
|
||||
assert w13_z is not None and w2_z is not None
|
||||
|
||||
x = torch.randn(M, K, dtype=dtype, device=device)
|
||||
topk_ids = torch.randint(0, E, (M, top_k), device=device, dtype=torch.int32)
|
||||
topk_w = torch.softmax(torch.randn(M, top_k, device=device), dim=-1).float()
|
||||
si, ei, ntp = moe_align_block_size(topk_ids, 1, E)
|
||||
|
||||
w1_out = torch.zeros(M * top_k, 2 * N_inter, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
x,
|
||||
w1_out,
|
||||
w13,
|
||||
w13_s,
|
||||
w13_z,
|
||||
torch.empty(0, device=device),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
top_k,
|
||||
1,
|
||||
False,
|
||||
0,
|
||||
)
|
||||
act_out = torch.empty(M * top_k, N_inter, dtype=dtype, device=device)
|
||||
apply_moe_activation(MoEActivation.SILU, act_out, w1_out)
|
||||
out = torch.zeros(M, K, dtype=dtype, device=device)
|
||||
ops.moe_gptq_gemm_rdna3(
|
||||
act_out,
|
||||
out,
|
||||
w2,
|
||||
w2_s,
|
||||
w2_z,
|
||||
topk_w.view(-1),
|
||||
si,
|
||||
ei,
|
||||
ntp,
|
||||
1,
|
||||
1,
|
||||
True,
|
||||
top_k,
|
||||
)
|
||||
|
||||
ref = torch.zeros(M, K, device=device)
|
||||
for m in range(M):
|
||||
for k in range(top_k):
|
||||
e = int(topk_ids[m, k])
|
||||
h = w13_ref[e] @ x[m].float()
|
||||
a = F.silu(h[:N_inter]) * h[N_inter:]
|
||||
ref[m] += (w2_ref[e] @ a) * topk_w[m, k]
|
||||
rel_l2 = (torch.norm(out.float() - ref) / torch.norm(ref)).item()
|
||||
assert rel_l2 < (0.02 if dtype == torch.float16 else 0.05), rel_l2
|
||||
|
||||
@@ -30,6 +30,8 @@ RDNAHybridW4A16LinearKernel = hybrid_module.RDNAHybridW4A16LinearKernel
|
||||
pack_int4_exllama_shuffle = hybrid_module.pack_int4_exllama_shuffle
|
||||
SUPPORTED_GROUP_SIZES = hybrid_module.SUPPORTED_GROUP_SIZES
|
||||
MAX_SKINNY_BATCH_SIZE = hybrid_module.MAX_SKINNY_BATCH_SIZE
|
||||
LDS_CAPACITY_ELEMENTS = hybrid_module.LDS_CAPACITY_ELEMENTS
|
||||
MEDIUM_SKINNY_LIMIT_ELEMENTS = hybrid_module.MEDIUM_SKINNY_LIMIT_ELEMENTS
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -100,8 +102,8 @@ def test_rdna_hybrid_w4a16_apply_matches_reference(dtype, group_size, has_zp, M)
|
||||
"""Smoke test the registered custom op for both decode and prefill batches.
|
||||
|
||||
Verifies the dispatch logic in `_rdna_hybrid_w4a16_apply_impl`:
|
||||
- M <= MAX_SKINNY_BATCH_SIZE: HIP wvSplitK_int4_g
|
||||
- M > MAX_SKINNY_BATCH_SIZE: Triton prefill kernel
|
||||
- supported M and K*M: HIP wvSplitK_int4_g
|
||||
- otherwise: Triton prefill kernel
|
||||
"""
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
@@ -563,3 +565,257 @@ def test_wvsplitk_int4_g_rejects_unpacked_zero_points():
|
||||
|
||||
with pytest.raises(RuntimeError, match="Zero points must be int32 or uint32"):
|
||||
ops.wvSplitK_int4_g(w, a, scales, num_compute_units(), G, zp_unpacked, None)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.skipif(
|
||||
not hasattr(torch.ops, "_rocm_C")
|
||||
or not hasattr(torch.ops._rocm_C, "wvSplitK_int4_g"),
|
||||
reason="wvSplitK_int4_g not built",
|
||||
)
|
||||
def test_wvsplitk_int4_g_rejects_undersized_weight_view():
|
||||
"""A sufficient row stride does not prove the row holds K/2 packed bytes.
|
||||
|
||||
Row padding makes stride(0) > K/2 legitimate, but the logical size still
|
||||
has to be checked: this view has a valid stride and half the required
|
||||
bytes, and without the size check the kernel reads past it.
|
||||
"""
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
K, N, G, M = 512, 256, 128, 1
|
||||
a = torch.randn((M, K), device=device, dtype=torch.float16)
|
||||
# Full-size backing allocation, so a missing check misreads rather than
|
||||
# running off the end of the buffer.
|
||||
backing = torch.randint(0, 255, (N, K // 2), device=device, dtype=torch.uint8).view(
|
||||
torch.int8
|
||||
)
|
||||
undersized = backing[:, : K // 4]
|
||||
assert undersized.stride(0) == K // 2
|
||||
scales = torch.rand((N, K // G), device=device, dtype=torch.float16)
|
||||
|
||||
with pytest.raises(RuntimeError, match=r"must contain M\*K/2 bytes"):
|
||||
ops.wvSplitK_int4_g(undersized, a, scales, num_compute_units(), G, None, None)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# gfx11 weight row-stride padding
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_weight_pad_bytes = hybrid_module._weight_pad_bytes
|
||||
_act_pad_bytes = hybrid_module._act_pad_bytes
|
||||
_WEIGHT_CLIFF_BYTES = hybrid_module._WEIGHT_CLIFF_BYTES
|
||||
_ACT_CLIFF_BYTES = hybrid_module._ACT_CLIFF_BYTES
|
||||
_STRIDE_PAD_BYTES = hybrid_module._STRIDE_PAD_BYTES
|
||||
|
||||
|
||||
def test_weight_pad_bytes_only_moves_strides_on_the_cliff():
|
||||
"""Pad 1024 B multiples, leave everything else dense."""
|
||||
# On the cliff (K % 2048 == 0): 1024 B multiples.
|
||||
for row_bytes in (1024, 2048, 4096, 5120, 6144, 7168, 8192):
|
||||
assert _weight_pad_bytes(row_bytes) == _STRIDE_PAD_BYTES
|
||||
|
||||
# Off it, including strides that are 512 B multiples but not 1024 B ones:
|
||||
# padding those measured as a real loss on gfx1151 (-14% at M=1 on a
|
||||
# 4864 B row).
|
||||
for row_bytes in (1280, 1536, 2560, 4864, 9472, 12800):
|
||||
assert _weight_pad_bytes(row_bytes) == 0
|
||||
|
||||
# The padded stride is never back on the cliff.
|
||||
for row_bytes in range(16, 16384, 16):
|
||||
padded = row_bytes + _weight_pad_bytes(row_bytes)
|
||||
assert padded % _WEIGHT_CLIFF_BYTES != 0 or row_bytes % _WEIGHT_CLIFF_BYTES
|
||||
|
||||
|
||||
def test_act_pad_bytes_only_moves_strides_on_the_cliff():
|
||||
"""Activations sit on a wider cliff than the packed weight: 2048 B."""
|
||||
# On the cliff (K % 1024 == 0 for a 2-byte dtype): 2048 B multiples.
|
||||
for row_bytes in (2048, 4096, 8192, 16384, 20480, 24576, 43008):
|
||||
assert _act_pad_bytes(row_bytes) == _STRIDE_PAD_BYTES
|
||||
|
||||
# Off it, including 1024 B multiples that are not 2048 B ones -- the
|
||||
# weight cliff must not be applied to activations.
|
||||
for row_bytes in (1024, 3072, 5120, 10752, 19456, 37888):
|
||||
assert _act_pad_bytes(row_bytes) == 0
|
||||
|
||||
# The padded stride is never back on the cliff.
|
||||
for row_bytes in range(16, 65536, 16):
|
||||
padded = row_bytes + _act_pad_bytes(row_bytes)
|
||||
assert padded % _ACT_CLIFF_BYTES != 0 or row_bytes % _ACT_CLIFF_BYTES
|
||||
|
||||
|
||||
def _row_padded_copy(t: torch.Tensor, pad_cols: int) -> torch.Tensor:
|
||||
"""Copy of ``t`` whose stride(0) is ``t.shape[1] + pad_cols``."""
|
||||
rows, cols = t.shape
|
||||
buf = torch.empty((rows, cols + pad_cols), dtype=t.dtype, device=t.device)
|
||||
buf[:, :cols].copy_(t)
|
||||
return buf[:, :cols]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("has_zp", [False, True])
|
||||
@pytest.mark.parametrize("M", [1, MAX_SKINNY_BATCH_SIZE + 1])
|
||||
def test_weight_row_padding_does_not_change_results(dtype, has_zp, M):
|
||||
"""Padding the weight rows is a pure layout change.
|
||||
|
||||
Both the HIP skinny kernel (M small) and the Triton prefill kernel take the
|
||||
weight row stride from the tensor, so a padded layout must produce exactly
|
||||
the same numbers as the dense one.
|
||||
"""
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
set_random_seed(0)
|
||||
K, N, G = 512, 256, 128
|
||||
|
||||
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
|
||||
w_q = pack_int4_exllama_shuffle(w_int4_nk).view(torch.int8)
|
||||
scales = (0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)).to(
|
||||
dtype
|
||||
)
|
||||
zp = (
|
||||
_pack_zp_rows_for_kernel(
|
||||
torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
|
||||
)
|
||||
if has_zp
|
||||
else None
|
||||
)
|
||||
|
||||
# 32 int32 columns = 128 B of pad, the production pad size.
|
||||
w_q_pad = _row_padded_copy(w_q.view(torch.int32), 32).view(torch.int8)
|
||||
assert w_q_pad.stride(0) == w_q.stride(0) + _STRIDE_PAD_BYTES
|
||||
|
||||
cu_count = num_compute_units()
|
||||
dense = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
a, w_q, scales, zp, None, cu_count, G
|
||||
)
|
||||
padded = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
a, w_q_pad, scales, zp, None, cu_count, G
|
||||
)
|
||||
torch.testing.assert_close(padded, dense, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not hybrid_module._on_gfx1151(),
|
||||
reason="row-stride padding is only enabled on gfx1151",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"K,expected_weight_stride",
|
||||
[
|
||||
(512, 256), # 256 B row: off the cliff, stored dense
|
||||
(8192, 4096 + 128), # 4096 B row: on the cliff, padded
|
||||
],
|
||||
)
|
||||
def test_process_weights_pads_cliff_rows(K, expected_weight_stride, dist_init):
|
||||
"""The stored weight row stride follows the padding rule."""
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
set_random_seed(0)
|
||||
N, G = 128, 128
|
||||
|
||||
w_int4_kn = torch.randint(0, 16, (K, N), device=device, dtype=torch.int32)
|
||||
layer = _build_dummy_layer(
|
||||
_pack_int4_along_k_to_ckpt(w_int4_kn),
|
||||
0.05 * torch.rand((N, K // G), device=device, dtype=torch.float16),
|
||||
zeros_ckpt=None,
|
||||
)
|
||||
config = MPLinearLayerConfig(
|
||||
full_weight_shape=(K, N),
|
||||
partition_weight_shape=(K, N),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=G,
|
||||
zero_points=False,
|
||||
)
|
||||
RDNAHybridW4A16LinearKernel(
|
||||
config,
|
||||
w_q_param_name="weight_packed",
|
||||
w_s_param_name="weight_scale",
|
||||
w_zp_param_name=None,
|
||||
).process_weights_after_loading(layer)
|
||||
|
||||
# Weight rows: stride is in int8 elements, i.e. bytes.
|
||||
assert layer.weight_packed.stride(0) == expected_weight_stride
|
||||
# The int32 view the Triton path uses survives the padded stride.
|
||||
assert tuple(layer.weight_packed.view(torch.int32).shape) == (N, K // 8)
|
||||
# Metadata is untouched by this change.
|
||||
assert layer.weight_scale.is_contiguous()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,expected_path",
|
||||
[
|
||||
(4, 8192, "hip"),
|
||||
(4, 9728, "hip"),
|
||||
(4, 9856, "triton"),
|
||||
(MAX_SKINNY_BATCH_SIZE + 1, 1024, "triton"),
|
||||
],
|
||||
ids=["regular_limit", "medium_range", "above_medium", "batch_too_large"],
|
||||
)
|
||||
def test_rdna_hybrid_w4a16_dispatch_boundaries(M, K, expected_path, monkeypatch):
|
||||
"""Route only supported regular and medium skinny shapes to HIP."""
|
||||
import vllm._custom_ops as ops
|
||||
|
||||
N, G = 16, 128
|
||||
x_mk = torch.empty((M, K), dtype=torch.float16)
|
||||
w_q = torch.empty((N, K // 2), dtype=torch.int8)
|
||||
scales = torch.empty((N, K // G), dtype=torch.float16)
|
||||
called = []
|
||||
|
||||
def fake_hip(*args, **kwargs):
|
||||
called.append("hip")
|
||||
return torch.empty((M, N), dtype=x_mk.dtype)
|
||||
|
||||
def fake_triton(*args, **kwargs):
|
||||
called.append("triton")
|
||||
return torch.empty((M, N), dtype=x_mk.dtype)
|
||||
|
||||
monkeypatch.setattr(ops, "wvSplitK_int4_g", fake_hip)
|
||||
monkeypatch.setattr(hybrid_module, "triton_w4a16_skinny_fmt_gemm", fake_triton)
|
||||
|
||||
output = hybrid_module._rdna_hybrid_w4a16_apply_impl(
|
||||
x_mk, w_q, scales, None, None, 40, G
|
||||
)
|
||||
|
||||
assert output.shape == (M, N)
|
||||
assert called == [expected_path]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("has_zp", [False, True])
|
||||
def test_rdna_hybrid_w4a16_medium_skinny_matches_reference(dtype, has_zp):
|
||||
"""Validate the HIP medium path against an FP32 dequantization reference."""
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("CUDA/HIP device not available")
|
||||
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
set_random_seed(0)
|
||||
M, K, N, G = 4, 9728, 256, 128
|
||||
assert LDS_CAPACITY_ELEMENTS < K * M <= MEDIUM_SKINNY_LIMIT_ELEMENTS
|
||||
|
||||
x_mk = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
|
||||
w_q = pack_int4_exllama_shuffle(w_int4_nk).contiguous().view(torch.int8)
|
||||
scales = (0.05 * torch.rand((N, K // G), device=device, dtype=torch.float32)).to(
|
||||
dtype
|
||||
)
|
||||
zp = (
|
||||
torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
|
||||
if has_zp
|
||||
else None
|
||||
)
|
||||
w_zp = _pack_zp_rows_for_kernel(zp) if zp is not None else None
|
||||
|
||||
output = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
x_mk, w_q, scales, w_zp, None, num_compute_units(), G
|
||||
)
|
||||
reference = _rdna_hybrid_w4a16_reference(x_mk, w_int4_nk, scales, zp, G, bias=None)
|
||||
|
||||
torch.testing.assert_close(output, reference, rtol=2e-2, atol=2e-2)
|
||||
|
||||
@@ -14,6 +14,8 @@ import torch
|
||||
|
||||
from vllm.config import KernelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
_POSSIBLE_KERNELS,
|
||||
_POSSIBLE_NVFP4_KERNELS,
|
||||
AiterInt8ScaledMMLinearKernel,
|
||||
CPUInt8ScaledMMLinearKernel,
|
||||
HummingFP8ScaledMMLinearKernel,
|
||||
@@ -26,6 +28,7 @@ from vllm.model_executor.kernels.linear import (
|
||||
init_int8_linear_kernel,
|
||||
register_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.humming import prioritize_humming
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
@@ -197,3 +200,21 @@ def test_register_oot_linear_kernel(platform_mock):
|
||||
assert isinstance(kernel, OOTInt8ScaledMMLinearKernel), (
|
||||
"init_int8_linear_kernel should return an instance of the registered kernel"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"possible_kernels", [_POSSIBLE_KERNELS, _POSSIBLE_NVFP4_KERNELS]
|
||||
)
|
||||
def test_humming_priority_only_reorders_humming_and_marlin(possible_kernels):
|
||||
"""Preferring Humming must not demote Marlin below unrelated kernels."""
|
||||
kernels = possible_kernels[PlatformEnum.CUDA]
|
||||
original = list(kernels)
|
||||
|
||||
assert prioritize_humming(kernels, compute_capability=80) == original
|
||||
|
||||
names = [k.__name__ for k in prioritize_humming(kernels, compute_capability=90)]
|
||||
humming = next(i for i, name in enumerate(names) if "Humming" in name)
|
||||
assert "Marlin" in names[humming + 1]
|
||||
del names[humming]
|
||||
assert names == [k.__name__ for k in original if "Humming" not in k.__name__]
|
||||
assert kernels == original
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the ROCm BF16x3 router GEMM.
|
||||
|
||||
The weight split and the eligibility policy are pure functions, so most of
|
||||
this runs device-free. Only the GEMM correctness tests need a gfx950 runner.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm as rocm_bf16x3,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm only")
|
||||
|
||||
# (hidden_size, num_experts) for the models that ship fp32 router weights.
|
||||
# (6144, 128) -> MiniMax-M3, (3072, 256) -> MiniMax-M2/M2.5
|
||||
ROUTER_SHAPES = [(6144, 128), (3072, 256)]
|
||||
ROUTER_WEIGHT_SCALE = 0.053
|
||||
|
||||
|
||||
def _requires_gfx950():
|
||||
if not rocm_bf16x3.platform_supported():
|
||||
pytest.skip("ROCm bf16x3 router GEMM requires gfx950")
|
||||
|
||||
|
||||
def _inputs(num_tokens, hidden_size, num_experts, seed=42, device="cuda"):
|
||||
torch.manual_seed(seed)
|
||||
x = torch.randn(num_tokens, hidden_size, device=device).to(torch.bfloat16)
|
||||
w = torch.randn(num_experts, hidden_size, dtype=torch.float32, device=device)
|
||||
return x, w * ROUTER_WEIGHT_SCALE
|
||||
|
||||
|
||||
def _top_set(logits, k=4):
|
||||
"""Sorted top-k indices; topk's order among near-ties is not meaningful."""
|
||||
return logits.topk(k, dim=-1).indices.sort(dim=-1).values
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("hidden_size", "num_experts"), ROUTER_SHAPES)
|
||||
def test_split_reconstructs_exactly(hidden_size: int, num_experts: int):
|
||||
torch.manual_seed(42)
|
||||
w = torch.randn(num_experts, hidden_size) * ROUTER_WEIGHT_SCALE
|
||||
split = rocm_bf16x3.split_bf16x3(w)
|
||||
|
||||
assert split.shape == (rocm_bf16x3.BF16X3_TERMS, num_experts, hidden_size)
|
||||
assert split.dtype == torch.bfloat16
|
||||
# router_gemm reinterprets this as (TERMS * E, K) without a copy.
|
||||
assert split.is_contiguous()
|
||||
assert torch.equal(split.float().sum(0), w)
|
||||
|
||||
|
||||
def test_split_is_exact_across_normal_exponents():
|
||||
exps = torch.arange(-100, 100, dtype=torch.float32)
|
||||
mantissas = torch.linspace(1.0, 1.999, 64, dtype=torch.float32)
|
||||
w = (mantissas[:, None] * torch.pow(2.0, exps)[None, :]).contiguous()
|
||||
w = torch.cat([w, -w], dim=0)
|
||||
assert torch.equal(rocm_bf16x3.split_bf16x3(w).float().sum(0), w)
|
||||
|
||||
|
||||
def test_split_rejects_non_finite():
|
||||
"""ValueError rather than assert: the caller downgrades to the fp32 path
|
||||
instead of failing model load, and this must survive ``python -O``."""
|
||||
for bad in (float("inf"), float("-inf"), float("nan")):
|
||||
with pytest.raises(ValueError):
|
||||
rocm_bf16x3.split_bf16x3(torch.full((2, 4), bad, dtype=torch.float32))
|
||||
|
||||
|
||||
def test_split_rejects_non_fp32():
|
||||
with pytest.raises(ValueError):
|
||||
rocm_bf16x3.split_bf16x3(torch.zeros(2, 4, dtype=torch.bfloat16))
|
||||
|
||||
|
||||
def test_split_does_not_mutate_input():
|
||||
w = torch.randn(8, 32, dtype=torch.float32)
|
||||
original = w.clone()
|
||||
rocm_bf16x3.split_bf16x3(w)
|
||||
assert torch.equal(w, original)
|
||||
|
||||
|
||||
def test_is_supported_rejects_ineligible_inputs():
|
||||
meta = torch.device("meta")
|
||||
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device=meta)
|
||||
w = torch.zeros(128, 6144, dtype=torch.float32, device=meta)
|
||||
assert rocm_bf16x3.is_supported(x, w)
|
||||
|
||||
assert not rocm_bf16x3.is_supported(x[: rocm_bf16x3.MIN_TOKENS - 1], w)
|
||||
assert not rocm_bf16x3.is_supported(x.float(), w)
|
||||
assert not rocm_bf16x3.is_supported(x, w.bfloat16())
|
||||
assert not rocm_bf16x3.is_supported(x, w[:, :128])
|
||||
assert not rocm_bf16x3.is_supported(x.T.contiguous().T, w)
|
||||
|
||||
|
||||
def test_is_supported_rejects_mismatched_devices():
|
||||
"""The weight can be offloaded while the activations are not."""
|
||||
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device="meta")
|
||||
w = torch.zeros(128, 6144, dtype=torch.float32)
|
||||
assert not rocm_bf16x3.is_supported(x, w)
|
||||
|
||||
|
||||
def test_is_supported_accepts_odd_expert_counts():
|
||||
"""Unlike a tl.arange-indexed kernel, torch.mm needs no power-of-two E."""
|
||||
meta = torch.device("meta")
|
||||
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device=meta)
|
||||
for num_experts in (17, 384, 512):
|
||||
w = torch.zeros(num_experts, 6144, dtype=torch.float32, device=meta)
|
||||
assert rocm_bf16x3.is_supported(x, w)
|
||||
|
||||
|
||||
# Measured on gfx950: this path lands at 3.4e-7 to 4.9e-7 and varies <0.2%
|
||||
# across token counts. Fixed bounds rather than a live comparison, because the
|
||||
# fp32 fallback swings 5.7e-7 to 1.4e-6 as torch picks different hipBLASLt
|
||||
# kernels at different M. _FP32_FALLBACK_BEST is the best the fallback was ever
|
||||
# observed to do, so beating it proves the claim without racing a moving target.
|
||||
_MAX_REL_L2 = 2e-6
|
||||
_FP32_FALLBACK_BEST = 5.7e-7
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("hidden_size", "num_experts"), ROUTER_SHAPES)
|
||||
@pytest.mark.parametrize("num_tokens", [2048, 8192, 16385])
|
||||
def test_matches_fp64_reference(hidden_size: int, num_experts: int, num_tokens: int):
|
||||
_requires_gfx950()
|
||||
x, w = _inputs(num_tokens, hidden_size, num_experts)
|
||||
out = rocm_bf16x3.bf16x3_router_gemm(x, rocm_bf16x3.split_bf16x3(w))
|
||||
ref = x.double() @ w.double().T
|
||||
|
||||
assert out.shape == (num_tokens, num_experts)
|
||||
assert out.dtype == torch.float32
|
||||
assert (out.double() - ref).norm() / ref.norm() < _MAX_REL_L2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts", [17, 384])
|
||||
def test_matches_fp64_for_non_power_of_two_experts(num_experts: int):
|
||||
"""Exercises the reduction's N mask, which the model shapes never hit."""
|
||||
_requires_gfx950()
|
||||
x, w = _inputs(8192, 6144, num_experts)
|
||||
out = rocm_bf16x3.bf16x3_router_gemm(x, rocm_bf16x3.split_bf16x3(w))
|
||||
ref = x.double() @ w.double().T
|
||||
|
||||
assert out.shape == (8192, num_experts)
|
||||
assert (out.double() - ref).norm() / ref.norm() < _MAX_REL_L2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("hidden_size", "num_experts"), ROUTER_SHAPES)
|
||||
def test_more_accurate_than_fp32_fallback(hidden_size: int, num_experts: int):
|
||||
"""The claim is not just accuracy but beating the fp32 GEMM it replaces."""
|
||||
_requires_gfx950()
|
||||
x, w = _inputs(16384, hidden_size, num_experts)
|
||||
ref = x.double() @ w.double().T
|
||||
out = rocm_bf16x3.bf16x3_router_gemm(x, rocm_bf16x3.split_bf16x3(w))
|
||||
assert (out.double() - ref).norm() / ref.norm() < _FP32_FALLBACK_BEST
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("hidden_size", "num_experts"), ROUTER_SHAPES)
|
||||
def test_expert_selection_matches_fp64(hidden_size: int, num_experts: int):
|
||||
"""Relative L2 is a proxy; the expert set is what reaches the model.
|
||||
|
||||
Measures zero misroutes on both shapes, where the fp32 fallback misroutes a
|
||||
nonzero number at some token counts.
|
||||
"""
|
||||
_requires_gfx950()
|
||||
num_tokens = 16384
|
||||
x, w = _inputs(num_tokens, hidden_size, num_experts)
|
||||
ref_set = _top_set(x.double() @ w.double().T)
|
||||
out = rocm_bf16x3.bf16x3_router_gemm(x, rocm_bf16x3.split_bf16x3(w))
|
||||
|
||||
misrouted = int((_top_set(out) != ref_set).any(-1).sum())
|
||||
assert misrouted / num_tokens < 1e-4
|
||||
|
||||
|
||||
def test_writes_into_supplied_out():
|
||||
_requires_gfx950()
|
||||
x, w = _inputs(8192, 6144, 128)
|
||||
split = rocm_bf16x3.split_bf16x3(w)
|
||||
out = torch.empty(8192, 128, dtype=torch.float32, device="cuda")
|
||||
|
||||
returned = rocm_bf16x3.bf16x3_router_gemm(x, split, out=out)
|
||||
assert returned is out
|
||||
assert torch.equal(out, rocm_bf16x3.bf16x3_router_gemm(x, split))
|
||||
|
||||
|
||||
def test_validates_layout_assumptions():
|
||||
"""The reduction assumes a contiguous (TERMS, E, K) split and a contiguous
|
||||
out; nothing in the kernel enforces either."""
|
||||
_requires_gfx950()
|
||||
x, w = _inputs(8192, 6144, 128)
|
||||
split = rocm_bf16x3.split_bf16x3(w)
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
rocm_bf16x3.bf16x3_router_gemm(x, split.transpose(1, 2))
|
||||
with pytest.raises(AssertionError):
|
||||
rocm_bf16x3.bf16x3_router_gemm(x.float(), split)
|
||||
with pytest.raises(AssertionError):
|
||||
rocm_bf16x3.bf16x3_router_gemm(x, split[:2])
|
||||
with pytest.raises(AssertionError):
|
||||
rocm_bf16x3.bf16x3_router_gemm(x[:, :512], split)
|
||||
wide = torch.empty(8192, 256, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(AssertionError):
|
||||
rocm_bf16x3.bf16x3_router_gemm(x, split, out=wide[:, :128])
|
||||
@@ -405,6 +405,80 @@ def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
|
||||
torch.testing.assert_close(cache_backing, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="needs a CUDA or ROCm device"
|
||||
)
|
||||
@pytest.mark.parametrize("gather_len", [None, 70, 200])
|
||||
@pytest.mark.parametrize("one_pass", [False, True])
|
||||
def test_v41_fp8_gather_is_grid_width_invariant(gather_len, one_pass, monkeypatch):
|
||||
"""The gather grid only partitions tokens, so its width cannot move the
|
||||
result, and the rows it writes must match a host-side dequantization.
|
||||
|
||||
Covers the fp8_ds_mla record (584 B); the MXFP8 one is covered below.
|
||||
Both dequantization forms are checked against the same host reference on
|
||||
whatever device is present: the wrapper picks the one-pass form on ROCm
|
||||
alone, so otherwise each platform would only ever see one of them.
|
||||
"""
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: one_pass)
|
||||
from vllm.models.deepseek_v41.common.ops import quantize_and_insert_k_cache
|
||||
from vllm.models.deepseek_v41.common.ops.cache_utils import (
|
||||
dequantize_and_gather_k_cache_triton,
|
||||
)
|
||||
|
||||
torch.manual_seed(7)
|
||||
device = "cuda"
|
||||
block_size, num_blocks = 64, 6
|
||||
num_tokens = block_size * num_blocks
|
||||
|
||||
k = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
# insert takes [num_blocks, block_bytes]; gather takes the record view.
|
||||
cache = torch.zeros(num_blocks, block_size * 584, dtype=torch.uint8, device=device)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
quantize_and_insert_k_cache(k, cache, slot_mapping, block_size=block_size)
|
||||
cache_view = cache.view(num_blocks, block_size, 584)
|
||||
|
||||
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
|
||||
gls = (
|
||||
None
|
||||
if gather_len is None
|
||||
else torch.tensor([gather_len], dtype=torch.int32, device=device)
|
||||
)
|
||||
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).view(1, -1)
|
||||
|
||||
def gather(bound):
|
||||
out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
dequantize_and_gather_k_cache_triton(
|
||||
out,
|
||||
cache_view,
|
||||
seq_lens,
|
||||
gls,
|
||||
block_table,
|
||||
block_size,
|
||||
0,
|
||||
max_gather_len=bound,
|
||||
)
|
||||
return out
|
||||
|
||||
# None keeps the historical 128; the rest make the wrapper pick other widths
|
||||
# for the same work, and the output must not notice.
|
||||
baseline = gather(None)
|
||||
for bound in (8, num_tokens, 1 << 20):
|
||||
torch.testing.assert_close(gather(bound), baseline, rtol=0, atol=0)
|
||||
|
||||
n = num_tokens if gather_len is None else gather_len
|
||||
start = num_tokens - n
|
||||
for i in (0, 1, n // 2, n - 1):
|
||||
values, scales = _ue8m0_reference(k[start + i, :448], 64, 448.0)
|
||||
nope = (values.to(torch.float32).view(7, 64) * scales.view(7, 1)).reshape(448)
|
||||
torch.testing.assert_close(
|
||||
baseline[0, i, :448], nope.to(torch.bfloat16), rtol=4e-3, atol=1e-6
|
||||
)
|
||||
# The RoPE dims are stored verbatim, so they come back exactly.
|
||||
torch.testing.assert_close(
|
||||
baseline[0, i, 448:], k[start + i, 448:], rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
def test_v41_mxfp8_cache_round_trip():
|
||||
"""quantize_and_insert -> dequantize_and_gather recovers the V4.1 record.
|
||||
|
||||
@@ -43,20 +43,20 @@ class TestGreedyArgmax:
|
||||
|
||||
|
||||
class TestFusedGumbelArgmax:
|
||||
@pytest.mark.parametrize("vocab_size", VOCAB_SIZES)
|
||||
def test_distribution_chi_squared(self, vocab_size: int):
|
||||
def test_distribution_chi_squared(self):
|
||||
"""Verify sampling distribution via chi-squared goodness of fit."""
|
||||
small_vocab = min(vocab_size, 100)
|
||||
# Small support is enough for chi-squared power; large-vocab bias
|
||||
# is covered by test_noise_not_sliding_window (#59786).
|
||||
small_vocab = 100
|
||||
logits = torch.randn(1, small_vocab, dtype=torch.float32)
|
||||
probs = logits.softmax(dim=-1).squeeze(0)
|
||||
|
||||
n_samples = 100_000
|
||||
counts = torch.zeros(small_vocab)
|
||||
for trial in range(n_samples):
|
||||
seed = torch.tensor([trial * 7 + 13], dtype=torch.long)
|
||||
tile = logits.expand(1, -1).contiguous()
|
||||
idx = torch.ops._C.fused_gumbel_argmax(tile, seed)
|
||||
counts[idx.item()] += 1
|
||||
seeds = torch.arange(n_samples, dtype=torch.long) * 7 + 13
|
||||
idx = torch.ops._C.fused_gumbel_argmax(
|
||||
logits.expand(n_samples, -1).contiguous(), seeds
|
||||
)
|
||||
counts = torch.bincount(idx, minlength=small_vocab).float()
|
||||
|
||||
expected = probs * n_samples
|
||||
mask = expected > 5
|
||||
@@ -70,6 +70,39 @@ class TestFusedGumbelArgmax:
|
||||
f"dof={dof}, p={p_value:.6f}"
|
||||
)
|
||||
|
||||
def test_noise_not_sliding_window(self):
|
||||
"""Table[(seed+i)&MASK] made consecutive seeds a 1-token shift.
|
||||
|
||||
On uniform logits that yields P(w(s+1) == w(s)-1) ~ 1-2/V. Independent
|
||||
per-token noise yields ~1/V. See #59786.
|
||||
"""
|
||||
vocab, n = 128, 2048
|
||||
logits = torch.zeros(n, vocab, dtype=torch.float32)
|
||||
seeds = torch.arange(n, dtype=torch.long)
|
||||
winners = torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
shifted = (winners[:-1] - 1) % vocab
|
||||
frac = (winners[1:] == shifted).float().mean().item()
|
||||
assert frac < 0.05, (
|
||||
f"consecutive-seed winners follow a table shift (frac={frac:.3f})"
|
||||
)
|
||||
|
||||
def test_zipf_head_matches_softmax(self):
|
||||
"""Zipf head probability should match softmax (#59786)."""
|
||||
vocab, n_samples, batch = 8192, 16384, 256
|
||||
logits = -1.1 * torch.arange(1, vocab + 1, dtype=torch.float32).log()
|
||||
p0 = torch.softmax(logits.double(), dim=-1)[0].item()
|
||||
row = logits.view(1, -1).expand(batch, vocab).contiguous()
|
||||
n_top = 0
|
||||
for start in range(0, n_samples, batch):
|
||||
seeds = torch.arange(start, start + batch, dtype=torch.long)
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(row, seeds)
|
||||
n_top += int((tokens == 0).sum())
|
||||
q0 = n_top / n_samples
|
||||
se = (p0 * (1.0 - p0) / n_samples) ** 0.5
|
||||
assert abs(q0 - p0) < 8.0 * se, (
|
||||
f"P(top-1) kernel={q0:.4f} exact={p0:.4f} se={se:.4f}"
|
||||
)
|
||||
|
||||
def test_deterministic_same_seed(self):
|
||||
"""Same seed produces same result."""
|
||||
logits = torch.randn(4, 32000, dtype=torch.float32)
|
||||
@@ -95,6 +128,108 @@ class TestFusedGumbelArgmax:
|
||||
assert result.min() >= 0
|
||||
assert result.max() < vocab_size
|
||||
|
||||
def test_samples_only_from_finite_logits(self):
|
||||
"""top-k/top-p mask with -inf; those IDs must never win."""
|
||||
allowed = torch.tensor([3, 17, 99, 1024, 8191], dtype=torch.long)
|
||||
logits = torch.full((64, 32000), float("-inf"), dtype=torch.float32)
|
||||
logits[:, allowed] = torch.randn(64, allowed.numel())
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(
|
||||
logits, torch.arange(64, dtype=torch.long)
|
||||
)
|
||||
assert torch.isin(tokens, allowed).all()
|
||||
|
||||
def test_single_unmasked_token(self):
|
||||
logits = torch.full((8, 4096), float("-inf"), dtype=torch.float32)
|
||||
logits[:, 123] = 0.0
|
||||
seeds = torch.tensor([0, -1, -(2**63), 2**63 - 1, 42, 99, 1000, 7])
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
assert (tokens == 123).all()
|
||||
|
||||
def test_tokens_beyond_gumbel_max_cannot_win(self):
|
||||
"""A gap larger than GUMBEL_MAX is unreachable."""
|
||||
logits = torch.zeros(256, 4, dtype=torch.float32)
|
||||
logits[:, 1:] = -80.0
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(
|
||||
logits, torch.arange(256, dtype=torch.long)
|
||||
)
|
||||
assert (tokens == 0).all()
|
||||
|
||||
def test_tied_huge_logits_not_index_biased(self):
|
||||
"""Noise must still break ties when logits sit at float32 max."""
|
||||
logits = torch.full(
|
||||
(4096, 2), torch.finfo(torch.float32).max, dtype=torch.float32
|
||||
)
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(
|
||||
logits, torch.arange(4096, dtype=torch.long)
|
||||
)
|
||||
frac0 = (tokens == 0).float().mean().item()
|
||||
assert set(tokens.tolist()) == {0, 1}, tokens.unique().tolist()
|
||||
assert 0.35 < frac0 < 0.65, f"P(index 0)={frac0:.3f} expected ~0.5"
|
||||
|
||||
def test_serial_batch1_matches_batched(self):
|
||||
"""OpenMP rows must match the batch=1 serial path."""
|
||||
logits = torch.randn(8, 2048, dtype=torch.float32)
|
||||
seeds = torch.arange(8, dtype=torch.long)
|
||||
batched = torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
serial = torch.stack(
|
||||
[
|
||||
torch.ops._C.fused_gumbel_argmax(logits[i : i + 1], seeds[i : i + 1])[0]
|
||||
for i in range(8)
|
||||
]
|
||||
)
|
||||
torch.testing.assert_close(batched, serial)
|
||||
|
||||
def test_noncontiguous_matches_contiguous(self):
|
||||
logits = torch.randn(8, 1024, dtype=torch.float32)[:, ::2]
|
||||
seeds = torch.arange(16, dtype=torch.long)[::2]
|
||||
assert not logits.is_contiguous() and not seeds.is_contiguous()
|
||||
expected = torch.ops._C.fused_gumbel_argmax(
|
||||
logits.contiguous(), seeds.contiguous()
|
||||
)
|
||||
actual = torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
def test_empty_batch(self):
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(
|
||||
torch.empty(0, 7), torch.empty(0, dtype=torch.long)
|
||||
)
|
||||
assert tokens.shape == (0,) and tokens.dtype == torch.int64
|
||||
|
||||
def test_masked_support_matches_softmax(self):
|
||||
"""Finite subset after -inf mask should match softmax."""
|
||||
from scipy.stats import chisquare
|
||||
|
||||
logits = torch.tensor(
|
||||
[0.0, -0.2, -0.7, -1.5, -3.0, float("-inf")], dtype=torch.float32
|
||||
)
|
||||
n_samples = 20_000
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(
|
||||
logits.expand(n_samples, -1).contiguous(),
|
||||
torch.arange(n_samples, dtype=torch.long),
|
||||
)
|
||||
counts = torch.bincount(tokens, minlength=logits.numel()).double()
|
||||
assert counts[-1] == 0
|
||||
probs = logits.double().softmax(-1)
|
||||
_, p_value = chisquare(counts[:-1].numpy(), (probs[:-1] * n_samples).numpy())
|
||||
assert p_value > 1e-7, f"masked-support p-value: {p_value}"
|
||||
|
||||
def test_zipf_tail_matches_softmax(self):
|
||||
"""Independent 53-bit Gumbel must not starve the Zipf tail (#59786)."""
|
||||
vocab, n_samples, batch = 8192, 16384, 256
|
||||
logits = -1.1 * torch.arange(1, vocab + 1, dtype=torch.float32).log()
|
||||
p_tail = torch.softmax(logits.double(), dim=-1)[vocab // 2 :].sum().item()
|
||||
row = logits.view(1, -1).expand(batch, vocab).contiguous()
|
||||
n_tail = 0
|
||||
for start in range(0, n_samples, batch):
|
||||
seeds = torch.arange(start, start + batch, dtype=torch.long)
|
||||
tokens = torch.ops._C.fused_gumbel_argmax(row, seeds)
|
||||
n_tail += int((tokens >= vocab // 2).sum())
|
||||
q_tail = n_tail / n_samples
|
||||
se = (p_tail * (1.0 - p_tail) / n_samples) ** 0.5
|
||||
assert abs(q_tail - p_tail) < 8.0 * se, (
|
||||
f"P(tail) kernel={q_tail:.4f} exact={p_tail:.4f} se={se:.4f}"
|
||||
)
|
||||
|
||||
|
||||
class TestMemory:
|
||||
def test_fused_no_intermediate_allocs(self):
|
||||
|
||||
@@ -3,25 +3,34 @@
|
||||
"""Layout plumbing for DeepSeek V4.1 mega attention.
|
||||
|
||||
The kernel's Q and O layouts are produced by permuting ``wq_b`` rows and
|
||||
``wo_a`` columns once at load, so what has to hold is that the permuted GEMMs
|
||||
agree with a reference that permutes the activation instead.
|
||||
``wo_a`` columns in their weight loaders, so what has to hold is that the
|
||||
permuted GEMMs agree with a reference that permutes the activation instead.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
from vllm.models.deepseek_v41.common.ops.fused_layout import (
|
||||
WV_GROUP_SIZE,
|
||||
o_fused_chunk_permutation,
|
||||
o_fused_permutation,
|
||||
permute_on_load,
|
||||
permute_q_to_fused,
|
||||
permute_wo_a_,
|
||||
permute_wq_b_,
|
||||
q_fused_permutation,
|
||||
)
|
||||
|
||||
HEAD_DIM = 512
|
||||
|
||||
|
||||
def _load(checkpoint: torch.Tensor, perm: torch.Tensor, dim: int) -> torch.Tensor:
|
||||
"""Load ``checkpoint`` into a fresh parameter through a permuting loader."""
|
||||
param = torch.nn.Parameter(torch.empty_like(checkpoint), requires_grad=False)
|
||||
param.weight_loader = default_weight_loader # type: ignore[attr-defined]
|
||||
permute_on_load(param, perm, dim)
|
||||
param.weight_loader(param, checkpoint) # type: ignore[attr-defined]
|
||||
return param.data
|
||||
|
||||
|
||||
def test_permuted_wq_b_gemm_matches_permuted_activation():
|
||||
"""Permuting wq_b's rows makes the Q GEMM emit the fused layout directly."""
|
||||
torch.manual_seed(0)
|
||||
@@ -32,18 +41,17 @@ def test_permuted_wq_b_gemm_matches_permuted_activation():
|
||||
0, 256, (num_heads * HEAD_DIM, q_lora_rank // 32), dtype=torch.uint8
|
||||
)
|
||||
qr = torch.randn(num_tokens, q_lora_rank)
|
||||
orig_weight, orig_scale = weight.clone(), scale.clone()
|
||||
|
||||
standard = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
|
||||
expected = permute_q_to_fused(standard)
|
||||
|
||||
permute_wq_b_(weight, scale, num_heads)
|
||||
fused = (qr @ weight.T).view(num_tokens, num_heads, HEAD_DIM)
|
||||
perm = q_fused_permutation(num_heads, HEAD_DIM)
|
||||
loaded_weight = _load(weight, perm, dim=0)
|
||||
fused = (qr @ loaded_weight.T).view(num_tokens, num_heads, HEAD_DIM)
|
||||
torch.testing.assert_close(fused, expected)
|
||||
# The scale has to follow its row, or dequant reads another row's scale.
|
||||
perm = q_fused_permutation(num_heads, HEAD_DIM)
|
||||
torch.testing.assert_close(weight, orig_weight[perm])
|
||||
torch.testing.assert_close(scale, orig_scale[perm])
|
||||
torch.testing.assert_close(loaded_weight, weight[perm])
|
||||
torch.testing.assert_close(_load(scale, perm, dim=0), scale[perm])
|
||||
|
||||
|
||||
def test_permuted_wo_a_consumes_fused_output():
|
||||
@@ -54,18 +62,18 @@ def test_permuted_wo_a_consumes_fused_output():
|
||||
weight = torch.randn(out_features, in_features)
|
||||
scale = torch.randint(0, 256, (out_features, in_features // 32), dtype=torch.uint8)
|
||||
standard_o = torch.randn(num_tokens, in_features)
|
||||
orig_weight, orig_scale = weight.clone(), scale.clone()
|
||||
|
||||
expected = standard_o @ weight.T
|
||||
# The kernel emits the same values in the fused chunk order.
|
||||
perm = o_fused_permutation(WV_GROUP_SIZE, HEAD_DIM)
|
||||
fused_o = standard_o[:, perm]
|
||||
|
||||
permute_wo_a_(weight, scale, WV_GROUP_SIZE)
|
||||
torch.testing.assert_close(weight, orig_weight[:, perm])
|
||||
torch.testing.assert_close(
|
||||
scale, orig_scale[:, o_fused_chunk_permutation(WV_GROUP_SIZE, HEAD_DIM)]
|
||||
)
|
||||
loaded_weight = _load(weight, perm, dim=1)
|
||||
torch.testing.assert_close(loaded_weight, weight[:, perm])
|
||||
chunk_perm = o_fused_chunk_permutation(WV_GROUP_SIZE, HEAD_DIM)
|
||||
torch.testing.assert_close(_load(scale, chunk_perm, dim=1), scale[:, chunk_perm])
|
||||
# Unlike wq_b, this permutation sits inside the 4096-term reduction, so the
|
||||
# summation order changes and the result differs in the last few ulps.
|
||||
torch.testing.assert_close(fused_o @ weight.T, expected, rtol=1e-3, atol=1e-3)
|
||||
torch.testing.assert_close(
|
||||
fused_o @ loaded_weight.T, expected, rtol=1e-3, atol=1e-3
|
||||
)
|
||||
|
||||
@@ -7,6 +7,9 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram,
|
||||
@@ -714,8 +717,9 @@ def test_engram_lookup_reuses_jit_across_token_shapes():
|
||||
)
|
||||
@pytest.mark.parametrize("cpu_offload", [False, True])
|
||||
@pytest.mark.parametrize("background", [False, True])
|
||||
@pytest.mark.parametrize("packed", [False, True])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 256])
|
||||
def test_engram_lookup_matches_torch(cpu_offload, background, num_tokens):
|
||||
def test_engram_lookup_matches_torch(cpu_offload, background, packed, num_tokens):
|
||||
"""The fused gather must be bit-exact with the torch dequant path it
|
||||
replaces, from HBM and from pinned host memory alike, and must contribute
|
||||
zeros for rows another TP rank owns."""
|
||||
@@ -729,8 +733,13 @@ def test_engram_lookup_matches_torch(cpu_offload, background, num_tokens):
|
||||
layer.vocab_start_idx,
|
||||
layer.vocab_end_idx,
|
||||
)
|
||||
out = torch.empty(num_tokens, cols, layer.dim, dtype=torch.bfloat16, device="cuda")
|
||||
width = layer.dim + layer.dim // 32 if packed else layer.dim
|
||||
dtype = torch.uint8 if packed else torch.bfloat16
|
||||
out = torch.empty(num_tokens, cols, width, dtype=dtype, device="cuda")
|
||||
layer.lookup(ids, out, background=background)
|
||||
if packed:
|
||||
values = out[..., : layer.dim].view(torch.float8_e4m3fn)
|
||||
out = dequant_mxfp8_to_bf16(values, out[..., layer.dim :])
|
||||
assert torch.equal(out, expected)
|
||||
|
||||
|
||||
|
||||
@@ -25,6 +25,8 @@ def require_flashinfer_bf16_cutedsl() -> None:
|
||||
(8, True, False),
|
||||
(16, False, True),
|
||||
(32, True, True),
|
||||
(64, True, True),
|
||||
(512, False, False),
|
||||
],
|
||||
)
|
||||
def test_flashinfer_bf16_cutedsl_correctness(
|
||||
|
||||
@@ -115,13 +115,13 @@ class MockLinearRequiresUnquantized(MockLinearFp8Static):
|
||||
requires_unquantized_input = True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 16, 128])
|
||||
@pytest.mark.parametrize("token_shape", [(1,), (16,), (128,), (1, 17), (2, 17)])
|
||||
@pytest.mark.parametrize("hidden_size", [128, 512, 1024])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@torch.inference_mode()
|
||||
def test_maybe_fused_act_quant_fp8_static(
|
||||
default_vllm_config,
|
||||
num_tokens: int,
|
||||
token_shape: tuple[int, ...],
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
@@ -133,26 +133,29 @@ def test_maybe_fused_act_quant_fp8_static(
|
||||
scale = torch.tensor([0.5], device=device, dtype=torch.float32)
|
||||
linear = MockLinearFp8Static(scale)
|
||||
|
||||
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device=device)
|
||||
x = torch.randn(*token_shape, hidden_size * 2, dtype=dtype, device=device)
|
||||
result = maybe_fused_act_quant(act_fn, x, linear)
|
||||
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
assert result.quant_key == kFp8StaticTensorSym
|
||||
assert result.data.dtype == current_platform.fp8_dtype()
|
||||
assert result.orig_dtype == dtype
|
||||
assert result.orig_shape == (num_tokens, hidden_size)
|
||||
assert result.orig_shape == (*token_shape, hidden_size)
|
||||
assert result.data.shape == result.orig_shape
|
||||
|
||||
ref_out = ref_impl(act_fn, x, scale)
|
||||
ref_out = ref_impl(act_fn, x.reshape(-1, hidden_size * 2), scale).reshape(
|
||||
result.orig_shape
|
||||
)
|
||||
torch.testing.assert_close(result.data.to(torch.float32), ref_out.to(torch.float32))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 16, 128])
|
||||
@pytest.mark.parametrize("token_shape", [(1,), (16,), (128,), (1, 17), (2, 17)])
|
||||
@pytest.mark.parametrize("hidden_size", [128, 512, 1024])
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@torch.inference_mode()
|
||||
def test_maybe_fused_act_quant_fp8_dynamic_block(
|
||||
default_vllm_config,
|
||||
num_tokens: int,
|
||||
token_shape: tuple[int, ...],
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
@@ -166,31 +169,34 @@ def test_maybe_fused_act_quant_fp8_dynamic_block(
|
||||
linear = MockLinearFp8Dynamic128()
|
||||
|
||||
scale = 1 / hidden_size
|
||||
x = torch.randn(num_tokens, hidden_size * 2, dtype=dtype, device=device) * scale
|
||||
x = torch.randn(*token_shape, hidden_size * 2, dtype=dtype, device=device) * scale
|
||||
result = maybe_fused_act_quant(act_fn, x, linear)
|
||||
|
||||
assert isinstance(result, QuantizedActivation)
|
||||
assert result.quant_key == kFp8Dynamic128Sym
|
||||
assert result.data.dtype == current_platform.fp8_dtype()
|
||||
assert result.orig_dtype == dtype
|
||||
assert result.orig_shape == (num_tokens, hidden_size)
|
||||
assert result.orig_shape == (*token_shape, hidden_size)
|
||||
assert result.data.shape == result.orig_shape
|
||||
|
||||
num_groups = hidden_size // group_size
|
||||
assert result.scale.shape == (num_tokens, num_groups)
|
||||
assert result.scale.shape == (*token_shape, num_groups)
|
||||
|
||||
gate, up = x.split(hidden_size, dim=-1)
|
||||
silu_out = F.silu(gate) * up
|
||||
ref_out, ref_scales = per_token_group_quant_fp8(
|
||||
silu_out, group_size=group_size, use_ue8m0=False
|
||||
silu_out.reshape(-1, hidden_size), group_size=group_size, use_ue8m0=False
|
||||
)
|
||||
ref_out = ref_out.reshape(result.orig_shape)
|
||||
ref_scales = ref_scales.reshape(*token_shape, num_groups)
|
||||
|
||||
torch.testing.assert_close(result.scale, ref_scales, rtol=1e-5, atol=1e-5)
|
||||
|
||||
ref_deq = ref_out.to(torch.float32) * ref_scales.repeat_interleave(
|
||||
group_size, dim=1
|
||||
group_size, dim=-1
|
||||
)
|
||||
result_deq = result.data.to(torch.float32) * result.scale.repeat_interleave(
|
||||
group_size, dim=1
|
||||
group_size, dim=-1
|
||||
)
|
||||
torch.testing.assert_close(ref_deq, result_deq, atol=5e-2, rtol=5e-2)
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@ The ROCm branch is guarded on ``not bias`` because ``torch.mm`` has no bias
|
||||
term; a biased gate must fall back so the bias is not silently dropped.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -168,3 +170,91 @@ def test_rocm_fp32_router_gemm_rejects_unsupported_configs(
|
||||
on_gfx950=on_gfx950,
|
||||
)
|
||||
assert not gate.allow_fp32_router_gemm
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ROCm BF16x3 tier (fp32 router weights)
|
||||
# ---------------------------------------------------------------------------
|
||||
def _make_bf16x3_gate(monkeypatch, *, supported=True, **kwargs):
|
||||
"""Build a gate with the ROCm bf16x3 platform check mocked."""
|
||||
from vllm.model_executor.layers.fused_moe.router import (
|
||||
bf16x3_router_gemm_rocm as rocm_bf16x3,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(rocm_bf16x3, "platform_supported", lambda: supported)
|
||||
kwargs.setdefault("params_dtype", torch.float32)
|
||||
return _make_gate(monkeypatch, is_rocm=True, **kwargs)
|
||||
|
||||
|
||||
def test_rocm_bf16x3_enabled_for_fp32_weight(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch)
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
# Built by process_weights_after_loading, not lazily in forward().
|
||||
# Registered up front but still None, so named_buffers() skips it.
|
||||
assert gate._bf16x3_weight is None
|
||||
assert "_bf16x3_weight" in gate._buffers
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_by_bias(monkeypatch):
|
||||
# torch.mm has no bias term.
|
||||
gate = _make_bf16x3_gate(monkeypatch, bias=True)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_for_bf16_weight(monkeypatch):
|
||||
# A bf16 weight belongs to the cuBLAS tier; there is nothing to split.
|
||||
gate = _make_bf16x3_gate(monkeypatch, params_dtype=torch.bfloat16)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_for_non_fp32_out_dtype(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_disabled_on_unsupported_arch(monkeypatch):
|
||||
# Only gfx950 is benchmarked; gfx942 has a different fp32:bf16 MFMA ratio.
|
||||
gate = _make_bf16x3_gate(monkeypatch, supported=False)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
assert not gate._rocm_bf16x3_weight_eligible
|
||||
|
||||
|
||||
def test_rocm_bf16x3_no_split_built_when_tier_disabled(monkeypatch):
|
||||
"""An ineligible out_dtype must not allocate a split that is never read;
|
||||
it is 1.5x the router weight on every layer."""
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=torch.bfloat16)
|
||||
gate.quant_method.process_weights_after_loading(gate)
|
||||
assert gate._bf16x3_weight is None
|
||||
|
||||
|
||||
def test_rocm_bf16x3_set_out_dtype_enables_tier(monkeypatch):
|
||||
gate = _make_bf16x3_gate(monkeypatch, out_dtype=None)
|
||||
assert not gate.allow_rocm_bf16x3_router_gemm
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
|
||||
def test_rocm_bf16x3_dispatch_is_a_custom_op():
|
||||
"""The num_tokens branch must stay opaque to Dynamo; see the op's impl."""
|
||||
assert hasattr(torch.ops.vllm, "rocm_bf16x3_router_gemm_dispatch")
|
||||
x = torch.zeros(8192, 6144, dtype=torch.bfloat16, device="meta")
|
||||
w = torch.zeros(128, 6144, dtype=torch.float32, device="meta")
|
||||
split = torch.zeros(3, 128, 6144, dtype=torch.bfloat16, device="meta")
|
||||
out = torch.ops.vllm.rocm_bf16x3_router_gemm_dispatch(x, w, split)
|
||||
assert out.shape == (8192, 128)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_rocm_bf16x3_is_reachable_for_low_m_shapes(monkeypatch):
|
||||
"""The gfx950 low-M fp32 tier claims these shapes too and returns
|
||||
unconditionally, so bf16x3 is only reachable if checked first."""
|
||||
gate = _make_bf16x3_gate(
|
||||
monkeypatch, input_size=6144, output_size=128, on_gfx950=True
|
||||
)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
assert gate.allow_rocm_bf16x3_router_gemm
|
||||
|
||||
source = inspect.getsource(type(gate).forward)
|
||||
assert source.index("allow_rocm_bf16x3_router_gemm") < source.index(
|
||||
"allow_fp32_router_gemm"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU-only integration tests for the UltraQuant cache dtype."""
|
||||
|
||||
from vllm.v1.attention.backends.ultraquant_attn import (
|
||||
UltraQuantAttentionBackend,
|
||||
)
|
||||
from vllm.v1.attention.ops.flydsl_ultraquant_decode import (
|
||||
ultraquant_flydsl_decode_eligible,
|
||||
)
|
||||
|
||||
|
||||
def test_ultraquant_prefers_block_size_64():
|
||||
# FlyDSL D=256 decode geometry requires a 64-token KV block.
|
||||
assert UltraQuantAttentionBackend.get_preferred_block_size(16) == 64
|
||||
assert UltraQuantAttentionBackend.get_preferred_block_size(128) == 64
|
||||
|
||||
|
||||
def test_ultraquant_flydsl_decode_routing():
|
||||
"""CPU check of FlyDSL vs unified-Triton decode eligibility."""
|
||||
eligible = dict(
|
||||
head_size=256,
|
||||
num_kv_groups=8,
|
||||
has_sinks=False,
|
||||
sliding_window=None,
|
||||
flydsl_loaded=True,
|
||||
)
|
||||
assert ultraquant_flydsl_decode_eligible(**eligible)
|
||||
for gqa in (6, 8, 16):
|
||||
assert ultraquant_flydsl_decode_eligible(**{**eligible, "num_kv_groups": gqa})
|
||||
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "flydsl_loaded": False})
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "head_size": 128})
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "num_kv_groups": 4})
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "has_sinks": True})
|
||||
assert not ultraquant_flydsl_decode_eligible(**{**eligible, "sliding_window": 4096})
|
||||
@@ -0,0 +1,111 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the UltraQuant Triton unified-attention fallback."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops.ultraquant.format import slot_size
|
||||
from vllm.v1.attention.ops.ultraquant.reference import reference_ultraquant_attention
|
||||
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
|
||||
from vllm.v1.attention.ops.ultraquant.triton_unified_attention import (
|
||||
ultraquant_unified_attention,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA/HIP device"
|
||||
)
|
||||
|
||||
|
||||
def _build_paged_cache(key, value, block_size):
|
||||
batch_size, seq_len, num_kv_heads, head_size = key.shape
|
||||
blocks_per_seq = (seq_len + block_size - 1) // block_size
|
||||
kv_cache = torch.zeros(
|
||||
batch_size * blocks_per_seq,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
slot_size(head_size),
|
||||
dtype=torch.uint8,
|
||||
device=key.device,
|
||||
)
|
||||
block_table = torch.arange(
|
||||
batch_size * blocks_per_seq,
|
||||
dtype=torch.int32,
|
||||
device=key.device,
|
||||
).view(batch_size, blocks_per_seq)
|
||||
slot_mapping = torch.arange(
|
||||
batch_size * blocks_per_seq * block_size,
|
||||
dtype=torch.int64,
|
||||
device=key.device,
|
||||
).view(batch_size, -1)[:, :seq_len]
|
||||
ultraquant_store(
|
||||
key.reshape(-1, num_kv_heads, head_size),
|
||||
value.reshape(-1, num_kv_heads, head_size),
|
||||
kv_cache,
|
||||
slot_mapping.reshape(-1),
|
||||
)
|
||||
return kv_cache, block_table
|
||||
|
||||
|
||||
def _attention_quality(actual: torch.Tensor, expected: torch.Tensor) -> dict:
|
||||
a = actual.float().reshape(-1)
|
||||
b = expected.float().reshape(-1)
|
||||
diff = (a - b).abs()
|
||||
return {
|
||||
"cos_sim": torch.nn.functional.cosine_similarity(a, b, dim=0).item(),
|
||||
"mean_abs": diff.mean().item(),
|
||||
"max_abs": diff.max().item(),
|
||||
"norm_ratio": (a.norm() / b.norm()).item(),
|
||||
}
|
||||
|
||||
|
||||
def _unified_decode(query, kv_cache, block_table, seq_lens, scale, sinks=None):
|
||||
batch_size = query.shape[0]
|
||||
return ultraquant_unified_attention(
|
||||
query=query,
|
||||
kv_cache=kv_cache,
|
||||
block_table=block_table,
|
||||
seq_lens=seq_lens,
|
||||
query_start_loc=torch.arange(
|
||||
batch_size + 1, dtype=torch.int32, device=query.device
|
||||
),
|
||||
scale=scale,
|
||||
max_query_len=1,
|
||||
max_seq_len=int(seq_lens.max().item()),
|
||||
sinks=sinks,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query_group_size", [6, 8])
|
||||
def test_unified_decode_d256_matches_reference(query_group_size):
|
||||
"""The D=256 fallback covers both target Qwen GQA layouts."""
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(0xD256 + query_group_size)
|
||||
|
||||
B, Hk, N, D = 2, 1, 256, 256
|
||||
Hq = query_group_size * Hk
|
||||
block_size = 32
|
||||
dtype = torch.bfloat16
|
||||
scale = D**-0.5
|
||||
|
||||
query = torch.randn(B, Hq, D, dtype=dtype, device=device)
|
||||
key = torch.randn(B, N, Hk, D, dtype=dtype, device=device)
|
||||
value = torch.randn_like(key)
|
||||
kv_cache, block_table = _build_paged_cache(key, value, block_size)
|
||||
seq_lens = torch.full((B,), N, dtype=torch.int32, device=device)
|
||||
|
||||
actual = _unified_decode(query, kv_cache, block_table, seq_lens, scale)
|
||||
expected = reference_ultraquant_attention(
|
||||
query=query,
|
||||
key=key,
|
||||
value=value,
|
||||
scale=scale,
|
||||
)
|
||||
|
||||
quality = _attention_quality(actual, expected)
|
||||
assert quality["cos_sim"] > 0.98, quality
|
||||
assert quality["mean_abs"] < 0.08, quality
|
||||
assert quality["max_abs"] < 0.8, quality
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the UltraQuant full-KV dequant kernel (continuation prefill).
|
||||
|
||||
The kernel reconstructs cached K/V from FP4 codes plus UE8M0 group scales.
|
||||
Because dequant is an exact table lookup times a power of two, the result
|
||||
must match the reference to within the output dtype's rounding.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops.ultraquant.format import slot_size
|
||||
from vllm.v1.attention.ops.ultraquant.reference import (
|
||||
ultraquant_dequant,
|
||||
ultraquant_encode,
|
||||
)
|
||||
from vllm.v1.attention.ops.ultraquant.triton_dequant import (
|
||||
ultraquant_full_dequant_kv,
|
||||
)
|
||||
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA/HIP device"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("head_dim", [128, 256])
|
||||
@pytest.mark.parametrize("num_kv_heads", [1, 4])
|
||||
@pytest.mark.parametrize("cached_len", [32, 96])
|
||||
def test_full_dequant_matches_reference(dtype, head_dim, num_kv_heads, cached_len):
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(0xD4)
|
||||
|
||||
block_size = 32
|
||||
alloc_len = cached_len
|
||||
num_blocks = alloc_len // block_size + 1
|
||||
|
||||
key = torch.randn(alloc_len, num_kv_heads, head_dim, dtype=dtype, device=device)
|
||||
value = torch.randn(alloc_len, num_kv_heads, head_dim, dtype=dtype, device=device)
|
||||
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
slot_size(head_dim),
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
# Leave block 0 unused so the block table exercises a non-identity mapping.
|
||||
slot_mapping = torch.arange(
|
||||
block_size, block_size + alloc_len, dtype=torch.int64, device=device
|
||||
)
|
||||
ultraquant_store(key, value, kv_cache, slot_mapping)
|
||||
|
||||
block_table = torch.arange(
|
||||
1, num_blocks, dtype=torch.int32, device=device
|
||||
).unsqueeze(0)
|
||||
|
||||
k_out = torch.zeros(
|
||||
1, num_kv_heads, alloc_len, head_dim, dtype=dtype, device=device
|
||||
)
|
||||
v_out = torch.zeros_like(k_out)
|
||||
ultraquant_full_dequant_kv(
|
||||
kv_cache=kv_cache,
|
||||
block_table=block_table,
|
||||
k_out=k_out,
|
||||
v_out=v_out,
|
||||
alloc_len=alloc_len,
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
# K is stored Hadamard-rotated, V unrotated; the kernel returns them as
|
||||
# stored, so the reference must not inverse-rotate either.
|
||||
ref_k = ultraquant_dequant(ultraquant_encode(key, rotate=True), head_dim)
|
||||
ref_v = ultraquant_dequant(ultraquant_encode(value, rotate=False), head_dim)
|
||||
|
||||
got_k = k_out[0].transpose(0, 1).to(torch.float32)
|
||||
got_v = v_out[0].transpose(0, 1).to(torch.float32)
|
||||
|
||||
# Reference is fp32; the kernel rounds to the output dtype once.
|
||||
tol = 1e-2 if dtype is torch.bfloat16 else 1e-3
|
||||
torch.testing.assert_close(got_k, ref_k.to(torch.float32), rtol=tol, atol=tol)
|
||||
torch.testing.assert_close(got_v, ref_v.to(torch.float32), rtol=tol, atol=tol)
|
||||
@@ -0,0 +1,149 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for the production UltraQuant D=256 FlyDSL decode."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops.flydsl_ultraquant_decode import (
|
||||
flydsl_ultraquant_decode_attention,
|
||||
is_ultraquant_flydsl_available,
|
||||
ultraquant_flydsl_decode_eligible,
|
||||
)
|
||||
from vllm.v1.attention.ops.ultraquant.format import slot_size
|
||||
from vllm.v1.attention.ops.ultraquant.reference import (
|
||||
hadamard_matrix,
|
||||
reference_ultraquant_attention,
|
||||
)
|
||||
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA/HIP device"
|
||||
)
|
||||
|
||||
|
||||
def _build_paged_cache(key, value, block_size):
|
||||
batch_size, seq_len, num_kv_heads, head_size = key.shape
|
||||
blocks_per_seq = (seq_len + block_size - 1) // block_size
|
||||
kv_cache = torch.zeros(
|
||||
batch_size * blocks_per_seq,
|
||||
block_size,
|
||||
num_kv_heads,
|
||||
slot_size(head_size),
|
||||
dtype=torch.uint8,
|
||||
device=key.device,
|
||||
)
|
||||
block_table = torch.arange(
|
||||
batch_size * blocks_per_seq,
|
||||
dtype=torch.int32,
|
||||
device=key.device,
|
||||
).view(batch_size, blocks_per_seq)
|
||||
slot_mapping = torch.arange(
|
||||
batch_size * blocks_per_seq * block_size,
|
||||
dtype=torch.int64,
|
||||
device=key.device,
|
||||
).view(batch_size, -1)[:, :seq_len]
|
||||
ultraquant_store(
|
||||
key.reshape(-1, num_kv_heads, head_size),
|
||||
value.reshape(-1, num_kv_heads, head_size),
|
||||
kv_cache,
|
||||
slot_mapping.reshape(-1),
|
||||
)
|
||||
return kv_cache, block_table
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch_size", "query_group_size", "block_size", "contiguous_query"),
|
||||
[
|
||||
(1, 8, 32, True),
|
||||
(8, 8, 64, True),
|
||||
(9, 8, 256, True),
|
||||
(1, 16, 32, True),
|
||||
(1, 6, 16, True),
|
||||
(1, 6, 32, True),
|
||||
(2, 6, 64, True),
|
||||
(1, 6, 128, True),
|
||||
(1, 6, 256, True),
|
||||
(1, 6, 32, False),
|
||||
],
|
||||
)
|
||||
def test_flydsl_decode_matches_reference(
|
||||
batch_size,
|
||||
query_group_size,
|
||||
block_size,
|
||||
contiguous_query,
|
||||
):
|
||||
if not ultraquant_flydsl_decode_eligible(
|
||||
head_size=256,
|
||||
num_kv_groups=query_group_size,
|
||||
has_sinks=False,
|
||||
sliding_window=None,
|
||||
flydsl_loaded=is_ultraquant_flydsl_available(),
|
||||
):
|
||||
pytest.skip("UltraQuant D=256 FlyDSL kernel is unavailable")
|
||||
|
||||
set_random_seed(0x57A1D + batch_size + block_size + query_group_size)
|
||||
device = torch.device("cuda")
|
||||
dtype = torch.bfloat16
|
||||
num_kv_heads = 1
|
||||
head_size = 256
|
||||
seq_len = 2048
|
||||
scale = head_size**-0.5
|
||||
|
||||
key = torch.randn(
|
||||
batch_size,
|
||||
seq_len,
|
||||
num_kv_heads,
|
||||
head_size,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
value = torch.randn_like(key)
|
||||
if contiguous_query:
|
||||
qkv = torch.randn(
|
||||
batch_size,
|
||||
query_group_size + 2 * num_kv_heads,
|
||||
head_size,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
query = qkv[:, :query_group_size]
|
||||
else:
|
||||
query_storage = torch.randn(
|
||||
batch_size,
|
||||
query_group_size,
|
||||
head_size * 2,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
)
|
||||
query = query_storage[..., ::2]
|
||||
kv_cache, block_table = _build_paged_cache(key, value, block_size)
|
||||
seq_lens = torch.full((batch_size,), seq_len, dtype=torch.int32, device=device)
|
||||
hadamard = hadamard_matrix(head_size, device)
|
||||
|
||||
actual = flydsl_ultraquant_decode_attention(
|
||||
query=query,
|
||||
kv_cache=kv_cache,
|
||||
block_table=block_table,
|
||||
seq_lens=seq_lens,
|
||||
scale=scale,
|
||||
PiT=hadamard,
|
||||
max_seq_len=seq_len,
|
||||
)
|
||||
expected = reference_ultraquant_attention(
|
||||
query=query,
|
||||
key=key,
|
||||
value=value,
|
||||
scale=scale,
|
||||
)
|
||||
|
||||
actual_f32 = actual.float()
|
||||
expected_f32 = expected.float()
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
actual_f32.flatten(), expected_f32.flatten(), dim=0
|
||||
)
|
||||
error = (actual_f32 - expected_f32).abs()
|
||||
assert cosine > 0.995
|
||||
assert error.mean() < 0.05
|
||||
assert error.max() < 0.8
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""UltraQuant inherits TurboQuant's forward/do_kv_cache_update orchestration.
|
||||
|
||||
UltraQuant only overrides the store/decode/prefill seams, so a TurboQuant-side
|
||||
change to a seam signature, to an inherited helper, or to the ``layer._tq_*``
|
||||
attributes ``forward`` reads will break UltraQuant at serve time without
|
||||
touching any UltraQuant kernel test. These checks pin that contract.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
|
||||
from vllm.v1.attention.backends import ultraquant_attn
|
||||
from vllm.v1.attention.backends.turboquant_attn import TurboQuantAttentionImpl
|
||||
from vllm.v1.attention.backends.ultraquant_attn import UltraQuantAttentionImpl
|
||||
|
||||
_SEAMS = ("_ensure_on_device", "_store_kv", "_decode_attention", "_prefill_attention")
|
||||
|
||||
|
||||
def _method_ast(cls: type, name: str) -> ast.AST:
|
||||
return ast.parse(textwrap.dedent(inspect.getsource(getattr(cls, name))))
|
||||
|
||||
|
||||
def _tq_attrs_read(cls: type, name: str) -> set[str]:
|
||||
"""``_tq_*`` attribute names read (not assigned) in ``cls.name``."""
|
||||
tree = _method_ast(cls, name)
|
||||
assigned = {
|
||||
t.attr
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Assign)
|
||||
for t in node.targets
|
||||
if isinstance(t, ast.Attribute)
|
||||
}
|
||||
read = {
|
||||
node.attr
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Attribute) and node.attr.startswith("_tq_")
|
||||
}
|
||||
return read - assigned
|
||||
|
||||
|
||||
def _tq_attrs_written(cls: type, name: str) -> set[str]:
|
||||
tree = _method_ast(cls, name)
|
||||
return {
|
||||
t.attr
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Assign)
|
||||
for t in node.targets
|
||||
if isinstance(t, ast.Attribute) and t.attr.startswith("_tq_")
|
||||
}
|
||||
|
||||
|
||||
def test_ultraquant_matches_turboquant_seam_contract():
|
||||
# 1. Each overridden seam accepts every parameter the base declares, so any
|
||||
# call the inherited orchestration makes stays valid for UltraQuant.
|
||||
for name in _SEAMS:
|
||||
base = list(
|
||||
inspect.signature(getattr(TurboQuantAttentionImpl, name)).parameters
|
||||
)
|
||||
override = list(
|
||||
inspect.signature(getattr(UltraQuantAttentionImpl, name)).parameters
|
||||
)
|
||||
assert override[: len(base)] == base, (
|
||||
f"UltraQuantAttentionImpl.{name} no longer accepts the arguments "
|
||||
f"TurboQuantAttentionImpl passes: base={base} override={override}"
|
||||
)
|
||||
|
||||
# 2. Keyword arguments UltraQuant passes to *inherited* TurboQuant helpers
|
||||
# must exist on those helpers (e.g. _flash_attn_varlen).
|
||||
own = vars(UltraQuantAttentionImpl)
|
||||
module = ast.parse(inspect.getsource(ultraquant_attn))
|
||||
for node in ast.walk(module):
|
||||
if not isinstance(node, ast.Call):
|
||||
continue
|
||||
fn = node.func
|
||||
if not (
|
||||
isinstance(fn, ast.Attribute)
|
||||
and isinstance(fn.value, ast.Name)
|
||||
and fn.value.id == "self"
|
||||
):
|
||||
continue
|
||||
target = getattr(TurboQuantAttentionImpl, fn.attr, None)
|
||||
if target is None or fn.attr in own or not callable(target):
|
||||
continue
|
||||
params = inspect.signature(target).parameters
|
||||
if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()):
|
||||
continue
|
||||
for kw in node.keywords:
|
||||
if kw.arg is None:
|
||||
continue
|
||||
assert kw.arg in params, (
|
||||
f"UltraQuant calls inherited {fn.attr}(..., {kw.arg}=...) but "
|
||||
f"TurboQuantAttentionImpl.{fn.attr} accepts {list(params)}"
|
||||
)
|
||||
|
||||
# 3. Every layer._tq_* attribute the inherited forward reads is set by
|
||||
# UltraQuant's _ensure_on_device.
|
||||
required = _tq_attrs_read(TurboQuantAttentionImpl, "forward")
|
||||
provided = _tq_attrs_written(UltraQuantAttentionImpl, "_ensure_on_device")
|
||||
assert required, "expected inherited forward to read layer._tq_* attributes"
|
||||
assert required <= provided, (
|
||||
f"UltraQuant._ensure_on_device does not set {sorted(required - provided)}, "
|
||||
f"which the inherited TurboQuant forward reads"
|
||||
)
|
||||
@@ -0,0 +1,149 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the Triton UltraQuant store kernel.
|
||||
|
||||
The Triton kernel must produce bit-identical codes and scale bytes vs
|
||||
the PyTorch reference (``reference.ultraquant_encode``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops.ultraquant.format import (
|
||||
k_codes_offset,
|
||||
k_scales_offset,
|
||||
n_groups,
|
||||
slot_size,
|
||||
v_codes_offset,
|
||||
v_scales_offset,
|
||||
)
|
||||
from vllm.v1.attention.ops.ultraquant.reference import ultraquant_encode
|
||||
from vllm.v1.attention.ops.ultraquant.triton_store import ultraquant_store
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="requires CUDA/HIP device"
|
||||
)
|
||||
|
||||
|
||||
def _make_cache(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
device: torch.device,
|
||||
) -> torch.Tensor:
|
||||
slot_b = slot_size(head_dim)
|
||||
return torch.zeros(
|
||||
num_blocks, block_size, num_kv_heads, slot_b, dtype=torch.uint8, device=device
|
||||
)
|
||||
|
||||
|
||||
def _read_codes_and_scales(
|
||||
kv_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
):
|
||||
"""Pull (k_codes, k_scale_bytes, v_codes, v_scale_bytes) for each
|
||||
(token, head) from the AoS slot layout. Scales come back as uint8
|
||||
(one E8M0 byte per group)."""
|
||||
block_size = kv_cache.shape[1]
|
||||
N = slot_mapping.shape[0]
|
||||
Gk = n_groups(head_dim)
|
||||
|
||||
k_codes_off = k_codes_offset(head_dim)
|
||||
k_scales_off = k_scales_offset(head_dim)
|
||||
v_codes_off = v_codes_offset(head_dim)
|
||||
v_scales_off = v_scales_offset(head_dim)
|
||||
codes_bytes = head_dim // 2
|
||||
|
||||
k_codes = torch.zeros(
|
||||
N, num_kv_heads, codes_bytes, dtype=torch.uint8, device=kv_cache.device
|
||||
)
|
||||
k_scale_bytes = torch.zeros(
|
||||
N, num_kv_heads, Gk, dtype=torch.uint8, device=kv_cache.device
|
||||
)
|
||||
v_codes = torch.zeros_like(k_codes)
|
||||
v_scale_bytes = torch.zeros_like(k_scale_bytes)
|
||||
|
||||
for ti in range(N):
|
||||
slot = int(slot_mapping[ti].item())
|
||||
if slot < 0:
|
||||
continue
|
||||
blk = slot // block_size
|
||||
off = slot % block_size
|
||||
for h in range(num_kv_heads):
|
||||
slot_bytes = kv_cache[blk, off, h] # [slot_size_aligned] uint8
|
||||
|
||||
k_codes[ti, h] = slot_bytes[k_codes_off : k_codes_off + codes_bytes]
|
||||
k_scale_bytes[ti, h] = slot_bytes[k_scales_off : k_scales_off + Gk]
|
||||
v_codes[ti, h] = slot_bytes[v_codes_off : v_codes_off + codes_bytes]
|
||||
v_scale_bytes[ti, h] = slot_bytes[v_scales_off : v_scales_off + Gk]
|
||||
|
||||
return k_codes, k_scale_bytes, v_codes, v_scale_bytes
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("head_dim", [128, 256])
|
||||
@pytest.mark.parametrize("num_kv_heads", [1, 4])
|
||||
@pytest.mark.parametrize("num_tokens", [1, 16, 17])
|
||||
def test_store_matches_reference(dtype, head_dim, num_kv_heads, num_tokens):
|
||||
"""Triton store output (codes + E8M0 scale bytes) matches the reference
|
||||
bit-exactly."""
|
||||
device = torch.device("cuda")
|
||||
set_random_seed(0xF8)
|
||||
|
||||
block_size = 32
|
||||
num_blocks = max(2, (num_tokens + block_size - 1) // block_size + 1)
|
||||
|
||||
key = torch.randn(num_tokens, num_kv_heads, head_dim, dtype=dtype, device=device)
|
||||
value = torch.randn(num_tokens, num_kv_heads, head_dim, dtype=dtype, device=device)
|
||||
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
if num_tokens >= 4:
|
||||
slot_mapping[1] = -1 # skip-path test
|
||||
|
||||
kv_cache = _make_cache(num_blocks, block_size, num_kv_heads, head_dim, device)
|
||||
|
||||
ultraquant_store(key, value, kv_cache, slot_mapping)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
ref_k = ultraquant_encode(key, rotate=True)
|
||||
ref_v = ultraquant_encode(value, rotate=False)
|
||||
|
||||
k_codes, k_scale_bytes, v_codes, v_scale_bytes = _read_codes_and_scales(
|
||||
kv_cache, slot_mapping, num_kv_heads, head_dim
|
||||
)
|
||||
|
||||
if num_tokens >= 4:
|
||||
assert (kv_cache[0, 1, :, :] == 0).all(), (
|
||||
"skipped slot's physical cache region was unexpectedly written"
|
||||
)
|
||||
|
||||
for ti in range(num_tokens):
|
||||
slot = int(slot_mapping[ti].item())
|
||||
if slot < 0:
|
||||
continue
|
||||
|
||||
assert torch.equal(k_codes[ti], ref_k.codes_packed[ti]), (
|
||||
f"K codes mismatch at token {ti}\n"
|
||||
f"got: {k_codes[ti].cpu().numpy().tolist()[:16]}\n"
|
||||
f"expected: {ref_k.codes_packed[ti].cpu().numpy().tolist()[:16]}"
|
||||
)
|
||||
assert torch.equal(k_scale_bytes[ti], ref_k.scale_bytes[ti]), (
|
||||
f"K scale bytes mismatch at token {ti}\n"
|
||||
f"got: {k_scale_bytes[ti].cpu().numpy().tolist()}\n"
|
||||
f"expected: {ref_k.scale_bytes[ti].cpu().numpy().tolist()}"
|
||||
)
|
||||
|
||||
assert torch.equal(v_codes[ti], ref_v.codes_packed[ti]), (
|
||||
f"V codes mismatch at token {ti}"
|
||||
)
|
||||
assert torch.equal(v_scale_bytes[ti], ref_v.scale_bytes[ti]), (
|
||||
f"V scale bytes mismatch at token {ti}\n"
|
||||
f"got: {v_scale_bytes[ti].cpu().numpy().tolist()}\n"
|
||||
f"expected: {ref_v.scale_bytes[ti].cpu().numpy().tolist()}"
|
||||
)
|
||||
Reference in New Issue
Block a user