Merge remote-tracking branch 'upstream/main' into patches

Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-03 21:59:03 +07:00
1012 changed files with 61873 additions and 14499 deletions
@@ -26,7 +26,6 @@ NUM_BLOCKS = 4321 # Arbitrary values for testing
PARTITION_SIZE_ROCM = 256
DTYPES = [torch.bfloat16, torch.float16]
NUM_GEN_SEQS = [7] # Arbitrary values for testing
NUM_PREFILL_SEQS = [3] # Arbitrary values for testing
NUM_HEADS = [(32, 8), (40, 40), (64, 8)] # Arbitrary values for testing
# Head sizes supported by the ROCm paged attention kernel.
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from contextlib import contextmanager
from typing import Literal
from unittest.mock import MagicMock, patch
import pytest
@@ -14,6 +15,7 @@ from vllm.config import (
VllmConfig,
set_current_vllm_config,
)
from vllm.config.cache import CacheDType
from vllm.platforms import current_platform
from vllm.platforms.cpu import CpuPlatform
from vllm.platforms.interface import DeviceCapability
@@ -26,7 +28,7 @@ else:
if current_platform.is_rocm():
from vllm.platforms.rocm import RocmPlatform
else:
RocmPlatform = None
RocmPlatform = None # type: ignore[misc] # Unavailable platform import.
from vllm.v1.attention.backend import AttentionType
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -406,7 +408,7 @@ def test_auto_backend_selection_behavior():
reason="Attention backend FA3 is not supported on ROCm. This test can't succeed.",
)
def test_per_head_quant_scales_backend_selection(
backend_name: str, flash_attn_version: int | None, should_succeed: bool
backend_name: str, flash_attn_version: Literal[2, 3, 4] | None, should_succeed: bool
):
"""Test backend selection when use_per_head_quant_scales=True."""
# Clear cache to ensure fresh backend selection
@@ -559,7 +561,7 @@ def test_non_causal_autoselect_backend():
"int8_per_token_head",
],
)
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: CacheDType):
"""FlashAttentionBackend must not claim support for kv_cache dtypes
that it cannot handle."""
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
@@ -569,7 +571,7 @@ def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"])
def test_flash_attn_accepts_handled_fp8_variants(
kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch
kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch
):
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
@@ -876,7 +878,9 @@ def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
config = EngineArgs(
model="google/gemma-4-31B-it",
dtype="bfloat16",
attention_config={"backend": "TRITON_FLASHINFER"},
attention_config=AttentionConfig(
backend=AttentionBackendEnum.TRITON_FLASHINFER
),
).create_engine_config()
with set_current_vllm_config(config):
assert (
@@ -151,7 +151,7 @@ def test_cascade(
max_seqlen_k=max_kv_len,
softmax_scale=scale,
causal=True,
window_size=window_size,
window_size=list(window_size),
block_table=block_tables,
softcap=soft_cap if soft_cap is not None else 0,
)
+2 -2
View File
@@ -187,7 +187,7 @@ def test_varlen_with_paged_kv(
max_seqlen_k=max_kv_len,
softmax_scale=scale,
causal=True,
window_size=window_size,
window_size=list(window_size),
block_table=block_tables,
softcap=soft_cap if soft_cap is not None else 0,
fa_version=fa_version,
@@ -274,7 +274,7 @@ def test_fa4_hd256_paged_call_shape(
max_seqlen_k=num_pages * block_size,
softmax_scale=scale,
causal=True,
window_size=window_size,
window_size=list(window_size),
block_table=block_tables[:, :num_pages],
num_splits=1,
fa_version=4,
@@ -197,11 +197,6 @@ def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> No
mixed_cache = initial_cache.clone()
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
kwargs = {
"ds_mla": cache_format == "fp8_ds_mla",
"q_scale_inv": scale_inv if cache_format == "fp8" else None,
"cache_scale_inv": scale_inv if cache_format == "fp8" else None,
}
output = fused_mla_decode_q_concat_kv_cache_insert(
inputs["ql_nope"],
@@ -210,7 +205,9 @@ def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> No
inputs["k_pe"],
mixed_cache,
slots,
**kwargs,
ds_mla=cache_format == "fp8_ds_mla",
q_scale_inv=scale_inv if cache_format == "fp8" else None,
cache_scale_inv=scale_inv if cache_format == "fp8" else None,
)
expected = _latent_query(inputs["ql_nope"], inputs["q_pe"])
+4 -4
View File
@@ -277,7 +277,7 @@ def test_mha_attn_varlen_forward_flashinfer(
# Override vllm config so get_vit_attn_backend returns FLASHINFER (simulates
# --mm-encoder-attn-backend=FLASHINFER).
vllm_config = get_current_vllm_config()
old_model_config = getattr(vllm_config, "model_config", None)
old_model_config = vllm_config.model_config
minimal_model_config = type(
"MinimalModelConfig",
(),
@@ -376,7 +376,7 @@ def test_mha_attn_varlen_forward_aiter_fp8(
torch.set_default_dtype(dtype)
vllm_config = get_current_vllm_config()
old_model_config = getattr(vllm_config, "model_config", None)
old_model_config = vllm_config.model_config
minimal_model_config = type(
"MinimalModelConfig",
(),
@@ -442,7 +442,7 @@ def test_mha_attn_varlen_forward_aiter_fp8(
)
def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config):
vllm_config = get_current_vllm_config()
old_model_config = getattr(vllm_config, "model_config", None)
old_model_config = vllm_config.model_config
vllm_config.model_config = type(
"MinimalModelConfig",
(),
@@ -468,7 +468,7 @@ def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config):
)
def test_mha_attn_fp8_rejects_wrong_backend(default_vllm_config):
vllm_config = get_current_vllm_config()
old_model_config = getattr(vllm_config, "model_config", None)
old_model_config = vllm_config.model_config
vllm_config.model_config = type(
"MinimalModelConfig",
(),
+27 -14
View File
@@ -614,14 +614,22 @@ def _reference_decode_index_score(
return out
def test_prefill_index_topk_correctness():
# 3600: 29 blocks over 12 splits of 3, so the last busy split has 2 blocks.
@pytest.mark.parametrize("prefix_len", [1024, 3600])
@pytest.mark.parametrize("force_split_k", [False, True])
def test_prefill_index_topk_correctness(monkeypatch, prefix_len, force_split_k):
if force_split_k:
# Take the SM12.0-only split-K launch on any CUDA GPU.
monkeypatch.setattr(
current_platform, "is_device_capability", lambda cap, *_: cap == (12, 0)
)
topk = 6
init_blocks = 0
local_blocks = 1
num_idx_heads = 2
head_dim = 16
q_lens = torch.tensor((4, 3), device="cuda", dtype=torch.int32)
prefix_lens = torch.tensor((0, 1024), device="cuda", dtype=torch.int32)
prefix_lens = torch.tensor((0, prefix_len), device="cuda", dtype=torch.int32)
seq_lens = prefix_lens + q_lens
batch = q_lens.numel()
max_seq_len = seq_lens.max().item()
@@ -1689,8 +1697,10 @@ def test_aiter_sparse_pa_layout_contract(monkeypatch, layout):
)
backend = sparse_attn_mod.MiniMaxM3SparseBackend
assert KVCacheLayout.LHBNC in backend.supported_kv_cache_layouts()
assert KVCacheLayout.LBHNC in backend.supported_kv_cache_layouts()
layouts = backend.supported_kv_cache_layouts()
assert layouts is not None
assert KVCacheLayout.LHBNC in layouts
assert KVCacheLayout.LBHNC in layouts
nb, h = 3, 1
cache_dtype = current_platform.fp8_dtype()
@@ -1772,19 +1782,22 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
import vllm.models.minimax_m3.amd.indexer_aiter as indexer_aiter_mod
monkeypatch.setattr(indexer_aiter_mod.current_platform, "is_rocm", lambda: True)
kwargs = dict(
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
num_index_heads=1,
index_head_dim=HEAD_DIM,
indexer_kv_dtype="fp8_e4m3",
max_model_len=8192,
)
def unsupported_reason() -> str | None:
return indexer_aiter_mod.aiter_indexer_unsupported_reason(
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
num_index_heads=1,
index_head_dim=HEAD_DIM,
# The implementation also accepts this legacy fp8 alias.
indexer_kv_dtype="fp8_e4m3",
max_model_len=8192,
)
monkeypatch.setattr(
indexer_aiter_mod, "_minimax_m3_aiter_sparse_pa_requested", lambda: False
)
reason = indexer_aiter_mod.aiter_indexer_unsupported_reason(**kwargs)
reason = unsupported_reason()
assert reason is not None and "AITER sparse PA attend" in reason
# With the attend asked for, the gate moves on to the kernel-contract
@@ -1792,7 +1805,7 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
monkeypatch.setattr(
indexer_aiter_mod, "_minimax_m3_aiter_sparse_pa_requested", lambda: True
)
reason = indexer_aiter_mod.aiter_indexer_unsupported_reason(**kwargs)
reason = unsupported_reason()
assert reason is None or "AITER sparse PA attend" not in reason
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Correctness tests for MiniMax M3 CUTLASS sparse decode."""
"""Correctness tests for MiniMax M3 MSA sparse attention (CUTLASS decode and
NVFP4 KV cache)."""
import math
from types import SimpleNamespace
@@ -8,6 +9,7 @@ from types import SimpleNamespace
import pytest
import torch
from tests.kernels.quantization.nvfp4_utils import dequant_nvfp4_kv_cache
from vllm import _custom_ops as ops
from vllm.config import AttentionConfig, CUDAGraphMode
from vllm.forward_context import ForwardContext, override_forward_context
@@ -18,6 +20,7 @@ from vllm.models.minimax_m3.common.sparse_attention import (
MiniMaxM3SparseBackend,
MiniMaxM3SparseMetadata,
MiniMaxM3SparseMetadataBuilder,
MiniMaxM3SparsePrefillMetadata,
MiniMaxM3SparseTritonImpl,
select_main_backend_and_impl_cls,
)
@@ -28,6 +31,7 @@ from vllm.models.minimax_m3.nvidia.model import MiniMaxM3SparseAttention
from vllm.models.minimax_m3.nvidia.msa_cutlass_sparse_decode import (
MSACutlassDecodePlanCache,
msa_cutlass_sparse_decode,
nvfp4_kv_cache_views,
prepare_decode_metadata,
should_prepare_decode_metadata,
)
@@ -36,6 +40,7 @@ from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
MiniMaxM3SparseMSADecodeMetadata,
MiniMaxM3SparseMSAImpl,
MiniMaxM3SparseMSAMetadataBuilder,
MiniMaxM3SparseMSANvfp4Backend,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -164,6 +169,53 @@ def test_msa_cutlass_decode_static_dispatch_requires_sm100(
)
@pytest.mark.parametrize("decode_backend", ["cutlass", "triton"])
def test_msa_cutlass_decode_static_dispatch_nvfp4_has_no_fallback(
decode_backend: str,
) -> None:
"""NVFP4 has no Triton decode, so it plans CUTLASS for any batch size."""
assert should_prepare_decode_metadata(
1,
DEFAULT_QUERY_LEN,
decode_backend=decode_backend, # type: ignore[arg-type]
num_q_heads=16,
num_kv_heads=1,
kv_cache_dtype="nvfp4",
page_size=BLOCK_SIZE,
topk_blocks=TOPK,
)
def test_nvfp4_selects_msa_nvfp4_backend_and_requires_sm100(
monkeypatch: pytest.MonkeyPatch,
) -> None:
backend_cls, impl_cls = select_main_backend_and_impl_cls(
topk_blocks=TOPK,
kv_cache_dtype="nvfp4",
num_kv_heads=1,
)
assert backend_cls is MiniMaxM3SparseMSANvfp4Backend
assert impl_cls is MiniMaxM3SparseMSAImpl
with pytest.raises(ValueError, match="nvfp4_4over6"):
select_main_backend_and_impl_cls(
topk_blocks=TOPK,
kv_cache_dtype="nvfp4_4over6",
num_kv_heads=1,
)
monkeypatch.setattr(
current_platform,
"is_device_capability_family",
lambda _: False,
)
with pytest.raises(ValueError, match="NVFP4"):
select_main_backend_and_impl_cls(
topk_blocks=TOPK,
kv_cache_dtype="nvfp4",
num_kv_heads=1,
)
@pytest.mark.parametrize(
("backend", "expected"),
[
@@ -387,7 +439,7 @@ def test_query_fp8_stays_valid_when_cutlass_plan_appears_on_replay(
assert capture.num_eager_breaks == 1
assert observed_ptrs == []
decode.msa_cutlass = object() # type: ignore[assignment]
decode.msa_cutlass = object()
for _ in range(3):
capture.replay()
stream.synchronize()
@@ -639,3 +691,391 @@ def test_msa_cutlass_decode_matches_triton_with_interleaved_cache(
graph.replay()
current_platform.synchronize()
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
def _make_paged_layout(
seq_lens_list: list[int],
) -> tuple[torch.Tensor, int]:
"""Block table over a shuffled physical page pool."""
pages_per_request = [math.ceil(seq_len / BLOCK_SIZE) for seq_len in seq_lens_list]
num_pages = sum(pages_per_request)
block_table = torch.zeros(
len(seq_lens_list), max(pages_per_request), dtype=torch.int32, device="cuda"
)
physical_pages = torch.randperm(num_pages, dtype=torch.int32, device="cuda")
offset = 0
for request, request_pages in enumerate(pages_per_request):
block_table[request, :request_pages] = physical_pages[
offset : offset + request_pages
]
offset += request_pages
return block_table, num_pages
def _make_nvfp4_kv_cache(
num_pages: int,
num_kv_heads: int,
kv_cache_dtype: str,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Write random K/V through the NVFP4 ``reshape_and_cache_flash`` writer,
whose pages hold every head's data then every head's scales per side, and
repack them into head slots.
Returns the packed HND cache ``[pages, 2 * Hkv, 128, 72]`` and its BF16
dequantization ``[pages, Hkv, 128, 2 * 128]`` for the Triton reference.
"""
num_slots = num_pages * BLOCK_SIZE
key = torch.randn(
num_slots, num_kv_heads, HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
value = torch.randn_like(key)
written = torch.zeros(
num_pages,
2 * num_kv_heads,
BLOCK_SIZE,
HEAD_DIM // 2 + HEAD_DIM // 16,
dtype=torch.uint8,
device="cuda",
)
k_cache, v_cache = written.transpose(1, 2).split(num_kv_heads, dim=-2)
ops.reshape_and_cache_flash(
key,
value,
k_cache,
v_cache,
torch.arange(num_slots, dtype=torch.int64, device="cuda"),
kv_cache_dtype,
k_scale,
v_scale,
)
kv_cache = torch.empty_like(written)
head_slot_views = iter(nvfp4_kv_cache_views(kv_cache))
dequantized = []
for side, global_scale, swizzled in (
(written[:, :num_kv_heads], k_scale, False),
(written[:, num_kv_heads:], v_scale, True),
):
data = side.as_strided(
(num_pages, num_kv_heads, BLOCK_SIZE, HEAD_DIM // 2),
(side.stride(0), BLOCK_SIZE * HEAD_DIM // 2, HEAD_DIM // 2, 1),
)
scale = side.as_strided(
(num_pages, num_kv_heads, BLOCK_SIZE, HEAD_DIM // 16),
(side.stride(0), BLOCK_SIZE * HEAD_DIM // 16, HEAD_DIM // 16, 1),
side.storage_offset() + num_kv_heads * BLOCK_SIZE * HEAD_DIM // 2,
)
next(head_slot_views).copy_(data)
next(head_slot_views).copy_(scale)
dequantized.append(
dequant_nvfp4_kv_cache(
data,
scale,
global_scale.item(),
HEAD_DIM,
BLOCK_SIZE,
swizzled_scales=swizzled,
)
)
return kv_cache, torch.cat(dequantized, dim=-1).to(torch.bfloat16)
def test_nvfp4_kv_cache_views_address_head_slots() -> None:
"""Slot 2h (K) and 2h + 1 (V) hold head h's data then scales, so each head
is one contiguous run of the page: a TP rank's page is a byte slice."""
num_kv_heads, slot_bytes = 4, BLOCK_SIZE * (HEAD_DIM // 2 + HEAD_DIM // 16)
kv_cache = torch.randint(
0,
256,
(3, 2 * num_kv_heads, BLOCK_SIZE, HEAD_DIM // 2 + HEAD_DIM // 16),
dtype=torch.uint8,
)
k_data, k_sf, v_data, v_sf = nvfp4_kv_cache_views(kv_cache)
runs = kv_cache.view(3, num_kv_heads, 2 * slot_bytes)
for head in range(num_kv_heads):
expected = torch.cat(
[v[:, head].flatten(1) for v in (k_data, k_sf, v_data, v_sf)], dim=1
)
assert torch.equal(runs[:, head], expected)
@pytest.mark.parametrize(
("num_q_heads", "num_kv_heads", "num_request_pairs", "query_len", "capture"),
[
pytest.param(64, 4, 1, 1, True, id="tp1-small-batch-query-len-1"),
pytest.param(64, 4, 8, 8, False, id="tp1-query-len-8"),
pytest.param(16, 1, 16, 4, True, id="tp4-query-len-4"),
],
)
def test_msa_cutlass_decode_nvfp4_matches_triton_on_dequantized_cache(
num_q_heads: int,
num_kv_heads: int,
num_request_pairs: int,
query_len: int,
capture: bool,
) -> None:
"""CUTLASS decode reads vLLM's packed NVFP4 pages, including swizzled V
scales and device global scales, as their exact dequantized values."""
torch.manual_seed(0)
seq_lens_list = [257, 513] * num_request_pairs
seq_lens_cpu = torch.tensor(seq_lens_list, dtype=torch.int32)
seq_lens = seq_lens_cpu.cuda()
block_table, num_pages = _make_paged_layout(seq_lens_list)
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
kv_cache, kv_reference = _make_nvfp4_kv_cache(
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
)
num_query_tokens = len(seq_lens_list) * query_len
query = torch.randn(
num_query_tokens, num_q_heads, HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
query_fp8 = query.to(torch.float8_e4m3fn)
topk = _make_topk(seq_lens_list, num_kv_heads, query_len)
expected = torch.empty_like(query)
minimax_m3_sparse_attn_decode(
query_fp8.to(torch.bfloat16),
kv_reference,
topk.transpose(0, 1),
block_table,
seq_lens,
num_kv_heads,
SM_SCALE,
expected,
query_len,
)
assert should_prepare_decode_metadata(
len(seq_lens_list),
query_len,
decode_backend="triton",
num_q_heads=num_q_heads,
num_kv_heads=num_kv_heads,
kv_cache_dtype="nvfp4",
page_size=BLOCK_SIZE,
topk_blocks=TOPK,
)
metadata = prepare_decode_metadata(
block_table,
seq_lens,
seq_lens_cpu,
query_len,
num_q_heads=num_q_heads,
num_kv_heads=num_kv_heads,
page_size=BLOCK_SIZE,
topk_blocks=TOPK,
)
actual = torch.zeros_like(query)
def run() -> None:
msa_cutlass_sparse_decode(
query_fp8,
kv_cache,
topk,
actual,
metadata,
scale=SM_SCALE,
q_scale_float=1.0,
k_scale_float=1.0,
v_scale_float=1.0,
k_scale=k_scale,
v_scale=v_scale,
)
if capture:
run()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
run()
actual.zero_()
graph.replay()
else:
run()
current_platform.synchronize()
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
@pytest.mark.parametrize(
("num_q_heads", "num_kv_heads"),
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
)
def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
num_q_heads: int,
num_kv_heads: int,
) -> None:
"""MSA prefill reads NVFP4 pages as their exact dequantized values,
including chunked prefill over cached context, and takes q from its fp8
copy (the only one the fused insert writes on this path)."""
torch.manual_seed(0)
q_lens = [300, 128, 77]
seq_lens_list = [300, 640, 1100]
block_table, num_pages = _make_paged_layout(seq_lens_list)
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
kv_cache, kv_reference = _make_nvfp4_kv_cache(
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
)
total_q = sum(q_lens)
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda")
context_lens = seq_lens - torch.tensor(q_lens, dtype=torch.int32, device="cuda")
cu_seqlens_q = torch.zeros(len(q_lens) + 1, dtype=torch.int32, device="cuda")
cu_seqlens_q[1:] = torch.tensor(q_lens, device="cuda").cumsum(0)
cu_seqlens_k = torch.zeros_like(cu_seqlens_q)
cu_seqlens_k[1:] = seq_lens.cumsum(0)
prefill = MiniMaxM3SparsePrefillMetadata(
cu_seqlens_q=cu_seqlens_q,
cu_seqlens_k=cu_seqlens_k,
seq_lens=seq_lens,
context_lens=context_lens,
block_table=block_table,
max_query_len=max(q_lens),
max_seq_len=max(seq_lens_list),
total_kv_blocks=sum(math.ceil(n / BLOCK_SIZE) for n in seq_lens_list),
)
metadata = MiniMaxM3SparseMetadata(
seq_lens=seq_lens,
max_seq_len=max(seq_lens_list),
slot_mapping=torch.empty(total_q, dtype=torch.int64, device="cuda"),
num_actual_tokens=total_q,
num_decodes=0,
num_decode_tokens=0,
num_prefills=len(q_lens),
num_prefill_tokens=total_q,
prefill=prefill,
)
# Every causally visible page (<= TOPK per query), ending with the local one.
topk = torch.full(
(total_q, num_kv_heads, TOPK), -1, dtype=torch.int32, device="cuda"
)
token = 0
for q_len, seq_len in zip(q_lens, seq_lens_list):
for local_q in range(q_len):
visible = (seq_len - q_len + local_q) // BLOCK_SIZE + 1
assert visible <= TOPK
topk[token, :, :visible] = torch.arange(visible, device="cuda")
token += 1
layer_name = "model.layers.0.self_attn.attn"
q_scale = 0.5
layer = SimpleNamespace(
layer_name=layer_name,
topk_indices_buffer=topk,
_q_scale_float=q_scale,
_k_scale=k_scale,
_v_scale=v_scale,
)
query_fp8 = (
torch.randn(
total_q, num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
/ q_scale
).to(torch.float8_e4m3fn)
query = query_fp8.to(torch.bfloat16) * q_scale
common = dict(
num_heads=num_q_heads,
head_size=HEAD_DIM,
scale=SM_SCALE,
num_kv_heads=num_kv_heads,
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
)
forward_context = ForwardContext(
no_compile_layers={},
attn_metadata={layer_name: metadata},
slot_mapping={},
)
with override_forward_context(forward_context):
expected = MiniMaxM3SparseTritonImpl(**common).forward(
layer, query, kv_reference, torch.empty_like(query)
)
actual = MiniMaxM3SparseMSAImpl(**common, kv_cache_dtype="nvfp4").forward(
layer,
torch.full_like(query, float("nan")),
kv_cache,
torch.zeros_like(query),
query_fp8=query_fp8,
)
current_platform.synchronize()
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
def test_msa_triton_decode_fallback_reads_dequantized_query_fp8() -> None:
"""With CUTLASS decode on but not planned for the batch, the Triton decode
fallback attends with the dequantized fp8 q, the only q written."""
torch.manual_seed(0)
num_q_heads, num_kv_heads, q_scale = 16, 1, 0.5
seq_lens_list = [257, 513, 129]
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda")
block_table, num_pages = _make_paged_layout(seq_lens_list)
kv_cache = (
torch.randn(
num_pages,
num_kv_heads,
BLOCK_SIZE,
2 * HEAD_DIM,
dtype=torch.bfloat16,
device="cuda",
)
* 0.25
).to(torch.float8_e4m3fn)
num_tokens = len(seq_lens_list)
metadata = MiniMaxM3SparseMetadata(
seq_lens=seq_lens,
max_seq_len=max(seq_lens_list),
slot_mapping=torch.empty(num_tokens, dtype=torch.int64, device="cuda"),
num_actual_tokens=num_tokens,
num_decodes=num_tokens,
num_decode_tokens=num_tokens,
num_prefills=0,
num_prefill_tokens=0,
decode=MiniMaxM3SparseMSADecodeMetadata(
seq_lens=seq_lens,
block_table=block_table,
decode_query_len=1,
msa_cutlass=None,
),
)
layer_name = "model.layers.0.self_attn.attn"
layer = SimpleNamespace(
layer_name=layer_name,
topk_indices_buffer=_make_topk(seq_lens_list, num_kv_heads, 1),
_q_scale_float=q_scale,
)
query_fp8 = torch.randn(
num_tokens, num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda"
).to(torch.float8_e4m3fn)
query = query_fp8.to(torch.bfloat16) * q_scale
common = dict(
num_heads=num_q_heads,
head_size=HEAD_DIM,
scale=SM_SCALE,
num_kv_heads=num_kv_heads,
kv_cache_dtype="fp8",
topk_blocks=TOPK,
sparse_block_size=BLOCK_SIZE,
)
impl = MiniMaxM3SparseMSAImpl(**common, msa_decode_backend="cutlass")
assert impl.use_cutlass_decode
forward_context = ForwardContext(
no_compile_layers={},
attn_metadata={layer_name: metadata},
slot_mapping={},
)
with override_forward_context(forward_context):
expected = MiniMaxM3SparseTritonImpl(**common).forward(
layer, query, kv_cache, torch.empty_like(query)
)
actual = impl.forward(
layer,
torch.full_like(query, float("nan")),
kv_cache,
torch.zeros_like(query),
query_fp8=query_fp8,
)
current_platform.synchronize()
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
@@ -783,6 +783,9 @@ def test_aiter_fa_shared_kv_matches_reference(
32 * BLOCK_SIZE, num_kv_heads, head_size, dtype=torch.bfloat16
)
value = torch.randn_like(key)
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionImpl
assert isinstance(target.impl, AiterFlashAttentionImpl)
target.impl.do_kv_cache_update(
target,
key,
@@ -82,6 +82,7 @@ def _run_verify_block():
)
from vllm.config import SpeculativeConfig
from vllm.config.vllm import set_current_vllm_config
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.workspace import init_workspace_manager
@@ -106,7 +107,7 @@ def _run_verify_block():
vllm_config.speculative_config = SpeculativeConfig(
method="ngram", num_speculative_tokens=QLEN - 1
)
vllm_config.model_config.get_num_attention_heads = types.MethodType(
vllm_config.model_config.get_num_attention_heads = types.MethodType( # type: ignore[method-assign]
lambda self, parallel_config, arch_config=None: NUM_QUERY_HEADS,
vllm_config.model_config,
)
@@ -202,6 +203,7 @@ def _run_verify_block():
lambda: spy,
),
):
assert isinstance(impl, AiterMLAImpl)
impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None)
return metadata, captured
@@ -167,7 +167,10 @@ def test_sparse_mla_sink_matches_ragged_reference(
impl.sinks = sinks
output, lse = impl._forward_mla(
SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale), q, kv, metadata
SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale),
q,
kv,
metadata,
)
kv_flat = kv_ref[:, 0]
references = []
@@ -0,0 +1,599 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DeepSeek V4.1's ROCm MXFP4 sparse indexer against a torch reference.
The indexer K cache is a view into vLLM's block-major KV pool: its pages are
a block stride apart with other layers' pages in between, so the tests put
it there. The reference reads the keys back from a second cache the same
writer fills in natural order, so it shares none of the preshuffled
addressing it checks.
"""
import itertools
import types
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_rocm():
pytest.skip("ROCm-only", allow_module_level=True)
from vllm.config import CUDAGraphMode
from vllm.models.deepseek_v41.amd.rocm import (
rocm_mxfp4_indexer_k_store,
rocm_mxfp4_indexer_q_quant,
)
from vllm.platforms.rocm import get_cdna_version
from vllm.utils.math_utils import cdiv
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerPrefillChunkMetadata,
)
from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import (
RocmMxfp4IndexerMetadata,
native_decode,
plan_gather_launches,
plan_prefill_chunks,
)
from vllm.v1.attention.ops import rocm_paged_mxfp4_indexer as ops
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
if get_cdna_version() != 4:
pytest.skip("CDNA4-only", allow_module_level=True)
HEADS, HEAD_DIM = 32, 128
WIDTH = HEAD_DIM // 2 + HEAD_DIM // 32
# V4.1's 8-token blocks, with the pool (2048 blocks) and top-k (512) scaled
# down so short test contexts still select.
CAND_BLOCK, POOL_BLOCKS, TOPK = 8, 16, 32
MAX_LEN = 4096
DEVICE = "cuda"
_E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
E2M1 = torch.cat([_E2M1, -_E2M1])
def _dequant(packed, e8m0):
"""[..., D/2] e2m1 pairs and [..., D/32] e8m0 to [..., D] fp32."""
table = E2M1.to(packed.device)
vals = torch.stack([table[(packed & 0xF).long()], table[(packed >> 4).long()]], -1)
scale = torch.pow(2.0, e8m0.float() - 127)
return (vals.flatten(-2).unflatten(-1, (-1, 32)) * scale[..., None]).flatten(-2)
@pytest.fixture(autouse=True)
def _workspace():
init_workspace_manager(torch.device(DEVICE))
yield
reset_workspace_manager()
class _Case:
"""Requests with their keys written into a block-major pool (ratio-2 and
ratio-1 pages side by side in every block) and into natural-order caches
the reference reads. The bf16 inputs are kept for the end-to-end test."""
def __init__(self, seq_lens, block):
from aiter.ops.triton.fusions.k_norm_rope_mxfp4_cache import (
k_norm_rope_mxfp4_cache,
)
self.seq_lens = seq_lens
num_blocks = sum(cdiv(n, block) for n in seq_lens) + 3
perm = torch.randperm(num_blocks)
width = max(cdiv(n, block) for n in seq_lens)
self.block_table = torch.zeros(len(seq_lens), width, dtype=torch.int32)
used = 0
for req, n in enumerate(seq_lens):
self.block_table[req, : cdiv(n, block)] = perm[used : used + cdiv(n, block)]
used += cdiv(n, block)
self.block_table = self.block_table.to(DEVICE)
def align(n):
return cdiv(n, 576) * 576
page2, page1 = block // 2 * WIDTH, block * WIDTH
stride = align(page2) + align(page1) + 576
self.pool = torch.zeros(num_blocks * stride, dtype=torch.uint8, device=DEVICE)
self.page_bytes = torch.zeros_like(self.pool, dtype=torch.bool)
for i in range(num_blocks):
base = i * stride
self.page_bytes[base : base + page2] = True
base += align(page2)
self.page_bytes[base : base + page1] = True
self.cache = {
2: torch.as_strided(
self.pool, (num_blocks, block // 2, WIDTH), (stride, WIDTH, 1), 0
),
1: torch.as_strided(
self.pool,
(num_blocks, block, WIDTH),
(stride, WIDTH, 1),
align(page2),
),
}
self.offsets = [0]
for n in seq_lens:
self.offsets.append(self.offsets[-1] + n)
# the same keys in natural order, one entry per page
self.natural = {
r: torch.zeros(self.offsets[-1], 1, WIDTH, dtype=torch.uint8, device=DEVICE)
for r in (1, 2)
}
self.cos_sin = cos_sin = torch.randn(MAX_LEN, 64, device=DEVICE)
self.norm = norm = torch.rand(HEAD_DIM, device=DEVICE) + 0.5
self.k_pre = {}
for req, n in enumerate(seq_lens):
if n == 0:
continue
k_pre = torch.randn(n, HEAD_DIM, device=DEVICE).to(torch.bfloat16)
self.k_pre[req] = k_pre
pos = torch.arange(n, device=DEVICE)
for ratio, cache in self.cache.items():
comp = pos // ratio
boundary = (pos + 1) % ratio == 0
entries = cache.shape[1]
page = self.block_table[req, comp // entries].long()
slot = torch.where(boundary, page * entries + comp % entries, -1)
nat = torch.where(boundary, self.offsets[req] + comp, -1)
rocm_mxfp4_indexer_k_store(
k_pre,
pos,
cos_sin,
norm,
1e-6,
cache,
slot,
ratio,
True,
num_heads=HEADS,
)
k_norm_rope_mxfp4_cache(
k_pre, pos, cos_sin, norm, 1e-6, self.natural[ratio], nat, ratio
)
def keys(self, ratio, req, n):
nat = self.natural[ratio][self.offsets[req] : self.offsets[req] + n, 0]
return _dequant(nat[:, : HEAD_DIM // 2], nat[:, HEAD_DIM // 2 :])
def _queries(num_rows):
q = torch.randint(0, 256, (num_rows, HEADS, HEAD_DIM // 2), dtype=torch.uint8)
scales = torch.randint(122, 130, (num_rows, HEADS, HEAD_DIM // 32))
q, scales = q.to(DEVICE), scales.to(torch.uint8).to(DEVICE)
weights = torch.randn(num_rows, HEADS, device=DEVICE)
# the fused Q kernel returns the scales as one int32 per head
return q, scales.view(torch.int32).squeeze(-1), weights, _dequant(q, scales)
def _scores(q, w, keys):
return (torch.relu(q @ keys.T) * w[:, None]).sum(0)
def _assert_topk(row, scores, k):
"""``row`` is a top-k of the entries of ``scores`` above -inf, up to the
fp32 reduce order."""
valid = scores > float("-inf")
want = min(k, int(valid.sum()))
sel = row[row >= 0].long()
assert sel.numel() == want and sel.unique().numel() == want
if want:
assert bool(valid[sel].all())
kth = scores[valid].topk(want).values[-1]
finite = scores[torch.isfinite(scores)]
tol = 1e-4 * float(finite.abs().max()) if finite.numel() else 0.0
assert bool((scores[sel] >= kth - tol).all())
def _block_maxima(scores):
end = scores.numel()
if end == 0:
return scores
padded = torch.full((cdiv(end, CAND_BLOCK) * CAND_BLOCK,), float("-inf"))
padded[:end] = scores.cpu()
out = padded.view(-1, CAND_BLOCK).amax(1)
out[(end - 1) // CAND_BLOCK] = float("inf")
return out.to(scores.device)
def _forward_context(monkeypatch, metadata):
context = types.SimpleNamespace(
attn_metadata={"indexer": metadata},
cudagraph_runtime_mode=CUDAGraphMode.NONE,
)
monkeypatch.setattr(ops, "get_forward_context", lambda: context)
def _decode_metadata(case, rows, ratio, query_lens):
lens = torch.tensor([(p + 1) // ratio for _, p in rows], dtype=torch.int32)
lens = lens.to(DEVICE)
block_table = case.block_table[[req for req, _ in rows]].contiguous()
entries = case.cache[ratio].shape[1]
next_n = max(query_lens)
context_lens = (torch.tensor(case.seq_lens, dtype=torch.int32) // ratio).to(DEVICE)
native = native_decode(lens, block_table, query_lens, next_n, context_lens)
schedule = torch.empty(
ops.rocm_mxfp4_decode_schedule_words(HEADS, HEAD_DIM, entries, next_n),
dtype=torch.int32,
device=DEVICE,
)
return RocmMxfp4IndexerMetadata(
seq_lens=None,
max_seq_len=max(case.seq_lens),
slot_mapping=None,
num_decodes=len(case.seq_lens),
num_decode_tokens=len(rows),
num_prefills=0,
num_prefill_tokens=0,
decode=types.SimpleNamespace(block_table=block_table, seq_lens=lens[:, None]),
decode_row_lens=lens,
decode_block_ends=(lens + CAND_BLOCK - 1) // CAND_BLOCK,
decode_native=native,
decode_schedule=ops.build_rocm_mxfp4_decode_schedule(
lens, HEADS, HEAD_DIM, entries, schedule, MAX_LEN // ratio, native
),
)
def _prefill_metadata(
case, rows, ratio, chunk_bounds, query_start_loc, gather_rows, gather
):
chunks = []
for req_lo, req_hi, t0, t1 in chunk_bounds:
ends = torch.tensor(
[(rows[t][1] + 1) // ratio for t in range(t0, t1)], dtype=torch.int32
).to(DEVICE)
chunks.append(
DeepseekV32IndexerPrefillChunkMetadata(
block_table=case.block_table[req_lo:req_hi],
cu_seqlen_ks=torch.zeros_like(ends),
cu_seqlen_ke=ends,
cu_seq_lens=torch.zeros(1, dtype=torch.int32, device=DEVICE),
token_to_seq=torch.zeros(1, dtype=torch.int32, device=DEVICE),
total_seq_lens=1,
token_start=t0,
token_end=t1,
num_reqs=req_hi - req_lo,
)
)
seq_lens = torch.tensor(case.seq_lens)
context_lens = (seq_lens // ratio).int().to(DEVICE)
plans = plan_prefill_chunks(
chunks,
query_start_loc,
seq_lens,
context_lens,
ratio,
CAND_BLOCK,
0.0 if gather else None,
torch.tensor(query_start_loc, dtype=torch.int32, device=DEVICE),
)
return RocmMxfp4IndexerMetadata(
seq_lens=None,
max_seq_len=max(case.seq_lens),
slot_mapping=None,
num_decodes=0,
num_decode_tokens=0,
num_prefills=len(case.seq_lens),
num_prefill_tokens=len(rows),
prefill=types.SimpleNamespace(chunks=chunks),
prefill_plans=plans,
gather_launches=plan_gather_launches(chunks, plans, gather_rows),
)
# The kernel prefers 128-token pages.
BLOCKS = pytest.mark.parametrize("block", [128, 64])
@BLOCKS
def test_k_store_stays_inside_its_pages(block):
"""The writer touches only its pages of the block-major pool. What they hold
is checked by the layer tests, which read them back through aiter's kernel."""
set_random_seed(0)
case = _Case([700, 333], block)
assert int(case.pool[~case.page_bytes].count_nonzero()) == 0
@pytest.mark.parametrize(
"rows, width, k",
# vLLM's kernel below 128 rows, aiter's from 128 rows and on every k=2048
[(8, 4096, 512), (128, 4096, 512), (8, 16384, 2048)],
)
def test_topk_matches_torch_on_both_kernels(rows, width, k):
set_random_seed(0)
logits = torch.randn(rows, width, device=DEVICE)
lengths = torch.randint(1, width + 1, (rows,), dtype=torch.int32, device=DEVICE)
lengths[0] = k // 2 # shorter than k, so the rest of the row pads with -1
out = torch.full((rows, k), 7, dtype=torch.int32, device=DEVICE)
ops._topk(logits, lengths, out, k)
for r in range(rows):
n = min(k, int(lengths[r]))
want = torch.topk(logits[r, : int(lengths[r])], n).indices.sort().values
got = out[r]
assert torch.equal(got[got >= 0].sort().values, want.int())
assert int((got == -1).sum()) == k - n
def _run_layers(monkeypatch, case, rows, metadata):
"""Ratio-2 dense layer, the candidate source, and a consumer both ways;
each checked against the torch reference."""
num_rows = len(rows)
q, q_scale, weights, q_ref = _queries(num_rows)
hidden = torch.empty(num_rows, 1, device=DEVICE)
def topk_buffer():
return torch.full((num_rows + 2, TOPK), 7, dtype=torch.int32, device=DEVICE)
_forward_context(monkeypatch, metadata(2))
out = topk_buffer()
ops.rocm_mxfp4_sparse_attn_indexer(
hidden,
"indexer",
case.cache[2],
q,
q_scale,
weights,
TOPK,
HEAD_DIM,
MAX_LEN // 2,
out,
compress_ratio=2,
)
for i, (req, pos) in enumerate(rows):
end = (pos + 1) // 2
keys = case.keys(2, req, end)
_assert_topk(out[i], _scores(q_ref[i], weights[i], keys), TOPK)
source = metadata(1)
_forward_context(monkeypatch, source)
pool = torch.full((num_rows + 2, POOL_BLOCKS), 5, dtype=torch.int32, device=DEVICE)
out = topk_buffer()
ops.rocm_mxfp4_sparse_attn_indexer(
hidden,
"indexer",
case.cache[1],
q,
q_scale,
weights,
TOPK,
HEAD_DIM,
MAX_LEN,
out,
compress_ratio=1,
candidate_blocks=pool,
candidate_block_size=CAND_BLOCK,
candidate_write=True,
)
allowed = []
for i, (req, pos) in enumerate(rows):
scores = _scores(q_ref[i], weights[i], case.keys(1, req, pos + 1))
_assert_topk(out[i], scores, TOPK)
_assert_topk(pool[i], _block_maxima(scores), POOL_BLOCKS)
mask = torch.zeros_like(scores, dtype=torch.bool)
for block in pool[i][pool[i] >= 0].tolist():
mask[block * CAND_BLOCK : (block + 1) * CAND_BLOCK] = True
allowed.append(torch.where(mask, scores, float("-inf")))
pa = ops._aiter()
build = pa.build_candidate_gather
builds = []
def counting_build(*args, **kwargs):
builds.append(1)
return build(*args, **kwargs)
monkeypatch.setattr(pa, "build_candidate_gather", counting_build)
for gather in (True, False):
consumer = metadata(1, gather)
consumer.decode_use_gather = gather
_forward_context(monkeypatch, consumer)
builds.clear()
# Two consumer layers of one step: the second reuses the first's pools.
for _ in range(2):
out = topk_buffer()
ops.rocm_mxfp4_sparse_mqa_indexer(
hidden,
"indexer",
case.cache[1],
q,
q_scale,
weights,
TOPK,
HEAD_DIM,
MAX_LEN,
out,
1,
pool,
CAND_BLOCK,
POOL_BLOCKS * CAND_BLOCK,
)
for i in range(num_rows):
_assert_topk(out[i], allowed[i], TOPK)
if gather:
launches = len(consumer.gather_launches) + (consumer.decode is not None)
assert len(builds) == launches
@BLOCKS
@pytest.mark.parametrize(
"query_lens",
# Uniform steps launch the dense layers on next_n-row sequences, also with
# cudagraph padding (query length 0) after them; a ragged step keeps a row
# per token.
[[6, 6, 6, 6], [2, 2, 2], [2, 2, 2, 0], [2, 1, 2]],
ids=["native6", "native2", "padded", "ragged"],
)
def test_decode_layers_match_reference(monkeypatch, block, query_lens):
set_random_seed(0)
next_n = max(query_lens)
case = _Case([q and n for q, n in zip(query_lens, [900, 333, 610, 1200])], block)
# a padding request has next_n rows of its own and no context
rows = [
(req, n - q + j if q else -1)
for req, (n, q) in enumerate(zip(case.seq_lens, query_lens))
for j in range(q or next_n)
]
_run_layers(
monkeypatch,
case,
rows,
lambda r, gather=False: _decode_metadata(case, rows, r, query_lens),
)
@pytest.mark.parametrize(
"seq_lens,new_tokens,chunks",
[
# one request past a cached prefix, then one sliced into two chunks
([760, 500], [300, 500], [(0, 1, 0, 300), (1, 2, 300, 560), (1, 2, 560, 800)]),
# three requests in one chunk, the first two with equal rows
([600, 450, 900], [200, 200, 120], [(0, 3, 0, 520)]),
# four requests with four different query lengths in one chunk
([900, 600, 450, 700], [150, 90, 200, 60], [(0, 4, 0, 500)]),
],
ids=["sliced", "batched", "ragged"],
)
# The consumers rejoin a request's rows across chunks, then cut them at most
# gather_rows a launch.
@pytest.mark.parametrize("gather_rows", [1 << 20, 128], ids=["joined", "cut"])
@BLOCKS
def test_prefill_layers_match_reference(
monkeypatch, seq_lens, new_tokens, chunks, gather_rows, block
):
set_random_seed(0)
case = _Case(seq_lens, block)
rows = [
(req, n - q + i)
for req, (n, q) in enumerate(zip(seq_lens, new_tokens))
for i in range(q)
]
query_start_loc = [0, *itertools.accumulate(new_tokens)]
_run_layers(
monkeypatch,
case,
rows,
lambda r, gather=True: _prefill_metadata(
case, rows, r, chunks, query_start_loc, gather_rows, gather
),
)
def _rope_gptj(x, cos_sin, positions):
"""GPT-J RoPE on the last 64 dims, as the indexer rotates Q and K: bf16 in
and out."""
x = x.bfloat16().float()
cos, sin = cos_sin[positions].chunk(2, dim=-1)
if x.dim() == 3:
cos, sin = cos[:, None], sin[:, None]
rot = x[..., -64:]
even, odd = rot[..., 0::2].clone(), rot[..., 1::2].clone()
rot[..., 0::2] = even * cos - odd * sin
rot[..., 1::2] = odd * cos + even * sin
return x.bfloat16()
def _mxfp4_round_trip(x):
"""MXFP4 round trip of x, rounded as the writers do: a scale of
2^ceil(log2(amax / 6)) per 32 values, then the nearest e2m1 value, ties to
even."""
blocks = x.float().unflatten(-1, (-1, 32))
amax = blocks.abs().amax(-1, keepdim=True).clamp(min=2**-126)
scale = torch.exp2(torch.ceil(torch.log2(amax / 6)))
y = (blocks / scale).clamp(-6, 6)
# Even codes first: argmin keeps the first of two equal distances.
grid = torch.cat([_E2M1[0::2], _E2M1[1::2]]).to(x.device)
idx = (y.abs()[..., None] - grid).abs().argmin(-1)
return (grid[idx] * y.sign() * scale).flatten(-2)
def _torch_keys(case, ratio, req):
"""A request's compressed keys in torch: k_norm of each group's last
token, rotated at the group's first position, through MXFP4."""
k = case.k_pre[req][ratio - 1 :: ratio].float()
k = k * torch.rsqrt(k.pow(2).mean(-1, keepdim=True) + 1e-6) * case.norm
pos = torch.arange(k.shape[0], device=DEVICE) * ratio
return _mxfp4_round_trip(_rope_gptj(k, case.cos_sin, pos))
def _topk_indices(q, weights, keys, ends, k):
"""Top-k of sum_h relu(q_h . key) * weights_h over each row's [0, end)."""
scores = (torch.einsum("thd,cd->thc", q, keys).relu() * weights[..., None]).sum(1)
cols = torch.arange(keys.shape[0], device=DEVICE)
return scores.masked_fill(cols >= ends[:, None], float("-inf")).topk(k).indices
def _recall(selected, reference):
"""Per row, the fraction of the reference top-k that ``selected`` picked."""
return (selected[:, :, None] == reference[:, None, :]).any(1).float().mean(1)
@BLOCKS
@pytest.mark.parametrize("ratio", [2, 1])
@pytest.mark.parametrize("step", ["decode", "prefill"])
def test_selection_from_bf16_matches_torch(monkeypatch, step, ratio, block):
"""End to end from bf16 K and Q: vLLM's MXFP4 K store and Q quant, the
paged pool, metadata from the planning functions the builder calls, and
the indexer op pick what the same MXFP4 math picks in torch."""
set_random_seed(0)
if step == "decode":
case = _Case([3000, 2600, 1900, 3500, 1200, 2200, 3900, 1600], block)
rows = [(req, n - 1) for req, n in enumerate(case.seq_lens)]
metadata = _decode_metadata(case, rows, ratio, [1] * len(rows))
else:
new_tokens = [300, 200, 250]
case = _Case([1500, 900, 2000], block)
rows = [
(req, n - q + i)
for req, (n, q) in enumerate(zip(case.seq_lens, new_tokens))
for i in range(q)
]
metadata = _prefill_metadata(
case,
rows,
ratio,
[(0, len(new_tokens), 0, len(rows))],
[0, *itertools.accumulate(new_tokens)],
1 << 20,
gather=False,
)
num_rows = len(rows)
q_bf16 = torch.randn(num_rows, HEADS, HEAD_DIM, device=DEVICE).to(torch.bfloat16)
w = torch.randn(num_rows, HEADS, device=DEVICE)
pos = torch.tensor([p for _, p in rows], device=DEVICE)
scales = (HEAD_DIM**-0.5, HEADS**-0.5)
(q, q_scale), weights = rocm_mxfp4_indexer_q_quant(
pos, q_bf16, case.cos_sin, w, *scales
)
_forward_context(monkeypatch, metadata)
out = torch.full((num_rows, TOPK), 7, dtype=torch.int32, device=DEVICE)
ops.rocm_mxfp4_sparse_attn_indexer(
torch.empty(num_rows, 1, device=DEVICE),
"indexer",
case.cache[ratio],
q,
q_scale,
weights,
TOPK,
HEAD_DIM,
MAX_LEN // ratio,
out,
compress_ratio=ratio,
)
ends = (pos + 1) // ratio
assert bool(((out >= 0) & (out < ends[:, None])).all())
assert all(row.unique().numel() == TOPK for row in out)
q_ref = _mxfp4_round_trip(_rope_gptj(q_bf16, case.cos_sin, pos))
w_ref = w * scales[0] * scales[1]
agree = []
for req in range(len(case.seq_lens)):
mine = torch.tensor([r == req for r, _ in rows], device=DEVICE)
ref = _topk_indices(
q_ref[mine], w_ref[mine], _torch_keys(case, ratio, req), ends[mine], TOPK
)
agree.append(_recall(out[mine], ref))
assert torch.cat(agree).mean().item() >= 0.99
@@ -153,6 +153,7 @@ def test_triton_unified_attn_diffkv_vs_reference(
else:
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func
assert fa_version is not None
# FA's TMA path needs aligned singleton strides (num_kv_heads == 1).
fa_k = canonicalize_singleton_dim_strides(key_cache)
fa_v = canonicalize_singleton_dim_strides(value_cache)
+1 -1
View File
@@ -79,7 +79,7 @@ class TestConfigSet:
config_set = ConfigSet("test_kernel")
with pytest.raises(KeyError, match="platform 'h100' not found"):
config_set.get_config("h100", "nonexistent")
config_set.get_config("h100", CaseKey({"batch": 32}))
config_data = {"num_warps": 8, "num_stages": 4}
data = {
@@ -81,7 +81,7 @@ class TestDynamicPerTokenScaledFp8QuantConfigPicker:
assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32})
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096)
selected_key = pick_config(args, config_keys)
@@ -127,7 +127,7 @@ class TestFusedQkNormRopeConfigPicker:
)
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
@@ -38,6 +38,7 @@ from torch._inductor.pattern_matcher import (
)
from torch.fx.experimental.proxy_tensor import make_fx
from vllm.kernels.helion.case_key import CaseKey
from vllm.kernels.helion.config_manager import ConfigManager
from vllm.kernels.helion.register import HelionKernelWrapper
@@ -45,7 +46,7 @@ from vllm.kernels.helion.register import HelionKernelWrapper
@contextlib.contextmanager
def _helion_mock_context():
configs = {
"default": helion.Config(block_sizes=[64], num_warps=2, num_stages=2),
CaseKey.default(): helion.Config(block_sizes=[64], num_warps=2, num_stages=2),
}
mock_config_manager = Mock(spec=ConfigManager)
mock_config_manager.get_platform_configs = Mock(return_value=configs)
@@ -88,7 +89,7 @@ class TestMakeFxHop:
raw_kernel_func=raw_add_scale,
op_name="test_make_fx",
fake_impl=lambda *a, **kw: None,
config_picker=lambda args, keys: "default",
config_picker=lambda args, keys: CaseKey.default(),
)
def fn(x, y):
@@ -145,7 +146,7 @@ class TestMakeFxHop:
raw_kernel_func=raw_silu_mul,
op_name="test_pm_silu_mul",
fake_impl=lambda *a, **kw: None,
config_picker=lambda args, keys: "default",
config_picker=lambda args, keys: CaseKey.default(),
)
def pattern(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
@@ -107,7 +107,7 @@ class TestPerTokenGroupFp8QuantConfigPicker:
)
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
+1
View File
@@ -397,6 +397,7 @@ class TestHelionKernelWrapper:
)
assert wrapper._disabled is True
assert wrapper._disabled_reason is not None
assert "No configs available" in wrapper._disabled_reason
def test_disabled_wrapper_raises_on_call(self, sample_kernel):
@@ -89,7 +89,7 @@ class TestRmsNormDynamicPerTokenQuantConfigPicker:
assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32})
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096)
selected_key = pick_config(args, config_keys)
@@ -112,7 +112,7 @@ class TestRmsNormPerBlockQuantConfigPicker:
)
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
@@ -96,7 +96,7 @@ class TestSiluAndMulPerBlockQuantConfigPicker:
)
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
args = _generate_fake_input(16, 4096, 128)
selected_key = pick_config(args, config_keys)
+2 -1
View File
@@ -83,7 +83,7 @@ class TestSiluMulFp8ConfigPicker:
assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 256})
def test_config_picker_no_configs(self):
config_keys: list[dict] = []
config_keys: list[CaseKey] = []
input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda")
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
@@ -355,6 +355,7 @@ class TestSiluMulFp8Integration:
registered_kernels = get_registered_kernels()
kernel_wrapper = registered_kernels["silu_mul_fp8"]
fake_impl = kernel_wrapper._fake_impl
assert fake_impl is not None
fake_output = fake_impl(input_tensor, scale)
+154 -102
View File
@@ -494,8 +494,10 @@ def _sd_conv_states(
return storage.transpose(1, 2)
def _maybe_pack_conv_weight(weight: torch.Tensor, is_vnni: bool) -> torch.Tensor:
return ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
def _maybe_pack_conv_weight(
weight: torch.Tensor, is_weight_packed: bool
) -> torch.Tensor:
return ops.causal_conv1d_weight_pack(weight) if is_weight_packed else weight
@torch.inference_mode()
@@ -523,7 +525,7 @@ def test_causal_conv1d_update_cpu_preserves_sd_state(
bias=None,
silu_activation=False,
conv_state_indices=state_indices,
is_vnni=False,
is_weight_packed=False,
)
assert out is x
@@ -666,7 +668,10 @@ def test_spec_aware_nonspec_materializes_state_indices(
@torch.inference_mode()
@pytest.mark.parametrize(("has_amx", "has_arm_bf16"), [(True, False), (False, True)])
def test_spec_forward_prepares_native_conv_metadata(
has_amx: bool,
has_arm_bf16: bool,
monkeypatch: pytest.MonkeyPatch,
) -> None:
block_table = torch.tensor([[0, 1], [4, 5]], dtype=torch.int32)
@@ -687,14 +692,17 @@ def test_spec_forward_prepares_native_conv_metadata(
)
forwarded_indices = None
forwarded_counts = None
forwarded_is_weight_packed = None
def causal_conv1d_update_cpu(**kwargs):
nonlocal forwarded_counts, forwarded_indices
nonlocal forwarded_counts, forwarded_indices, forwarded_is_weight_packed
forwarded_indices = kwargs["conv_state_indices"]
forwarded_counts = kwargs["num_accepted_tokens"]
forwarded_is_weight_packed = kwargs["is_weight_packed"]
return kwargs["x"]
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True)
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: has_amx)
monkeypatch.setattr(gdn_attention, "is_arm_bf16", lambda: has_arm_bf16)
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
monkeypatch.setattr(
gdn_attention.ops, "causal_conv1d_update_cpu", causal_conv1d_update_cpu
@@ -732,6 +740,7 @@ def test_spec_forward_prepares_native_conv_metadata(
assert actual is not None
assert actual.is_contiguous()
assert actual.dtype == torch.int32
assert forwarded_is_weight_packed is True
torch.testing.assert_close(actual, reference)
@@ -792,15 +801,15 @@ def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> No
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
reason="native causal conv1d requires AMX or Arm BF16 support",
)
@torch.inference_mode()
def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
state_len = CONV_KERNEL - 1
wide_state_len = state_len + 5
batch_size = 3
is_vnni = True
is_weight_packed = True
x, weight, bias = _conv_inputs(batch_size)
conv_state_indices = torch.tensor([2, 0, 1], dtype=torch.int32)
@@ -813,7 +822,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
wide_state[:, :, state_len:].fill_(7)
wide_tail = wide_state[:, :, state_len:].clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
out_narrow = ops.causal_conv1d_update_cpu(
x=x,
conv_states=narrow_state,
@@ -821,7 +830,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
bias=bias,
silu_activation=True,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
out_wide = ops.causal_conv1d_update_cpu(
x=x,
@@ -830,7 +839,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
bias=bias,
silu_activation=True,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
@@ -872,11 +881,18 @@ def _ref_causal_conv1d_update_cpu_multi(
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
reason="native causal conv1d requires AMX or Arm BF16 support",
)
@pytest.mark.parametrize(
("batch_size, seq_len, accepted_counts, has_bias, silu_activation, is_vnni"),
(
"batch_size, "
"seq_len, "
"accepted_counts, "
"has_bias, "
"silu_activation, "
"is_weight_packed"
),
[
(1, 1, [1], False, False, False),
(1, 1, [1], True, True, True),
@@ -893,7 +909,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
accepted_counts: list[int],
has_bias: bool,
silu_activation: bool,
is_vnni: bool,
is_weight_packed: bool,
) -> None:
dim = 96
state_len = seq_len + 2
@@ -911,7 +927,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
)
conv_states = conv_states_ref.clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
out = ops.causal_conv1d_update_cpu(
x=x,
conv_states=conv_states,
@@ -919,7 +935,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
bias=bias,
silu_activation=silu_activation,
conv_state_indices=conv_state_indices,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
num_accepted_tokens=num_accepted_tokens,
)
ref_out = _ref_causal_conv1d_update_cpu_multi(
@@ -937,8 +953,8 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
reason="native causal conv1d requires AMX or Arm BF16 support",
)
@pytest.mark.parametrize("num_accepted", [0, 17])
@torch.inference_mode()
@@ -961,14 +977,15 @@ def test_causal_conv1d_update_cpu_rejects_invalid_accepted_count(
bias=None,
silu_activation=True,
conv_state_indices=torch.tensor([0], dtype=torch.int32),
is_vnni=False,
is_weight_packed=False,
num_accepted_tokens=torch.tensor([num_accepted], dtype=torch.int32),
)
@pytest.mark.skipif(
not torch.cpu._is_avx512_bf16_supported(),
reason="causal_conv1d_fwd_cpu requires AVX-512BF16 (Intel Xeon or AMD EPYC)",
not (torch.cpu._is_avx512_bf16_supported() or gdn_attention.is_arm_bf16()),
reason="causal_conv1d_fwd_cpu requires AVX-512BF16"
" (Intel Xeon or AMD EPYC) or Arm BF16 support",
)
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
@torch.inference_mode()
@@ -977,13 +994,13 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
matches the single-call result.
Regression test for ``causal_conv1d_fwd_varlen_kernel_impl`` (``conv.cpp``)
ignoring the carried conv state on continued chunks. Runs on any
AVX-512BF16 CPU since conv.cpp uses VDPBF16PS, not AMX tiles.
ignoring the carried conv state on continued chunks. Runs on aarch64 cpus with
bf16 support or any AVX-512BF16 CPU since conv.cpp uses VDPBF16PS, not AMX tiles.
"""
state_len = CONV_KERNEL - 1
x, weight, bias = _conv_inputs(total_tokens)
def amx(x_seg, conv_states, has_init):
def native(x_seg, conv_states, has_init):
seq = x_seg.shape[0]
return ops.causal_conv1d_fwd_cpu(
x=x_seg.transpose(0, 1), # [dim, seq]; stride(-2)==1 (view of [seq,dim])
@@ -994,30 +1011,30 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
cache_indices=torch.tensor([0], dtype=torch.int32),
has_initial_state=torch.tensor([has_init]),
silu_activation=True,
is_vnni=False,
is_weight_packed=False,
).contiguous()
# conv_state layout passed by the AMX branch: [num_slots, dim, state_len].
cs_full = torch.zeros(1, CONV_DIM, state_len, dtype=x.dtype)
out_full = amx(x, cs_full, False)
out_full = native(x, cs_full, False)
cs_split = torch.zeros(1, CONV_DIM, state_len, dtype=x.dtype)
out1 = amx(x[:split], cs_split, False)
out2 = amx(x[split:], cs_split, True)
out1 = native(x[:split], cs_split, False)
out2 = native(x[split:], cs_split, True)
out_split = torch.cat([out1, out2], dim=1)
torch.testing.assert_close(out_split, out_full, atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not torch.cpu._is_amx_tile_supported(),
reason="requires AMX support",
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
reason="native causal conv1d requires AMX or Arm BF16 support",
)
@torch.inference_mode()
def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
state_len = CONV_KERNEL - 1
wide_state_len = state_len + 5
is_vnni = True
is_weight_packed = True
seq_lens = [CHUNK_SIZE - 1, CHUNK_SIZE + 5]
total_tokens = sum(seq_lens)
x, weight, bias = _conv_inputs(total_tokens)
@@ -1034,7 +1051,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
wide_state[:, :, state_len:].fill_(7)
wide_tail = wide_state[:, :, state_len:].clone()
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
out_narrow = ops.causal_conv1d_fwd_cpu(
x=x.transpose(0, 1),
weight=conv_weight,
@@ -1044,7 +1061,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
out_wide = ops.causal_conv1d_fwd_cpu(
x=x.transpose(0, 1),
@@ -1055,7 +1072,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
cache_indices=cache_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
@@ -1100,15 +1117,23 @@ def test_batch_memcpy_cpu_fallback() -> None:
# ---------------------------------------------------------------------------
# C++ conv (conv.cpp) uses VDPBF16PS, not AMX tiles, so it runs on any
# AVX-512BF16 CPU; weight is VNNI-packed on this same predicate at load time.
# C++ conv (conv.cpp) uses AArch64BF16 or VDPBF16PS, not AMX tiles, so it can
# run on any AVX-512BF16 CPU; weight is packed on this same predicate at load time.
# ---------------------------------------------------------------------------
_HAS_AVX512_BF16 = torch.cpu._is_avx512_bf16_supported()
_STATE_LEN = CONV_KERNEL - 1
CONV_EQUIV_SEQ_LENS = [[1], [7], [64], [65], [1, 2, 3], [63, 64, 65], [128, 129]]
CONV_EQUIV_SEQ_LENS = [
[1],
[7],
[64],
[65],
[1, 2, 3],
[63, 64, 65],
[128, 129],
]
def _conv_fp32_oracle(x, weight, bias, seq_lens, activation="silu"):
@@ -1153,10 +1178,10 @@ def _run_prefill_torch(x, weight, bias, seq_lens):
return out.transpose(0, 1).contiguous(), conv_states
def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False):
def _run_prefill_cpp(x, weight, bias, seq_lens, is_weight_packed=False):
num_seqs = len(seq_lens)
packed_w = ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
if is_vnni:
packed_w = ops.causal_conv1d_weight_pack(weight) if is_weight_packed else weight
if is_weight_packed:
# C++-branch layout: kv-cache "SD" [slots, state_len, dim] transposed to
# [slots, dim, state_len] (a non-contiguous view).
conv_state = torch.zeros(
@@ -1176,7 +1201,7 @@ def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False):
cache_indices=torch.arange(num_seqs, dtype=torch.int32),
has_initial_state=torch.zeros(num_seqs, dtype=torch.bool),
silu_activation=True,
is_vnni=is_vnni,
is_weight_packed=is_weight_packed,
)
return out.transpose(0, 1).contiguous(), conv_state
@@ -1245,7 +1270,7 @@ def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state(
cache_indices=state_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=False,
is_weight_packed=False,
)
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True)
out_ds = gdn_attention._causal_conv1d_fwd_cpu(
@@ -1257,7 +1282,7 @@ def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state(
cache_indices=state_indices,
has_initial_state=has_initial_state,
silu_activation=True,
is_vnni=False,
is_weight_packed=False,
)
torch.testing.assert_close(out_ds, out_sd)
@@ -1302,7 +1327,7 @@ def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback(
bias=None,
silu_activation=False,
conv_state_indices=state_indices,
is_vnni=False,
is_weight_packed=False,
num_accepted_tokens=num_accepted_tokens,
)
@@ -1321,7 +1346,8 @@ def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback(
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
@torch.inference_mode()
@@ -1335,7 +1361,8 @@ def test_conv_cpp_matches_torch(seq_lens):
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
@torch.inference_mode()
@@ -1354,89 +1381,114 @@ def test_conv_cpp_no_worse_than_torch_vs_fp32(seq_lens):
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
@torch.inference_mode()
def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
"""The exact runtime prefill sequence (VNNI-packed weight + SD-layout
conv_state view + is_vnni=True) must match the torch fallback. Validates
the packing + layout handoff on any AVX-512BF16 CPU."""
def test_conv_cpp_weight_packed_matches_torch(seq_lens):
"""The exact runtime prefill sequence (packed weight + SD-layout
conv_state view + is_weight_packed=True) must match the torch fallback. Validates
the native packing + layout handoff."""
x, weight, bias = _conv_inputs(sum(seq_lens))
out_torch, _ = _run_prefill_torch(x, weight, bias, seq_lens)
out_vnni, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
torch.testing.assert_close(out_vnni, out_torch, atol=1e-2, rtol=1e-2)
out_weight_packed, _ = _run_prefill_cpp(
x, weight, bias, seq_lens, is_weight_packed=True
)
torch.testing.assert_close(out_weight_packed, out_torch, atol=1e-2, rtol=1e-2)
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@pytest.mark.parametrize("batch", DECODE_BATCH_SIZES)
@pytest.mark.parametrize(
("dim", "has_bias", "silu_activation", "is_weight_packed"),
[
(32, False, False, False),
(32, True, True, True),
(96, False, True, True),
(96, True, False, False),
(8192, False, True, False),
(8192, True, True, True),
],
)
@torch.inference_mode()
def test_conv_update_cpp_matches_torch(batch):
"""Decode conv: causal_conv1d_update_cpu matches causal_conv1d_update_torch,
including the in-place conv_state update (the next-step handoff)."""
def test_conv_update_cpp_matches_torch(
batch, dim, has_bias, silu_activation, is_weight_packed
):
"""Decode preserves indexed, padded state across successive signed inputs."""
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
causal_conv1d_update_torch,
)
x = tensor_cache(batch * CONV_DIM, torch.bfloat16).view(batch, CONV_DIM)
weight = tensor_cache(CONV_DIM * CONV_KERNEL, torch.bfloat16).view(
CONV_DIM, CONV_KERNEL
)
bias = tensor_cache(CONV_DIM, torch.bfloat16)
conv_state = tensor_cache(batch * CONV_DIM * _STATE_LEN, torch.bfloat16).view(
batch, CONV_DIM, _STATE_LEN
)
torch.manual_seed(0)
weight = torch.randn(dim, CONV_KERNEL, dtype=torch.bfloat16)
bias = torch.randn(dim, dtype=torch.bfloat16) if has_bias else None
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
indices = torch.arange(batch, 0, -1, dtype=torch.int32)
storage_ref = torch.randn(batch + 2, _STATE_LEN + 2, dim, dtype=torch.bfloat16)
storage_cpp = storage_ref.clone()
cs_cpp = storage_cpp[:, :_STATE_LEN].transpose(1, 2)
cs_torch = conv_state.clone()
out_torch = causal_conv1d_update_torch(
x=x.unsqueeze(-1),
conv_state=cs_torch,
weight=weight,
bias=bias,
activation="silu",
).squeeze(-1)
cs_cpp = conv_state.clone()
out_cpp = ops.causal_conv1d_update_cpu(
x=x.contiguous(),
conv_states=cs_cpp,
weight=weight,
bias=bias,
silu_activation=True,
conv_state_indices=torch.arange(batch, dtype=torch.int32),
is_vnni=False,
)
torch.testing.assert_close(out_cpp, out_torch, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(cs_cpp, cs_torch, atol=1e-2, rtol=1e-2)
for _ in range(3):
x = torch.randn(batch, dim, dtype=torch.bfloat16)
cs_torch = storage_ref[indices, :_STATE_LEN].transpose(1, 2).contiguous()
out_torch = causal_conv1d_update_torch(
x=x.unsqueeze(-1),
conv_state=cs_torch,
weight=weight,
bias=bias,
activation="silu" if silu_activation else None,
).squeeze(-1)
storage_ref[indices, :_STATE_LEN] = cs_torch.transpose(1, 2)
out_cpp = ops.causal_conv1d_update_cpu(
x=x,
conv_states=cs_cpp,
weight=conv_weight,
bias=bias,
silu_activation=silu_activation,
conv_state_indices=indices,
is_weight_packed=is_weight_packed,
)
torch.testing.assert_close(out_cpp, out_torch, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(storage_cpp, storage_ref, atol=0, rtol=0)
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
)
@torch.inference_mode()
def test_conv_weight_pack_roundtrip_unpacked_matches():
"""`causal_conv1d_weight_pack` repacks the (dim, width) weight, so the C++ conv op
with packed weight + is_vnni=True must equal unpacked weight + is_vnni=False.
Guards the spec-decode contract: the runtime VNNI-packs `layer.conv1d.weight`
in place and stashes the original as `_cpu_unpacked_conv_weight` for the torch
spec-decode path, so both must produce identical math.
with packed weight + is_weight_packed=True must equal unpacked weight
+ is_weight_packed=False. Guards the spec-decode contract: the runtime packs
`layer.conv1d.weight` in place and stashes the original as
`_cpu_unpacked_conv_weight` for the torch spec-decode path, so both must produce
identical math.
"""
seq_lens = [7, 64, 65]
x, weight, bias = _conv_inputs(sum(seq_lens))
out_unpacked, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False)
out_packed, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
torch.testing.assert_close(out_packed, out_unpacked, atol=1e-2, rtol=1e-2)
out_unpacked, state_unpacked = _run_prefill_cpp(
x, weight, bias, seq_lens, is_weight_packed=False
)
out_packed, state_packed = _run_prefill_cpp(
x, weight, bias, seq_lens, is_weight_packed=True
)
torch.testing.assert_close(out_packed, out_unpacked, atol=0, rtol=0)
torch.testing.assert_close(state_packed, state_unpacked, atol=0, rtol=0)
@pytest.mark.skipif(
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
reason="C++ causal_conv1d requires AVX-512BF16 or AArch64 BF16",
)
@torch.inference_mode()
def test_spec_decode_unpacked_conv_weight_stash():
"""Spec-decode correctness: AVX-512BF16 CPUs VNNI-pack ``conv1d.weight`` in place
at load time and stash the original ``(dim, width)`` tensor as
"""Spec-decode correctness: AVX-512BF16 and AArch64 BF16 CPUs pack ``conv1d.weight``
in place at load time and stash the original ``(dim, width)`` tensor as
``_cpu_unpacked_conv_weight``; the spec-decode path must use that stash since
reading the packed weight directly produces garbage. Verifies the recovered
weight equals the original and that torch F.conv1d agrees with the C++ conv
@@ -1460,7 +1512,7 @@ def test_spec_decode_unpacked_conv_weight_stash():
# 1. The stash must exist and equal the original (dim, width) weight.
assert hasattr(conv, "_cpu_unpacked_conv_weight"), (
"dispatch_cpu_unquantized_gemm did not stash _cpu_unpacked_conv_weight "
"on an AVX-512BF16 CPU"
"on an AArch64-BF16 or AVX-512BF16 CPU"
)
stash = conv._cpu_unpacked_conv_weight
torch.testing.assert_close(stash, orig_2d, atol=0, rtol=0)
@@ -1469,10 +1521,10 @@ def test_spec_decode_unpacked_conv_weight_stash():
recovered = getattr(conv, "_cpu_unpacked_conv_weight", None)
assert recovered is not None
# conv.weight is now packed; using it directly (the bug) would differ.
packed_weight = conv.weight # [dim, 1, width], VNNI-packed contents
packed_weight = conv.weight # [dim, 1, width], packed contents
# 3. Spec-path torch conv with the recovered unpacked weight must match the
# nonspec C++ conv with the packed weight (is_vnni=True).
# nonspec C++ conv with the packed weight (is_weight_packed=True).
seq_lens = [7, 65]
total = sum(seq_lens)
x = tensor_cache(total * CONV_DIM, torch.bfloat16).view(total, CONV_DIM)
@@ -1494,7 +1546,7 @@ def test_spec_decode_unpacked_conv_weight_stash():
cache_indices=torch.arange(num_seqs, dtype=torch.int32),
has_initial_state=torch.zeros(num_seqs, dtype=torch.bool),
silu_activation=True,
is_vnni=True,
is_weight_packed=True,
)
.transpose(0, 1)
.contiguous()
@@ -88,11 +88,6 @@ def test_short_conv_forward_native_prefill(vllm_config):
state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32),
num_accepted_tokens=None,
query_start_loc_d=None,
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
num_computed_tokens_p=None,
seq_lens=torch.tensor([5]),
)
@@ -145,11 +140,6 @@ def test_short_conv_forward_native_decode(vllm_config):
state_indices_tensor_d=state_indices_tensor_d,
num_accepted_tokens=None,
query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32),
block_idx_last_scheduled_token=None,
block_idx_first_scheduled_token_p=None,
block_idx_last_computed_token=None,
block_idx_last_scheduled_token_prev_step=None,
num_computed_tokens_p=None,
seq_lens=torch.tensor([1, 1]),
)
@@ -176,7 +176,7 @@ def test_forward_core_split_matches_unified(
builder = GDNAttentionMetadataBuilder(
# GDNAttentionMetadataBuilder declares kv_cache_spec as AttentionSpec but
# asserts isinstance(kv_cache_spec, MambaSpec). MambaSpec is correct here.
kv_cache_spec=MambaSpec( # type: ignore[arg-type]
kv_cache_spec=MambaSpec(
block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.float16,)
),
layer_names=[PREFIX],
+388 -5
View File
@@ -1,24 +1,30 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from vllm.config.mamba import MambaBackendEnum, MambaConfig, MambaSSUAlgorithm
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateShapeCalculator
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
FlashInferSSUBackend,
TritonSSUBackend,
commit_replayssm_ring_trackers,
get_mamba_ssu_backend,
initialize_mamba_ssu_backend,
reset_replayssm_ring_trackers,
selective_state_update,
selective_state_update_replayssm_flashinfer,
update_replayssm_ring_trackers,
)
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
@@ -32,6 +38,13 @@ try:
except ImportError:
HAS_FLASHINFER = False
try:
from flashinfer.mamba.checkpointing_ssu import CheckpointingSSURunner
HAS_FLASHINFER_CHECKPOINTING_SSU = CheckpointingSSURunner is not None
except ImportError:
HAS_FLASHINFER_CHECKPOINTING_SSU = False
@pytest.fixture(autouse=True)
def restore_backend_state():
@@ -47,6 +60,7 @@ def restore_backend_state():
def test_flashinfer_replayssm_ring_tracker_lifecycle():
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda")
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
observed = []
@@ -54,6 +68,7 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle():
update_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
logical_window=16,
ring_buffer_len=17,
@@ -69,9 +84,205 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle():
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
)
assert (ring_start[1].item(), prev_num_accepted[1].item()) == (0, 0)
assert (
ring_start[1].item(),
prev_num_accepted[1].item(),
prev_query_len[1].item(),
) == (0, 0, 0)
@pytest.mark.parametrize(
("accepted_sequence", "expected"),
[
pytest.param(
[4] * 22,
[
(0, 0, 4),
(0, 4, 4),
(0, 8, 4),
(0, 12, 4),
(0, 16, 4),
(16, 4, 4),
(16, 8, 4),
(16, 12, 4),
(16, 16, 4),
(12, 4, 4),
(12, 8, 4),
(12, 12, 4),
(12, 16, 4),
(8, 4, 4),
(8, 8, 4),
(8, 12, 4),
(8, 16, 4),
(4, 4, 4),
(4, 8, 4),
(4, 12, 4),
(4, 16, 4),
(0, 4, 4),
],
id="all-accepted",
),
pytest.param(
[4, 4, 0, 3, 4, 2, 4, 1],
[
(0, 0, 4),
(0, 4, 4),
(0, 4, 4),
(0, 7, 4),
(0, 11, 4),
(0, 13, 4),
(13, 4, 4),
(13, 5, 4),
],
id="mixed",
),
],
)
def test_replayssm_commit_tracker_acceptance_sequence(accepted_sequence, expected):
logical_window = 16
num_speculative_tokens = 3
query_len = 1 + num_speculative_tokens
ring_buffer_len = logical_window + 1 + num_speculative_tokens
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda")
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
query_start_loc = torch.tensor([0, query_len], dtype=torch.int32, device="cuda")
observed = []
for accepted in accepted_sequence:
commit_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
torch.tensor([accepted], dtype=torch.int32, device="cuda"),
query_start_loc,
logical_window,
ring_buffer_len,
)
snapshot = (
ring_start[1].item(),
prev_num_accepted[1].item(),
prev_query_len[1].item(),
)
observed.append(snapshot)
assert snapshot[1] + snapshot[2] <= ring_buffer_len
assert observed == expected
def test_replayssm_resume_resets_commit_history():
ring_start = torch.tensor([0, 13], dtype=torch.int32, device="cuda")
prev_num_accepted = torch.tensor([0, 13], dtype=torch.int32, device="cuda")
prev_query_len = torch.tensor([0, 4], dtype=torch.int32, device="cuda")
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
)
assert (
ring_start[1].item(),
prev_num_accepted[1].item(),
prev_query_len[1].item(),
) == (0, 0, 0)
commit_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
torch.tensor([3], dtype=torch.int32, device="cuda"),
torch.tensor([0, 4], dtype=torch.int32, device="cuda"),
logical_window=16,
ring_buffer_len=20,
)
assert (
ring_start[1].item(),
prev_num_accepted[1].item(),
prev_query_len[1].item(),
) == (0, 0, 4)
def test_replayssm_commit_tracker_ragged_query_lengths():
ring_start = torch.zeros(3, dtype=torch.int32, device="cuda")
prev_num_accepted = torch.zeros(3, dtype=torch.int32, device="cuda")
prev_query_len = torch.zeros(3, dtype=torch.int32, device="cuda")
state_batch_indices = torch.tensor([1, 2], dtype=torch.int32, device="cuda")
query_start_loc = torch.tensor([0, 4, 6], dtype=torch.int32, device="cuda")
observed = []
for accepted in ([4, 2], [3, 1]):
commit_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
torch.tensor(accepted, dtype=torch.int32, device="cuda"),
query_start_loc,
logical_window=16,
ring_buffer_len=20,
)
observed.append(
[
(
ring_start[slot].item(),
prev_num_accepted[slot].item(),
prev_query_len[slot].item(),
)
for slot in (1, 2)
]
)
assert observed == [[(0, 0, 4), (0, 0, 2)], [(0, 3, 4), (0, 1, 2)]]
@pytest.mark.parametrize("operation", ["commit", "reset"])
def test_replayssm_tracker_kernels_mask_invalid_slots(operation):
num_states = 3
ring_start = torch.tensor([11, 2, 33], dtype=torch.int32, device="cuda")
prev_num_accepted = torch.tensor([11, 3, 33], dtype=torch.int32, device="cuda")
prev_query_len = torch.tensor([11, 4, 33], dtype=torch.int32, device="cuda")
state_batch_indices = torch.tensor(
[-1, num_states, NULL_BLOCK_ID, 1], dtype=torch.int32, device="cuda"
)
if operation == "commit":
commit_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
torch.tensor([4, 4, 4, 2], dtype=torch.int32, device="cuda"),
torch.tensor([0, 4, 8, 12, 16], dtype=torch.int32, device="cuda"),
logical_window=16,
ring_buffer_len=20,
)
expected_valid = (2, 5, 4)
else:
reset_replayssm_ring_trackers(
ring_start,
prev_num_accepted,
prev_query_len,
state_batch_indices,
)
expected_valid = (0, 0, 0)
assert (
ring_start.tolist(),
prev_num_accepted.tolist(),
prev_query_len.tolist(),
) == (
[11, expected_valid[0], 33],
[11, expected_valid[1], 33],
[11, expected_valid[2], 33],
)
def _kv_cache_config_with_ssu(
@@ -231,14 +442,185 @@ def test_triton_basic_call():
assert not torch.isnan(out).any()
@pytest.mark.parametrize("layout", ["packed", "dense"])
def test_replayssm_flashinfer_call_forwards_mtp_layout(monkeypatch, layout):
import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod
kernel = Mock(return_value=torch.empty(0))
monkeypatch.setattr(mod, "_flashinfer_replayssm_kernel", kernel)
batch, max_seqlen, nheads, dim, dstate, ngroups = 2, 4, 2, 4, 8, 1
state = torch.empty(2, nheads, dim, dstate)
x_shape: tuple[int, ...]
B_shape: tuple[int, ...]
expected_x_shape: tuple[int, ...]
expected_B_shape: tuple[int, ...]
if layout == "packed":
x_shape = (6, nheads, dim)
B_shape = (6, ngroups, dstate)
expected_x_shape = (1, 6, nheads, dim)
expected_B_shape = (1, 6, ngroups, dstate)
cu_seqlens = torch.tensor([0, 4, 6], dtype=torch.int32)
kernel_max_seqlen = max_seqlen
else:
x_shape = (batch, max_seqlen, nheads, dim)
B_shape = (batch, max_seqlen, ngroups, dstate)
expected_x_shape = x_shape
expected_B_shape = B_shape
cu_seqlens = None
kernel_max_seqlen = None
x = torch.empty(x_shape)
dt = torch.empty_like(x)
A = torch.empty(nheads, dim, dstate)
B = torch.empty(B_shape)
C = torch.empty_like(B)
out = torch.empty_like(x)
x_cache = torch.empty(2, nheads, 20, dim)
dt_cache = torch.empty(2, nheads, 20)
B_cache = torch.empty(2, ngroups, 20, dstate)
ring_start = torch.zeros(2, dtype=torch.int32)
prev_num_accepted = torch.zeros(2, dtype=torch.int32)
prev_query_len = torch.zeros(2, dtype=torch.int32)
selective_state_update_replayssm_flashinfer(
state,
x,
dt,
A,
B,
C,
out,
x_cache,
B_cache,
dt_cache,
ring_start,
prev_num_accepted,
prev_query_len,
logical_window=16,
state_batch_indices=torch.tensor([0, 1], dtype=torch.int32),
cu_seqlens=cu_seqlens,
max_seqlen=kernel_max_seqlen,
update_trackers=False,
)
args = kernel.call_args.args
assert args[6].shape == expected_x_shape
assert args[7].shape == expected_x_shape
assert args[9].shape == expected_B_shape
assert args[10].shape == expected_B_shape
assert args[11].shape == expected_x_shape
assert kernel.call_args.kwargs["cu_seqlens"] is cu_seqlens
assert kernel.call_args.kwargs["max_seqlen"] == kernel_max_seqlen
@pytest.mark.parametrize(
("backend", "expected_ring_len"),
("query_start_loc", "expected_shape", "expected_max_seqlen"),
[
(MambaBackendEnum.TRITON, 16),
(MambaBackendEnum.FLASHINFER, 17),
pytest.param([0, 4, 8], (2, 4, 2, 4), None, id="dense"),
pytest.param([0, 4, 6], (6, 2, 4), 4, id="packed"),
],
)
def test_replayssm_physical_ring_shape(backend, expected_ring_len):
def test_replayssm_mixer_selects_mtp_layout(
monkeypatch, query_start_loc, expected_shape, expected_max_seqlen
):
import vllm.model_executor.layers.mamba.mamba_mixer2 as mod
mixer = MambaMixer2.__new__(MambaMixer2)
torch.nn.Module.__init__(mixer)
mixer.prefix = "mixer"
mixer.tped_intermediate_size = 0
mixer.tped_conv_size = 1
mixer.tped_dt_size = 2
mixer.num_heads = 2
mixer.head_dim = 4
mixer.n_groups = mixer.tp_size = 1
mixer.ssm_state_size = 8
mixer.num_spec = 3
mixer.use_replayssm = True
mixer.replayssm_buffer_len = 16
mixer._commits_replayssm_trackers = True
mixer._updates_replayssm_trackers = False
mixer.mamba_config = MambaConfig(backend=MambaBackendEnum.FLASHINFER)
mixer.cache_config = SimpleNamespace(mamba_block_size=16, mamba_cache_mode="none")
mixer.conv_weights = torch.empty(0)
mixer.conv1d = SimpleNamespace(bias=None)
mixer.activation = "silu"
mixer.A = torch.empty(2)
mixer.dt_bias = torch.empty(2)
mixer.D = torch.empty(2)
mixer._replayssm_ring_start = torch.zeros(3, dtype=torch.int32)
mixer._replayssm_prev_num_accepted = torch.zeros(3, dtype=torch.int32)
mixer._replayssm_prev_query_len = torch.zeros(3, dtype=torch.int32)
mixer.kv_cache = (
torch.empty(3, 1),
torch.empty(3, 2, 4, 8),
torch.empty(3, 2, 20, 4),
torch.empty(3, 2, 20),
torch.empty(3, 1, 20, 8),
)
num_decode_tokens = query_start_loc[-1]
query_start_loc_d = torch.tensor(query_start_loc, dtype=torch.int32)
metadata = Mamba2AttentionMetadata(
num_prefills=0,
num_prefill_tokens=0,
num_decodes=2,
num_decode_tokens=num_decode_tokens,
num_reqs=2,
has_initial_states_p=None,
query_start_loc_p=None,
state_indices_tensor_p=None,
state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32),
query_start_loc_d=query_start_loc_d,
num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32),
seq_lens=torch.tensor([104, 102], dtype=torch.int32),
replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)),
replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32),
)
def split_hidden_states_B_C(values):
tokens = values.size(0)
return (
torch.empty(tokens, 8),
torch.empty(tokens, 8),
torch.empty(tokens, 8),
)
mixer.split_hidden_states_B_C_fn = split_hidden_states_B_C
kernel = Mock()
monkeypatch.setattr(
mod,
"get_forward_context",
lambda: SimpleNamespace(attn_metadata={mixer.prefix: metadata}),
)
monkeypatch.setattr(mod, "commit_replayssm_ring_trackers", Mock())
monkeypatch.setattr(
mod, "causal_conv1d_update", lambda values, *args, **kwargs: values
)
monkeypatch.setattr(mod, "selective_state_update_replayssm_flashinfer", kernel)
mixer.conv_ssm_forward(
torch.empty(num_decode_tokens, 3), torch.empty(num_decode_tokens, 8)
)
assert kernel.call_args.args[1].shape == expected_shape
if expected_max_seqlen is None:
assert kernel.call_args.kwargs["cu_seqlens"] is None
else:
assert kernel.call_args.kwargs["cu_seqlens"] is query_start_loc_d
assert kernel.call_args.kwargs["max_seqlen"] == expected_max_seqlen
@pytest.mark.parametrize(
("backend", "num_speculative_tokens", "expected_ring_len"),
[
(MambaBackendEnum.TRITON, 0, 16),
(MambaBackendEnum.FLASHINFER, 0, 17),
(MambaBackendEnum.FLASHINFER, 3, 20),
],
)
def test_replayssm_physical_ring_shape(
backend, num_speculative_tokens, expected_ring_len
):
base_shapes = ((64, 3), (8, 4, 16))
shapes = MambaStateShapeCalculator.append_replayssm_ring(
@@ -247,6 +629,7 @@ def test_replayssm_physical_ring_shape(backend, expected_ring_len):
tp_world_size=2,
logical_window=16,
backend=backend,
num_speculative_tokens=num_speculative_tokens,
)
assert shapes[2:] == (
+1 -1
View File
@@ -538,7 +538,7 @@ def run_8_bit(
assert num_local_experts is not None
return run_with_expert_maps(
num_experts,
num_local_experts, # type: ignore[arg-type]
num_local_experts,
quant_config,
**kwargs,
)
@@ -29,14 +29,20 @@ from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig,
FusedMoEParallelConfig,
RoutingMethodType,
nvfp4_moe_quant_config,
)
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutedsl_moe import (
FlashInferCuteDSLExperts,
)
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
NvFp4MoeBackend,
make_nvfp4_moe_quant_config,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
)
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
quantize_nvfp4_per_token_input,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_cutedsl_moe_nvfp4
from vllm.utils.math_utils import next_power_of_2
@@ -159,8 +165,12 @@ def _dequantize_nvfp4_linear(
return (tensor_f32 * tensor_sf.unsqueeze(-1)).reshape(m, k).to(dtype)
@pytest.mark.parametrize("m,n,k,e,topk", [(16, 128, 512, 4, 2)])
@pytest.mark.parametrize(
"m,n,k,e,topk",
[(16, 128, 512, 4, 2), (16, 192, 512, 4, 2), (16, 192, 2688, 4, 2)],
)
@pytest.mark.parametrize("activation,alpha,beta,limit", _ACT_CASES)
@pytest.mark.parametrize("per_token_activation", [False, True])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@torch.inference_mode()
def test_flashinfer_cutedsl_fp4_moe(
@@ -173,6 +183,7 @@ def test_flashinfer_cutedsl_fp4_moe(
alpha: float | None,
beta: float | None,
limit: float | None,
per_token_activation: bool,
dtype: torch.dtype,
workspace_init,
):
@@ -181,6 +192,10 @@ def test_flashinfer_cutedsl_fp4_moe(
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
):
hidden_states = torch.randn((m, k), device="cuda", dtype=dtype) / 10
# Keep GEMM1 activation variance stable as K changes, so the fixed
# tolerance against the BF16 GEMM2 reference measures comparable FP4
# quantization error at each hidden width.
hidden_states *= (512 / k) ** 0.5
w1_rows = 2 * n if activation.is_gated else n
w1 = torch.randn((e, w1_rows, k), device="cuda", dtype=dtype) / 15
@@ -193,7 +208,6 @@ def test_flashinfer_cutedsl_fp4_moe(
hidden_states, score, topk, renormalize=False
)
activation = MoEActivation.RELU2_NO_MUL
moe_config = FusedMoEConfig(
num_experts=e,
experts_per_token=topk,
@@ -231,19 +245,30 @@ def test_flashinfer_cutedsl_fp4_moe(
w2_scale_2=(1.0 / w2_global_scale),
a2_scale=a2_scale,
)
quant_config = nvfp4_moe_quant_config(
g1_alphas=w1_alpha,
g2_alphas=w2_alpha,
a1_gscale=(1.0 / a1_scale),
a2_gscale=(1.0 / a2_scale),
w1_scale=w1_scale_cutedsl,
# I=192 must remain unpadded for gated experts, but become 256 for
# non-gated experts. The reference below uses the checkpoint shapes.
expected_n = n if activation.is_gated else (n + 127) // 128 * 128
assert moe_config.intermediate_size_per_partition == expected_n
assert w1_cutedsl.shape[1] == expected_n * (2 if activation.is_gated else 1)
assert w2_cutedsl.shape[2] * 2 == expected_n
expected_k = (k + 255) // 256 * 256
assert moe_config.hidden_dim == expected_k
assert moe_config.hidden_dim_unpadded == k
assert w1_cutedsl.shape[2] * 2 == expected_k
assert w2_cutedsl.shape[1] == expected_k
quant_config = make_nvfp4_moe_quant_config(
backend=NvFp4MoeBackend.FLASHINFER_CUTEDSL,
w13_scale_2=w1_alpha,
w2_scale_2=w2_alpha,
a13_scale=a1_scale,
a2_scale=a2_scale,
w13_scale=w1_scale_cutedsl,
w2_scale=w2_scale_cutedsl,
is_scale_swizzled=False,
# Unset params must be omitted rather than forwarded as None into
# the kernel's float-typed SwiGLU arguments.
gemm1_alpha=alpha,
gemm1_beta=beta,
gemm1_clamp_limit=limit,
swiglu_alpha=alpha,
swiglu_beta=beta,
swiglu_limit=limit,
)
cutedsl_experts = mk.FusedMoEKernel(
@@ -256,11 +281,13 @@ def test_flashinfer_cutedsl_fp4_moe(
FlashInferCuteDSLExperts(
moe_config=moe_config,
quant_config=quant_config,
per_token_activation=per_token_activation,
),
)
cutedsl_output = cutedsl_experts.apply(
hidden_states=hidden_states,
# The model runner pads inputs and trims outputs around this kernel.
hidden_states=torch.nn.functional.pad(hidden_states, (0, expected_k - k)),
w1=w1_cutedsl,
w2=w2_cutedsl,
topk_weights=topk_weights,
@@ -270,19 +297,25 @@ def test_flashinfer_cutedsl_fp4_moe(
expert_map=None,
apply_router_weight_on_input=False,
)
cutedsl_output = cutedsl_output[:, :k]
a_global_scale = torch.ones(1, device="cuda", dtype=torch.float32)
a_q, a_scale = ops.scaled_fp4_quant(
hidden_states,
a_global_scale,
is_sf_swizzled_layout=False,
)
if per_token_activation:
a_q, a_scale, a_row_scale = quantize_nvfp4_per_token_input(hidden_states)
else:
a_q, a_scale = ops.scaled_fp4_quant(
hidden_states,
a_global_scale,
is_sf_swizzled_layout=False,
)
a_in_dtype = _dequantize_nvfp4_linear(
a_q,
a_scale,
a_global_scale,
dtype=dtype,
)
if per_token_activation:
a_in_dtype = (a_in_dtype.float() * a_row_scale.view(-1, 1)).to(dtype)
w1_d = torch.empty((e, w1_rows, k), device="cuda", dtype=dtype)
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
@@ -0,0 +1,128 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib
import sys
import types
from types import SimpleNamespace
import torch
def _describe(tensor):
if tensor is None:
return "None (the op was never called)"
return f"a different tensor of shape {tuple(tensor.shape)}"
def _import_mori_prepare_finalize(monkeypatch):
# This is a contract test for vLLM's adapter; it does not need a real MoRI
# installation or GPU kernels.
fake_mori = types.ModuleType("mori")
fake_mori.ops = SimpleNamespace( # type: ignore[attr-defined]
EpDispatchCombineOp=object
)
monkeypatch.setitem(sys.modules, "mori", fake_mori)
module_name = "vllm.model_executor.layers.fused_moe.prepare_finalize.mori"
sys.modules.pop(module_name, None)
return importlib.import_module(module_name)
class _FakeMoriOp:
def __init__(self):
self.dispatch_indices = None
self.combine_indices = None
def dispatch(self, a1, weights, scale, indices):
self.dispatch_indices = indices
# Model the global dispatched layout seen with EP=16: the experts need
# these IDs, but passing them to combine violates MoRI's contract.
dispatch_indices = indices.repeat(16, 1)
dispatch_a1 = a1.repeat(16, 1)
dispatch_weights = weights.repeat(16, 1)
recv_token_num = torch.tensor([dispatch_a1.shape[0]])
return (
dispatch_a1,
dispatch_weights,
scale,
dispatch_indices,
recv_token_num,
)
def combine(self, fused_expert_output, weights, indices):
self.combine_indices = indices
return (fused_expert_output,)
def test_mori_combine_uses_original_per_rank_topk_ids(monkeypatch):
mori_module = _import_mori_prepare_finalize(monkeypatch)
mori_op = _FakeMoriOp()
adapter = mori_module.MoriPrepareAndFinalize(
mori_op=mori_op,
max_tokens_per_rank=16,
num_dispatchers=1,
)
hidden_states = torch.randn(2, 4)
router_topk_ids = torch.tensor([[0, 1], [2, 3]], dtype=torch.int32)
topk_weights = torch.ones(2, 2)
quant_config = SimpleNamespace(
is_block_quantized=False,
is_per_act_token=False,
)
(
dispatch_a1,
_,
_,
dispatch_ids,
dispatch_weights,
) = adapter.prepare(
hidden_states,
topk_weights,
router_topk_ids,
num_experts=4,
expert_map=None,
apply_router_weight_on_input=False,
quant_config=quant_config,
)
# Read the shapes before the identity asserts below, which narrow these
# names to the fake op's Optional attribute type.
router_shape = tuple(router_topk_ids.shape)
dispatch_shape = tuple(dispatch_ids.shape)
expected_rows = router_shape[0] * 16
assert dispatch_shape[0] == expected_rows, (
"prepare() must return the ids dispatch() produced: EP=16 expands "
f"{router_shape[0]} tokens to {expected_rows} rows, but prepare() "
f"returned {dispatch_shape[0]} rows"
)
assert mori_op.dispatch_indices is router_topk_ids, (
"dispatch() must be handed this rank's own pre-dispatch routing of "
f"shape {router_shape}, but it received "
f"{_describe(mori_op.dispatch_indices)}"
)
output = torch.empty_like(hidden_states)
adapter.finalize(
output,
dispatch_a1,
dispatch_weights,
dispatch_ids,
apply_router_weight_on_input=False,
weight_and_reduce_impl=SimpleNamespace(),
)
assert mori_op.combine_indices is router_topk_ids, (
"combine() reduces over this rank's own routing, so it must be handed "
f"the pre-dispatch tensor of shape {router_shape}, but it received "
f"{_describe(mori_op.combine_indices)}. finalize() most likely "
"forwarded its own topk_ids, which the modular kernel has already "
"rebound to the ids dispatch() returned (ROCm/mori#475)"
)
assert mori_op.combine_indices is not dispatch_ids, (
f"combine() was handed the dispatched ids of shape {dispatch_shape} "
"instead of this rank's pre-dispatch routing, which silently corrupts "
"the reduction (ROCm/mori#475)"
)
+1 -1
View File
@@ -1724,7 +1724,7 @@ def test_rocm_mxfp4_moe_oracle(
w13_conv, w2_conv, w13_scale_conv, w2_scale_conv, w13_bias_conv, w2_bias_conv = (
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
mxfp4_backend=backend,
layer=layer, # type: ignore[arg-type]
layer=layer,
w13_weight=w13_quant,
w2_weight=w2_quant,
w13_weight_scale=w13_scale,
+359 -1
View File
@@ -30,7 +30,7 @@ import torch.nn.functional as F
from tests.kernels.utils import _assert_deterministic
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_gfx942, on_gfx950
from vllm.platforms.rocm import on_cdna, on_gfx942, on_gfx950
from vllm.utils.torch_utils import set_random_seed
pytestmark = pytest.mark.skipif(
@@ -638,6 +638,115 @@ def test_aiter_moe_padding_env_var(
assert envs.VLLM_ROCM_MOE_PADDING is moe_padding
# a4w4 (FP4 activation) opt-in gating test ---------------------------------
@pytest.mark.parametrize(
("env_value", "model_type", "expected"),
[
# Unset/"0" is opt-in-only: a4w4 never turns on by itself, even on
# a validated model type.
(None, "deepseek_v41", False),
("0", "deepseek_v41", False),
("1", "deepseek_v41", True),
("1", "deepseek_v41_text", True),
],
)
def test_aiter_moe_a4w4_dsv4_is_opt_in(
env_value: str | None,
model_type: str,
expected: bool,
monkeypatch: pytest.MonkeyPatch,
):
"""vllm-project/vllm#58819's a4w4 MoE activation path must stay a plain
explicit opt-in (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1), with no
default-on behavior, but is still only permitted on the model types
this flag has been validated for."""
import types
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
fake_vllm_config = types.SimpleNamespace(
model_config=types.SimpleNamespace(
hf_config=types.SimpleNamespace(model_type=model_type)
)
)
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
if env_value is None:
mp.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
else:
mp.setenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", env_value)
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
with mock.patch(
"vllm.config.get_current_vllm_config_or_none",
return_value=fake_vllm_config,
):
moe_config = make_dummy_moe_config(
num_experts=4,
experts_per_token=2,
hidden_dim=256,
intermediate_size=512,
in_dtype=torch.bfloat16,
)
assert moe_config.use_mxfp4_w4a4_dsv4 is expected
@pytest.mark.parametrize("model_type", ["deepseek_v4", "gpt_oss", None])
def test_aiter_moe_a4w4_dsv4_rejects_unvalidated_model_type(
model_type: str | None,
monkeypatch: pytest.MonkeyPatch,
):
"""Forcing a4w4 on a model type this flag hasn't been GSM8K/AgentX
validated for must raise: it also flips the MXFP4 weight-shuffle layout
(ATOM SEPARATED vs INTERLEAVE), which silently produces garbled output
on the wrong model rather than just being a slower path — the same bug
class vllm-project/vllm#58819's own second commit fixed for DeepSeek
V4.1 itself."""
import types
from tests.kernels.moe.utils import make_dummy_moe_config
from vllm._aiter_ops import rocm_aiter_ops
_assert_aiter_supported()
fake_vllm_config = types.SimpleNamespace(
model_config=types.SimpleNamespace(
hf_config=types.SimpleNamespace(model_type=model_type)
)
)
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
mp.setenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", "1")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
with (
mock.patch(
"vllm.config.get_current_vllm_config_or_none",
return_value=fake_vllm_config,
),
pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4"),
):
make_dummy_moe_config(
num_experts=4,
experts_per_token=2,
hidden_dim=256,
intermediate_size=512,
in_dtype=torch.bfloat16,
)
# Dispatch-policy forwarding test ------------------------------------------
@@ -1208,3 +1317,252 @@ def test_aiter_fused_moe_mi3xx_fp8_accuracy():
pass_rate=1.0,
max_violation_factor=1.5,
)
# Weight alignment tests --------------------------------------------------
#
# AITER's CK 2stages MoE kernel rejects an intermediate size not divisible by
# its tile width (64 at or below inter_dim 192, 128 above). Some model + TP
# splits land on an unaligned size (e.g. 1792 / TP=8 = 224), so the AITER path
# rounds the intermediate dim up in ``maybe_roundup_sizes`` and allocates the
# weights at the padded size.
ALIGNMENT_HIDDEN = 64
ALIGNMENT_NUM_EXPERTS = 2
def _make_alignment_moe_config(intermediate: int):
from tests.kernels.moe.utils import make_dummy_moe_config
return make_dummy_moe_config(
num_experts=ALIGNMENT_NUM_EXPERTS,
hidden_dim=ALIGNMENT_HIDDEN,
intermediate_size=intermediate,
)
def _make_aiter_method(moe_config):
"""Build the unquantized method with the backend pinned to AITER.
``select_unquantized_moe_backend`` needs a real ROCm + AITER runtime, so
stub it out and set the backend directly.
"""
from unittest.mock import patch
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
)
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
UnquantizedFusedMoEMethod,
)
with patch(
"vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method"
".select_unquantized_moe_backend",
return_value=(UnquantizedMoeBackend.AITER, None),
):
return UnquantizedFusedMoEMethod(moe_config)
def _roundup(method, moe_config, intermediate):
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
return method.maybe_roundup_sizes(
hidden_size=ALIGNMENT_HIDDEN,
intermediate_size_per_partition=intermediate,
act_dtype=moe_config.in_dtype,
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
)
@pytest.mark.parametrize(
("intermediate", "expected_padded"),
[
(64, 64), # already aligned, untouched
(96, 128),
(160, 192),
(192, 192), # must stay 192: it is valid and has tuned configs
(224, 256), # K2-Horizon-375B at TP=8
(256, 256),
(448, 512), # K2-Horizon-375B at TP=4
(4096, 4096),
],
)
def test_aiter_moe_roundup_pads_intermediate(
intermediate,
expected_padded,
default_vllm_config,
):
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
aiter_moe_intermediate_alignment,
)
moe_config = _make_alignment_moe_config(intermediate)
method = _make_aiter_method(moe_config)
hidden, padded = _roundup(method, moe_config, intermediate)
assert padded == expected_padded
assert hidden == ALIGNMENT_HIDDEN
# Padding must not move a size across the <= 192 threshold, or the
# alignment we picked is not the one the kernel dispatches the padded
# shape to.
assert aiter_moe_intermediate_alignment(padded) == (
aiter_moe_intermediate_alignment(intermediate)
)
@pytest.mark.parametrize("backend_name", ["TRITON", "FLASHINFER_CUTLASS"])
def test_aiter_moe_roundup_is_not_applied_to_other_backends(
backend_name,
default_vllm_config,
):
"""Backends with no alignment requirement keep the unaligned size."""
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
UnquantizedMoeBackend,
)
intermediate = 224
moe_config = _make_alignment_moe_config(intermediate)
method = _make_aiter_method(moe_config)
method.unquantized_backend = UnquantizedMoeBackend[backend_name]
_, padded = _roundup(method, moe_config, intermediate)
assert padded == intermediate
@pytest.mark.parametrize("intermediate", [224, 448])
def test_aiter_moe_reload_zeroes_intermediate_padding(
intermediate,
monkeypatch,
default_vllm_config,
):
"""A reload must re-zero pad lanes left dirty in the weight storage.
The reload writes back only the logical slices, so the pad lanes hold
garbage (NaN here) that the AITER conversion must zero. Two passes also
cover reloading into storage that already holds a previous conversion.
"""
from unittest.mock import MagicMock
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
aiter_moe_intermediate_alignment,
)
alignment = aiter_moe_intermediate_alignment(intermediate)
padded = -(-intermediate // alignment) * alignment
assert padded != intermediate
moe_config = _make_alignment_moe_config(intermediate)
assert moe_config.is_act_and_mul
method = _make_aiter_method(moe_config)
# RoutedExperts records the rounded-up size after maybe_roundup_sizes.
moe_config.intermediate_size_per_partition = padded
layer = torch.nn.Module()
layer.moe_config = moe_config
method.create_weights(
layer=layer,
num_experts=ALIGNMENT_NUM_EXPERTS,
hidden_size=ALIGNMENT_HIDDEN,
intermediate_size_per_partition=padded,
params_dtype=torch.float32,
)
# Identity shuffle keeps pad lanes sliceable; both stubs need real ROCm.
monkeypatch.setattr(
"vllm.model_executor.layers.fused_moe.oracle.unquantized"
".rocm_aiter_ops.shuffle_weights",
lambda w13, w2: (w13, w2),
)
monkeypatch.setattr(
method,
"_init_moe_kernel",
lambda _layer: setattr(method, "moe_kernel", MagicMock()),
)
up = padded # up projection starts one padded block into the fused rows
for _ in range(2):
gate = torch.randn(ALIGNMENT_NUM_EXPERTS, intermediate, ALIGNMENT_HIDDEN)
up_proj = torch.randn(ALIGNMENT_NUM_EXPERTS, intermediate, ALIGNMENT_HIDDEN)
down = torch.randn(ALIGNMENT_NUM_EXPERTS, ALIGNMENT_HIDDEN, intermediate)
# NaN the pads, then write only the logical slices like the loader.
layer.w13_weight.data.fill_(float("nan"))
layer.w2_weight.data.fill_(float("nan"))
layer.w13_weight.data[:, :intermediate].copy_(gate)
layer.w13_weight.data[:, up : up + intermediate].copy_(up_proj)
layer.w2_weight.data[:, :, :intermediate].copy_(down)
method.process_weights_after_loading(layer)
w13 = layer.w13_weight
w2 = layer.w2_weight
assert torch.equal(w13[:, :intermediate], gate)
assert torch.equal(w13[:, up : up + intermediate], up_proj)
assert torch.equal(w2[:, :, :intermediate], down)
assert torch.all(w13[:, intermediate:up] == 0)
assert torch.all(w13[:, up + intermediate :] == 0)
assert torch.all(w2[:, :, intermediate:] == 0)
def _aiter_accepts_intermediate(intermediate: int, num_tokens: int) -> bool:
"""Run the real AITER CK MoE kernel and report whether it dispatched."""
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
ActivationMethod,
QuantMethod,
)
case = _make_moe_case(
num_tokens=num_tokens,
hidden_dim=1024,
intermediate_dim=intermediate,
num_experts=8,
topk=2,
seed=0,
)
try:
_run_fused_moe(
case["hidden_states"],
case["w1"],
case["w2"],
case["topk_weights"],
case["topk_ids"],
activation_method=int(ActivationMethod.SILU),
quant_method=int(QuantMethod.NO),
)
except RuntimeError:
return False
return True
@pytest.mark.skipif(
not on_cdna(),
reason="CDNA ROCm only",
)
@pytest.mark.parametrize("intermediate", [192, 224, 256])
# With topk=2 and 8 experts, these token counts select block_m 32 / 64 / 128
# respectively (AITER's get_block_size_M, on both 256- and 304-CU parts).
@pytest.mark.parametrize("num_tokens", [1024, 2560, 5120])
def test_aiter_moe_alignment_rule_holds_across_block_m(intermediate, num_tokens):
"""The alignment rule must hold at every reachable ``block_m``.
Stage 1 checks ``NPerBlock`` and stage 2 ``KPerBlock``; both vary with
``block_m``, so a size validated at one tile size says nothing about the
others.
"""
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
aiter_moe_intermediate_alignment,
)
_assert_aiter_supported()
alignment = aiter_moe_intermediate_alignment(intermediate)
padded = -(-intermediate // alignment) * alignment
assert _aiter_accepts_intermediate(intermediate, num_tokens) == (
intermediate % alignment == 0
)
assert _aiter_accepts_intermediate(padded, num_tokens)
@@ -0,0 +1,136 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the AITER biased_grouped_topk expert-group count.
The AITER kernel is only instantiated for ``NUM_GRP`` in {1, 2, 4, 8}, so the
group count derived from ``num_experts`` has to be rounded to a supported
value that still divides ``num_experts``. Grouping is a no-op on this path
(``topk_group == num_expert_group``), so any such value routes identically;
the constraint is purely about which kernel exists.
"""
import pytest
import torch
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
AITER_MAX_EXPERTS_PER_GROUP as MAX_EXPERTS_PER_GROUP,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
AITER_SUPPORTED_NUM_GRP as SUPPORTED_NUM_GRP,
)
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
_aiter_can_use_biased_grouped_topk,
_aiter_get_num_expert_group,
)
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
grouped_topk,
)
from vllm.platforms import current_platform
from vllm.platforms.rocm import on_cdna
pytestmark = pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm-specific tests"
)
@pytest.mark.parametrize(
("num_experts", "expected"),
[
(8, 1), # fits in a single group
(32, 1), # exactly at the per-group limit
(64, 2),
(128, 4),
(256, 8),
(72, 8), # naive ceil gives 3 -> unsupported, rounds up to 8
(96, 8), # naive ceil gives 3 -> unsupported, rounds up to 8
(160, 8), # naive ceil gives 5 -> unsupported, rounds up to 8
(192, 8), # naive ceil gives 6 -> unsupported, rounds up to 8
# No supported NUM_GRP divides these within the group-size limit, so
# the naive value is kept and the call site declines the kernel.
(33, 3),
(129, 43),
(257, 257),
(320, 10),
(384, 12),
],
)
def test_group_count_and_guard(num_experts, expected):
g = _aiter_get_num_expert_group(num_experts)
assert g == expected
# The router asserts both unconditionally; rounding must never break them.
assert num_experts % g == 0
assert num_experts // g <= MAX_EXPERTS_PER_GROUP
supported = g in SUPPORTED_NUM_GRP
# Ample topk, so only the membership check can decline.
assert _aiter_can_use_biased_grouped_topk(num_experts, topk=64) == supported
if supported:
assert _aiter_can_use_biased_grouped_topk(num_experts, topk=g)
assert not _aiter_can_use_biased_grouped_topk(num_experts, topk=g - 1)
@pytest.mark.skipif(not on_cdna(), reason="CDNA ROCm only")
@pytest.mark.parametrize(
"num_experts",
[
64, # g == 2, not rounded -- covers the no-op claim itself
128, # g == 4, not rounded
256, # g == 8, not rounded
96, # naive 3 -> rounded to 8
192, # naive 6 -> rounded to 8
],
)
def test_rounded_group_count_routes_like_the_reference(num_experts):
"""The rounded group count must route identically to ``grouped_topk``.
Gating is continuous random, so exact score ties -- the one place the two
may legitimately differ -- have measure zero.
"""
torch.manual_seed(num_experts)
device = "cuda"
topk = 8
num_tokens = 83 # not a multiple of any warp/tile size
g = _aiter_get_num_expert_group(num_experts)
assert _aiter_can_use_biased_grouped_topk(num_experts, topk)
gating = torch.randn(num_tokens, num_experts, dtype=torch.float32, device=device)
bias = torch.randn(num_experts, dtype=torch.float32, device=device)
ref_weights, ref_ids = grouped_topk(
hidden_states=torch.empty(num_tokens, 1, device=device),
gating_output=gating,
topk=topk,
renormalize=True,
num_expert_group=g,
topk_group=g,
scoring_func="sigmoid",
e_score_correction_bias=bias,
)
weights = torch.empty(num_tokens, topk, dtype=torch.float32, device=device)
ids = torch.empty(num_tokens, topk, dtype=torch.int32, device=device)
rocm_aiter_ops.biased_grouped_topk(
gating,
bias,
weights,
ids,
num_expert_group=g,
topk_group=g,
need_renorm=True,
)
# Neither side promises an order within a token's top-k, so sort by expert
# id and carry the weights through the same permutation -- sorting the two
# independently would not catch a weight attached to the wrong expert.
order = ids.argsort(dim=-1)
ref_order = ref_ids.argsort(dim=-1)
torch.testing.assert_close(
ids.gather(1, order), ref_ids.to(torch.int32).gather(1, ref_order)
)
torch.testing.assert_close(
weights.gather(1, order),
ref_weights.gather(1, ref_order),
atol=1e-4,
rtol=1e-4,
)
+3 -2
View File
@@ -224,8 +224,9 @@ def test_zen_int8_dispatch_contract(
call = mock_zentorch_fused_moe["fused_moe"]
assert call["w13_scales"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE)
assert call["w2_scales"].shape == (EXPERT_NUM, HIDDEN_SIZE)
assert call["w13_scales"].dtype == torch.bfloat16
assert call["w2_scales"].dtype == torch.bfloat16
# f32 scales keep the layer on zentorch's fast routed MoE executor.
assert call["w13_scales"].dtype == torch.float32
assert call["w2_scales"].dtype == torch.float32
assert call["act"] == act.value.lower()
assert call["skip_weighted"] is False
assert call["topk_weights"].dtype == torch.float32
-92
View File
@@ -7,7 +7,6 @@ import torch
import vllm._custom_ops as ops
from tests.kernels.quant_utils import per_block_cast_to_int8
from tests.kernels.quantization.nvfp4_utils import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.all2all_utils import (
maybe_make_prepare_finalize,
@@ -552,97 +551,6 @@ class TestMLP(torch.nn.Module):
return x
class RealMLP(torch.nn.Module):
def __init__(
self,
hidden_size: int,
intermediate_size: int,
w1: torch.Tensor,
w2: torch.Tensor,
hidden_act: str = "silu",
quant_config=None,
reduce_results: bool = True,
prefix: str = "",
w1_s: torch.Tensor | None = None,
w2_s: torch.Tensor | None = None,
) -> None:
from vllm.model_executor.layers.linear import (
MergedColumnParallelLinear,
RowParallelLinear,
)
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
hidden_size,
[intermediate_size] * 2,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.gate_up_proj",
)
self.gate_up_proj.register_parameter(
"weight", torch.nn.Parameter(w1, requires_grad=False)
)
self.gate_up_proj.register_parameter(
"weight_scale", torch.nn.Parameter(w1_s, requires_grad=False)
)
self.gate_up_proj.register_parameter(
"input_scale", None
) # torch.nn.Parameter(None, requires_grad=False))
self.down_proj = RowParallelLinear(
intermediate_size,
hidden_size,
bias=False,
quant_config=quant_config,
reduce_results=reduce_results,
prefix=f"{prefix}.down_proj",
)
self.down_proj.register_parameter(
"weight", torch.nn.Parameter(w2, requires_grad=False)
)
self.down_proj.register_parameter(
"weight_scale", torch.nn.Parameter(w2_s, requires_grad=False)
)
self.down_proj.register_parameter(
"input_scale", None
) # torch.nn.Parameter(None, requires_grad=False))
if hidden_act != "silu":
raise ValueError(
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
)
self.act_fn = SiluAndMul()
def forward(self, x):
gate_up, _ = self.gate_up_proj(x)
x = self.act_fn(gate_up)
x, _ = self.down_proj(x)
return x
def make_shared_experts_with_weights(
N: int,
K: int,
in_dtype: torch.dtype,
w1: torch.Tensor,
w2: torch.Tensor,
w1_s: torch.Tensor | None = None,
w2_s: torch.Tensor | None = None,
quant_dtype: torch.dtype | str | None = None,
) -> torch.nn.Module:
old_dtype = torch.get_default_dtype()
try:
torch.set_default_dtype(in_dtype)
if quant_dtype == torch.float8_e4m3fn:
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
quant_config = Fp8Config()
else:
quant_config = None
return RealMLP(K, N, w1, w2, "silu", quant_config, w1_s=w1_s, w2_s=w2_s)
finally:
torch.set_default_dtype(old_dtype)
def modular_triton_fused_moe(
moe_config: FusedMoEConfig,
quant_config: FusedMoEQuantConfig,
+2 -1
View File
@@ -37,7 +37,8 @@ from vllm.utils.flashinfer import (
)
from vllm.utils.import_utils import has_deep_gemm
if current_platform.get_device_capability() < (9, 0):
capability = current_platform.get_device_capability()
if capability is None or capability < (9, 0):
pytest.skip("FP8 Triton requires CUDA 9.0 or higher", allow_module_level=True)
vllm_config = VllmConfig()
@@ -62,7 +62,8 @@ PER_TOKEN_GROUP_SHAPE = (1, -1)
PER_OUT_CH_GROUP_SHAPE = (-1, 1)
capability = current_platform.get_device_capability()
capability = capability[0] * 10 + capability[1]
# Fall back to 0 so import doesn't fail on unknown capability
capability_int = (capability[0] * 10 + capability[1]) if capability is not None else 0
def rand_int8(shape: tuple, device: str = "cuda"):
@@ -639,7 +640,7 @@ def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool):
def test_cutlass_support_opcheck():
opcheck(torch.ops._C.cutlass_scaled_mm_supports_fp8, (capability,))
opcheck(torch.ops._C.cutlass_scaled_mm_supports_fp8, (capability_int,))
@pytest.mark.parametrize("num_experts", [8, 64])
@@ -29,7 +29,8 @@ if not current_platform.is_cuda():
# `is_quant_method_supported` conflates kernels with quantization methods
# an assumption which is breaking down as quantizations methods can have
# have kernels and some kernels support multiple quantization methods.
IS_SUPPORTED_BY_GPU = current_platform.get_device_capability()[0] >= 9
capability = current_platform.get_device_capability()
IS_SUPPORTED_BY_GPU = capability is not None and capability[0] >= 9
MNK_SHAPES = [
(1, 128, 128),
@@ -17,8 +17,9 @@ from vllm.platforms import current_platform
from vllm.scalar_type import ScalarType, scalar_types
from vllm.utils.torch_utils import set_random_seed
capability = current_platform.get_device_capability()
IS_SUPPORTED_BY_GPU = (
current_platform.is_cuda() and current_platform.get_device_capability()[0] >= 9
current_platform.is_cuda() and capability is not None and capability[0] >= 9
)
+6 -2
View File
@@ -12,6 +12,7 @@ from tests.kernels.quant_utils import (
)
from tests.kernels.utils import opcheck
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
scaled_quantize,
)
from vllm.platforms import current_platform
@@ -208,7 +209,10 @@ def test_static_fp8_quant_group_2d(
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
ref_out, scale = scaled_quantize(
x, group_shape, current_platform.fp8_dtype(), compute_dtype=torch.float32
x,
GroupShape(*group_shape),
current_platform.fp8_dtype(),
compute_dtype=torch.float32,
)
ops_out, ops_scale = ops.scaled_fp8_quant(x, scale=scale, group_shape=group_shape)
@@ -236,7 +240,7 @@ def test_static_fp8_quant_1d_scale(
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
ref_out, scale_2d = scaled_quantize(
x, group_shape, FP8_DTYPE, compute_dtype=torch.float32
x, GroupShape(*group_shape), FP8_DTYPE, compute_dtype=torch.float32
)
# Flatten scale to 1D for testing 1D scale path
@@ -38,7 +38,8 @@ CUDA_DEVICES = [
# `is_quant_method_supported` conflates kernels with quantization methods
# an assumption which is breaking down as quantizations methods can have
# have kernels and some kernels support multiple quantization methods.
IS_SUPPORTED_BY_GPU = current_platform.get_device_capability()[0] >= 9
capability = current_platform.get_device_capability()
IS_SUPPORTED_BY_GPU = capability is not None and capability[0] >= 9
MNK_SHAPES = [
(1, 128, 128),
@@ -305,7 +306,7 @@ def test_machete_all_schedules(shape, types: TypeConfig):
if types.group_scale_type is None:
group_sizes = [None]
else:
group_sizes = query_machete_supported_group_sizes(types.act_type)
group_sizes = list(query_machete_supported_group_sizes(types.act_type))
for group_size in group_sizes:
if not group_size_valid(shape, group_size):
@@ -334,7 +335,7 @@ def test_machete_heuristic(shape, types: TypeConfig):
if types.group_scale_type is None:
group_sizes = [None]
else:
group_sizes = query_machete_supported_group_sizes(types.act_type)
group_sizes = list(query_machete_supported_group_sizes(types.act_type))
for group_size in group_sizes:
if not group_size_valid(shape, group_size):
@@ -342,7 +342,7 @@ def marlin_generate_valid_test_cases():
[quant_test_config["b_type"]],
quant_test_config.get("c_type", f16_types),
quant_test_config["group_blocks"],
)
) # type: ignore[call-overload]
for sub_case in inner_combinations:
if (
@@ -11,6 +11,7 @@ import pytest
import torch
from vllm import _custom_ops as ops
from vllm.model_executor.layers.linear import LinearBase
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
GPTQ_MARLIN_TILE,
apply_gptq_marlin_linear,
@@ -524,7 +525,7 @@ def test_check_marlin_supports_layer_allow_tile_padding():
)
# Tile-misaligned but group-aligned: rejected strictly, allowed w/ padding
layer = _FakeLinear(4640, 512, input_size=2048)
layer: LinearBase = _FakeLinear(4640, 512, input_size=2048)
assert not check_marlin_supports_layer(layer, 128)
assert check_marlin_supports_layer(layer, 128, allow_tile_padding=True)
assert check_marlin_supports_layer(layer, -1, allow_tile_padding=True)
@@ -35,7 +35,7 @@ pytestmark = pytest.mark.cpu_test
def test_auto_linear_backend_is_unchanged_without_overrides():
kernels = [object]
kernels: list[type] = [object]
quantizations = (
"fp8_block_w8a8",
"fp8_w8a8",
@@ -0,0 +1,661 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the zentorch DA8W4 (W4A8) int4 linear and MoE paths on Zen CPUs.
The zentorch ops are mocked with reference implementations when zentorch is not
installed, so the layout and dispatch contracts are covered in CI.
"""
import dataclasses
import pytest
import torch
from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32
from tests.kernels.quant_utils import ref_dynamic_per_token_quant
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
MPLinearLayerConfig,
)
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
ZentorchWNA16LinearKernel,
_import_unpack_from_int32,
)
from vllm.model_executor.layers.fused_moe.oracle import int_wna16
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import WNA16MoEBackend
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
GROUP_SIZE = 128
IN_FEATURES = 512
OUT_FEATURES = 128
def _unpack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor:
"""int8 [N, K/2] (or int32 [N, K/8]) packed s4 -> float32 [N, K]."""
words = packed.view(torch.int32)
out = torch.zeros(words.shape[0], in_features, dtype=torch.float32)
for i in range(8):
nibble = (words >> (4 * i)) & 0xF
out[:, i::8] = torch.where(nibble > 7, nibble - 16, nibble).float()
return out
def _repack_s4(unpacked: torch.Tensor) -> torch.Tensor:
"""Reference zentorch_woq_repack_weight: int8 [N, K] -> int32 [N, K/8]."""
n, k = unpacked.shape
values = unpacked.to(torch.int32).reshape(n, k // 8, 8)
out = torch.zeros(n, k // 8, dtype=torch.int32)
for i in range(8):
out |= (values[:, :, i] & 0xF) << (4 * i)
return out
def _quantize_per_group(
weight: torch.Tensor, group_size: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""Symmetric int4 per-group quantization -> (int8 in [-8, 7], scale [N, G])."""
out_features, in_features = weight.shape
grouped = weight.float().reshape(
out_features, in_features // group_size, group_size
)
scale = grouped.abs().amax(dim=-1) / 8.0
quantized = (grouped / scale.unsqueeze(-1)).round().clamp(-8, 7).to(torch.int8)
return quantized.reshape(out_features, in_features), scale.to(torch.bfloat16)
def _dequantize_per_group(
quantized: torch.Tensor, scale: torch.Tensor, group_size: int
) -> torch.Tensor:
"""Dequantize with [N, G] scales into a float32 [N, K] weight."""
return quantized.float() * scale.float().repeat_interleave(group_size, dim=1)
def _dynamic_quant_matmul(
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None
) -> torch.Tensor:
"""DA8W4 reference: per-token int8 activation quant against a bf16 weight."""
x_q, scale = ref_dynamic_per_token_quant(x.float(), torch.int8)
out = (x_q.float() @ weight.t()) * scale
if bias is not None:
out = out + bias.float()
return out
@pytest.fixture
def mock_zentorch_ops():
"""Register reference zentorch ops when zentorch is not installed."""
if hasattr(torch.ops.zentorch, "zentorch_dynamic_qlinear"):
yield None
return
calls: dict[str, tuple] = {}
def _dynamic_qlinear(input, weight, weight_scales, bias=None, zentorch_op_name=""):
calls["dynamic_qlinear"] = (input, weight, weight_scales, bias)
in_features = weight.view(torch.int32).shape[1] * 8
group_size = in_features // weight_scales.shape[0]
# weight_scales is [G, N]; expand to a [N, K] dequantized weight.
scales = weight_scales.float().t().repeat_interleave(group_size, dim=1)
weight_deq = _unpack_s4(weight, in_features) * scales
out = _dynamic_quant_matmul(input, weight_deq, bias)
return out.to(input.dtype)
def _fused_moe(
output,
input,
w13,
w2,
w13_bias,
w2_bias,
topk_weights,
topk_id,
skip_weighted,
act,
w13_scales=None,
w2_scales=None,
zentorch_op_name="",
):
calls["fused_moe"] = (
input,
w13,
w2,
topk_weights,
topk_id,
skip_weighted,
act,
w13_scales,
w2_scales,
)
output.zero_()
lib_def = torch.library.Library("zentorch", "DEF")
lib_def.define("zentorch_woq_repack_weight(Tensor unpacked_weight) -> Tensor")
lib_def.define(
"zentorch_dynamic_qlinear(Tensor input, Tensor weight, "
"Tensor weight_scales, Tensor? bias=None, *, "
"str zentorch_op_name='zentorch::zentorch_dynamic_qlinear') -> Tensor"
)
lib_def.define(
"zentorch_fused_moe(Tensor(a!) output, Tensor input, Tensor w13, "
"Tensor w2, Tensor? w13_bias, Tensor? w2_bias, Tensor topk_weights, "
"Tensor topk_id, bool skip_weighted, str act, "
"Tensor? w13_scales=None, Tensor? w2_scales=None, *, "
"str zentorch_op_name='zentorch::zentorch_fused_moe') -> ()"
)
lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
lib_impl.impl("zentorch_woq_repack_weight", _repack_s4)
lib_impl.impl("zentorch_dynamic_qlinear", _dynamic_qlinear)
lib_impl.impl("zentorch_fused_moe", _fused_moe)
yield calls
lib_impl._destroy()
lib_def._destroy()
def _make_config(
act_type: torch.dtype = torch.bfloat16,
weight_type=scalar_types.uint4b8,
zero_points: bool = False,
group_size: int = GROUP_SIZE,
) -> MPLinearLayerConfig:
return MPLinearLayerConfig(
full_weight_shape=(IN_FEATURES, OUT_FEATURES),
partition_weight_shape=(IN_FEATURES, OUT_FEATURES),
weight_type=weight_type,
act_type=act_type,
group_size=group_size,
zero_points=zero_points,
)
def _make_layer(group_size: int = GROUP_SIZE) -> tuple[torch.nn.Module, torch.Tensor]:
"""Build a layer holding a CT-packed int4 weight, plus its dequantized form."""
torch.manual_seed(0)
weight = torch.randn(OUT_FEATURES, IN_FEATURES, dtype=torch.bfloat16)
quantized, scale = _quantize_per_group(weight, group_size)
# compressed-tensors stores [N, K//8] int32, packed along the input dim.
packed = pack_to_int32(quantized, 4, packed_dim=1)
layer = torch.nn.Module()
layer.weight_packed = torch.nn.Parameter(packed, requires_grad=False)
layer.weight_packed.input_dim = 1
layer.weight_packed.packed_dim = 1
layer.weight_scale = torch.nn.Parameter(scale, requires_grad=False)
layer.weight_zero_point = None
return layer, _dequantize_per_group(quantized, scale, group_size)
def _make_kernel(config: MPLinearLayerConfig) -> ZentorchWNA16LinearKernel:
return ZentorchWNA16LinearKernel(
config, "weight_packed", "weight_scale", "weight_zero_point"
)
# ---------------------------------------------------------------------------
# Dense DA8W4
# ---------------------------------------------------------------------------
def test_da8w4_eligible_for_symmetric_bf16_layer(mock_zentorch_ops, monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config())
assert kernel._maybe_process_da8w4_weights(layer)
@pytest.mark.parametrize(
"config_kwargs,reason",
[
({"act_type": torch.float32}, "f32 activations are rejected"),
(
{"weight_type": scalar_types.uint4, "zero_points": True},
"asymmetric int4 is unsupported",
),
],
)
def test_da8w4_not_eligible(mock_zentorch_ops, monkeypatch, config_kwargs, reason):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config(**config_kwargs))
assert not kernel._maybe_process_da8w4_weights(layer), reason
def test_da8w4_not_eligible_when_env_disabled(mock_zentorch_ops, monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"envs.VLLM_CPU_INT4_W4A8",
False,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config())
assert not kernel._maybe_process_da8w4_weights(layer)
def test_da8w4_process_weights_layout(mock_zentorch_ops, monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config())
kernel.process_weights_after_loading(layer)
assert layer._zentorch_da8w4
assert layer._zentorch_kind == "compressed_tensors_w4a8_da8w4"
# Packed s4 holds two nibbles per byte, and scales transpose to {G, N}.
assert layer._zentorch_da8w4_packed.dtype == torch.int8
assert layer._zentorch_da8w4_packed.shape == (OUT_FEATURES, IN_FEATURES // 2)
assert layer._zentorch_da8w4_scale.dtype == torch.bfloat16
assert layer._zentorch_da8w4_scale.shape == (
IN_FEATURES // GROUP_SIZE,
OUT_FEATURES,
)
assert layer._zentorch_da8w4_packed.is_contiguous()
assert layer._zentorch_da8w4_scale.is_contiguous()
# The checkpoint parameters are released once repacked.
assert layer.weight_packed.numel() == 0
assert layer.weight_scale.numel() == 0
def test_da8w4_apply_weights_matches_dequantized_reference(
mock_zentorch_ops, monkeypatch
):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
layer, weight_deq = _make_layer()
kernel = _make_kernel(_make_config())
kernel.process_weights_after_loading(layer)
x = torch.randn(4, IN_FEATURES, dtype=torch.bfloat16)
bias = torch.randn(OUT_FEATURES, dtype=torch.bfloat16)
out = kernel.apply_weights(layer, x, bias)
expected = _dynamic_quant_matmul(x, weight_deq, bias)
assert out.dtype == torch.bfloat16
torch.testing.assert_close(out.float(), expected, rtol=2e-2, atol=2e-2)
def test_da8w4_falls_back_to_w4a16_when_op_missing(monkeypatch):
"""Without zentorch_dynamic_qlinear the layer must not take the DA8W4 path."""
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
lambda ops: "zentorch_dynamic_qlinear" not in ops,
)
layer, _ = _make_layer()
kernel = _make_kernel(_make_config())
assert not kernel._maybe_process_da8w4_weights(layer)
def test_can_implement_requires_zen_cpu(monkeypatch):
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: False,
)
ok, reason = ZentorchWNA16LinearKernel.can_implement(_make_config())
assert not ok
assert reason is not None
def test_can_implement_accepts_shapes_cpuwna16_rejects(monkeypatch):
"""CPUWNA16's N/K % 32 rule is a oneDNN packing constraint zentorch lacks."""
from vllm.model_executor.kernels.linear.mixed_precision.cpu import (
CPUWNA16LinearKernel,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.cpu."
"current_platform.is_cpu",
lambda: True,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
lambda ops: True,
)
unaligned = dataclasses.replace(
_make_config(), partition_weight_shape=(IN_FEATURES, OUT_FEATURES - 16)
)
assert unaligned.partition_weight_shape[1] % 32 != 0
ok, _ = CPUWNA16LinearKernel.can_implement(unaligned)
assert not ok, "the parent is expected to turn this shape away"
ok, reason = ZentorchWNA16LinearKernel.can_implement(unaligned)
assert ok, reason
@pytest.mark.parametrize(
"group_size,expected_ok",
[
(-1, True), # per-channel, one group spanning K
(GROUP_SIZE, True),
(IN_FEATURES, True),
(6, False), # even, so the parent's multiple-of-2 rule would allow it
(0, False),
],
)
def test_can_implement_group_size_must_divide_k(monkeypatch, group_size, expected_ok):
"""ZenDNN WOQ groups along K, so a group that straddles the end is unusable."""
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
"current_platform.is_zen_cpu",
lambda: True,
)
monkeypatch.setattr(
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
lambda ops: True,
)
ok, reason = ZentorchWNA16LinearKernel.can_implement(
_make_config(group_size=group_size)
)
assert ok is expected_ok, reason
if not expected_ok:
assert reason is not None
assert "must divide input size" in reason
# ---------------------------------------------------------------------------
# MoE DA8W4
# ---------------------------------------------------------------------------
NUM_EXPERTS = 4
HIDDEN = 256
INTERMEDIATE = 128
def _make_moe_weights(group_size: int = 32):
"""Build CT-layout MoE weights: w13 [E, 2I, H//8], w2 [E, H, I//8]."""
torch.manual_seed(0)
w13_q, w13_s, w2_q, w2_s = [], [], [], []
for _ in range(NUM_EXPERTS):
w13 = torch.randn(2 * INTERMEDIATE, HIDDEN, dtype=torch.bfloat16)
q13, s13 = _quantize_per_group(w13, group_size)
# CT is N-first and packs along the input dim: [N, K//8] per expert.
w13_q.append(pack_to_int32(q13, 4, packed_dim=1))
w13_s.append(s13)
w2 = torch.randn(HIDDEN, INTERMEDIATE, dtype=torch.bfloat16)
q2, s2 = _quantize_per_group(w2, group_size)
w2_q.append(pack_to_int32(q2, 4, packed_dim=1))
w2_s.append(s2)
return (
torch.stack(w13_q),
torch.stack(w2_q),
torch.stack(w13_s),
torch.stack(w2_s),
)
def test_zen_cpu_first_in_cpu_backend_priority(monkeypatch):
monkeypatch.setattr(int_wna16.current_platform, "is_cpu", lambda: True)
backends = int_wna16._get_priority_backends()
assert backends[0] == WNA16MoEBackend.ZEN_CPU
# The generic CPU backend stays available as a fallback.
assert WNA16MoEBackend.CPU in backends
def test_zen_cpu_process_weights_layout(mock_zentorch_ops):
group_size = 32
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
assert w13.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8)
assert w2.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8)
converted = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)
w13_out, w2_out, w13_s_out, w2_s_out = converted[:4]
# zentorch consumes [E, N, K/8] packed s4 with per-group [E, G, N] scales.
assert w13_out.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8)
assert w2_out.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8)
assert w13_out.dtype == w2_out.dtype == torch.int32
assert w13_s_out.shape == (NUM_EXPERTS, HIDDEN // group_size, 2 * INTERMEDIATE)
assert w2_s_out.shape == (NUM_EXPERTS, INTERMEDIATE // group_size, HIDDEN)
assert w13_s_out.dtype == w2_s_out.dtype == torch.bfloat16
# Symmetric checkpoints carry no zero points into the kernel.
assert converted[8] is None and converted[9] is None
def test_zen_cpu_repack_is_value_exact(mock_zentorch_ops):
"""The repacked weight must dequantize back to the checkpoint values."""
group_size = 32
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
w13_out = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0]
expected = _import_unpack_from_int32()(
w13,
4,
torch.Size([NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN]),
packed_dim=1,
)
for expert in range(NUM_EXPERTS):
torch.testing.assert_close(
_unpack_s4(w13_out[expert], HIDDEN),
expected[expert].float(),
)
@pytest.mark.parametrize(
"group_size,expected_ok",
[(32, True), (128, True), (-1, False), (2, False)],
)
def test_zen_cpu_group_size_gating(group_size, expected_ok):
quant_config = type("Args", (), {"group_size": group_size})()
reason = int_wna16._backend_incompatibility_reason(
WNA16MoEBackend.ZEN_CPU,
moe_config=None,
quant_config=quant_config,
may_have_zp=False,
may_have_bias=False,
allow_tile_padding=True,
)
assert (reason is None) == expected_ok
@pytest.mark.parametrize(
"may_have_zp,may_have_bias,expected_ok",
[(True, False, False), (False, True, True), (True, True, False)],
)
def test_zen_cpu_rejects_zero_points_but_takes_bias(
may_have_zp, may_have_bias, expected_ok
):
quant_config = type("Args", (), {"group_size": 128})()
reason = int_wna16._backend_incompatibility_reason(
WNA16MoEBackend.ZEN_CPU,
moe_config=None,
quant_config=quant_config,
may_have_zp=may_have_zp,
may_have_bias=may_have_bias,
allow_tile_padding=True,
)
assert (reason is None) == expected_ok
def test_zen_cpu_rejects_moe_wna16_layout():
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
quant_config = MoeWNA16Config(
linear_quant_method="gptq",
weight_bits=4,
group_size=128,
has_zp=False,
lm_head_quantized=False,
modules_to_not_convert=None,
full_config={},
)
reason = int_wna16._backend_incompatibility_reason(
WNA16MoEBackend.ZEN_CPU,
moe_config=None,
quant_config=quant_config,
may_have_zp=False,
may_have_bias=False,
allow_tile_padding=True,
)
assert reason is not None
def test_zen_cpu_disabled_by_env(monkeypatch):
monkeypatch.setattr(int_wna16.envs, "VLLM_CPU_INT4_W4A8", False)
quant_config = type("Args", (), {"group_size": 128})()
reason = int_wna16._backend_incompatibility_reason(
WNA16MoEBackend.ZEN_CPU,
moe_config=None,
quant_config=quant_config,
may_have_zp=False,
may_have_bias=False,
allow_tile_padding=True,
)
assert reason is not None
def test_zen_cpu_backend_maps_to_experts_class():
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
assert int_wna16.backend_to_kernel_cls(WNA16MoEBackend.ZEN_CPU) == [
ZentorchExpertsInt4
]
def test_zen_experts_support_predicates():
from vllm.model_executor.kernels.linear.zentorch_utils import (
_ZENTORCH_MOE_ACTIVATIONS,
)
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kInt4Static,
kInt4Static32,
kInt8StaticChannelSym,
)
assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static, None)
assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static32, None)
assert not ZentorchExpertsInt4._supports_quant_scheme(kInt8StaticChannelSym, None)
for act in MoEActivation:
expected = act.value in _ZENTORCH_MOE_ACTIVATIONS
assert ZentorchExpertsInt4._supports_activation(act) == expected
assert ZentorchExpertsInt4.requires_interleaved_w13
assert not ZentorchExpertsInt4._supports_no_act_and_mul()
assert ZentorchExpertsInt4._supports_parallel_config(
type("Par", (), {"use_ep": False})()
)
assert not ZentorchExpertsInt4._supports_parallel_config(
type("Par", (), {"use_ep": True})()
)
def test_zen_experts_take_custom_routing():
"""Models with their own router (gemma-4) reach select_experts through the
config captured off the layer, since apply() cannot be handed the callable."""
from types import SimpleNamespace
from tests.kernels.moe.test_cpu_fused_moe import _StubMoELayer
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
assert ZentorchExpertsInt4._supports_routing_method(
RoutingMethodType.Custom, None, None
)
def routing_fn(**kwargs):
raise AssertionError("not called")
layer = _StubMoELayer(
torch.zeros(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN),
torch.zeros(NUM_EXPERTS, HIDDEN, INTERMEDIATE),
MoEActivation.SILU,
)
layer.renormalize = True
layer.custom_routing_function = routing_fn
experts = SimpleNamespace(renormalize=False, custom_routing_function=None)
ZentorchExpertsInt4.process_weights_after_loading(experts, layer)
assert experts.custom_routing_function is routing_fn
assert experts.renormalize is True
def _swigluoai_perm(activation, experts_cls=None):
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
swigluoai_w13_interleave_perm,
)
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
ZentorchExpertsInt4,
)
return swigluoai_w13_interleave_perm(
experts_cls or ZentorchExpertsInt4,
activation,
2 * INTERMEDIATE,
torch.device("cpu"),
)
def test_swigluoai_perm_only_for_zen_swigluoai():
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
assert _swigluoai_perm(MoEActivation.SWIGLUOAI) is not None
# Other activations keep the half-split layout the loader leaves.
assert _swigluoai_perm(MoEActivation.SILU) is None
# object stands in for any kernel that never set requires_interleaved_w13.
assert _swigluoai_perm(MoEActivation.SWIGLUOAI, experts_cls=object) is None
def test_swigluoai_perm_interleaves_weights_scales_and_bias(mock_zentorch_ops):
"""The permutation is a gather on w13's output-channel axis, which is dim 1
for the N-first packed weights and their group scales."""
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
group_size = 32
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
bias = torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, dtype=torch.bfloat16)
perm = _swigluoai_perm(MoEActivation.SWIGLUOAI)
baseline = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0]
permuted = int_wna16._process_weights_zen_cpu(
w13[:, perm].contiguous(), w2, w13_scale[:, perm].contiguous(), w2_scale
)[0]
for expert in range(NUM_EXPERTS):
before = _unpack_s4(baseline[expert], HIDDEN)
after = _unpack_s4(permuted[expert], HIDDEN)
torch.testing.assert_close(after[0::2], before[:INTERMEDIATE])
torch.testing.assert_close(after[1::2], before[INTERMEDIATE:])
torch.testing.assert_close(bias[:, perm][:, 0::2], bias[:, :INTERMEDIATE])
torch.testing.assert_close(bias[:, perm][:, 1::2], bias[:, INTERMEDIATE:])
+5 -4
View File
@@ -203,7 +203,7 @@ class TestInt4ScaledMmCpu:
print(f" [PASS] INT4 GEMM with bias: M={M}")
def test_gemm_3d_input(self):
"""apply() reshapes 3D input [B, S, K] -> [B*S, K] -> back to 3D."""
"""int4_scaled_mm_cpu flattens 3D [B, S, K] to 2D and reshapes back."""
K, N, group_size = 256, 128, 128
(packed_qweight, packed_qzeros, scales, float_ref, _, _) = (
make_awq_checkpoint_data(K, N, group_size)
@@ -220,12 +220,13 @@ class TestInt4ScaledMmCpu:
x_3d = torch.randn(B, S, K, dtype=torch.bfloat16)
x_2d = x_3d.reshape(-1, K)
out_3d = int4_scaled_mm_cpu(x_3d, blocked_w, blocked_zp, blocked_s, None)
out_2d = int4_scaled_mm_cpu(x_2d, blocked_w, blocked_zp, blocked_s, None)
out_3d = out_2d.reshape(B, S, N)
ref_out = torch.mm(x_2d.float(), float_ref).reshape(B, S, N)
assert out_3d.shape == (B, S, N)
torch.testing.assert_close(out_3d.reshape(-1, N), out_2d)
ref_out = torch.mm(x_2d.float(), float_ref).reshape(B, S, N)
abs_diff = (out_3d.float() - ref_out).abs()
mean_abs = abs_diff.mean().item()
ref_mag = ref_out.abs().mean().item() + 1e-6
+33
View File
@@ -85,6 +85,12 @@ QWEN4_EXP_SM90_CASES = [
for num_tokens, config in plans.items()
]
QWEN4_EXP_SM100_CASES = [
(n, k, num_tokens, config)
for (n, k), plans in qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS.items()
for num_tokens, config in plans.items()
]
EXPECTED_CUTE_CONFIGS = {
(3072, 7168, 1): (224, 3, 4, 8),
(3072, 7168, 2): (128, 3, 2, 8),
@@ -269,6 +275,9 @@ def test_kda_overlap_configs_match_measured_table() -> None:
}
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="KDA CuTeDSL helper requires CUDA"
)
@pytest.mark.parametrize(
"compute_capability,supported",
[((9, 0), False), ((10, 0), True), ((10, 3), True), ((12, 0), True)],
@@ -714,6 +723,7 @@ def test_qwen4_exp_hopper_plans_are_valid() -> None:
"capability,expected_plans",
[
((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS),
((10, 0), qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS),
((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS),
((8, 0), {}),
],
@@ -906,6 +916,29 @@ def test_qwen4_exp_sm90_selected_shapes(
assert cosine > 0.999
@pytest.mark.parametrize("n,k,num_tokens,config", QWEN4_EXP_SM100_CASES)
def test_qwen4_exp_sm100_selected_shapes(
n: int,
k: int,
num_tokens: int,
config: SkinnyGemmConfig,
) -> None:
_require_capability_and_cute((10, 0))
torch.manual_seed(42 + num_tokens)
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
selected = qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS[(n, k)][num_tokens]
assert selected == config
output = qwen4_exp_gemm._qwen4_exp_low_latency_gemm(x, weight)
reference = torch.nn.functional.linear(x, weight)
cosine = torch.nn.functional.cosine_similarity(
output.float().flatten(), reference.float().flatten(), dim=0
).item()
assert cosine > 0.999
def test_glm52_q_b_nonpacked_single_row_falls_back() -> None:
_require_sm103_and_cute()
spec = glm52_gemm.GLM52_Q_B_PROJECTION
+6 -2
View File
@@ -2182,7 +2182,9 @@ def test_v41_rocm_csa2_full_pipeline():
)
q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
assert torch.equal(q_out, q)
assert torch.equal(
q_out, q.view(num_tokens, attention.n_local_heads, attention.head_dim)
)
assert torch.equal(kv_out, kv)
assert torch.equal(index_q_out, index_q)
assert torch.equal(index_q_scale_out, index_q_scale)
@@ -2273,7 +2275,9 @@ def test_v41_rocm_csa2_reindex_pipeline():
)
q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
assert torch.equal(q_out, q)
assert torch.equal(
q_out, q.view(num_tokens, attention.n_local_heads, attention.head_dim)
)
assert torch.equal(kv_out, kv)
assert torch.equal(index_q_out, index_q)
assert torch.equal(index_q_scale_out, index_q_scale)
+120
View File
@@ -0,0 +1,120 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tracemalloc
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_cpu():
pytest.skip("skipping CPU-only tests", allow_module_level=True)
import vllm._C # noqa: F401, E402
VOCAB_SIZES = [32000, 49152, 128256]
BATCH_SIZES = [1, 4, 16]
def reference_greedy_sample(logits: torch.Tensor) -> torch.Tensor:
return logits.argmax(dim=-1).view(-1)
class TestGreedyArgmax:
@pytest.mark.parametrize("vocab_size", VOCAB_SIZES)
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
def test_exact_match(self, vocab_size: int, batch_size: int):
logits = torch.randn(batch_size, vocab_size, dtype=torch.float32)
expected = reference_greedy_sample(logits)
result = torch.ops._C.greedy_argmax(logits)
torch.testing.assert_close(result, expected)
def test_single_dominant(self):
logits = torch.full((1, 50000), -1e9, dtype=torch.float32)
logits[0, 42] = 100.0
assert torch.ops._C.greedy_argmax(logits).item() == 42
def test_negative_logits(self):
logits = torch.randn(8, 32000, dtype=torch.float32) - 10.0
expected = reference_greedy_sample(logits)
result = torch.ops._C.greedy_argmax(logits)
torch.testing.assert_close(result, expected)
class TestFusedGumbelArgmax:
@pytest.mark.parametrize("vocab_size", VOCAB_SIZES)
def test_distribution_chi_squared(self, vocab_size: int):
"""Verify sampling distribution via chi-squared goodness of fit."""
small_vocab = min(vocab_size, 100)
logits = torch.randn(1, small_vocab, dtype=torch.float32)
probs = logits.softmax(dim=-1).squeeze(0)
n_samples = 100_000
counts = torch.zeros(small_vocab)
for trial in range(n_samples):
seed = torch.tensor([trial * 7 + 13], dtype=torch.long)
tile = logits.expand(1, -1).contiguous()
idx = torch.ops._C.fused_gumbel_argmax(tile, seed)
counts[idx.item()] += 1
expected = probs * n_samples
mask = expected > 5
chi2 = ((counts[mask] - expected[mask]) ** 2 / expected[mask]).sum()
dof = mask.sum().item() - 1
from scipy.stats import chi2 as chi2_dist
p_value = 1.0 - chi2_dist.cdf(chi2.item(), dof)
assert p_value > 0.001, (
f"Chi-squared test failed: chi2={chi2.item():.1f}, "
f"dof={dof}, p={p_value:.6f}"
)
def test_deterministic_same_seed(self):
"""Same seed produces same result."""
logits = torch.randn(4, 32000, dtype=torch.float32)
seeds = torch.tensor([42, 123, 456, 789], dtype=torch.long)
r1 = torch.ops._C.fused_gumbel_argmax(logits, seeds)
r2 = torch.ops._C.fused_gumbel_argmax(logits, seeds)
torch.testing.assert_close(r1, r2)
def test_different_seeds_differ(self):
logits = torch.zeros(16, 50000, dtype=torch.float32)
seeds_a = torch.arange(16, dtype=torch.long)
seeds_b = torch.arange(16, dtype=torch.long) + 1_000_000
r_a = torch.ops._C.fused_gumbel_argmax(logits, seeds_a)
r_b = torch.ops._C.fused_gumbel_argmax(logits, seeds_b)
assert not torch.equal(r_a, r_b)
@pytest.mark.parametrize("vocab_size", VOCAB_SIZES)
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
def test_output_in_range(self, vocab_size: int, batch_size: int):
logits = torch.randn(batch_size, vocab_size, dtype=torch.float32)
seeds = torch.arange(batch_size, dtype=torch.long)
result = torch.ops._C.fused_gumbel_argmax(logits, seeds)
assert result.min() >= 0
assert result.max() < vocab_size
class TestMemory:
def test_fused_no_intermediate_allocs(self):
"""Verify that the fused kernel does not allocate large intermediates."""
logits = torch.randn(16, 128256, dtype=torch.float32)
seeds = torch.arange(16, dtype=torch.long)
torch.ops._C.fused_gumbel_argmax(logits, seeds)
tracemalloc.start()
snap_before = tracemalloc.take_snapshot()
for _ in range(50):
torch.ops._C.fused_gumbel_argmax(logits, seeds)
snap_after = tracemalloc.take_snapshot()
tracemalloc.stop()
diff = snap_after.compare_to(snap_before, "lineno")
total_new_bytes = sum(s.size_diff for s in diff if s.size_diff > 0)
vocab_bytes = 16 * 128256 * 4
assert total_new_bytes < vocab_bytes, (
f"Fused kernel allocated {total_new_bytes} bytes, "
f"expected < {vocab_bytes} (one intermediate tensor)"
)
+25 -25
View File
@@ -9,13 +9,10 @@ import torch
from vllm.models.deepseek_v41.common import engram as engram_ops
from vllm.models.deepseek_v41.common.engram import (
Engram as CommonEngram,
)
from vllm.models.deepseek_v41.common.engram import (
Engram,
NgramHashState,
ParallelEngramEmbedding,
)
from vllm.models.deepseek_v41.nvidia import engram as nvidia_engram_ops
from vllm.models.deepseek_v41.nvidia.engram import Engram, ParallelEngramEmbedding
from vllm.platforms import current_platform
@@ -111,7 +108,7 @@ def test_fused_engram_post_wkv_matches_reference(
)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: tp_rank)
module = CommonEngram.__new__(CommonEngram)
module = Engram.__new__(Engram)
torch.nn.Module.__init__(module)
module.dim = dim
module.hc_mult = hc_mult
@@ -121,6 +118,7 @@ def test_fused_engram_post_wkv_matches_reference(
module.embed_tokens = torch.nn.Identity()
# `forward` reads rows staged by `prepare_embeddings`, so inject kv there.
module.embed_tokens.tp_size = module.embed_tokens.n_hash_cols = 1
module.embed_tokens.dp_size = 1
module.staged_rows = kv.unsqueeze(1)
if use_sequence_parallel:
padded = torch.nn.functional.pad(kv, (0, 0, 0, (-num_kv_tokens) % tp_size))
@@ -592,9 +590,8 @@ def test_engram_rejects_empty_head_shards(tp_size, dp_size, n_heads, monkeypatch
monkeypatch.setattr(
engram_ops, "get_tensor_model_parallel_world_size", lambda: tp_size
)
monkeypatch.setattr(nvidia_engram_ops, "get_engram_dp_size", lambda: dp_size)
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: dp_size)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(nvidia_engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
with pytest.raises(AssertionError, match="ranks without hash heads"):
ParallelEngramEmbedding(n_heads * 17, 64, (17,) * n_heads)
@@ -602,10 +599,19 @@ def test_engram_rejects_empty_head_shards(tp_size, dp_size, n_heads, monkeypatch
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="CUDA or ROCm required"
)
@pytest.mark.parametrize("cpu_offload", [False, True])
@pytest.mark.parametrize(
"cpu_offload,sort_rows", [(False, False), (True, False), (True, True)]
)
@pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
def test_engram_head_shards_reconstruct_checkpoint(cpu_offload, tp_size, monkeypatch):
"""Keep complete buckets and reconstruct head order, including TP padding."""
def test_engram_head_shards_reconstruct_checkpoint(
cpu_offload, sort_rows, tp_size, monkeypatch
):
"""Keep complete buckets and reconstruct head order, including TP padding,
whether or not the host lookup sorts its rows by table offset."""
if sort_rows:
monkeypatch.setattr(
engram_ops, "_engram_lookup_thresholds", lambda _: (0, None)
)
head_sizes = (17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73)
num_rows, dim = sum(head_sizes), 64
torch.manual_seed(0)
@@ -796,10 +802,8 @@ def test_engram_shared_prefetch_stream_waits_per_layer():
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
@pytest.mark.parametrize(
"backend,cpu_offload", [("nvidia", None), ("nvidia", False), ("common", False)]
)
def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload):
@pytest.mark.parametrize("cpu_offload", [None, False])
def test_engram_constructor_honors_offload(monkeypatch, cpu_offload):
"""Default offload uses pinned storage and the given stream; False uses HBM."""
from vllm.config import EngramConfig
@@ -817,23 +821,18 @@ def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload):
else EngramConfig(cpu_offload=cpu_offload),
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
)
offloaded = backend == "nvidia" and vllm_config.engram_config.cpu_offload
if backend == "common":
vllm_config.engram_config = None
offloaded = vllm_config.engram_config.cpu_offload
monkeypatch.setattr(engram_ops, "get_current_vllm_config", lambda: vllm_config)
monkeypatch.setattr(
nvidia_engram_ops, "get_current_vllm_config", lambda: vllm_config
)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_world_size", lambda: 1)
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
monkeypatch.setattr(
engram_ops, "ColumnParallelLinear", lambda *a, **k: torch.nn.Identity()
)
stream = torch.cuda.Stream()
kwargs = {"prefetch_stream": stream} if backend == "nvidia" else {}
with torch.device("cuda"):
cls = Engram if backend == "nvidia" else CommonEngram
module = cls(config, None, layout, 0, False, "engram", **kwargs)
module = Engram(
config, None, layout, 0, False, "engram", prefetch_stream=stream
)
layer = module.embed_tokens
if offloaded:
assert layer.weight.device.type == "cpu" and layer.weight.is_pinned()
@@ -897,7 +896,8 @@ def test_engram_prefetch_detects_missing_dependency(monkeypatch, missing_depende
"The allocator reuse negative control requires the native allocator"
)
def start(self, hash_ids, rows, stream):
def start(self, hash_ids, rows):
stream = self._prefetch_stream
if missing_dependency != "producer_wait":
stream.wait_stream(torch.cuda.current_stream())
if missing_dependency != "record_stream":
@@ -24,6 +24,7 @@ outputs are checked within 1 representable-step (ULP); bf16 norm/RoPE outputs us
rtol/atol=1e-2 (the tolerance the sibling deepseek_v4 fused-kernel test uses).
"""
from functools import partial
from typing import NamedTuple
import pytest
@@ -880,22 +881,36 @@ def test_fused_q_no_indexer(num_tokens: int, cfg: ModelConfig):
@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
@pytest.mark.parametrize("num_tokens", [1, 17, 512])
@pytest.mark.parametrize("has_indexer", [True, False])
def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig):
@pytest.mark.parametrize("use_mha", [True, False])
@pytest.mark.parametrize(
"index_interleave,cache_dtype", [(False, torch.float32), (True, torch.bfloat16)]
)
@pytest.mark.parametrize("capture", [False, True])
def test_fused_q_bf16_query(
num_tokens: int,
has_indexer: bool,
use_mha: bool,
index_interleave: bool,
cache_dtype: torch.dtype,
capture: bool,
cfg: ModelConfig,
):
"""bf16-query path (FlashMLA sparse, SM90/SM100): only the RoPE'd q_pe is
produced (bf16, unquantized); ql_nope is consumed directly by the caller."""
produced; MHA does not need an absorbed ql_nope projection."""
torch.manual_seed(6)
dev = "cuda"
max_pos = 8192
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
# Attention takes q_pe as a strided slice of the full Q projection.
q_pe = torch.randn(
num_tokens, cfg.num_heads, cfg.rope_dim, device=dev, dtype=torch.bfloat16
)
num_tokens, cfg.num_heads, 4 * cfg.rope_dim, device=dev, dtype=torch.bfloat16
)[..., -cfg.rope_dim :]
ql_nope = torch.randn(
num_tokens, cfg.num_heads, cfg.kv_lora, device=dev, dtype=torch.bfloat16
)
q_scale = torch.tensor([0.37], device=dev, dtype=torch.float32)
q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev).to(cache_dtype)
index_q = index_w = idx_cos_sin = None
if has_indexer:
@@ -909,24 +924,35 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig
index_w = torch.randn(
num_tokens, cfg.index_heads, device=dev, dtype=torch.float32
)
idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev).to(cache_dtype)
iq_fp8, iw_out, q_pe_out = K.fused_q(
run = partial(
K.fused_q,
pos,
q_pe,
q_cos_sin,
index_q,
idx_cos_sin,
ql_nope,
q_pe if use_mha else ql_nope,
q_scale,
index_w,
cfg.index_head_dim**-0.5,
cfg.index_heads**-0.5,
has_indexer=has_indexer,
index_rope_interleave=False,
index_rope_interleave=index_interleave,
quantize_mqa=False,
)
iq_fp8, iw_out, q_pe_out = run()
if capture:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
iq_fp8, iw_out, q_pe_out = run()
q_pe.mul_(0.5)
if index_q is not None:
index_q.mul_(1.5)
graph.replay()
# MQA query: only the RoPE'd q_pe, bf16, unquantized.
assert q_pe_out.dtype == torch.bfloat16
assert q_pe_out.shape == (num_tokens, cfg.num_heads, cfg.rope_dim)
@@ -945,7 +971,7 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig
index_q.float(),
pos.unsqueeze(-1).expand(num_tokens, cfg.index_heads),
idx_cos_sin,
interleave=False,
interleave=index_interleave,
)
q_ref, scale_ref = ue8m0_quant(iq_ref)
assert_fp8(iq_fp8, q_ref, "indexer-Q fp8 (bf16-query path)")
@@ -320,6 +320,38 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
index_cache.view(-1, HEAD_DIM), expected_index_cache, rtol=0, atol=0
)
# Without q_out, q is emitted only in fp8: the same bytes, while qkv keeps
# its raw q and every other output matches the run above.
outputs = [qkv_orig.clone(), torch.empty_like(q_fp8), torch.empty_like(index_q)]
caches = [torch.zeros_like(kv_cache), torch.zeros_like(index_cache)]
ops.fused_minimax_m3_qknorm_rope_kv_insert(
outputs[0],
q_w,
k_w,
cos_sin,
positions,
num_heads,
num_kv_heads,
ROTARY_DIM,
eps,
iq_w,
ik_w,
num_idx_heads,
slot_mapping,
index_slot_mapping,
*caches,
block_size,
None,
outputs[2],
kv_cache_dtype,
q_fp8_out=outputs[1],
q_fp8_scale=0.5,
)
for actual, expected in zip(
outputs + caches, [qkv, q_fp8, index_q, kv_cache, index_cache]
):
assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8))
@pytest.mark.parametrize("num_tokens", [1, 64, 513])
@pytest.mark.parametrize("block_size", [16, 64])
@@ -441,8 +473,8 @@ def test_sparse_skip_index_branch(num_tokens, block_size, kv_cache_dtype):
# ── Test 4: fp8 (e4m3) index outputs ─────────────────────────────────────────
# The fp8 score path stores index_q and the index-K cache as e4m3 while q/k/v +
# q_out stay bf16. Asserts: (1) q/k/v/q_out are bit-identical to the bf16 run
# (the index dtype must not perturb the main branch), and (2) the e4m3 index
# outputs dequantize close to the bf16 reference.
# (the index dtype must not perturb the main branch), and (2) index_q is the
# e4m3 cast of the bf16 run's index_q, and the index-K cache is within fp8 ulp.
@pytest.mark.skipif(
@@ -528,6 +560,196 @@ def test_sparse_full_fp8_index(num_tokens, block_size):
torch.testing.assert_close(qkv_fp, qkv_bf, rtol=0, atol=0)
torch.testing.assert_close(kvc_fp, kvc_bf, rtol=0, atol=0)
# (2) Dequantized e4m3 index outputs match the bf16 reference within fp8 ulp.
torch.testing.assert_close(iq_fp.float(), iq_bf.float(), rtol=0.13, atol=0.05)
# (2) index_q is e4m3(bf16(q)); the index-K cache is within fp8 ulp.
assert torch.equal(
iq_fp.view(torch.uint8), iq_bf.to(torch.float8_e4m3fn).view(torch.uint8)
)
torch.testing.assert_close(idxc_fp.float(), idxc_bf.float(), rtol=0.13, atol=0.05)
# ── Test 5: sparse mode with an NVFP4 main KV cache ──────────────────────────
# The NVFP4 inserts quantize x * sf_scale, where sf_scale = 1 / the global
# scale: per 16-value block,
# scale = min(448, max(E4M3(max(amax, 1e-12) / 6), 1/512)) and
# code = sign | RNE E2M1 of min(|x / scale|, 6), with zero stored as +0.
E2M1_MIDPOINTS = (0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0)
def _div(a, b):
# Division by a Python scalar would become a reciprocal multiply.
return a / torch.as_tensor(b, dtype=a.dtype, device=a.device)
def nvfp4_ref(x, sf_scale):
"""Returns the E2M1 codes [..., 128] and E4M3 scale bytes [..., 8] of x."""
blocks = x.float().unflatten(-1, (-1, 16))
amax = blocks.abs().amax(-1, keepdim=True).clamp_min(1e-12)
scale = (sf_scale * _div(amax, 6.0)).clamp(max=448.0)
scale = scale.to(torch.float8_e4m3fn).float().clamp(min=2.0**-9)
q = _div(blocks * sf_scale, scale)
# Alternating > / >= rounds exact midpoints to the even code.
mag = sum(
(q.abs() >= m) if i % 2 else (q.abs() > m) for i, m in enumerate(E2M1_MIDPOINTS)
).to(torch.uint8)
codes = torch.where(mag > 0, mag | (torch.signbit(q).to(torch.uint8) << 3), 0)
scale_bytes = scale.squeeze(-1).to(torch.float8_e4m3fn).view(torch.uint8)
return codes.flatten(-2), scale_bytes
def nvfp4_pages(sides, slot_mapping, num_blocks, block_size):
"""Expected NVFP4 pages [nb, len(sides) * nkv, bs, 72] for the K (then V)
side given as (x [N, nkv, 128], sf_scale). Slot h * len(sides) + side is
head h's packed E2M1 data [bs, 64] (low nibble = even element), then its
E4M3 scales [bs, 8]: linear for K, 4x4 token-quad swizzled for V."""
nkv, ns, device = sides[0][0].shape[1], HEAD_DIM // 16, slot_mapping.device
row_bytes, data_bytes = HEAD_DIM // 2 + ns, block_size * HEAD_DIM // 2
pages = torch.zeros(
num_blocks,
nkv,
len(sides),
block_size * row_bytes,
dtype=torch.uint8,
device=device,
)
valid = slot_mapping >= 0
blk, tok = slot_mapping[valid] // block_size, slot_mapping[valid] % block_size
g, t = torch.arange(ns, device=device), tok[:, None]
scale_offsets = (t * ns + g, ((t // 4) * 4 + g // 2) * ns + (g % 2) * 4 + t % 4)
for side, (x, sf_scale) in enumerate(sides):
codes, scales = nvfp4_ref(x[valid], sf_scale)
data = pages[:, :, side, :data_bytes].view(num_blocks, nkv, block_size, -1)
data[blk, :, tok] = codes[..., 0::2] | (codes[..., 1::2] << 4)
scale_region = pages[:, :, side, data_bytes:]
for h in range(nkv):
scale_region[blk[:, None], h, scale_offsets[side]] = scales[:, h]
return pages.view(num_blocks, -1, block_size, row_bytes)
def nvfp4_edge_values(num_rows, global_scale):
"""[num_rows, 128] bf16 whose 16-value blocks, divided by global_scale, hit
the quantizer's corner cases: exact E2M1 midpoints under non-power-of-2 scales
(blocks 0-1), amax under the 1/512 scale floor (2), all -0 (3), and
saturation with |x| > 4096 up to bf16 max (4) or with inf (5)."""
g = torch.Generator().manual_seed(0)
x = torch.randn(num_rows, 8, 16, generator=g)
sign = torch.where(torch.rand(x.shape, generator=g) < 0.5, -1.0, 1.0)
steps = torch.tensor(E2M1_MIDPOINTS + (0.5, 1.0, 3.0))
mant = 1 + torch.randint(1, 8, (num_rows, 2, 1), generator=g) / 8
scale = mant * 2.0 ** torch.randint(-6, 3, (num_rows, 2, 1), generator=g)
x[:, :2] = steps[torch.randint(len(steps), (num_rows, 2, 16), generator=g)] * scale
x[:, :2, :1] = 6 * scale # amax = 6 * scale, so the block scale is `scale`
x[:, 2] = torch.rand(num_rows, 16, generator=g) * 2.0**-8
x[:, 4] *= 5000
x = x * sign * global_scale
x[:, 3] = -0.0
x[:, 4, 0] = sign[:, 4, 0] * torch.finfo(torch.bfloat16).max
x[:, 5, 0] = sign[:, 5, 0] * float("inf")
return x.flatten(1).to(torch.bfloat16)
@pytest.mark.skipif(
not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10,
reason="NVFP4 KV caches need SM100",
)
@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513])
@pytest.mark.parametrize("num_kv_heads", [1, 4])
@pytest.mark.parametrize("block_size", [16, 128])
@pytest.mark.parametrize("kv_scales", [(0.5, 2.0), (1.0, 1.0)])
def test_sparse_full_nvfp4(num_tokens, num_kv_heads, block_size, kv_scales):
"""The fused NVFP4 insert writes the reference quantizer's bytes (packed
E2M1, linear K / swizzled V E4M3 block scales) for the kernel's own
post-RoPE k and for a raw v that hits the quantizer's corner cases, and
skips padded slots. Unit global scales take the kernel's separate fast
path."""
kv_cache_dtype = "nvfp4"
torch.manual_seed(5)
device, dtype, eps = "cuda", torch.bfloat16, 1e-6
base, max_pos = 5_000_000.0, 4096
num_heads, num_idx_heads = 4 * num_kv_heads, 1
weights = [
torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1 for _ in range(4)
]
cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
positions = torch.randint(
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
)
qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
iqsz = num_idx_heads * HEAD_DIM
qkv = torch.randn(
num_tokens, qsz + 2 * kvsz + iqsz + HEAD_DIM, dtype=dtype, device=device
)
k_scale, v_scale = (
torch.tensor(s, dtype=torch.float32, device=device) for s in kv_scales
)
qkv[:, qsz + kvsz : qsz + 2 * kvsz] = nvfp4_edge_values(
num_tokens * num_kv_heads, v_scale.item()
).view(num_tokens, kvsz)
qkv_orig = qkv.clone()
num_blocks = (num_tokens + block_size - 1) // block_size + 1
kv_cache = torch.zeros(
num_blocks,
2 * num_kv_heads,
block_size,
HEAD_DIM // 2 + HEAD_DIM // 16,
dtype=torch.uint8,
device=device,
)
index_cache = torch.zeros(
num_blocks, block_size, HEAD_DIM, dtype=dtype, device=device
)
slot_mapping = torch.randperm(
num_blocks * block_size, dtype=torch.int64, device=device
)[:num_tokens]
slot_mapping[::5] = -1 # padded / unscheduled tokens are not written
index_slot_mapping = torch.randperm(
num_blocks * block_size, dtype=torch.int64, device=device
)[:num_tokens]
ops.fused_minimax_m3_qknorm_rope_kv_insert(
qkv,
*weights[:2],
cos_sin,
positions,
num_heads,
num_kv_heads,
ROTARY_DIM,
eps,
*weights[2:],
num_idx_heads,
slot_mapping,
index_slot_mapping,
kv_cache,
index_cache,
block_size,
torch.empty(num_tokens, qsz, dtype=dtype, device=device),
torch.empty(num_tokens, iqsz, dtype=dtype, device=device),
kv_cache_dtype,
kv_k_scale=k_scale,
kv_v_scale=v_scale,
)
_, k_out, v_out, _, index_k = qkv.split([qsz, kvsz, kvsz, iqsz, HEAD_DIM], dim=-1)
torch.testing.assert_close(
v_out, qkv_orig.split([qsz, kvsz, kvsz, iqsz, HEAD_DIM], dim=-1)[2]
)
expected_kv_cache = nvfp4_pages(
[
(k_out.reshape(num_tokens, num_kv_heads, HEAD_DIM), 1 / k_scale),
(v_out.reshape(num_tokens, num_kv_heads, HEAD_DIM), 1 / v_scale),
],
slot_mapping,
num_blocks,
block_size,
)
assert torch.equal(kv_cache, expected_kv_cache), (
f"{int((kv_cache != expected_kv_cache).sum())} NVFP4 cache bytes differ"
)
expected_index_cache = torch.zeros_like(index_cache).view(-1, HEAD_DIM)
expected_index_cache[index_slot_mapping] = index_k
torch.testing.assert_close(
index_cache.view(-1, HEAD_DIM), expected_index_cache, rtol=0, atol=0
)
@@ -233,6 +233,26 @@ def _assert_eq(r_ref, r_kern):
)
def _cache_pool_bytes(kv_cache: torch.Tensor, pool_slot: int) -> torch.Tensor:
"""Read a logical pool's K and scale bytes from the platform cache layout."""
page_size = kv_cache.shape[1]
head_dim = kv_cache.shape[2] - 4
page_idx, token_offset = divmod(pool_slot, page_size)
flat = kv_cache[page_idx].reshape(-1)
dims = torch.arange(head_dim, device=kv_cache.device)
if current_platform.is_rocm() and page_size > 1:
k_offsets = (
(token_offset // 16) * 16 * head_dim
+ (dims // 16) * 16 * 16
+ (token_offset % 16) * 16
+ dims % 16
)
else:
k_offsets = token_offset * head_dim + dims
scale_offset = page_size * head_dim + 4 * token_offset
return torch.cat((flat[k_offsets], flat[scale_offset : scale_offset + 4]))
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
def test_amd_prefill_writer_uses_preshuffled_cache_layout():
from vllm.models.glm5next.amd.ops.kpool_compress import (
@@ -343,7 +363,7 @@ def test_decode_writer_matches_prefill_writer(pool_size, ring_pools):
p
for p in range(n_pools)
if not torch.equal(
kv_prefill[p // page, p % page], kv_decode[p // page, p % page]
_cache_pool_bytes(kv_prefill, p), _cache_pool_bytes(kv_decode, p)
)
]
assert not differing, (
@@ -379,13 +399,6 @@ def test_rejected_draft_redo_needs_ring_slots(ring_pools):
kv = torch.zeros_like(kv_ref)
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
def pool_bytes(cache, p):
# Page layout: [page * HEAD_DIM bytes of K rows | page * 4 bytes of scales].
flat = cache[0].reshape(-1)
k_bytes = flat[p * HEAD_DIM : (p + 1) * HEAD_DIM]
s_bytes = flat[page * HEAD_DIM + 4 * p : page * HEAD_DIM + 4 * (p + 1)]
return torch.cat([k_bytes, s_bytes])
def step(positions, keys, scores):
pos = torch.tensor([positions], dtype=torch.int32, device=dev)
slots = [(p // pool) if p % pool == pool - 1 else -1 for p in positions]
@@ -415,7 +428,7 @@ def test_rejected_draft_redo_needs_ring_slots(ring_pools):
)
step([8, 9, 10, 11], k[8:12], score[8:12]) # all drafts rejected
for p in (1, 2):
assert torch.equal(pool_bytes(kv, p), pool_bytes(kv_ref, p)), p
assert torch.equal(_cache_pool_bytes(kv, p), _cache_pool_bytes(kv_ref, p)), p
# Draft 7 completes pool 1 and is rejected. With a one-pool ring, drafts
# 8 and 9 overwrite the slots of positions 4 and 5, which are read by the
@@ -430,7 +443,7 @@ def test_rejected_draft_redo_needs_ring_slots(ring_pools):
torch.cat([score[6:7], draft_scores]),
)
step([7, 8, 9, 10], k[7:11], score[7:11])
pool1_ok = torch.equal(pool_bytes(kv, 1), pool_bytes(kv_ref, 1))
pool1_ok = torch.equal(_cache_pool_bytes(kv, 1), _cache_pool_bytes(kv_ref, 1))
if ring_pools == 1:
assert not pool1_ok, "expected a one-pool ring to corrupt pool 1"
else:
+230
View File
@@ -0,0 +1,230 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""ROCm projection/collective contract corresponding to the SM100 GEMM-RS/AR.
Exercise the production RowParallelLinear and sequence-parallel reduction
paths: native BF16/MXFP8 GEMM, AITER all-reduce, and RCCL reduce-scatter. This
checks the same observable contract, not NVIDIA's fused GEMM/communication
implementation or its performance.
"""
import pytest
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from tests.kernels.test_gemm_rs_ar import _N, _SHAPES
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
from vllm.config.quantization import QuantizationConfigArgs
from vllm.distributed import cleanup_dist_env_and_memory, get_tp_group
from vllm.distributed.parallel_state import (
graph_capture,
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.linear import RowParallelLinear
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
_mxfp8_e4m3_quantize_torch,
dequant_mxfp8_to_bf16,
)
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
# Retain the expanded parent's shape transitions, plus fully padded RS ranks.
_ROCM_SHAPES = ((1, 512), (7, 768), *_SHAPES)
def _reference(
x: torch.Tensor,
layer: RowParallelLinear,
all_reduce: bool,
) -> torch.Tensor:
weight = layer.weight
if hasattr(layer, "weight_scale"):
# The production ROCm path uses the Triton quantizer and dot_scaled;
# use the independent torch quantizer and FP32 matmul as the oracle.
x_q, x_scale = _mxfp8_e4m3_quantize_torch(x)
x = dequant_mxfp8_to_bf16(x_q, x_scale)
weight = dequant_mxfp8_to_bf16(weight, layer.weight_scale)
partial = (x.float() @ weight.float().T).to(torch.bfloat16).float()
group = get_tp_group()
# Always use FP32 RCCL AR for the oracle, including the RS candidate.
dist.all_reduce(partial, group=group.device_group)
expected = partial.to(torch.bfloat16)
if all_reduce:
return expected
padding = -x.shape[0] % group.world_size
expected = torch.nn.functional.pad(expected, (0, 0, 0, padding))
return expected.chunk(group.world_size)[group.rank_in_group]
def _apply(x: torch.Tensor, layer: RowParallelLinear) -> torch.Tensor:
output, _ = layer(x)
return output if layer.reduce_results else sp_reduce_scatter(output)
def _assert_output(
actual: torch.Tensor, expected: torch.Tensor, M: int, all_reduce: bool
) -> None:
assert actual.dtype == torch.bfloat16
assert actual.shape == expected.shape
assert torch.isfinite(actual).all()
# Match the parent's tolerance at 0.1 weight scale, then bound total error.
# BF16 partial/reduction rounding can dominate near a cancelled output.
torch.testing.assert_close(actual, expected, rtol=5e-2, atol=0.4)
error = (actual.float() - expected.float()).norm()
assert error <= 5e-3 * expected.float().norm() + 1e-6
if not all_reduce:
rank_start = get_tp_group().rank_in_group * actual.shape[0]
valid_rows = min(max(M - rank_start, 0), actual.shape[0])
assert not actual[valid_rows:].any(), "RS padding must remain zero"
def _run_mode(
device: torch.device, backend: str, all_reduce: bool, world_size: int
) -> None:
group = get_tp_group()
generator = torch.Generator(device=device)
quant_config = (
OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
if backend == "mxfp8"
else None
)
layers = {}
for K in sorted({K for _, K in _ROCM_SHAPES}):
with torch.device(device):
layer = RowParallelLinear(
K * world_size,
_N,
bias=False,
params_dtype=torch.bfloat16,
quant_config=quant_config,
reduce_results=all_reduce,
)
generator.manual_seed(1000 + group.rank_in_group * 100 + K)
weight = (
torch.randn(_N, K, dtype=torch.bfloat16, device=device, generator=generator)
* 0.1
)
layer.weight = torch.nn.Parameter(weight, requires_grad=False)
layer.quant_method.process_weights_after_loading(layer)
if backend == "mxfp8":
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
RocmDotScaledMxfp8LinearKernel,
)
assert isinstance(layer.quant_method.kernel, RocmDotScaledMxfp8LinearKernel)
assert layer.weight.dtype == torch.float8_e4m3fn
layers[K] = layer
def make_input(M: int, K: int, seed: int) -> torch.Tensor:
generator.manual_seed(seed + group.rank_in_group * 100)
return torch.randn(
M, K, dtype=torch.bfloat16, device=device, generator=generator
)
comm = group.device_communicator
assert comm is not None
assert comm.use_aiter_allreduce
assert comm.aiter_ar_comm is not None and not comm.aiter_ar_comm.disabled
assert comm.qr_comm is None or comm.qr_comm.disabled
assert comm.pynccl_comm is not None and not comm.pynccl_comm.disabled
for M, K in (*_ROCM_SHAPES, *_ROCM_SHAPES[::-1]):
x = make_input(M, K, 2000 + M + K)
if all_reduce:
probe = torch.empty(M, _N, dtype=x.dtype, device=device)
assert comm.aiter_ar_comm.should_custom_ar(probe)
expected = _reference(x, layers[K], all_reduce)
actual = _apply(x, layers[K])
_assert_output(actual, expected, M, all_reduce)
# Retained outputs must survive the next call's collective buffer reuse.
x = make_input(257, 768, 2500)
first = _apply(x, layers[768])
snapshot = first.clone()
second = _apply(-x, layers[768])
torch.accelerator.synchronize(device)
assert first.data_ptr() != second.data_ptr()
torch.testing.assert_close(first, snapshot, rtol=0, atol=0)
M, K = 1025, 4224
graph_x = make_input(M, K, 3000)
for _ in range(3):
_apply(graph_x, layers[K])
torch.accelerator.synchronize(device)
dist.barrier(group=group.device_group)
graph = torch.cuda.CUDAGraph()
# vLLM's context registers the AITER IPC addresses used by graph replay.
with (
graph_capture(device=device) as context,
torch.cuda.graph(graph, stream=context.stream),
):
graph_output = _apply(graph_x, layers[K])
torch.cuda.current_stream().wait_stream(context.stream)
dist.barrier(group=group.device_group)
for replay in range(3):
graph_x.copy_(make_input(M, K, 3001 + replay))
expected = _reference(graph_x, layers[K], all_reduce)
graph.replay()
torch.accelerator.synchronize(device)
_assert_output(graph_output, expected, M, all_reduce)
dist.barrier(group=group.device_group)
def _worker(rank: int, world_size: int, port: int, backend: str) -> None:
device = torch.device("cuda", rank)
torch.accelerator.set_device_index(device)
update_environment_variables(
{
"RANK": str(rank),
"LOCAL_RANK": str(rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": str(port),
}
)
config = VllmConfig()
config.model_config = ModelConfig(dtype="bfloat16")
try:
with set_current_vllm_config(config), torch.inference_mode():
init_distributed_environment()
initialize_model_parallel(tensor_model_parallel_size=world_size)
for all_reduce in (False, True):
_run_mode(device, backend, all_reduce, world_size)
finally:
cleanup_dist_env_and_memory()
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm GEMM-RS/AR contract")
@pytest.mark.parametrize("world_size", [2, 4])
@pytest.mark.parametrize(
"backend",
[
"bf16",
pytest.param(
"mxfp8",
marks=pytest.mark.skipif(
not current_platform.supports_mx(), reason="native MXFP8 requires CDNA4"
),
),
],
)
@pytest.mark.distributed(num_gpus=4)
def test_rocm_gemm_rs_ar(
monkeypatch: pytest.MonkeyPatch, world_size: int, backend: str
) -> None:
"""Preserve rank reduction, tails, shape reuse, lifetime, and graph replay."""
if torch.accelerator.device_count() < world_size:
pytest.skip(f"GEMM-RS/AR requires {world_size} GPUs")
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1")
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", "NONE")
try:
mp.spawn(
_worker, args=(world_size, get_open_port(), backend), nprocs=world_size
)
finally:
cleanup_dist_env_and_memory()