Merge remote-tracking branch 'upstream/main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -26,7 +26,6 @@ NUM_BLOCKS = 4321 # Arbitrary values for testing
|
||||
PARTITION_SIZE_ROCM = 256
|
||||
DTYPES = [torch.bfloat16, torch.float16]
|
||||
NUM_GEN_SEQS = [7] # Arbitrary values for testing
|
||||
NUM_PREFILL_SEQS = [3] # Arbitrary values for testing
|
||||
NUM_HEADS = [(32, 8), (40, 40), (64, 8)] # Arbitrary values for testing
|
||||
|
||||
# Head sizes supported by the ROCm paged attention kernel.
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from contextlib import contextmanager
|
||||
from typing import Literal
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -14,6 +15,7 @@ from vllm.config import (
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.cpu import CpuPlatform
|
||||
from vllm.platforms.interface import DeviceCapability
|
||||
@@ -26,7 +28,7 @@ else:
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import RocmPlatform
|
||||
else:
|
||||
RocmPlatform = None
|
||||
RocmPlatform = None # type: ignore[misc] # Unavailable platform import.
|
||||
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -406,7 +408,7 @@ def test_auto_backend_selection_behavior():
|
||||
reason="Attention backend FA3 is not supported on ROCm. This test can't succeed.",
|
||||
)
|
||||
def test_per_head_quant_scales_backend_selection(
|
||||
backend_name: str, flash_attn_version: int | None, should_succeed: bool
|
||||
backend_name: str, flash_attn_version: Literal[2, 3, 4] | None, should_succeed: bool
|
||||
):
|
||||
"""Test backend selection when use_per_head_quant_scales=True."""
|
||||
# Clear cache to ensure fresh backend selection
|
||||
@@ -559,7 +561,7 @@ def test_non_causal_autoselect_backend():
|
||||
"int8_per_token_head",
|
||||
],
|
||||
)
|
||||
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
|
||||
def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: CacheDType):
|
||||
"""FlashAttentionBackend must not claim support for kv_cache dtypes
|
||||
that it cannot handle."""
|
||||
from vllm.v1.attention.backends.flash_attn import FlashAttentionBackend
|
||||
@@ -569,7 +571,7 @@ def test_flash_attn_rejects_unhandled_kv_cache_dtypes(kv_cache_dtype: str):
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "fp8_e4m3"])
|
||||
def test_flash_attn_accepts_handled_fp8_variants(
|
||||
kv_cache_dtype: str, monkeypatch: pytest.MonkeyPatch
|
||||
kv_cache_dtype: CacheDType, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
"""FlashAttentionBackend must accept the two fp8 dtypes it can actually
|
||||
handle: 'fp8' (alias for fp8_e4m3fn) and 'fp8_e4m3'."""
|
||||
@@ -876,7 +878,9 @@ def test_rswa_selection_does_not_reuse_causal_result(blackwell_selection):
|
||||
config = EngineArgs(
|
||||
model="google/gemma-4-31B-it",
|
||||
dtype="bfloat16",
|
||||
attention_config={"backend": "TRITON_FLASHINFER"},
|
||||
attention_config=AttentionConfig(
|
||||
backend=AttentionBackendEnum.TRITON_FLASHINFER
|
||||
),
|
||||
).create_engine_config()
|
||||
with set_current_vllm_config(config):
|
||||
assert (
|
||||
|
||||
@@ -151,7 +151,7 @@ def test_cascade(
|
||||
max_seqlen_k=max_kv_len,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
window_size=list(window_size),
|
||||
block_table=block_tables,
|
||||
softcap=soft_cap if soft_cap is not None else 0,
|
||||
)
|
||||
|
||||
@@ -187,7 +187,7 @@ def test_varlen_with_paged_kv(
|
||||
max_seqlen_k=max_kv_len,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
window_size=list(window_size),
|
||||
block_table=block_tables,
|
||||
softcap=soft_cap if soft_cap is not None else 0,
|
||||
fa_version=fa_version,
|
||||
@@ -274,7 +274,7 @@ def test_fa4_hd256_paged_call_shape(
|
||||
max_seqlen_k=num_pages * block_size,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=window_size,
|
||||
window_size=list(window_size),
|
||||
block_table=block_tables[:, :num_pages],
|
||||
num_splits=1,
|
||||
fa_version=4,
|
||||
|
||||
@@ -197,11 +197,6 @@ def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> No
|
||||
mixed_cache = initial_cache.clone()
|
||||
reference_cache = _reference_cache(cache_format, inputs, initial_cache, slots)
|
||||
scale_inv = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
kwargs = {
|
||||
"ds_mla": cache_format == "fp8_ds_mla",
|
||||
"q_scale_inv": scale_inv if cache_format == "fp8" else None,
|
||||
"cache_scale_inv": scale_inv if cache_format == "fp8" else None,
|
||||
}
|
||||
|
||||
output = fused_mla_decode_q_concat_kv_cache_insert(
|
||||
inputs["ql_nope"],
|
||||
@@ -210,7 +205,9 @@ def test_decode_concat_ignores_negative_slots_for_cache(cache_format: str) -> No
|
||||
inputs["k_pe"],
|
||||
mixed_cache,
|
||||
slots,
|
||||
**kwargs,
|
||||
ds_mla=cache_format == "fp8_ds_mla",
|
||||
q_scale_inv=scale_inv if cache_format == "fp8" else None,
|
||||
cache_scale_inv=scale_inv if cache_format == "fp8" else None,
|
||||
)
|
||||
|
||||
expected = _latent_query(inputs["ql_nope"], inputs["q_pe"])
|
||||
|
||||
@@ -277,7 +277,7 @@ def test_mha_attn_varlen_forward_flashinfer(
|
||||
# Override vllm config so get_vit_attn_backend returns FLASHINFER (simulates
|
||||
# --mm-encoder-attn-backend=FLASHINFER).
|
||||
vllm_config = get_current_vllm_config()
|
||||
old_model_config = getattr(vllm_config, "model_config", None)
|
||||
old_model_config = vllm_config.model_config
|
||||
minimal_model_config = type(
|
||||
"MinimalModelConfig",
|
||||
(),
|
||||
@@ -376,7 +376,7 @@ def test_mha_attn_varlen_forward_aiter_fp8(
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
vllm_config = get_current_vllm_config()
|
||||
old_model_config = getattr(vllm_config, "model_config", None)
|
||||
old_model_config = vllm_config.model_config
|
||||
minimal_model_config = type(
|
||||
"MinimalModelConfig",
|
||||
(),
|
||||
@@ -442,7 +442,7 @@ def test_mha_attn_varlen_forward_aiter_fp8(
|
||||
)
|
||||
def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config):
|
||||
vllm_config = get_current_vllm_config()
|
||||
old_model_config = getattr(vllm_config, "model_config", None)
|
||||
old_model_config = vllm_config.model_config
|
||||
vllm_config.model_config = type(
|
||||
"MinimalModelConfig",
|
||||
(),
|
||||
@@ -468,7 +468,7 @@ def test_mha_attn_aiter_fp8_rejects_unsupported_arch(default_vllm_config):
|
||||
)
|
||||
def test_mha_attn_fp8_rejects_wrong_backend(default_vllm_config):
|
||||
vllm_config = get_current_vllm_config()
|
||||
old_model_config = getattr(vllm_config, "model_config", None)
|
||||
old_model_config = vllm_config.model_config
|
||||
vllm_config.model_config = type(
|
||||
"MinimalModelConfig",
|
||||
(),
|
||||
|
||||
@@ -614,14 +614,22 @@ def _reference_decode_index_score(
|
||||
return out
|
||||
|
||||
|
||||
def test_prefill_index_topk_correctness():
|
||||
# 3600: 29 blocks over 12 splits of 3, so the last busy split has 2 blocks.
|
||||
@pytest.mark.parametrize("prefix_len", [1024, 3600])
|
||||
@pytest.mark.parametrize("force_split_k", [False, True])
|
||||
def test_prefill_index_topk_correctness(monkeypatch, prefix_len, force_split_k):
|
||||
if force_split_k:
|
||||
# Take the SM12.0-only split-K launch on any CUDA GPU.
|
||||
monkeypatch.setattr(
|
||||
current_platform, "is_device_capability", lambda cap, *_: cap == (12, 0)
|
||||
)
|
||||
topk = 6
|
||||
init_blocks = 0
|
||||
local_blocks = 1
|
||||
num_idx_heads = 2
|
||||
head_dim = 16
|
||||
q_lens = torch.tensor((4, 3), device="cuda", dtype=torch.int32)
|
||||
prefix_lens = torch.tensor((0, 1024), device="cuda", dtype=torch.int32)
|
||||
prefix_lens = torch.tensor((0, prefix_len), device="cuda", dtype=torch.int32)
|
||||
seq_lens = prefix_lens + q_lens
|
||||
batch = q_lens.numel()
|
||||
max_seq_len = seq_lens.max().item()
|
||||
@@ -1689,8 +1697,10 @@ def test_aiter_sparse_pa_layout_contract(monkeypatch, layout):
|
||||
)
|
||||
|
||||
backend = sparse_attn_mod.MiniMaxM3SparseBackend
|
||||
assert KVCacheLayout.LHBNC in backend.supported_kv_cache_layouts()
|
||||
assert KVCacheLayout.LBHNC in backend.supported_kv_cache_layouts()
|
||||
layouts = backend.supported_kv_cache_layouts()
|
||||
assert layouts is not None
|
||||
assert KVCacheLayout.LHBNC in layouts
|
||||
assert KVCacheLayout.LBHNC in layouts
|
||||
|
||||
nb, h = 3, 1
|
||||
cache_dtype = current_platform.fp8_dtype()
|
||||
@@ -1772,19 +1782,22 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
|
||||
import vllm.models.minimax_m3.amd.indexer_aiter as indexer_aiter_mod
|
||||
|
||||
monkeypatch.setattr(indexer_aiter_mod.current_platform, "is_rocm", lambda: True)
|
||||
kwargs = dict(
|
||||
topk_blocks=TOPK,
|
||||
sparse_block_size=BLOCK_SIZE,
|
||||
num_index_heads=1,
|
||||
index_head_dim=HEAD_DIM,
|
||||
indexer_kv_dtype="fp8_e4m3",
|
||||
max_model_len=8192,
|
||||
)
|
||||
|
||||
def unsupported_reason() -> str | None:
|
||||
return indexer_aiter_mod.aiter_indexer_unsupported_reason(
|
||||
topk_blocks=TOPK,
|
||||
sparse_block_size=BLOCK_SIZE,
|
||||
num_index_heads=1,
|
||||
index_head_dim=HEAD_DIM,
|
||||
# The implementation also accepts this legacy fp8 alias.
|
||||
indexer_kv_dtype="fp8_e4m3",
|
||||
max_model_len=8192,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
indexer_aiter_mod, "_minimax_m3_aiter_sparse_pa_requested", lambda: False
|
||||
)
|
||||
reason = indexer_aiter_mod.aiter_indexer_unsupported_reason(**kwargs)
|
||||
reason = unsupported_reason()
|
||||
assert reason is not None and "AITER sparse PA attend" in reason
|
||||
|
||||
# With the attend asked for, the gate moves on to the kernel-contract
|
||||
@@ -1792,7 +1805,7 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
indexer_aiter_mod, "_minimax_m3_aiter_sparse_pa_requested", lambda: True
|
||||
)
|
||||
reason = indexer_aiter_mod.aiter_indexer_unsupported_reason(**kwargs)
|
||||
reason = unsupported_reason()
|
||||
assert reason is None or "AITER sparse PA attend" not in reason
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness tests for MiniMax M3 CUTLASS sparse decode."""
|
||||
"""Correctness tests for MiniMax M3 MSA sparse attention (CUTLASS decode and
|
||||
NVFP4 KV cache)."""
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
@@ -8,6 +9,7 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.quantization.nvfp4_utils import dequant_nvfp4_kv_cache
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import AttentionConfig, CUDAGraphMode
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
@@ -18,6 +20,7 @@ from vllm.models.minimax_m3.common.sparse_attention import (
|
||||
MiniMaxM3SparseBackend,
|
||||
MiniMaxM3SparseMetadata,
|
||||
MiniMaxM3SparseMetadataBuilder,
|
||||
MiniMaxM3SparsePrefillMetadata,
|
||||
MiniMaxM3SparseTritonImpl,
|
||||
select_main_backend_and_impl_cls,
|
||||
)
|
||||
@@ -28,6 +31,7 @@ from vllm.models.minimax_m3.nvidia.model import MiniMaxM3SparseAttention
|
||||
from vllm.models.minimax_m3.nvidia.msa_cutlass_sparse_decode import (
|
||||
MSACutlassDecodePlanCache,
|
||||
msa_cutlass_sparse_decode,
|
||||
nvfp4_kv_cache_views,
|
||||
prepare_decode_metadata,
|
||||
should_prepare_decode_metadata,
|
||||
)
|
||||
@@ -36,6 +40,7 @@ from vllm.models.minimax_m3.nvidia.sparse_attention_msa import (
|
||||
MiniMaxM3SparseMSADecodeMetadata,
|
||||
MiniMaxM3SparseMSAImpl,
|
||||
MiniMaxM3SparseMSAMetadataBuilder,
|
||||
MiniMaxM3SparseMSANvfp4Backend,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -164,6 +169,53 @@ def test_msa_cutlass_decode_static_dispatch_requires_sm100(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("decode_backend", ["cutlass", "triton"])
|
||||
def test_msa_cutlass_decode_static_dispatch_nvfp4_has_no_fallback(
|
||||
decode_backend: str,
|
||||
) -> None:
|
||||
"""NVFP4 has no Triton decode, so it plans CUTLASS for any batch size."""
|
||||
assert should_prepare_decode_metadata(
|
||||
1,
|
||||
DEFAULT_QUERY_LEN,
|
||||
decode_backend=decode_backend, # type: ignore[arg-type]
|
||||
num_q_heads=16,
|
||||
num_kv_heads=1,
|
||||
kv_cache_dtype="nvfp4",
|
||||
page_size=BLOCK_SIZE,
|
||||
topk_blocks=TOPK,
|
||||
)
|
||||
|
||||
|
||||
def test_nvfp4_selects_msa_nvfp4_backend_and_requires_sm100(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
backend_cls, impl_cls = select_main_backend_and_impl_cls(
|
||||
topk_blocks=TOPK,
|
||||
kv_cache_dtype="nvfp4",
|
||||
num_kv_heads=1,
|
||||
)
|
||||
assert backend_cls is MiniMaxM3SparseMSANvfp4Backend
|
||||
assert impl_cls is MiniMaxM3SparseMSAImpl
|
||||
with pytest.raises(ValueError, match="nvfp4_4over6"):
|
||||
select_main_backend_and_impl_cls(
|
||||
topk_blocks=TOPK,
|
||||
kv_cache_dtype="nvfp4_4over6",
|
||||
num_kv_heads=1,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"is_device_capability_family",
|
||||
lambda _: False,
|
||||
)
|
||||
with pytest.raises(ValueError, match="NVFP4"):
|
||||
select_main_backend_and_impl_cls(
|
||||
topk_blocks=TOPK,
|
||||
kv_cache_dtype="nvfp4",
|
||||
num_kv_heads=1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "expected"),
|
||||
[
|
||||
@@ -387,7 +439,7 @@ def test_query_fp8_stays_valid_when_cutlass_plan_appears_on_replay(
|
||||
assert capture.num_eager_breaks == 1
|
||||
assert observed_ptrs == []
|
||||
|
||||
decode.msa_cutlass = object() # type: ignore[assignment]
|
||||
decode.msa_cutlass = object()
|
||||
for _ in range(3):
|
||||
capture.replay()
|
||||
stream.synchronize()
|
||||
@@ -639,3 +691,391 @@ def test_msa_cutlass_decode_matches_triton_with_interleaved_cache(
|
||||
graph.replay()
|
||||
current_platform.synchronize()
|
||||
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
|
||||
|
||||
|
||||
def _make_paged_layout(
|
||||
seq_lens_list: list[int],
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
"""Block table over a shuffled physical page pool."""
|
||||
pages_per_request = [math.ceil(seq_len / BLOCK_SIZE) for seq_len in seq_lens_list]
|
||||
num_pages = sum(pages_per_request)
|
||||
block_table = torch.zeros(
|
||||
len(seq_lens_list), max(pages_per_request), dtype=torch.int32, device="cuda"
|
||||
)
|
||||
physical_pages = torch.randperm(num_pages, dtype=torch.int32, device="cuda")
|
||||
offset = 0
|
||||
for request, request_pages in enumerate(pages_per_request):
|
||||
block_table[request, :request_pages] = physical_pages[
|
||||
offset : offset + request_pages
|
||||
]
|
||||
offset += request_pages
|
||||
return block_table, num_pages
|
||||
|
||||
|
||||
def _make_nvfp4_kv_cache(
|
||||
num_pages: int,
|
||||
num_kv_heads: int,
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Write random K/V through the NVFP4 ``reshape_and_cache_flash`` writer,
|
||||
whose pages hold every head's data then every head's scales per side, and
|
||||
repack them into head slots.
|
||||
|
||||
Returns the packed HND cache ``[pages, 2 * Hkv, 128, 72]`` and its BF16
|
||||
dequantization ``[pages, Hkv, 128, 2 * 128]`` for the Triton reference.
|
||||
"""
|
||||
num_slots = num_pages * BLOCK_SIZE
|
||||
key = torch.randn(
|
||||
num_slots, num_kv_heads, HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
value = torch.randn_like(key)
|
||||
written = torch.zeros(
|
||||
num_pages,
|
||||
2 * num_kv_heads,
|
||||
BLOCK_SIZE,
|
||||
HEAD_DIM // 2 + HEAD_DIM // 16,
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
)
|
||||
k_cache, v_cache = written.transpose(1, 2).split(num_kv_heads, dim=-2)
|
||||
ops.reshape_and_cache_flash(
|
||||
key,
|
||||
value,
|
||||
k_cache,
|
||||
v_cache,
|
||||
torch.arange(num_slots, dtype=torch.int64, device="cuda"),
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
)
|
||||
|
||||
kv_cache = torch.empty_like(written)
|
||||
head_slot_views = iter(nvfp4_kv_cache_views(kv_cache))
|
||||
dequantized = []
|
||||
for side, global_scale, swizzled in (
|
||||
(written[:, :num_kv_heads], k_scale, False),
|
||||
(written[:, num_kv_heads:], v_scale, True),
|
||||
):
|
||||
data = side.as_strided(
|
||||
(num_pages, num_kv_heads, BLOCK_SIZE, HEAD_DIM // 2),
|
||||
(side.stride(0), BLOCK_SIZE * HEAD_DIM // 2, HEAD_DIM // 2, 1),
|
||||
)
|
||||
scale = side.as_strided(
|
||||
(num_pages, num_kv_heads, BLOCK_SIZE, HEAD_DIM // 16),
|
||||
(side.stride(0), BLOCK_SIZE * HEAD_DIM // 16, HEAD_DIM // 16, 1),
|
||||
side.storage_offset() + num_kv_heads * BLOCK_SIZE * HEAD_DIM // 2,
|
||||
)
|
||||
next(head_slot_views).copy_(data)
|
||||
next(head_slot_views).copy_(scale)
|
||||
dequantized.append(
|
||||
dequant_nvfp4_kv_cache(
|
||||
data,
|
||||
scale,
|
||||
global_scale.item(),
|
||||
HEAD_DIM,
|
||||
BLOCK_SIZE,
|
||||
swizzled_scales=swizzled,
|
||||
)
|
||||
)
|
||||
return kv_cache, torch.cat(dequantized, dim=-1).to(torch.bfloat16)
|
||||
|
||||
|
||||
def test_nvfp4_kv_cache_views_address_head_slots() -> None:
|
||||
"""Slot 2h (K) and 2h + 1 (V) hold head h's data then scales, so each head
|
||||
is one contiguous run of the page: a TP rank's page is a byte slice."""
|
||||
num_kv_heads, slot_bytes = 4, BLOCK_SIZE * (HEAD_DIM // 2 + HEAD_DIM // 16)
|
||||
kv_cache = torch.randint(
|
||||
0,
|
||||
256,
|
||||
(3, 2 * num_kv_heads, BLOCK_SIZE, HEAD_DIM // 2 + HEAD_DIM // 16),
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
k_data, k_sf, v_data, v_sf = nvfp4_kv_cache_views(kv_cache)
|
||||
runs = kv_cache.view(3, num_kv_heads, 2 * slot_bytes)
|
||||
for head in range(num_kv_heads):
|
||||
expected = torch.cat(
|
||||
[v[:, head].flatten(1) for v in (k_data, k_sf, v_data, v_sf)], dim=1
|
||||
)
|
||||
assert torch.equal(runs[:, head], expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_q_heads", "num_kv_heads", "num_request_pairs", "query_len", "capture"),
|
||||
[
|
||||
pytest.param(64, 4, 1, 1, True, id="tp1-small-batch-query-len-1"),
|
||||
pytest.param(64, 4, 8, 8, False, id="tp1-query-len-8"),
|
||||
pytest.param(16, 1, 16, 4, True, id="tp4-query-len-4"),
|
||||
],
|
||||
)
|
||||
def test_msa_cutlass_decode_nvfp4_matches_triton_on_dequantized_cache(
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
num_request_pairs: int,
|
||||
query_len: int,
|
||||
capture: bool,
|
||||
) -> None:
|
||||
"""CUTLASS decode reads vLLM's packed NVFP4 pages, including swizzled V
|
||||
scales and device global scales, as their exact dequantized values."""
|
||||
torch.manual_seed(0)
|
||||
seq_lens_list = [257, 513] * num_request_pairs
|
||||
seq_lens_cpu = torch.tensor(seq_lens_list, dtype=torch.int32)
|
||||
seq_lens = seq_lens_cpu.cuda()
|
||||
block_table, num_pages = _make_paged_layout(seq_lens_list)
|
||||
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
|
||||
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
|
||||
kv_cache, kv_reference = _make_nvfp4_kv_cache(
|
||||
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
|
||||
)
|
||||
|
||||
num_query_tokens = len(seq_lens_list) * query_len
|
||||
query = torch.randn(
|
||||
num_query_tokens, num_q_heads, HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
query_fp8 = query.to(torch.float8_e4m3fn)
|
||||
topk = _make_topk(seq_lens_list, num_kv_heads, query_len)
|
||||
expected = torch.empty_like(query)
|
||||
minimax_m3_sparse_attn_decode(
|
||||
query_fp8.to(torch.bfloat16),
|
||||
kv_reference,
|
||||
topk.transpose(0, 1),
|
||||
block_table,
|
||||
seq_lens,
|
||||
num_kv_heads,
|
||||
SM_SCALE,
|
||||
expected,
|
||||
query_len,
|
||||
)
|
||||
|
||||
assert should_prepare_decode_metadata(
|
||||
len(seq_lens_list),
|
||||
query_len,
|
||||
decode_backend="triton",
|
||||
num_q_heads=num_q_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
kv_cache_dtype="nvfp4",
|
||||
page_size=BLOCK_SIZE,
|
||||
topk_blocks=TOPK,
|
||||
)
|
||||
metadata = prepare_decode_metadata(
|
||||
block_table,
|
||||
seq_lens,
|
||||
seq_lens_cpu,
|
||||
query_len,
|
||||
num_q_heads=num_q_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
page_size=BLOCK_SIZE,
|
||||
topk_blocks=TOPK,
|
||||
)
|
||||
actual = torch.zeros_like(query)
|
||||
|
||||
def run() -> None:
|
||||
msa_cutlass_sparse_decode(
|
||||
query_fp8,
|
||||
kv_cache,
|
||||
topk,
|
||||
actual,
|
||||
metadata,
|
||||
scale=SM_SCALE,
|
||||
q_scale_float=1.0,
|
||||
k_scale_float=1.0,
|
||||
v_scale_float=1.0,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
|
||||
if capture:
|
||||
run()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run()
|
||||
actual.zero_()
|
||||
graph.replay()
|
||||
else:
|
||||
run()
|
||||
current_platform.synchronize()
|
||||
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_q_heads", "num_kv_heads"),
|
||||
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
|
||||
)
|
||||
def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
) -> None:
|
||||
"""MSA prefill reads NVFP4 pages as their exact dequantized values,
|
||||
including chunked prefill over cached context, and takes q from its fp8
|
||||
copy (the only one the fused insert writes on this path)."""
|
||||
torch.manual_seed(0)
|
||||
q_lens = [300, 128, 77]
|
||||
seq_lens_list = [300, 640, 1100]
|
||||
block_table, num_pages = _make_paged_layout(seq_lens_list)
|
||||
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
|
||||
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
|
||||
kv_cache, kv_reference = _make_nvfp4_kv_cache(
|
||||
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
|
||||
)
|
||||
|
||||
total_q = sum(q_lens)
|
||||
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda")
|
||||
context_lens = seq_lens - torch.tensor(q_lens, dtype=torch.int32, device="cuda")
|
||||
cu_seqlens_q = torch.zeros(len(q_lens) + 1, dtype=torch.int32, device="cuda")
|
||||
cu_seqlens_q[1:] = torch.tensor(q_lens, device="cuda").cumsum(0)
|
||||
cu_seqlens_k = torch.zeros_like(cu_seqlens_q)
|
||||
cu_seqlens_k[1:] = seq_lens.cumsum(0)
|
||||
prefill = MiniMaxM3SparsePrefillMetadata(
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
cu_seqlens_k=cu_seqlens_k,
|
||||
seq_lens=seq_lens,
|
||||
context_lens=context_lens,
|
||||
block_table=block_table,
|
||||
max_query_len=max(q_lens),
|
||||
max_seq_len=max(seq_lens_list),
|
||||
total_kv_blocks=sum(math.ceil(n / BLOCK_SIZE) for n in seq_lens_list),
|
||||
)
|
||||
metadata = MiniMaxM3SparseMetadata(
|
||||
seq_lens=seq_lens,
|
||||
max_seq_len=max(seq_lens_list),
|
||||
slot_mapping=torch.empty(total_q, dtype=torch.int64, device="cuda"),
|
||||
num_actual_tokens=total_q,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_prefills=len(q_lens),
|
||||
num_prefill_tokens=total_q,
|
||||
prefill=prefill,
|
||||
)
|
||||
|
||||
# Every causally visible page (<= TOPK per query), ending with the local one.
|
||||
topk = torch.full(
|
||||
(total_q, num_kv_heads, TOPK), -1, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
token = 0
|
||||
for q_len, seq_len in zip(q_lens, seq_lens_list):
|
||||
for local_q in range(q_len):
|
||||
visible = (seq_len - q_len + local_q) // BLOCK_SIZE + 1
|
||||
assert visible <= TOPK
|
||||
topk[token, :, :visible] = torch.arange(visible, device="cuda")
|
||||
token += 1
|
||||
|
||||
layer_name = "model.layers.0.self_attn.attn"
|
||||
q_scale = 0.5
|
||||
layer = SimpleNamespace(
|
||||
layer_name=layer_name,
|
||||
topk_indices_buffer=topk,
|
||||
_q_scale_float=q_scale,
|
||||
_k_scale=k_scale,
|
||||
_v_scale=v_scale,
|
||||
)
|
||||
query_fp8 = (
|
||||
torch.randn(
|
||||
total_q, num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
/ q_scale
|
||||
).to(torch.float8_e4m3fn)
|
||||
query = query_fp8.to(torch.bfloat16) * q_scale
|
||||
common = dict(
|
||||
num_heads=num_q_heads,
|
||||
head_size=HEAD_DIM,
|
||||
scale=SM_SCALE,
|
||||
num_kv_heads=num_kv_heads,
|
||||
topk_blocks=TOPK,
|
||||
sparse_block_size=BLOCK_SIZE,
|
||||
)
|
||||
forward_context = ForwardContext(
|
||||
no_compile_layers={},
|
||||
attn_metadata={layer_name: metadata},
|
||||
slot_mapping={},
|
||||
)
|
||||
with override_forward_context(forward_context):
|
||||
expected = MiniMaxM3SparseTritonImpl(**common).forward(
|
||||
layer, query, kv_reference, torch.empty_like(query)
|
||||
)
|
||||
actual = MiniMaxM3SparseMSAImpl(**common, kv_cache_dtype="nvfp4").forward(
|
||||
layer,
|
||||
torch.full_like(query, float("nan")),
|
||||
kv_cache,
|
||||
torch.zeros_like(query),
|
||||
query_fp8=query_fp8,
|
||||
)
|
||||
current_platform.synchronize()
|
||||
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
|
||||
|
||||
|
||||
def test_msa_triton_decode_fallback_reads_dequantized_query_fp8() -> None:
|
||||
"""With CUTLASS decode on but not planned for the batch, the Triton decode
|
||||
fallback attends with the dequantized fp8 q, the only q written."""
|
||||
torch.manual_seed(0)
|
||||
num_q_heads, num_kv_heads, q_scale = 16, 1, 0.5
|
||||
seq_lens_list = [257, 513, 129]
|
||||
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda")
|
||||
block_table, num_pages = _make_paged_layout(seq_lens_list)
|
||||
kv_cache = (
|
||||
torch.randn(
|
||||
num_pages,
|
||||
num_kv_heads,
|
||||
BLOCK_SIZE,
|
||||
2 * HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
* 0.25
|
||||
).to(torch.float8_e4m3fn)
|
||||
num_tokens = len(seq_lens_list)
|
||||
metadata = MiniMaxM3SparseMetadata(
|
||||
seq_lens=seq_lens,
|
||||
max_seq_len=max(seq_lens_list),
|
||||
slot_mapping=torch.empty(num_tokens, dtype=torch.int64, device="cuda"),
|
||||
num_actual_tokens=num_tokens,
|
||||
num_decodes=num_tokens,
|
||||
num_decode_tokens=num_tokens,
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
decode=MiniMaxM3SparseMSADecodeMetadata(
|
||||
seq_lens=seq_lens,
|
||||
block_table=block_table,
|
||||
decode_query_len=1,
|
||||
msa_cutlass=None,
|
||||
),
|
||||
)
|
||||
layer_name = "model.layers.0.self_attn.attn"
|
||||
layer = SimpleNamespace(
|
||||
layer_name=layer_name,
|
||||
topk_indices_buffer=_make_topk(seq_lens_list, num_kv_heads, 1),
|
||||
_q_scale_float=q_scale,
|
||||
)
|
||||
query_fp8 = torch.randn(
|
||||
num_tokens, num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
).to(torch.float8_e4m3fn)
|
||||
query = query_fp8.to(torch.bfloat16) * q_scale
|
||||
common = dict(
|
||||
num_heads=num_q_heads,
|
||||
head_size=HEAD_DIM,
|
||||
scale=SM_SCALE,
|
||||
num_kv_heads=num_kv_heads,
|
||||
kv_cache_dtype="fp8",
|
||||
topk_blocks=TOPK,
|
||||
sparse_block_size=BLOCK_SIZE,
|
||||
)
|
||||
impl = MiniMaxM3SparseMSAImpl(**common, msa_decode_backend="cutlass")
|
||||
assert impl.use_cutlass_decode
|
||||
forward_context = ForwardContext(
|
||||
no_compile_layers={},
|
||||
attn_metadata={layer_name: metadata},
|
||||
slot_mapping={},
|
||||
)
|
||||
with override_forward_context(forward_context):
|
||||
expected = MiniMaxM3SparseTritonImpl(**common).forward(
|
||||
layer, query, kv_cache, torch.empty_like(query)
|
||||
)
|
||||
actual = impl.forward(
|
||||
layer,
|
||||
torch.full_like(query, float("nan")),
|
||||
kv_cache,
|
||||
torch.zeros_like(query),
|
||||
query_fp8=query_fp8,
|
||||
)
|
||||
current_platform.synchronize()
|
||||
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
|
||||
|
||||
@@ -783,6 +783,9 @@ def test_aiter_fa_shared_kv_matches_reference(
|
||||
32 * BLOCK_SIZE, num_kv_heads, head_size, dtype=torch.bfloat16
|
||||
)
|
||||
value = torch.randn_like(key)
|
||||
from vllm.v1.attention.backends.rocm_aiter_fa import AiterFlashAttentionImpl
|
||||
|
||||
assert isinstance(target.impl, AiterFlashAttentionImpl)
|
||||
target.impl.do_kv_cache_update(
|
||||
target,
|
||||
key,
|
||||
|
||||
@@ -82,6 +82,7 @@ def _run_verify_block():
|
||||
)
|
||||
from vllm.config import SpeculativeConfig
|
||||
from vllm.config.vllm import set_current_vllm_config
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla import AiterMLAImpl
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
@@ -106,7 +107,7 @@ def _run_verify_block():
|
||||
vllm_config.speculative_config = SpeculativeConfig(
|
||||
method="ngram", num_speculative_tokens=QLEN - 1
|
||||
)
|
||||
vllm_config.model_config.get_num_attention_heads = types.MethodType(
|
||||
vllm_config.model_config.get_num_attention_heads = types.MethodType( # type: ignore[method-assign]
|
||||
lambda self, parallel_config, arch_config=None: NUM_QUERY_HEADS,
|
||||
vllm_config.model_config,
|
||||
)
|
||||
@@ -202,6 +203,7 @@ def _run_verify_block():
|
||||
lambda: spy,
|
||||
),
|
||||
):
|
||||
assert isinstance(impl, AiterMLAImpl)
|
||||
impl.forward_mqa((q_nope, q_pe), kv_cache, metadata, layer=None)
|
||||
|
||||
return metadata, captured
|
||||
|
||||
@@ -167,7 +167,10 @@ def test_sparse_mla_sink_matches_ragged_reference(
|
||||
impl.sinks = sinks
|
||||
|
||||
output, lse = impl._forward_mla(
|
||||
SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale), q, kv, metadata
|
||||
SimpleNamespace(_q_scale=q_scale, _k_scale=kv_scale),
|
||||
q,
|
||||
kv,
|
||||
metadata,
|
||||
)
|
||||
kv_flat = kv_ref[:, 0]
|
||||
references = []
|
||||
|
||||
@@ -0,0 +1,599 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""DeepSeek V4.1's ROCm MXFP4 sparse indexer against a torch reference.
|
||||
|
||||
The indexer K cache is a view into vLLM's block-major KV pool: its pages are
|
||||
a block stride apart with other layers' pages in between, so the tests put
|
||||
it there. The reference reads the keys back from a second cache the same
|
||||
writer fills in natural order, so it shares none of the preshuffled
|
||||
addressing it checks.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
import types
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip("ROCm-only", allow_module_level=True)
|
||||
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
rocm_mxfp4_indexer_k_store,
|
||||
rocm_mxfp4_indexer_q_quant,
|
||||
)
|
||||
from vllm.platforms.rocm import get_cdna_version
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
DeepseekV32IndexerPrefillChunkMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import (
|
||||
RocmMxfp4IndexerMetadata,
|
||||
native_decode,
|
||||
plan_gather_launches,
|
||||
plan_prefill_chunks,
|
||||
)
|
||||
from vllm.v1.attention.ops import rocm_paged_mxfp4_indexer as ops
|
||||
from vllm.v1.worker.workspace import init_workspace_manager, reset_workspace_manager
|
||||
|
||||
if get_cdna_version() != 4:
|
||||
pytest.skip("CDNA4-only", allow_module_level=True)
|
||||
|
||||
HEADS, HEAD_DIM = 32, 128
|
||||
WIDTH = HEAD_DIM // 2 + HEAD_DIM // 32
|
||||
# V4.1's 8-token blocks, with the pool (2048 blocks) and top-k (512) scaled
|
||||
# down so short test contexts still select.
|
||||
CAND_BLOCK, POOL_BLOCKS, TOPK = 8, 16, 32
|
||||
MAX_LEN = 4096
|
||||
DEVICE = "cuda"
|
||||
|
||||
_E2M1 = torch.tensor([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0])
|
||||
E2M1 = torch.cat([_E2M1, -_E2M1])
|
||||
|
||||
|
||||
def _dequant(packed, e8m0):
|
||||
"""[..., D/2] e2m1 pairs and [..., D/32] e8m0 to [..., D] fp32."""
|
||||
table = E2M1.to(packed.device)
|
||||
vals = torch.stack([table[(packed & 0xF).long()], table[(packed >> 4).long()]], -1)
|
||||
scale = torch.pow(2.0, e8m0.float() - 127)
|
||||
return (vals.flatten(-2).unflatten(-1, (-1, 32)) * scale[..., None]).flatten(-2)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _workspace():
|
||||
init_workspace_manager(torch.device(DEVICE))
|
||||
yield
|
||||
reset_workspace_manager()
|
||||
|
||||
|
||||
class _Case:
|
||||
"""Requests with their keys written into a block-major pool (ratio-2 and
|
||||
ratio-1 pages side by side in every block) and into natural-order caches
|
||||
the reference reads. The bf16 inputs are kept for the end-to-end test."""
|
||||
|
||||
def __init__(self, seq_lens, block):
|
||||
from aiter.ops.triton.fusions.k_norm_rope_mxfp4_cache import (
|
||||
k_norm_rope_mxfp4_cache,
|
||||
)
|
||||
|
||||
self.seq_lens = seq_lens
|
||||
num_blocks = sum(cdiv(n, block) for n in seq_lens) + 3
|
||||
perm = torch.randperm(num_blocks)
|
||||
width = max(cdiv(n, block) for n in seq_lens)
|
||||
self.block_table = torch.zeros(len(seq_lens), width, dtype=torch.int32)
|
||||
used = 0
|
||||
for req, n in enumerate(seq_lens):
|
||||
self.block_table[req, : cdiv(n, block)] = perm[used : used + cdiv(n, block)]
|
||||
used += cdiv(n, block)
|
||||
self.block_table = self.block_table.to(DEVICE)
|
||||
|
||||
def align(n):
|
||||
return cdiv(n, 576) * 576
|
||||
|
||||
page2, page1 = block // 2 * WIDTH, block * WIDTH
|
||||
stride = align(page2) + align(page1) + 576
|
||||
self.pool = torch.zeros(num_blocks * stride, dtype=torch.uint8, device=DEVICE)
|
||||
self.page_bytes = torch.zeros_like(self.pool, dtype=torch.bool)
|
||||
for i in range(num_blocks):
|
||||
base = i * stride
|
||||
self.page_bytes[base : base + page2] = True
|
||||
base += align(page2)
|
||||
self.page_bytes[base : base + page1] = True
|
||||
self.cache = {
|
||||
2: torch.as_strided(
|
||||
self.pool, (num_blocks, block // 2, WIDTH), (stride, WIDTH, 1), 0
|
||||
),
|
||||
1: torch.as_strided(
|
||||
self.pool,
|
||||
(num_blocks, block, WIDTH),
|
||||
(stride, WIDTH, 1),
|
||||
align(page2),
|
||||
),
|
||||
}
|
||||
self.offsets = [0]
|
||||
for n in seq_lens:
|
||||
self.offsets.append(self.offsets[-1] + n)
|
||||
# the same keys in natural order, one entry per page
|
||||
self.natural = {
|
||||
r: torch.zeros(self.offsets[-1], 1, WIDTH, dtype=torch.uint8, device=DEVICE)
|
||||
for r in (1, 2)
|
||||
}
|
||||
self.cos_sin = cos_sin = torch.randn(MAX_LEN, 64, device=DEVICE)
|
||||
self.norm = norm = torch.rand(HEAD_DIM, device=DEVICE) + 0.5
|
||||
self.k_pre = {}
|
||||
for req, n in enumerate(seq_lens):
|
||||
if n == 0:
|
||||
continue
|
||||
k_pre = torch.randn(n, HEAD_DIM, device=DEVICE).to(torch.bfloat16)
|
||||
self.k_pre[req] = k_pre
|
||||
pos = torch.arange(n, device=DEVICE)
|
||||
for ratio, cache in self.cache.items():
|
||||
comp = pos // ratio
|
||||
boundary = (pos + 1) % ratio == 0
|
||||
entries = cache.shape[1]
|
||||
page = self.block_table[req, comp // entries].long()
|
||||
slot = torch.where(boundary, page * entries + comp % entries, -1)
|
||||
nat = torch.where(boundary, self.offsets[req] + comp, -1)
|
||||
rocm_mxfp4_indexer_k_store(
|
||||
k_pre,
|
||||
pos,
|
||||
cos_sin,
|
||||
norm,
|
||||
1e-6,
|
||||
cache,
|
||||
slot,
|
||||
ratio,
|
||||
True,
|
||||
num_heads=HEADS,
|
||||
)
|
||||
k_norm_rope_mxfp4_cache(
|
||||
k_pre, pos, cos_sin, norm, 1e-6, self.natural[ratio], nat, ratio
|
||||
)
|
||||
|
||||
def keys(self, ratio, req, n):
|
||||
nat = self.natural[ratio][self.offsets[req] : self.offsets[req] + n, 0]
|
||||
return _dequant(nat[:, : HEAD_DIM // 2], nat[:, HEAD_DIM // 2 :])
|
||||
|
||||
|
||||
def _queries(num_rows):
|
||||
q = torch.randint(0, 256, (num_rows, HEADS, HEAD_DIM // 2), dtype=torch.uint8)
|
||||
scales = torch.randint(122, 130, (num_rows, HEADS, HEAD_DIM // 32))
|
||||
q, scales = q.to(DEVICE), scales.to(torch.uint8).to(DEVICE)
|
||||
weights = torch.randn(num_rows, HEADS, device=DEVICE)
|
||||
# the fused Q kernel returns the scales as one int32 per head
|
||||
return q, scales.view(torch.int32).squeeze(-1), weights, _dequant(q, scales)
|
||||
|
||||
|
||||
def _scores(q, w, keys):
|
||||
return (torch.relu(q @ keys.T) * w[:, None]).sum(0)
|
||||
|
||||
|
||||
def _assert_topk(row, scores, k):
|
||||
"""``row`` is a top-k of the entries of ``scores`` above -inf, up to the
|
||||
fp32 reduce order."""
|
||||
valid = scores > float("-inf")
|
||||
want = min(k, int(valid.sum()))
|
||||
sel = row[row >= 0].long()
|
||||
assert sel.numel() == want and sel.unique().numel() == want
|
||||
if want:
|
||||
assert bool(valid[sel].all())
|
||||
kth = scores[valid].topk(want).values[-1]
|
||||
finite = scores[torch.isfinite(scores)]
|
||||
tol = 1e-4 * float(finite.abs().max()) if finite.numel() else 0.0
|
||||
assert bool((scores[sel] >= kth - tol).all())
|
||||
|
||||
|
||||
def _block_maxima(scores):
|
||||
end = scores.numel()
|
||||
if end == 0:
|
||||
return scores
|
||||
padded = torch.full((cdiv(end, CAND_BLOCK) * CAND_BLOCK,), float("-inf"))
|
||||
padded[:end] = scores.cpu()
|
||||
out = padded.view(-1, CAND_BLOCK).amax(1)
|
||||
out[(end - 1) // CAND_BLOCK] = float("inf")
|
||||
return out.to(scores.device)
|
||||
|
||||
|
||||
def _forward_context(monkeypatch, metadata):
|
||||
context = types.SimpleNamespace(
|
||||
attn_metadata={"indexer": metadata},
|
||||
cudagraph_runtime_mode=CUDAGraphMode.NONE,
|
||||
)
|
||||
monkeypatch.setattr(ops, "get_forward_context", lambda: context)
|
||||
|
||||
|
||||
def _decode_metadata(case, rows, ratio, query_lens):
|
||||
lens = torch.tensor([(p + 1) // ratio for _, p in rows], dtype=torch.int32)
|
||||
lens = lens.to(DEVICE)
|
||||
block_table = case.block_table[[req for req, _ in rows]].contiguous()
|
||||
entries = case.cache[ratio].shape[1]
|
||||
next_n = max(query_lens)
|
||||
context_lens = (torch.tensor(case.seq_lens, dtype=torch.int32) // ratio).to(DEVICE)
|
||||
native = native_decode(lens, block_table, query_lens, next_n, context_lens)
|
||||
schedule = torch.empty(
|
||||
ops.rocm_mxfp4_decode_schedule_words(HEADS, HEAD_DIM, entries, next_n),
|
||||
dtype=torch.int32,
|
||||
device=DEVICE,
|
||||
)
|
||||
return RocmMxfp4IndexerMetadata(
|
||||
seq_lens=None,
|
||||
max_seq_len=max(case.seq_lens),
|
||||
slot_mapping=None,
|
||||
num_decodes=len(case.seq_lens),
|
||||
num_decode_tokens=len(rows),
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
decode=types.SimpleNamespace(block_table=block_table, seq_lens=lens[:, None]),
|
||||
decode_row_lens=lens,
|
||||
decode_block_ends=(lens + CAND_BLOCK - 1) // CAND_BLOCK,
|
||||
decode_native=native,
|
||||
decode_schedule=ops.build_rocm_mxfp4_decode_schedule(
|
||||
lens, HEADS, HEAD_DIM, entries, schedule, MAX_LEN // ratio, native
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _prefill_metadata(
|
||||
case, rows, ratio, chunk_bounds, query_start_loc, gather_rows, gather
|
||||
):
|
||||
chunks = []
|
||||
for req_lo, req_hi, t0, t1 in chunk_bounds:
|
||||
ends = torch.tensor(
|
||||
[(rows[t][1] + 1) // ratio for t in range(t0, t1)], dtype=torch.int32
|
||||
).to(DEVICE)
|
||||
chunks.append(
|
||||
DeepseekV32IndexerPrefillChunkMetadata(
|
||||
block_table=case.block_table[req_lo:req_hi],
|
||||
cu_seqlen_ks=torch.zeros_like(ends),
|
||||
cu_seqlen_ke=ends,
|
||||
cu_seq_lens=torch.zeros(1, dtype=torch.int32, device=DEVICE),
|
||||
token_to_seq=torch.zeros(1, dtype=torch.int32, device=DEVICE),
|
||||
total_seq_lens=1,
|
||||
token_start=t0,
|
||||
token_end=t1,
|
||||
num_reqs=req_hi - req_lo,
|
||||
)
|
||||
)
|
||||
seq_lens = torch.tensor(case.seq_lens)
|
||||
context_lens = (seq_lens // ratio).int().to(DEVICE)
|
||||
plans = plan_prefill_chunks(
|
||||
chunks,
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
context_lens,
|
||||
ratio,
|
||||
CAND_BLOCK,
|
||||
0.0 if gather else None,
|
||||
torch.tensor(query_start_loc, dtype=torch.int32, device=DEVICE),
|
||||
)
|
||||
return RocmMxfp4IndexerMetadata(
|
||||
seq_lens=None,
|
||||
max_seq_len=max(case.seq_lens),
|
||||
slot_mapping=None,
|
||||
num_decodes=0,
|
||||
num_decode_tokens=0,
|
||||
num_prefills=len(case.seq_lens),
|
||||
num_prefill_tokens=len(rows),
|
||||
prefill=types.SimpleNamespace(chunks=chunks),
|
||||
prefill_plans=plans,
|
||||
gather_launches=plan_gather_launches(chunks, plans, gather_rows),
|
||||
)
|
||||
|
||||
|
||||
# The kernel prefers 128-token pages.
|
||||
BLOCKS = pytest.mark.parametrize("block", [128, 64])
|
||||
|
||||
|
||||
@BLOCKS
|
||||
def test_k_store_stays_inside_its_pages(block):
|
||||
"""The writer touches only its pages of the block-major pool. What they hold
|
||||
is checked by the layer tests, which read them back through aiter's kernel."""
|
||||
set_random_seed(0)
|
||||
case = _Case([700, 333], block)
|
||||
assert int(case.pool[~case.page_bytes].count_nonzero()) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"rows, width, k",
|
||||
# vLLM's kernel below 128 rows, aiter's from 128 rows and on every k=2048
|
||||
[(8, 4096, 512), (128, 4096, 512), (8, 16384, 2048)],
|
||||
)
|
||||
def test_topk_matches_torch_on_both_kernels(rows, width, k):
|
||||
set_random_seed(0)
|
||||
logits = torch.randn(rows, width, device=DEVICE)
|
||||
lengths = torch.randint(1, width + 1, (rows,), dtype=torch.int32, device=DEVICE)
|
||||
lengths[0] = k // 2 # shorter than k, so the rest of the row pads with -1
|
||||
out = torch.full((rows, k), 7, dtype=torch.int32, device=DEVICE)
|
||||
ops._topk(logits, lengths, out, k)
|
||||
for r in range(rows):
|
||||
n = min(k, int(lengths[r]))
|
||||
want = torch.topk(logits[r, : int(lengths[r])], n).indices.sort().values
|
||||
got = out[r]
|
||||
assert torch.equal(got[got >= 0].sort().values, want.int())
|
||||
assert int((got == -1).sum()) == k - n
|
||||
|
||||
|
||||
def _run_layers(monkeypatch, case, rows, metadata):
|
||||
"""Ratio-2 dense layer, the candidate source, and a consumer both ways;
|
||||
each checked against the torch reference."""
|
||||
num_rows = len(rows)
|
||||
q, q_scale, weights, q_ref = _queries(num_rows)
|
||||
hidden = torch.empty(num_rows, 1, device=DEVICE)
|
||||
|
||||
def topk_buffer():
|
||||
return torch.full((num_rows + 2, TOPK), 7, dtype=torch.int32, device=DEVICE)
|
||||
|
||||
_forward_context(monkeypatch, metadata(2))
|
||||
out = topk_buffer()
|
||||
ops.rocm_mxfp4_sparse_attn_indexer(
|
||||
hidden,
|
||||
"indexer",
|
||||
case.cache[2],
|
||||
q,
|
||||
q_scale,
|
||||
weights,
|
||||
TOPK,
|
||||
HEAD_DIM,
|
||||
MAX_LEN // 2,
|
||||
out,
|
||||
compress_ratio=2,
|
||||
)
|
||||
for i, (req, pos) in enumerate(rows):
|
||||
end = (pos + 1) // 2
|
||||
keys = case.keys(2, req, end)
|
||||
_assert_topk(out[i], _scores(q_ref[i], weights[i], keys), TOPK)
|
||||
|
||||
source = metadata(1)
|
||||
_forward_context(monkeypatch, source)
|
||||
pool = torch.full((num_rows + 2, POOL_BLOCKS), 5, dtype=torch.int32, device=DEVICE)
|
||||
out = topk_buffer()
|
||||
ops.rocm_mxfp4_sparse_attn_indexer(
|
||||
hidden,
|
||||
"indexer",
|
||||
case.cache[1],
|
||||
q,
|
||||
q_scale,
|
||||
weights,
|
||||
TOPK,
|
||||
HEAD_DIM,
|
||||
MAX_LEN,
|
||||
out,
|
||||
compress_ratio=1,
|
||||
candidate_blocks=pool,
|
||||
candidate_block_size=CAND_BLOCK,
|
||||
candidate_write=True,
|
||||
)
|
||||
allowed = []
|
||||
for i, (req, pos) in enumerate(rows):
|
||||
scores = _scores(q_ref[i], weights[i], case.keys(1, req, pos + 1))
|
||||
_assert_topk(out[i], scores, TOPK)
|
||||
_assert_topk(pool[i], _block_maxima(scores), POOL_BLOCKS)
|
||||
mask = torch.zeros_like(scores, dtype=torch.bool)
|
||||
for block in pool[i][pool[i] >= 0].tolist():
|
||||
mask[block * CAND_BLOCK : (block + 1) * CAND_BLOCK] = True
|
||||
allowed.append(torch.where(mask, scores, float("-inf")))
|
||||
|
||||
pa = ops._aiter()
|
||||
build = pa.build_candidate_gather
|
||||
builds = []
|
||||
|
||||
def counting_build(*args, **kwargs):
|
||||
builds.append(1)
|
||||
return build(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(pa, "build_candidate_gather", counting_build)
|
||||
for gather in (True, False):
|
||||
consumer = metadata(1, gather)
|
||||
consumer.decode_use_gather = gather
|
||||
_forward_context(monkeypatch, consumer)
|
||||
builds.clear()
|
||||
# Two consumer layers of one step: the second reuses the first's pools.
|
||||
for _ in range(2):
|
||||
out = topk_buffer()
|
||||
ops.rocm_mxfp4_sparse_mqa_indexer(
|
||||
hidden,
|
||||
"indexer",
|
||||
case.cache[1],
|
||||
q,
|
||||
q_scale,
|
||||
weights,
|
||||
TOPK,
|
||||
HEAD_DIM,
|
||||
MAX_LEN,
|
||||
out,
|
||||
1,
|
||||
pool,
|
||||
CAND_BLOCK,
|
||||
POOL_BLOCKS * CAND_BLOCK,
|
||||
)
|
||||
for i in range(num_rows):
|
||||
_assert_topk(out[i], allowed[i], TOPK)
|
||||
if gather:
|
||||
launches = len(consumer.gather_launches) + (consumer.decode is not None)
|
||||
assert len(builds) == launches
|
||||
|
||||
|
||||
@BLOCKS
|
||||
@pytest.mark.parametrize(
|
||||
"query_lens",
|
||||
# Uniform steps launch the dense layers on next_n-row sequences, also with
|
||||
# cudagraph padding (query length 0) after them; a ragged step keeps a row
|
||||
# per token.
|
||||
[[6, 6, 6, 6], [2, 2, 2], [2, 2, 2, 0], [2, 1, 2]],
|
||||
ids=["native6", "native2", "padded", "ragged"],
|
||||
)
|
||||
def test_decode_layers_match_reference(monkeypatch, block, query_lens):
|
||||
set_random_seed(0)
|
||||
next_n = max(query_lens)
|
||||
case = _Case([q and n for q, n in zip(query_lens, [900, 333, 610, 1200])], block)
|
||||
# a padding request has next_n rows of its own and no context
|
||||
rows = [
|
||||
(req, n - q + j if q else -1)
|
||||
for req, (n, q) in enumerate(zip(case.seq_lens, query_lens))
|
||||
for j in range(q or next_n)
|
||||
]
|
||||
_run_layers(
|
||||
monkeypatch,
|
||||
case,
|
||||
rows,
|
||||
lambda r, gather=False: _decode_metadata(case, rows, r, query_lens),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"seq_lens,new_tokens,chunks",
|
||||
[
|
||||
# one request past a cached prefix, then one sliced into two chunks
|
||||
([760, 500], [300, 500], [(0, 1, 0, 300), (1, 2, 300, 560), (1, 2, 560, 800)]),
|
||||
# three requests in one chunk, the first two with equal rows
|
||||
([600, 450, 900], [200, 200, 120], [(0, 3, 0, 520)]),
|
||||
# four requests with four different query lengths in one chunk
|
||||
([900, 600, 450, 700], [150, 90, 200, 60], [(0, 4, 0, 500)]),
|
||||
],
|
||||
ids=["sliced", "batched", "ragged"],
|
||||
)
|
||||
# The consumers rejoin a request's rows across chunks, then cut them at most
|
||||
# gather_rows a launch.
|
||||
@pytest.mark.parametrize("gather_rows", [1 << 20, 128], ids=["joined", "cut"])
|
||||
@BLOCKS
|
||||
def test_prefill_layers_match_reference(
|
||||
monkeypatch, seq_lens, new_tokens, chunks, gather_rows, block
|
||||
):
|
||||
set_random_seed(0)
|
||||
case = _Case(seq_lens, block)
|
||||
rows = [
|
||||
(req, n - q + i)
|
||||
for req, (n, q) in enumerate(zip(seq_lens, new_tokens))
|
||||
for i in range(q)
|
||||
]
|
||||
query_start_loc = [0, *itertools.accumulate(new_tokens)]
|
||||
_run_layers(
|
||||
monkeypatch,
|
||||
case,
|
||||
rows,
|
||||
lambda r, gather=True: _prefill_metadata(
|
||||
case, rows, r, chunks, query_start_loc, gather_rows, gather
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _rope_gptj(x, cos_sin, positions):
|
||||
"""GPT-J RoPE on the last 64 dims, as the indexer rotates Q and K: bf16 in
|
||||
and out."""
|
||||
x = x.bfloat16().float()
|
||||
cos, sin = cos_sin[positions].chunk(2, dim=-1)
|
||||
if x.dim() == 3:
|
||||
cos, sin = cos[:, None], sin[:, None]
|
||||
rot = x[..., -64:]
|
||||
even, odd = rot[..., 0::2].clone(), rot[..., 1::2].clone()
|
||||
rot[..., 0::2] = even * cos - odd * sin
|
||||
rot[..., 1::2] = odd * cos + even * sin
|
||||
return x.bfloat16()
|
||||
|
||||
|
||||
def _mxfp4_round_trip(x):
|
||||
"""MXFP4 round trip of x, rounded as the writers do: a scale of
|
||||
2^ceil(log2(amax / 6)) per 32 values, then the nearest e2m1 value, ties to
|
||||
even."""
|
||||
blocks = x.float().unflatten(-1, (-1, 32))
|
||||
amax = blocks.abs().amax(-1, keepdim=True).clamp(min=2**-126)
|
||||
scale = torch.exp2(torch.ceil(torch.log2(amax / 6)))
|
||||
y = (blocks / scale).clamp(-6, 6)
|
||||
# Even codes first: argmin keeps the first of two equal distances.
|
||||
grid = torch.cat([_E2M1[0::2], _E2M1[1::2]]).to(x.device)
|
||||
idx = (y.abs()[..., None] - grid).abs().argmin(-1)
|
||||
return (grid[idx] * y.sign() * scale).flatten(-2)
|
||||
|
||||
|
||||
def _torch_keys(case, ratio, req):
|
||||
"""A request's compressed keys in torch: k_norm of each group's last
|
||||
token, rotated at the group's first position, through MXFP4."""
|
||||
k = case.k_pre[req][ratio - 1 :: ratio].float()
|
||||
k = k * torch.rsqrt(k.pow(2).mean(-1, keepdim=True) + 1e-6) * case.norm
|
||||
pos = torch.arange(k.shape[0], device=DEVICE) * ratio
|
||||
return _mxfp4_round_trip(_rope_gptj(k, case.cos_sin, pos))
|
||||
|
||||
|
||||
def _topk_indices(q, weights, keys, ends, k):
|
||||
"""Top-k of sum_h relu(q_h . key) * weights_h over each row's [0, end)."""
|
||||
scores = (torch.einsum("thd,cd->thc", q, keys).relu() * weights[..., None]).sum(1)
|
||||
cols = torch.arange(keys.shape[0], device=DEVICE)
|
||||
return scores.masked_fill(cols >= ends[:, None], float("-inf")).topk(k).indices
|
||||
|
||||
|
||||
def _recall(selected, reference):
|
||||
"""Per row, the fraction of the reference top-k that ``selected`` picked."""
|
||||
return (selected[:, :, None] == reference[:, None, :]).any(1).float().mean(1)
|
||||
|
||||
|
||||
@BLOCKS
|
||||
@pytest.mark.parametrize("ratio", [2, 1])
|
||||
@pytest.mark.parametrize("step", ["decode", "prefill"])
|
||||
def test_selection_from_bf16_matches_torch(monkeypatch, step, ratio, block):
|
||||
"""End to end from bf16 K and Q: vLLM's MXFP4 K store and Q quant, the
|
||||
paged pool, metadata from the planning functions the builder calls, and
|
||||
the indexer op pick what the same MXFP4 math picks in torch."""
|
||||
set_random_seed(0)
|
||||
if step == "decode":
|
||||
case = _Case([3000, 2600, 1900, 3500, 1200, 2200, 3900, 1600], block)
|
||||
rows = [(req, n - 1) for req, n in enumerate(case.seq_lens)]
|
||||
metadata = _decode_metadata(case, rows, ratio, [1] * len(rows))
|
||||
else:
|
||||
new_tokens = [300, 200, 250]
|
||||
case = _Case([1500, 900, 2000], block)
|
||||
rows = [
|
||||
(req, n - q + i)
|
||||
for req, (n, q) in enumerate(zip(case.seq_lens, new_tokens))
|
||||
for i in range(q)
|
||||
]
|
||||
metadata = _prefill_metadata(
|
||||
case,
|
||||
rows,
|
||||
ratio,
|
||||
[(0, len(new_tokens), 0, len(rows))],
|
||||
[0, *itertools.accumulate(new_tokens)],
|
||||
1 << 20,
|
||||
gather=False,
|
||||
)
|
||||
num_rows = len(rows)
|
||||
q_bf16 = torch.randn(num_rows, HEADS, HEAD_DIM, device=DEVICE).to(torch.bfloat16)
|
||||
w = torch.randn(num_rows, HEADS, device=DEVICE)
|
||||
pos = torch.tensor([p for _, p in rows], device=DEVICE)
|
||||
scales = (HEAD_DIM**-0.5, HEADS**-0.5)
|
||||
|
||||
(q, q_scale), weights = rocm_mxfp4_indexer_q_quant(
|
||||
pos, q_bf16, case.cos_sin, w, *scales
|
||||
)
|
||||
_forward_context(monkeypatch, metadata)
|
||||
out = torch.full((num_rows, TOPK), 7, dtype=torch.int32, device=DEVICE)
|
||||
ops.rocm_mxfp4_sparse_attn_indexer(
|
||||
torch.empty(num_rows, 1, device=DEVICE),
|
||||
"indexer",
|
||||
case.cache[ratio],
|
||||
q,
|
||||
q_scale,
|
||||
weights,
|
||||
TOPK,
|
||||
HEAD_DIM,
|
||||
MAX_LEN // ratio,
|
||||
out,
|
||||
compress_ratio=ratio,
|
||||
)
|
||||
ends = (pos + 1) // ratio
|
||||
assert bool(((out >= 0) & (out < ends[:, None])).all())
|
||||
assert all(row.unique().numel() == TOPK for row in out)
|
||||
|
||||
q_ref = _mxfp4_round_trip(_rope_gptj(q_bf16, case.cos_sin, pos))
|
||||
w_ref = w * scales[0] * scales[1]
|
||||
agree = []
|
||||
for req in range(len(case.seq_lens)):
|
||||
mine = torch.tensor([r == req for r, _ in rows], device=DEVICE)
|
||||
ref = _topk_indices(
|
||||
q_ref[mine], w_ref[mine], _torch_keys(case, ratio, req), ends[mine], TOPK
|
||||
)
|
||||
agree.append(_recall(out[mine], ref))
|
||||
assert torch.cat(agree).mean().item() >= 0.99
|
||||
@@ -153,6 +153,7 @@ def test_triton_unified_attn_diffkv_vs_reference(
|
||||
else:
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func
|
||||
|
||||
assert fa_version is not None
|
||||
# FA's TMA path needs aligned singleton strides (num_kv_heads == 1).
|
||||
fa_k = canonicalize_singleton_dim_strides(key_cache)
|
||||
fa_v = canonicalize_singleton_dim_strides(value_cache)
|
||||
|
||||
@@ -79,7 +79,7 @@ class TestConfigSet:
|
||||
config_set = ConfigSet("test_kernel")
|
||||
|
||||
with pytest.raises(KeyError, match="platform 'h100' not found"):
|
||||
config_set.get_config("h100", "nonexistent")
|
||||
config_set.get_config("h100", CaseKey({"batch": 32}))
|
||||
|
||||
config_data = {"num_warps": 8, "num_stages": 4}
|
||||
data = {
|
||||
|
||||
@@ -81,7 +81,7 @@ class TestDynamicPerTokenScaledFp8QuantConfigPicker:
|
||||
assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32})
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
|
||||
@@ -127,7 +127,7 @@ class TestFusedQkNormRopeConfigPicker:
|
||||
)
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
|
||||
@@ -38,6 +38,7 @@ from torch._inductor.pattern_matcher import (
|
||||
)
|
||||
from torch.fx.experimental.proxy_tensor import make_fx
|
||||
|
||||
from vllm.kernels.helion.case_key import CaseKey
|
||||
from vllm.kernels.helion.config_manager import ConfigManager
|
||||
from vllm.kernels.helion.register import HelionKernelWrapper
|
||||
|
||||
@@ -45,7 +46,7 @@ from vllm.kernels.helion.register import HelionKernelWrapper
|
||||
@contextlib.contextmanager
|
||||
def _helion_mock_context():
|
||||
configs = {
|
||||
"default": helion.Config(block_sizes=[64], num_warps=2, num_stages=2),
|
||||
CaseKey.default(): helion.Config(block_sizes=[64], num_warps=2, num_stages=2),
|
||||
}
|
||||
mock_config_manager = Mock(spec=ConfigManager)
|
||||
mock_config_manager.get_platform_configs = Mock(return_value=configs)
|
||||
@@ -88,7 +89,7 @@ class TestMakeFxHop:
|
||||
raw_kernel_func=raw_add_scale,
|
||||
op_name="test_make_fx",
|
||||
fake_impl=lambda *a, **kw: None,
|
||||
config_picker=lambda args, keys: "default",
|
||||
config_picker=lambda args, keys: CaseKey.default(),
|
||||
)
|
||||
|
||||
def fn(x, y):
|
||||
@@ -145,7 +146,7 @@ class TestMakeFxHop:
|
||||
raw_kernel_func=raw_silu_mul,
|
||||
op_name="test_pm_silu_mul",
|
||||
fake_impl=lambda *a, **kw: None,
|
||||
config_picker=lambda args, keys: "default",
|
||||
config_picker=lambda args, keys: CaseKey.default(),
|
||||
)
|
||||
|
||||
def pattern(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
@@ -107,7 +107,7 @@ class TestPerTokenGroupFp8QuantConfigPicker:
|
||||
)
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
|
||||
@@ -397,6 +397,7 @@ class TestHelionKernelWrapper:
|
||||
)
|
||||
|
||||
assert wrapper._disabled is True
|
||||
assert wrapper._disabled_reason is not None
|
||||
assert "No configs available" in wrapper._disabled_reason
|
||||
|
||||
def test_disabled_wrapper_raises_on_call(self, sample_kernel):
|
||||
|
||||
@@ -89,7 +89,7 @@ class TestRmsNormDynamicPerTokenQuantConfigPicker:
|
||||
assert selected_key == CaseKey({"hidden_size": 2048, "num_tokens": 32})
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
|
||||
@@ -112,7 +112,7 @@ class TestRmsNormPerBlockQuantConfigPicker:
|
||||
)
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
|
||||
@@ -96,7 +96,7 @@ class TestSiluAndMulPerBlockQuantConfigPicker:
|
||||
)
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
args = _generate_fake_input(16, 4096, 128)
|
||||
selected_key = pick_config(args, config_keys)
|
||||
|
||||
@@ -83,7 +83,7 @@ class TestSiluMulFp8ConfigPicker:
|
||||
assert selected_key == CaseKey({"intermediate": 4096, "numtokens": 256})
|
||||
|
||||
def test_config_picker_no_configs(self):
|
||||
config_keys: list[dict] = []
|
||||
config_keys: list[CaseKey] = []
|
||||
|
||||
input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
@@ -355,6 +355,7 @@ class TestSiluMulFp8Integration:
|
||||
registered_kernels = get_registered_kernels()
|
||||
kernel_wrapper = registered_kernels["silu_mul_fp8"]
|
||||
fake_impl = kernel_wrapper._fake_impl
|
||||
assert fake_impl is not None
|
||||
|
||||
fake_output = fake_impl(input_tensor, scale)
|
||||
|
||||
|
||||
@@ -494,8 +494,10 @@ def _sd_conv_states(
|
||||
return storage.transpose(1, 2)
|
||||
|
||||
|
||||
def _maybe_pack_conv_weight(weight: torch.Tensor, is_vnni: bool) -> torch.Tensor:
|
||||
return ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
|
||||
def _maybe_pack_conv_weight(
|
||||
weight: torch.Tensor, is_weight_packed: bool
|
||||
) -> torch.Tensor:
|
||||
return ops.causal_conv1d_weight_pack(weight) if is_weight_packed else weight
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@@ -523,7 +525,7 @@ def test_causal_conv1d_update_cpu_preserves_sd_state(
|
||||
bias=None,
|
||||
silu_activation=False,
|
||||
conv_state_indices=state_indices,
|
||||
is_vnni=False,
|
||||
is_weight_packed=False,
|
||||
)
|
||||
|
||||
assert out is x
|
||||
@@ -666,7 +668,10 @@ def test_spec_aware_nonspec_materializes_state_indices(
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize(("has_amx", "has_arm_bf16"), [(True, False), (False, True)])
|
||||
def test_spec_forward_prepares_native_conv_metadata(
|
||||
has_amx: bool,
|
||||
has_arm_bf16: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
block_table = torch.tensor([[0, 1], [4, 5]], dtype=torch.int32)
|
||||
@@ -687,14 +692,17 @@ def test_spec_forward_prepares_native_conv_metadata(
|
||||
)
|
||||
forwarded_indices = None
|
||||
forwarded_counts = None
|
||||
forwarded_is_weight_packed = None
|
||||
|
||||
def causal_conv1d_update_cpu(**kwargs):
|
||||
nonlocal forwarded_counts, forwarded_indices
|
||||
nonlocal forwarded_counts, forwarded_indices, forwarded_is_weight_packed
|
||||
forwarded_indices = kwargs["conv_state_indices"]
|
||||
forwarded_counts = kwargs["num_accepted_tokens"]
|
||||
forwarded_is_weight_packed = kwargs["is_weight_packed"]
|
||||
return kwargs["x"]
|
||||
|
||||
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: True)
|
||||
monkeypatch.setattr(torch.cpu, "_is_amx_tile_supported", lambda: has_amx)
|
||||
monkeypatch.setattr(gdn_attention, "is_arm_bf16", lambda: has_arm_bf16)
|
||||
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: False)
|
||||
monkeypatch.setattr(
|
||||
gdn_attention.ops, "causal_conv1d_update_cpu", causal_conv1d_update_cpu
|
||||
@@ -732,6 +740,7 @@ def test_spec_forward_prepares_native_conv_metadata(
|
||||
assert actual is not None
|
||||
assert actual.is_contiguous()
|
||||
assert actual.dtype == torch.int32
|
||||
assert forwarded_is_weight_packed is True
|
||||
torch.testing.assert_close(actual, reference)
|
||||
|
||||
|
||||
@@ -792,15 +801,15 @@ def test_causal_conv1d_torch_two_call_split(total_tokens: int, split: int) -> No
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="requires AMX support",
|
||||
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
|
||||
reason="native causal conv1d requires AMX or Arm BF16 support",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
|
||||
state_len = CONV_KERNEL - 1
|
||||
wide_state_len = state_len + 5
|
||||
batch_size = 3
|
||||
is_vnni = True
|
||||
is_weight_packed = True
|
||||
x, weight, bias = _conv_inputs(batch_size)
|
||||
conv_state_indices = torch.tensor([2, 0, 1], dtype=torch.int32)
|
||||
|
||||
@@ -813,7 +822,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
|
||||
wide_state[:, :, state_len:].fill_(7)
|
||||
wide_tail = wide_state[:, :, state_len:].clone()
|
||||
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
|
||||
out_narrow = ops.causal_conv1d_update_cpu(
|
||||
x=x,
|
||||
conv_states=narrow_state,
|
||||
@@ -821,7 +830,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
|
||||
bias=bias,
|
||||
silu_activation=True,
|
||||
conv_state_indices=conv_state_indices,
|
||||
is_vnni=is_vnni,
|
||||
is_weight_packed=is_weight_packed,
|
||||
)
|
||||
out_wide = ops.causal_conv1d_update_cpu(
|
||||
x=x,
|
||||
@@ -830,7 +839,7 @@ def test_causal_conv1d_update_cpu_accepts_wide_state() -> None:
|
||||
bias=bias,
|
||||
silu_activation=True,
|
||||
conv_state_indices=conv_state_indices,
|
||||
is_vnni=is_vnni,
|
||||
is_weight_packed=is_weight_packed,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
|
||||
@@ -872,11 +881,18 @@ def _ref_causal_conv1d_update_cpu_multi(
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="requires AMX support",
|
||||
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
|
||||
reason="native causal conv1d requires AMX or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("batch_size, seq_len, accepted_counts, has_bias, silu_activation, is_vnni"),
|
||||
(
|
||||
"batch_size, "
|
||||
"seq_len, "
|
||||
"accepted_counts, "
|
||||
"has_bias, "
|
||||
"silu_activation, "
|
||||
"is_weight_packed"
|
||||
),
|
||||
[
|
||||
(1, 1, [1], False, False, False),
|
||||
(1, 1, [1], True, True, True),
|
||||
@@ -893,7 +909,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
|
||||
accepted_counts: list[int],
|
||||
has_bias: bool,
|
||||
silu_activation: bool,
|
||||
is_vnni: bool,
|
||||
is_weight_packed: bool,
|
||||
) -> None:
|
||||
dim = 96
|
||||
state_len = seq_len + 2
|
||||
@@ -911,7 +927,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
|
||||
)
|
||||
conv_states = conv_states_ref.clone()
|
||||
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
|
||||
out = ops.causal_conv1d_update_cpu(
|
||||
x=x,
|
||||
conv_states=conv_states,
|
||||
@@ -919,7 +935,7 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
|
||||
bias=bias,
|
||||
silu_activation=silu_activation,
|
||||
conv_state_indices=conv_state_indices,
|
||||
is_vnni=is_vnni,
|
||||
is_weight_packed=is_weight_packed,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
)
|
||||
ref_out = _ref_causal_conv1d_update_cpu_multi(
|
||||
@@ -937,8 +953,8 @@ def test_causal_conv1d_update_cpu_multi_token_matches_python(
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="requires AMX support",
|
||||
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
|
||||
reason="native causal conv1d requires AMX or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize("num_accepted", [0, 17])
|
||||
@torch.inference_mode()
|
||||
@@ -961,14 +977,15 @@ def test_causal_conv1d_update_cpu_rejects_invalid_accepted_count(
|
||||
bias=None,
|
||||
silu_activation=True,
|
||||
conv_state_indices=torch.tensor([0], dtype=torch.int32),
|
||||
is_vnni=False,
|
||||
is_weight_packed=False,
|
||||
num_accepted_tokens=torch.tensor([num_accepted], dtype=torch.int32),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_avx512_bf16_supported(),
|
||||
reason="causal_conv1d_fwd_cpu requires AVX-512BF16 (Intel Xeon or AMD EPYC)",
|
||||
not (torch.cpu._is_avx512_bf16_supported() or gdn_attention.is_arm_bf16()),
|
||||
reason="causal_conv1d_fwd_cpu requires AVX-512BF16"
|
||||
" (Intel Xeon or AMD EPYC) or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize("total_tokens, split", TWO_CALL_SPLITS)
|
||||
@torch.inference_mode()
|
||||
@@ -977,13 +994,13 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
|
||||
matches the single-call result.
|
||||
|
||||
Regression test for ``causal_conv1d_fwd_varlen_kernel_impl`` (``conv.cpp``)
|
||||
ignoring the carried conv state on continued chunks. Runs on any
|
||||
AVX-512BF16 CPU since conv.cpp uses VDPBF16PS, not AMX tiles.
|
||||
ignoring the carried conv state on continued chunks. Runs on aarch64 cpus with
|
||||
bf16 support or any AVX-512BF16 CPU since conv.cpp uses VDPBF16PS, not AMX tiles.
|
||||
"""
|
||||
state_len = CONV_KERNEL - 1
|
||||
x, weight, bias = _conv_inputs(total_tokens)
|
||||
|
||||
def amx(x_seg, conv_states, has_init):
|
||||
def native(x_seg, conv_states, has_init):
|
||||
seq = x_seg.shape[0]
|
||||
return ops.causal_conv1d_fwd_cpu(
|
||||
x=x_seg.transpose(0, 1), # [dim, seq]; stride(-2)==1 (view of [seq,dim])
|
||||
@@ -994,30 +1011,30 @@ def test_causal_conv1d_fwd_cpu_two_call_split(total_tokens: int, split: int) ->
|
||||
cache_indices=torch.tensor([0], dtype=torch.int32),
|
||||
has_initial_state=torch.tensor([has_init]),
|
||||
silu_activation=True,
|
||||
is_vnni=False,
|
||||
is_weight_packed=False,
|
||||
).contiguous()
|
||||
|
||||
# conv_state layout passed by the AMX branch: [num_slots, dim, state_len].
|
||||
cs_full = torch.zeros(1, CONV_DIM, state_len, dtype=x.dtype)
|
||||
out_full = amx(x, cs_full, False)
|
||||
out_full = native(x, cs_full, False)
|
||||
|
||||
cs_split = torch.zeros(1, CONV_DIM, state_len, dtype=x.dtype)
|
||||
out1 = amx(x[:split], cs_split, False)
|
||||
out2 = amx(x[split:], cs_split, True)
|
||||
out1 = native(x[:split], cs_split, False)
|
||||
out2 = native(x[split:], cs_split, True)
|
||||
out_split = torch.cat([out1, out2], dim=1)
|
||||
|
||||
torch.testing.assert_close(out_split, out_full, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cpu._is_amx_tile_supported(),
|
||||
reason="requires AMX support",
|
||||
not (torch.cpu._is_amx_tile_supported() or gdn_attention.is_arm_bf16()),
|
||||
reason="native causal conv1d requires AMX or Arm BF16 support",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
|
||||
state_len = CONV_KERNEL - 1
|
||||
wide_state_len = state_len + 5
|
||||
is_vnni = True
|
||||
is_weight_packed = True
|
||||
seq_lens = [CHUNK_SIZE - 1, CHUNK_SIZE + 5]
|
||||
total_tokens = sum(seq_lens)
|
||||
x, weight, bias = _conv_inputs(total_tokens)
|
||||
@@ -1034,7 +1051,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
|
||||
wide_state[:, :, state_len:].fill_(7)
|
||||
wide_tail = wide_state[:, :, state_len:].clone()
|
||||
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_vnni)
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
|
||||
out_narrow = ops.causal_conv1d_fwd_cpu(
|
||||
x=x.transpose(0, 1),
|
||||
weight=conv_weight,
|
||||
@@ -1044,7 +1061,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
|
||||
cache_indices=cache_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
silu_activation=True,
|
||||
is_vnni=is_vnni,
|
||||
is_weight_packed=is_weight_packed,
|
||||
)
|
||||
out_wide = ops.causal_conv1d_fwd_cpu(
|
||||
x=x.transpose(0, 1),
|
||||
@@ -1055,7 +1072,7 @@ def test_causal_conv1d_fwd_cpu_accepts_wide_state() -> None:
|
||||
cache_indices=cache_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
silu_activation=True,
|
||||
is_vnni=is_vnni,
|
||||
is_weight_packed=is_weight_packed,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out_wide, out_narrow, atol=1e-2, rtol=1e-2)
|
||||
@@ -1100,15 +1117,23 @@ def test_batch_memcpy_cpu_fallback() -> None:
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# C++ conv (conv.cpp) uses VDPBF16PS, not AMX tiles, so it runs on any
|
||||
# AVX-512BF16 CPU; weight is VNNI-packed on this same predicate at load time.
|
||||
# C++ conv (conv.cpp) uses AArch64BF16 or VDPBF16PS, not AMX tiles, so it can
|
||||
# run on any AVX-512BF16 CPU; weight is packed on this same predicate at load time.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_HAS_AVX512_BF16 = torch.cpu._is_avx512_bf16_supported()
|
||||
|
||||
_STATE_LEN = CONV_KERNEL - 1
|
||||
|
||||
CONV_EQUIV_SEQ_LENS = [[1], [7], [64], [65], [1, 2, 3], [63, 64, 65], [128, 129]]
|
||||
CONV_EQUIV_SEQ_LENS = [
|
||||
[1],
|
||||
[7],
|
||||
[64],
|
||||
[65],
|
||||
[1, 2, 3],
|
||||
[63, 64, 65],
|
||||
[128, 129],
|
||||
]
|
||||
|
||||
|
||||
def _conv_fp32_oracle(x, weight, bias, seq_lens, activation="silu"):
|
||||
@@ -1153,10 +1178,10 @@ def _run_prefill_torch(x, weight, bias, seq_lens):
|
||||
return out.transpose(0, 1).contiguous(), conv_states
|
||||
|
||||
|
||||
def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False):
|
||||
def _run_prefill_cpp(x, weight, bias, seq_lens, is_weight_packed=False):
|
||||
num_seqs = len(seq_lens)
|
||||
packed_w = ops.causal_conv1d_weight_pack(weight) if is_vnni else weight
|
||||
if is_vnni:
|
||||
packed_w = ops.causal_conv1d_weight_pack(weight) if is_weight_packed else weight
|
||||
if is_weight_packed:
|
||||
# C++-branch layout: kv-cache "SD" [slots, state_len, dim] transposed to
|
||||
# [slots, dim, state_len] (a non-contiguous view).
|
||||
conv_state = torch.zeros(
|
||||
@@ -1176,7 +1201,7 @@ def _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False):
|
||||
cache_indices=torch.arange(num_seqs, dtype=torch.int32),
|
||||
has_initial_state=torch.zeros(num_seqs, dtype=torch.bool),
|
||||
silu_activation=True,
|
||||
is_vnni=is_vnni,
|
||||
is_weight_packed=is_weight_packed,
|
||||
)
|
||||
return out.transpose(0, 1).contiguous(), conv_state
|
||||
|
||||
@@ -1245,7 +1270,7 @@ def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state(
|
||||
cache_indices=state_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
silu_activation=True,
|
||||
is_vnni=False,
|
||||
is_weight_packed=False,
|
||||
)
|
||||
monkeypatch.setattr(gdn_attention, "is_conv_state_dim_first", lambda: True)
|
||||
out_ds = gdn_attention._causal_conv1d_fwd_cpu(
|
||||
@@ -1257,7 +1282,7 @@ def test_causal_conv1d_fwd_cpu_ds_adapter_varlen_mixed_initial_state(
|
||||
cache_indices=state_indices,
|
||||
has_initial_state=has_initial_state,
|
||||
silu_activation=True,
|
||||
is_vnni=False,
|
||||
is_weight_packed=False,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out_ds, out_sd)
|
||||
@@ -1302,7 +1327,7 @@ def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback(
|
||||
bias=None,
|
||||
silu_activation=False,
|
||||
conv_state_indices=state_indices,
|
||||
is_vnni=False,
|
||||
is_weight_packed=False,
|
||||
num_accepted_tokens=num_accepted_tokens,
|
||||
)
|
||||
|
||||
@@ -1321,7 +1346,8 @@ def test_causal_conv1d_update_cpu_ds_adapter_wide_multi_token_copyback(
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
|
||||
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
|
||||
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
|
||||
@torch.inference_mode()
|
||||
@@ -1335,7 +1361,8 @@ def test_conv_cpp_matches_torch(seq_lens):
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
|
||||
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
|
||||
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
|
||||
@torch.inference_mode()
|
||||
@@ -1354,89 +1381,114 @@ def test_conv_cpp_no_worse_than_torch_vs_fp32(seq_lens):
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
|
||||
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
|
||||
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize("seq_lens", CONV_EQUIV_SEQ_LENS)
|
||||
@torch.inference_mode()
|
||||
def test_conv_cpp_vnni_packed_matches_torch(seq_lens):
|
||||
"""The exact runtime prefill sequence (VNNI-packed weight + SD-layout
|
||||
conv_state view + is_vnni=True) must match the torch fallback. Validates
|
||||
the packing + layout handoff on any AVX-512BF16 CPU."""
|
||||
def test_conv_cpp_weight_packed_matches_torch(seq_lens):
|
||||
"""The exact runtime prefill sequence (packed weight + SD-layout
|
||||
conv_state view + is_weight_packed=True) must match the torch fallback. Validates
|
||||
the native packing + layout handoff."""
|
||||
x, weight, bias = _conv_inputs(sum(seq_lens))
|
||||
out_torch, _ = _run_prefill_torch(x, weight, bias, seq_lens)
|
||||
out_vnni, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
|
||||
torch.testing.assert_close(out_vnni, out_torch, atol=1e-2, rtol=1e-2)
|
||||
out_weight_packed, _ = _run_prefill_cpp(
|
||||
x, weight, bias, seq_lens, is_weight_packed=True
|
||||
)
|
||||
torch.testing.assert_close(out_weight_packed, out_torch, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
|
||||
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
|
||||
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
|
||||
)
|
||||
@pytest.mark.parametrize("batch", DECODE_BATCH_SIZES)
|
||||
@pytest.mark.parametrize(
|
||||
("dim", "has_bias", "silu_activation", "is_weight_packed"),
|
||||
[
|
||||
(32, False, False, False),
|
||||
(32, True, True, True),
|
||||
(96, False, True, True),
|
||||
(96, True, False, False),
|
||||
(8192, False, True, False),
|
||||
(8192, True, True, True),
|
||||
],
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_conv_update_cpp_matches_torch(batch):
|
||||
"""Decode conv: causal_conv1d_update_cpu matches causal_conv1d_update_torch,
|
||||
including the in-place conv_state update (the next-step handoff)."""
|
||||
def test_conv_update_cpp_matches_torch(
|
||||
batch, dim, has_bias, silu_activation, is_weight_packed
|
||||
):
|
||||
"""Decode preserves indexed, padded state across successive signed inputs."""
|
||||
from vllm.model_executor.layers.mamba.ops.cpu.causal_conv1d import (
|
||||
causal_conv1d_update_torch,
|
||||
)
|
||||
|
||||
x = tensor_cache(batch * CONV_DIM, torch.bfloat16).view(batch, CONV_DIM)
|
||||
weight = tensor_cache(CONV_DIM * CONV_KERNEL, torch.bfloat16).view(
|
||||
CONV_DIM, CONV_KERNEL
|
||||
)
|
||||
bias = tensor_cache(CONV_DIM, torch.bfloat16)
|
||||
conv_state = tensor_cache(batch * CONV_DIM * _STATE_LEN, torch.bfloat16).view(
|
||||
batch, CONV_DIM, _STATE_LEN
|
||||
)
|
||||
torch.manual_seed(0)
|
||||
weight = torch.randn(dim, CONV_KERNEL, dtype=torch.bfloat16)
|
||||
bias = torch.randn(dim, dtype=torch.bfloat16) if has_bias else None
|
||||
conv_weight = _maybe_pack_conv_weight(weight, is_weight_packed)
|
||||
indices = torch.arange(batch, 0, -1, dtype=torch.int32)
|
||||
storage_ref = torch.randn(batch + 2, _STATE_LEN + 2, dim, dtype=torch.bfloat16)
|
||||
storage_cpp = storage_ref.clone()
|
||||
cs_cpp = storage_cpp[:, :_STATE_LEN].transpose(1, 2)
|
||||
|
||||
cs_torch = conv_state.clone()
|
||||
out_torch = causal_conv1d_update_torch(
|
||||
x=x.unsqueeze(-1),
|
||||
conv_state=cs_torch,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
activation="silu",
|
||||
).squeeze(-1)
|
||||
|
||||
cs_cpp = conv_state.clone()
|
||||
out_cpp = ops.causal_conv1d_update_cpu(
|
||||
x=x.contiguous(),
|
||||
conv_states=cs_cpp,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
silu_activation=True,
|
||||
conv_state_indices=torch.arange(batch, dtype=torch.int32),
|
||||
is_vnni=False,
|
||||
)
|
||||
torch.testing.assert_close(out_cpp, out_torch, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(cs_cpp, cs_torch, atol=1e-2, rtol=1e-2)
|
||||
for _ in range(3):
|
||||
x = torch.randn(batch, dim, dtype=torch.bfloat16)
|
||||
cs_torch = storage_ref[indices, :_STATE_LEN].transpose(1, 2).contiguous()
|
||||
out_torch = causal_conv1d_update_torch(
|
||||
x=x.unsqueeze(-1),
|
||||
conv_state=cs_torch,
|
||||
weight=weight,
|
||||
bias=bias,
|
||||
activation="silu" if silu_activation else None,
|
||||
).squeeze(-1)
|
||||
storage_ref[indices, :_STATE_LEN] = cs_torch.transpose(1, 2)
|
||||
out_cpp = ops.causal_conv1d_update_cpu(
|
||||
x=x,
|
||||
conv_states=cs_cpp,
|
||||
weight=conv_weight,
|
||||
bias=bias,
|
||||
silu_activation=silu_activation,
|
||||
conv_state_indices=indices,
|
||||
is_weight_packed=is_weight_packed,
|
||||
)
|
||||
torch.testing.assert_close(out_cpp, out_torch, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(storage_cpp, storage_ref, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
|
||||
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
|
||||
reason="C++ causal_conv1d requires AVX-512BF16 or Arm BF16 support",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_conv_weight_pack_roundtrip_unpacked_matches():
|
||||
"""`causal_conv1d_weight_pack` repacks the (dim, width) weight, so the C++ conv op
|
||||
with packed weight + is_vnni=True must equal unpacked weight + is_vnni=False.
|
||||
Guards the spec-decode contract: the runtime VNNI-packs `layer.conv1d.weight`
|
||||
in place and stashes the original as `_cpu_unpacked_conv_weight` for the torch
|
||||
spec-decode path, so both must produce identical math.
|
||||
with packed weight + is_weight_packed=True must equal unpacked weight
|
||||
+ is_weight_packed=False. Guards the spec-decode contract: the runtime packs
|
||||
`layer.conv1d.weight` in place and stashes the original as
|
||||
`_cpu_unpacked_conv_weight` for the torch spec-decode path, so both must produce
|
||||
identical math.
|
||||
"""
|
||||
seq_lens = [7, 64, 65]
|
||||
x, weight, bias = _conv_inputs(sum(seq_lens))
|
||||
out_unpacked, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=False)
|
||||
out_packed, _ = _run_prefill_cpp(x, weight, bias, seq_lens, is_vnni=True)
|
||||
torch.testing.assert_close(out_packed, out_unpacked, atol=1e-2, rtol=1e-2)
|
||||
out_unpacked, state_unpacked = _run_prefill_cpp(
|
||||
x, weight, bias, seq_lens, is_weight_packed=False
|
||||
)
|
||||
out_packed, state_packed = _run_prefill_cpp(
|
||||
x, weight, bias, seq_lens, is_weight_packed=True
|
||||
)
|
||||
torch.testing.assert_close(out_packed, out_unpacked, atol=0, rtol=0)
|
||||
torch.testing.assert_close(state_packed, state_unpacked, atol=0, rtol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not _HAS_AVX512_BF16, reason="C++ causal_conv1d requires AVX-512BF16"
|
||||
not (_HAS_AVX512_BF16 or gdn_attention.is_arm_bf16()),
|
||||
reason="C++ causal_conv1d requires AVX-512BF16 or AArch64 BF16",
|
||||
)
|
||||
@torch.inference_mode()
|
||||
def test_spec_decode_unpacked_conv_weight_stash():
|
||||
"""Spec-decode correctness: AVX-512BF16 CPUs VNNI-pack ``conv1d.weight`` in place
|
||||
at load time and stash the original ``(dim, width)`` tensor as
|
||||
"""Spec-decode correctness: AVX-512BF16 and AArch64 BF16 CPUs pack ``conv1d.weight``
|
||||
in place at load time and stash the original ``(dim, width)`` tensor as
|
||||
``_cpu_unpacked_conv_weight``; the spec-decode path must use that stash since
|
||||
reading the packed weight directly produces garbage. Verifies the recovered
|
||||
weight equals the original and that torch F.conv1d agrees with the C++ conv
|
||||
@@ -1460,7 +1512,7 @@ def test_spec_decode_unpacked_conv_weight_stash():
|
||||
# 1. The stash must exist and equal the original (dim, width) weight.
|
||||
assert hasattr(conv, "_cpu_unpacked_conv_weight"), (
|
||||
"dispatch_cpu_unquantized_gemm did not stash _cpu_unpacked_conv_weight "
|
||||
"on an AVX-512BF16 CPU"
|
||||
"on an AArch64-BF16 or AVX-512BF16 CPU"
|
||||
)
|
||||
stash = conv._cpu_unpacked_conv_weight
|
||||
torch.testing.assert_close(stash, orig_2d, atol=0, rtol=0)
|
||||
@@ -1469,10 +1521,10 @@ def test_spec_decode_unpacked_conv_weight_stash():
|
||||
recovered = getattr(conv, "_cpu_unpacked_conv_weight", None)
|
||||
assert recovered is not None
|
||||
# conv.weight is now packed; using it directly (the bug) would differ.
|
||||
packed_weight = conv.weight # [dim, 1, width], VNNI-packed contents
|
||||
packed_weight = conv.weight # [dim, 1, width], packed contents
|
||||
|
||||
# 3. Spec-path torch conv with the recovered unpacked weight must match the
|
||||
# nonspec C++ conv with the packed weight (is_vnni=True).
|
||||
# nonspec C++ conv with the packed weight (is_weight_packed=True).
|
||||
seq_lens = [7, 65]
|
||||
total = sum(seq_lens)
|
||||
x = tensor_cache(total * CONV_DIM, torch.bfloat16).view(total, CONV_DIM)
|
||||
@@ -1494,7 +1546,7 @@ def test_spec_decode_unpacked_conv_weight_stash():
|
||||
cache_indices=torch.arange(num_seqs, dtype=torch.int32),
|
||||
has_initial_state=torch.zeros(num_seqs, dtype=torch.bool),
|
||||
silu_activation=True,
|
||||
is_vnni=True,
|
||||
is_weight_packed=True,
|
||||
)
|
||||
.transpose(0, 1)
|
||||
.contiguous()
|
||||
|
||||
@@ -88,11 +88,6 @@ def test_short_conv_forward_native_prefill(vllm_config):
|
||||
state_indices_tensor_d=torch.empty((0, 1), dtype=torch.int32),
|
||||
num_accepted_tokens=None,
|
||||
query_start_loc_d=None,
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
num_computed_tokens_p=None,
|
||||
seq_lens=torch.tensor([5]),
|
||||
)
|
||||
|
||||
@@ -145,11 +140,6 @@ def test_short_conv_forward_native_decode(vllm_config):
|
||||
state_indices_tensor_d=state_indices_tensor_d,
|
||||
num_accepted_tokens=None,
|
||||
query_start_loc_d=torch.tensor([0, 1, 2], dtype=torch.int32),
|
||||
block_idx_last_scheduled_token=None,
|
||||
block_idx_first_scheduled_token_p=None,
|
||||
block_idx_last_computed_token=None,
|
||||
block_idx_last_scheduled_token_prev_step=None,
|
||||
num_computed_tokens_p=None,
|
||||
seq_lens=torch.tensor([1, 1]),
|
||||
)
|
||||
|
||||
|
||||
@@ -176,7 +176,7 @@ def test_forward_core_split_matches_unified(
|
||||
builder = GDNAttentionMetadataBuilder(
|
||||
# GDNAttentionMetadataBuilder declares kv_cache_spec as AttentionSpec but
|
||||
# asserts isinstance(kv_cache_spec, MambaSpec). MambaSpec is correct here.
|
||||
kv_cache_spec=MambaSpec( # type: ignore[arg-type]
|
||||
kv_cache_spec=MambaSpec(
|
||||
block_size=BLOCK_SIZE, shapes=((16, 64),), dtypes=(torch.float16,)
|
||||
),
|
||||
layer_names=[PREFIX],
|
||||
|
||||
@@ -1,24 +1,30 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.mamba import MambaBackendEnum, MambaConfig, MambaSSUAlgorithm
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.model_executor.layers.mamba.mamba_utils import MambaStateShapeCalculator
|
||||
from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
|
||||
FlashInferSSUBackend,
|
||||
TritonSSUBackend,
|
||||
commit_replayssm_ring_trackers,
|
||||
get_mamba_ssu_backend,
|
||||
initialize_mamba_ssu_backend,
|
||||
reset_replayssm_ring_trackers,
|
||||
selective_state_update,
|
||||
selective_state_update_replayssm_flashinfer,
|
||||
update_replayssm_ring_trackers,
|
||||
)
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.mamba2_attn import Mamba2AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import MambaAttentionBackendEnum
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
@@ -32,6 +38,13 @@ try:
|
||||
except ImportError:
|
||||
HAS_FLASHINFER = False
|
||||
|
||||
try:
|
||||
from flashinfer.mamba.checkpointing_ssu import CheckpointingSSURunner
|
||||
|
||||
HAS_FLASHINFER_CHECKPOINTING_SSU = CheckpointingSSURunner is not None
|
||||
except ImportError:
|
||||
HAS_FLASHINFER_CHECKPOINTING_SSU = False
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def restore_backend_state():
|
||||
@@ -47,6 +60,7 @@ def restore_backend_state():
|
||||
def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
@@ -54,6 +68,7 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
update_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
logical_window=16,
|
||||
ring_buffer_len=17,
|
||||
@@ -69,9 +84,205 @@ def test_flashinfer_replayssm_ring_tracker_lifecycle():
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
)
|
||||
assert (ring_start[1].item(), prev_num_accepted[1].item()) == (0, 0)
|
||||
assert (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
) == (0, 0, 0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("accepted_sequence", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
[4] * 22,
|
||||
[
|
||||
(0, 0, 4),
|
||||
(0, 4, 4),
|
||||
(0, 8, 4),
|
||||
(0, 12, 4),
|
||||
(0, 16, 4),
|
||||
(16, 4, 4),
|
||||
(16, 8, 4),
|
||||
(16, 12, 4),
|
||||
(16, 16, 4),
|
||||
(12, 4, 4),
|
||||
(12, 8, 4),
|
||||
(12, 12, 4),
|
||||
(12, 16, 4),
|
||||
(8, 4, 4),
|
||||
(8, 8, 4),
|
||||
(8, 12, 4),
|
||||
(8, 16, 4),
|
||||
(4, 4, 4),
|
||||
(4, 8, 4),
|
||||
(4, 12, 4),
|
||||
(4, 16, 4),
|
||||
(0, 4, 4),
|
||||
],
|
||||
id="all-accepted",
|
||||
),
|
||||
pytest.param(
|
||||
[4, 4, 0, 3, 4, 2, 4, 1],
|
||||
[
|
||||
(0, 0, 4),
|
||||
(0, 4, 4),
|
||||
(0, 4, 4),
|
||||
(0, 7, 4),
|
||||
(0, 11, 4),
|
||||
(0, 13, 4),
|
||||
(13, 4, 4),
|
||||
(13, 5, 4),
|
||||
],
|
||||
id="mixed",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_replayssm_commit_tracker_acceptance_sequence(accepted_sequence, expected):
|
||||
logical_window = 16
|
||||
num_speculative_tokens = 3
|
||||
query_len = 1 + num_speculative_tokens
|
||||
ring_buffer_len = logical_window + 1 + num_speculative_tokens
|
||||
ring_start = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.zeros(2, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
query_start_loc = torch.tensor([0, query_len], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
for accepted in accepted_sequence:
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor([accepted], dtype=torch.int32, device="cuda"),
|
||||
query_start_loc,
|
||||
logical_window,
|
||||
ring_buffer_len,
|
||||
)
|
||||
snapshot = (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
)
|
||||
observed.append(snapshot)
|
||||
assert snapshot[1] + snapshot[2] <= ring_buffer_len
|
||||
|
||||
assert observed == expected
|
||||
|
||||
|
||||
def test_replayssm_resume_resets_commit_history():
|
||||
ring_start = torch.tensor([0, 13], dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.tensor([0, 13], dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.tensor([0, 4], dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1], dtype=torch.int32, device="cuda")
|
||||
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
)
|
||||
assert (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
) == (0, 0, 0)
|
||||
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor([3], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([0, 4], dtype=torch.int32, device="cuda"),
|
||||
logical_window=16,
|
||||
ring_buffer_len=20,
|
||||
)
|
||||
assert (
|
||||
ring_start[1].item(),
|
||||
prev_num_accepted[1].item(),
|
||||
prev_query_len[1].item(),
|
||||
) == (0, 0, 4)
|
||||
|
||||
|
||||
def test_replayssm_commit_tracker_ragged_query_lengths():
|
||||
ring_start = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.zeros(3, dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor([1, 2], dtype=torch.int32, device="cuda")
|
||||
query_start_loc = torch.tensor([0, 4, 6], dtype=torch.int32, device="cuda")
|
||||
|
||||
observed = []
|
||||
for accepted in ([4, 2], [3, 1]):
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor(accepted, dtype=torch.int32, device="cuda"),
|
||||
query_start_loc,
|
||||
logical_window=16,
|
||||
ring_buffer_len=20,
|
||||
)
|
||||
observed.append(
|
||||
[
|
||||
(
|
||||
ring_start[slot].item(),
|
||||
prev_num_accepted[slot].item(),
|
||||
prev_query_len[slot].item(),
|
||||
)
|
||||
for slot in (1, 2)
|
||||
]
|
||||
)
|
||||
|
||||
assert observed == [[(0, 0, 4), (0, 0, 2)], [(0, 3, 4), (0, 1, 2)]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("operation", ["commit", "reset"])
|
||||
def test_replayssm_tracker_kernels_mask_invalid_slots(operation):
|
||||
num_states = 3
|
||||
ring_start = torch.tensor([11, 2, 33], dtype=torch.int32, device="cuda")
|
||||
prev_num_accepted = torch.tensor([11, 3, 33], dtype=torch.int32, device="cuda")
|
||||
prev_query_len = torch.tensor([11, 4, 33], dtype=torch.int32, device="cuda")
|
||||
state_batch_indices = torch.tensor(
|
||||
[-1, num_states, NULL_BLOCK_ID, 1], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
if operation == "commit":
|
||||
commit_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
torch.tensor([4, 4, 4, 2], dtype=torch.int32, device="cuda"),
|
||||
torch.tensor([0, 4, 8, 12, 16], dtype=torch.int32, device="cuda"),
|
||||
logical_window=16,
|
||||
ring_buffer_len=20,
|
||||
)
|
||||
expected_valid = (2, 5, 4)
|
||||
else:
|
||||
reset_replayssm_ring_trackers(
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
state_batch_indices,
|
||||
)
|
||||
expected_valid = (0, 0, 0)
|
||||
|
||||
assert (
|
||||
ring_start.tolist(),
|
||||
prev_num_accepted.tolist(),
|
||||
prev_query_len.tolist(),
|
||||
) == (
|
||||
[11, expected_valid[0], 33],
|
||||
[11, expected_valid[1], 33],
|
||||
[11, expected_valid[2], 33],
|
||||
)
|
||||
|
||||
|
||||
def _kv_cache_config_with_ssu(
|
||||
@@ -231,14 +442,185 @@ def test_triton_basic_call():
|
||||
assert not torch.isnan(out).any()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("layout", ["packed", "dense"])
|
||||
def test_replayssm_flashinfer_call_forwards_mtp_layout(monkeypatch, layout):
|
||||
import vllm.model_executor.layers.mamba.ops.ssu_dispatch as mod
|
||||
|
||||
kernel = Mock(return_value=torch.empty(0))
|
||||
monkeypatch.setattr(mod, "_flashinfer_replayssm_kernel", kernel)
|
||||
|
||||
batch, max_seqlen, nheads, dim, dstate, ngroups = 2, 4, 2, 4, 8, 1
|
||||
state = torch.empty(2, nheads, dim, dstate)
|
||||
x_shape: tuple[int, ...]
|
||||
B_shape: tuple[int, ...]
|
||||
expected_x_shape: tuple[int, ...]
|
||||
expected_B_shape: tuple[int, ...]
|
||||
if layout == "packed":
|
||||
x_shape = (6, nheads, dim)
|
||||
B_shape = (6, ngroups, dstate)
|
||||
expected_x_shape = (1, 6, nheads, dim)
|
||||
expected_B_shape = (1, 6, ngroups, dstate)
|
||||
cu_seqlens = torch.tensor([0, 4, 6], dtype=torch.int32)
|
||||
kernel_max_seqlen = max_seqlen
|
||||
else:
|
||||
x_shape = (batch, max_seqlen, nheads, dim)
|
||||
B_shape = (batch, max_seqlen, ngroups, dstate)
|
||||
expected_x_shape = x_shape
|
||||
expected_B_shape = B_shape
|
||||
cu_seqlens = None
|
||||
kernel_max_seqlen = None
|
||||
x = torch.empty(x_shape)
|
||||
dt = torch.empty_like(x)
|
||||
A = torch.empty(nheads, dim, dstate)
|
||||
B = torch.empty(B_shape)
|
||||
C = torch.empty_like(B)
|
||||
out = torch.empty_like(x)
|
||||
x_cache = torch.empty(2, nheads, 20, dim)
|
||||
dt_cache = torch.empty(2, nheads, 20)
|
||||
B_cache = torch.empty(2, ngroups, 20, dstate)
|
||||
ring_start = torch.zeros(2, dtype=torch.int32)
|
||||
prev_num_accepted = torch.zeros(2, dtype=torch.int32)
|
||||
prev_query_len = torch.zeros(2, dtype=torch.int32)
|
||||
selective_state_update_replayssm_flashinfer(
|
||||
state,
|
||||
x,
|
||||
dt,
|
||||
A,
|
||||
B,
|
||||
C,
|
||||
out,
|
||||
x_cache,
|
||||
B_cache,
|
||||
dt_cache,
|
||||
ring_start,
|
||||
prev_num_accepted,
|
||||
prev_query_len,
|
||||
logical_window=16,
|
||||
state_batch_indices=torch.tensor([0, 1], dtype=torch.int32),
|
||||
cu_seqlens=cu_seqlens,
|
||||
max_seqlen=kernel_max_seqlen,
|
||||
update_trackers=False,
|
||||
)
|
||||
|
||||
args = kernel.call_args.args
|
||||
assert args[6].shape == expected_x_shape
|
||||
assert args[7].shape == expected_x_shape
|
||||
assert args[9].shape == expected_B_shape
|
||||
assert args[10].shape == expected_B_shape
|
||||
assert args[11].shape == expected_x_shape
|
||||
assert kernel.call_args.kwargs["cu_seqlens"] is cu_seqlens
|
||||
assert kernel.call_args.kwargs["max_seqlen"] == kernel_max_seqlen
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "expected_ring_len"),
|
||||
("query_start_loc", "expected_shape", "expected_max_seqlen"),
|
||||
[
|
||||
(MambaBackendEnum.TRITON, 16),
|
||||
(MambaBackendEnum.FLASHINFER, 17),
|
||||
pytest.param([0, 4, 8], (2, 4, 2, 4), None, id="dense"),
|
||||
pytest.param([0, 4, 6], (6, 2, 4), 4, id="packed"),
|
||||
],
|
||||
)
|
||||
def test_replayssm_physical_ring_shape(backend, expected_ring_len):
|
||||
def test_replayssm_mixer_selects_mtp_layout(
|
||||
monkeypatch, query_start_loc, expected_shape, expected_max_seqlen
|
||||
):
|
||||
import vllm.model_executor.layers.mamba.mamba_mixer2 as mod
|
||||
|
||||
mixer = MambaMixer2.__new__(MambaMixer2)
|
||||
torch.nn.Module.__init__(mixer)
|
||||
mixer.prefix = "mixer"
|
||||
mixer.tped_intermediate_size = 0
|
||||
mixer.tped_conv_size = 1
|
||||
mixer.tped_dt_size = 2
|
||||
mixer.num_heads = 2
|
||||
mixer.head_dim = 4
|
||||
mixer.n_groups = mixer.tp_size = 1
|
||||
mixer.ssm_state_size = 8
|
||||
mixer.num_spec = 3
|
||||
mixer.use_replayssm = True
|
||||
mixer.replayssm_buffer_len = 16
|
||||
mixer._commits_replayssm_trackers = True
|
||||
mixer._updates_replayssm_trackers = False
|
||||
mixer.mamba_config = MambaConfig(backend=MambaBackendEnum.FLASHINFER)
|
||||
mixer.cache_config = SimpleNamespace(mamba_block_size=16, mamba_cache_mode="none")
|
||||
mixer.conv_weights = torch.empty(0)
|
||||
mixer.conv1d = SimpleNamespace(bias=None)
|
||||
mixer.activation = "silu"
|
||||
mixer.A = torch.empty(2)
|
||||
mixer.dt_bias = torch.empty(2)
|
||||
mixer.D = torch.empty(2)
|
||||
mixer._replayssm_ring_start = torch.zeros(3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_num_accepted = torch.zeros(3, dtype=torch.int32)
|
||||
mixer._replayssm_prev_query_len = torch.zeros(3, dtype=torch.int32)
|
||||
mixer.kv_cache = (
|
||||
torch.empty(3, 1),
|
||||
torch.empty(3, 2, 4, 8),
|
||||
torch.empty(3, 2, 20, 4),
|
||||
torch.empty(3, 2, 20),
|
||||
torch.empty(3, 1, 20, 8),
|
||||
)
|
||||
|
||||
num_decode_tokens = query_start_loc[-1]
|
||||
query_start_loc_d = torch.tensor(query_start_loc, dtype=torch.int32)
|
||||
metadata = Mamba2AttentionMetadata(
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
num_decodes=2,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
num_reqs=2,
|
||||
has_initial_states_p=None,
|
||||
query_start_loc_p=None,
|
||||
state_indices_tensor_p=None,
|
||||
state_indices_tensor_d=torch.tensor([[1], [2]], dtype=torch.int32),
|
||||
query_start_loc_d=query_start_loc_d,
|
||||
num_accepted_tokens=torch.tensor([4, 2], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([104, 102], dtype=torch.int32),
|
||||
replayssm_scratch=(torch.empty(0), torch.empty(0), torch.empty(0)),
|
||||
replayssm_state_indices_d=torch.tensor([1, 2], dtype=torch.int32),
|
||||
)
|
||||
|
||||
def split_hidden_states_B_C(values):
|
||||
tokens = values.size(0)
|
||||
return (
|
||||
torch.empty(tokens, 8),
|
||||
torch.empty(tokens, 8),
|
||||
torch.empty(tokens, 8),
|
||||
)
|
||||
|
||||
mixer.split_hidden_states_B_C_fn = split_hidden_states_B_C
|
||||
kernel = Mock()
|
||||
monkeypatch.setattr(
|
||||
mod,
|
||||
"get_forward_context",
|
||||
lambda: SimpleNamespace(attn_metadata={mixer.prefix: metadata}),
|
||||
)
|
||||
monkeypatch.setattr(mod, "commit_replayssm_ring_trackers", Mock())
|
||||
monkeypatch.setattr(
|
||||
mod, "causal_conv1d_update", lambda values, *args, **kwargs: values
|
||||
)
|
||||
monkeypatch.setattr(mod, "selective_state_update_replayssm_flashinfer", kernel)
|
||||
|
||||
mixer.conv_ssm_forward(
|
||||
torch.empty(num_decode_tokens, 3), torch.empty(num_decode_tokens, 8)
|
||||
)
|
||||
|
||||
assert kernel.call_args.args[1].shape == expected_shape
|
||||
if expected_max_seqlen is None:
|
||||
assert kernel.call_args.kwargs["cu_seqlens"] is None
|
||||
else:
|
||||
assert kernel.call_args.kwargs["cu_seqlens"] is query_start_loc_d
|
||||
assert kernel.call_args.kwargs["max_seqlen"] == expected_max_seqlen
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "num_speculative_tokens", "expected_ring_len"),
|
||||
[
|
||||
(MambaBackendEnum.TRITON, 0, 16),
|
||||
(MambaBackendEnum.FLASHINFER, 0, 17),
|
||||
(MambaBackendEnum.FLASHINFER, 3, 20),
|
||||
],
|
||||
)
|
||||
def test_replayssm_physical_ring_shape(
|
||||
backend, num_speculative_tokens, expected_ring_len
|
||||
):
|
||||
base_shapes = ((64, 3), (8, 4, 16))
|
||||
|
||||
shapes = MambaStateShapeCalculator.append_replayssm_ring(
|
||||
@@ -247,6 +629,7 @@ def test_replayssm_physical_ring_shape(backend, expected_ring_len):
|
||||
tp_world_size=2,
|
||||
logical_window=16,
|
||||
backend=backend,
|
||||
num_speculative_tokens=num_speculative_tokens,
|
||||
)
|
||||
|
||||
assert shapes[2:] == (
|
||||
|
||||
@@ -538,7 +538,7 @@ def run_8_bit(
|
||||
assert num_local_experts is not None
|
||||
return run_with_expert_maps(
|
||||
num_experts,
|
||||
num_local_experts, # type: ignore[arg-type]
|
||||
num_local_experts,
|
||||
quant_config,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -29,14 +29,20 @@ from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
FusedMoEParallelConfig,
|
||||
RoutingMethodType,
|
||||
nvfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutedsl_moe import (
|
||||
FlashInferCuteDSLExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.nvfp4 import (
|
||||
NvFp4MoeBackend,
|
||||
make_nvfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
|
||||
prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
|
||||
quantize_nvfp4_per_token_input,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_cutedsl_moe_nvfp4
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
@@ -159,8 +165,12 @@ def _dequantize_nvfp4_linear(
|
||||
return (tensor_f32 * tensor_sf.unsqueeze(-1)).reshape(m, k).to(dtype)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("m,n,k,e,topk", [(16, 128, 512, 4, 2)])
|
||||
@pytest.mark.parametrize(
|
||||
"m,n,k,e,topk",
|
||||
[(16, 128, 512, 4, 2), (16, 192, 512, 4, 2), (16, 192, 2688, 4, 2)],
|
||||
)
|
||||
@pytest.mark.parametrize("activation,alpha,beta,limit", _ACT_CASES)
|
||||
@pytest.mark.parametrize("per_token_activation", [False, True])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@torch.inference_mode()
|
||||
def test_flashinfer_cutedsl_fp4_moe(
|
||||
@@ -173,6 +183,7 @@ def test_flashinfer_cutedsl_fp4_moe(
|
||||
alpha: float | None,
|
||||
beta: float | None,
|
||||
limit: float | None,
|
||||
per_token_activation: bool,
|
||||
dtype: torch.dtype,
|
||||
workspace_init,
|
||||
):
|
||||
@@ -181,6 +192,10 @@ def test_flashinfer_cutedsl_fp4_moe(
|
||||
VllmConfig(parallel_config=ParallelConfig(pipeline_parallel_size=1))
|
||||
):
|
||||
hidden_states = torch.randn((m, k), device="cuda", dtype=dtype) / 10
|
||||
# Keep GEMM1 activation variance stable as K changes, so the fixed
|
||||
# tolerance against the BF16 GEMM2 reference measures comparable FP4
|
||||
# quantization error at each hidden width.
|
||||
hidden_states *= (512 / k) ** 0.5
|
||||
|
||||
w1_rows = 2 * n if activation.is_gated else n
|
||||
w1 = torch.randn((e, w1_rows, k), device="cuda", dtype=dtype) / 15
|
||||
@@ -193,7 +208,6 @@ def test_flashinfer_cutedsl_fp4_moe(
|
||||
hidden_states, score, topk, renormalize=False
|
||||
)
|
||||
|
||||
activation = MoEActivation.RELU2_NO_MUL
|
||||
moe_config = FusedMoEConfig(
|
||||
num_experts=e,
|
||||
experts_per_token=topk,
|
||||
@@ -231,19 +245,30 @@ def test_flashinfer_cutedsl_fp4_moe(
|
||||
w2_scale_2=(1.0 / w2_global_scale),
|
||||
a2_scale=a2_scale,
|
||||
)
|
||||
quant_config = nvfp4_moe_quant_config(
|
||||
g1_alphas=w1_alpha,
|
||||
g2_alphas=w2_alpha,
|
||||
a1_gscale=(1.0 / a1_scale),
|
||||
a2_gscale=(1.0 / a2_scale),
|
||||
w1_scale=w1_scale_cutedsl,
|
||||
# I=192 must remain unpadded for gated experts, but become 256 for
|
||||
# non-gated experts. The reference below uses the checkpoint shapes.
|
||||
expected_n = n if activation.is_gated else (n + 127) // 128 * 128
|
||||
assert moe_config.intermediate_size_per_partition == expected_n
|
||||
assert w1_cutedsl.shape[1] == expected_n * (2 if activation.is_gated else 1)
|
||||
assert w2_cutedsl.shape[2] * 2 == expected_n
|
||||
expected_k = (k + 255) // 256 * 256
|
||||
assert moe_config.hidden_dim == expected_k
|
||||
assert moe_config.hidden_dim_unpadded == k
|
||||
assert w1_cutedsl.shape[2] * 2 == expected_k
|
||||
assert w2_cutedsl.shape[1] == expected_k
|
||||
quant_config = make_nvfp4_moe_quant_config(
|
||||
backend=NvFp4MoeBackend.FLASHINFER_CUTEDSL,
|
||||
w13_scale_2=w1_alpha,
|
||||
w2_scale_2=w2_alpha,
|
||||
a13_scale=a1_scale,
|
||||
a2_scale=a2_scale,
|
||||
w13_scale=w1_scale_cutedsl,
|
||||
w2_scale=w2_scale_cutedsl,
|
||||
is_scale_swizzled=False,
|
||||
# Unset params must be omitted rather than forwarded as None into
|
||||
# the kernel's float-typed SwiGLU arguments.
|
||||
gemm1_alpha=alpha,
|
||||
gemm1_beta=beta,
|
||||
gemm1_clamp_limit=limit,
|
||||
swiglu_alpha=alpha,
|
||||
swiglu_beta=beta,
|
||||
swiglu_limit=limit,
|
||||
)
|
||||
|
||||
cutedsl_experts = mk.FusedMoEKernel(
|
||||
@@ -256,11 +281,13 @@ def test_flashinfer_cutedsl_fp4_moe(
|
||||
FlashInferCuteDSLExperts(
|
||||
moe_config=moe_config,
|
||||
quant_config=quant_config,
|
||||
per_token_activation=per_token_activation,
|
||||
),
|
||||
)
|
||||
|
||||
cutedsl_output = cutedsl_experts.apply(
|
||||
hidden_states=hidden_states,
|
||||
# The model runner pads inputs and trims outputs around this kernel.
|
||||
hidden_states=torch.nn.functional.pad(hidden_states, (0, expected_k - k)),
|
||||
w1=w1_cutedsl,
|
||||
w2=w2_cutedsl,
|
||||
topk_weights=topk_weights,
|
||||
@@ -270,19 +297,25 @@ def test_flashinfer_cutedsl_fp4_moe(
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
cutedsl_output = cutedsl_output[:, :k]
|
||||
|
||||
a_global_scale = torch.ones(1, device="cuda", dtype=torch.float32)
|
||||
a_q, a_scale = ops.scaled_fp4_quant(
|
||||
hidden_states,
|
||||
a_global_scale,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
if per_token_activation:
|
||||
a_q, a_scale, a_row_scale = quantize_nvfp4_per_token_input(hidden_states)
|
||||
else:
|
||||
a_q, a_scale = ops.scaled_fp4_quant(
|
||||
hidden_states,
|
||||
a_global_scale,
|
||||
is_sf_swizzled_layout=False,
|
||||
)
|
||||
a_in_dtype = _dequantize_nvfp4_linear(
|
||||
a_q,
|
||||
a_scale,
|
||||
a_global_scale,
|
||||
dtype=dtype,
|
||||
)
|
||||
if per_token_activation:
|
||||
a_in_dtype = (a_in_dtype.float() * a_row_scale.view(-1, 1)).to(dtype)
|
||||
|
||||
w1_d = torch.empty((e, w1_rows, k), device="cuda", dtype=dtype)
|
||||
w2_d = torch.empty((e, k, n), device="cuda", dtype=dtype)
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
import types
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def _describe(tensor):
|
||||
if tensor is None:
|
||||
return "None (the op was never called)"
|
||||
return f"a different tensor of shape {tuple(tensor.shape)}"
|
||||
|
||||
|
||||
def _import_mori_prepare_finalize(monkeypatch):
|
||||
# This is a contract test for vLLM's adapter; it does not need a real MoRI
|
||||
# installation or GPU kernels.
|
||||
fake_mori = types.ModuleType("mori")
|
||||
fake_mori.ops = SimpleNamespace( # type: ignore[attr-defined]
|
||||
EpDispatchCombineOp=object
|
||||
)
|
||||
monkeypatch.setitem(sys.modules, "mori", fake_mori)
|
||||
module_name = "vllm.model_executor.layers.fused_moe.prepare_finalize.mori"
|
||||
sys.modules.pop(module_name, None)
|
||||
return importlib.import_module(module_name)
|
||||
|
||||
|
||||
class _FakeMoriOp:
|
||||
def __init__(self):
|
||||
self.dispatch_indices = None
|
||||
self.combine_indices = None
|
||||
|
||||
def dispatch(self, a1, weights, scale, indices):
|
||||
self.dispatch_indices = indices
|
||||
# Model the global dispatched layout seen with EP=16: the experts need
|
||||
# these IDs, but passing them to combine violates MoRI's contract.
|
||||
dispatch_indices = indices.repeat(16, 1)
|
||||
dispatch_a1 = a1.repeat(16, 1)
|
||||
dispatch_weights = weights.repeat(16, 1)
|
||||
recv_token_num = torch.tensor([dispatch_a1.shape[0]])
|
||||
return (
|
||||
dispatch_a1,
|
||||
dispatch_weights,
|
||||
scale,
|
||||
dispatch_indices,
|
||||
recv_token_num,
|
||||
)
|
||||
|
||||
def combine(self, fused_expert_output, weights, indices):
|
||||
self.combine_indices = indices
|
||||
return (fused_expert_output,)
|
||||
|
||||
|
||||
def test_mori_combine_uses_original_per_rank_topk_ids(monkeypatch):
|
||||
mori_module = _import_mori_prepare_finalize(monkeypatch)
|
||||
mori_op = _FakeMoriOp()
|
||||
adapter = mori_module.MoriPrepareAndFinalize(
|
||||
mori_op=mori_op,
|
||||
max_tokens_per_rank=16,
|
||||
num_dispatchers=1,
|
||||
)
|
||||
|
||||
hidden_states = torch.randn(2, 4)
|
||||
router_topk_ids = torch.tensor([[0, 1], [2, 3]], dtype=torch.int32)
|
||||
topk_weights = torch.ones(2, 2)
|
||||
quant_config = SimpleNamespace(
|
||||
is_block_quantized=False,
|
||||
is_per_act_token=False,
|
||||
)
|
||||
|
||||
(
|
||||
dispatch_a1,
|
||||
_,
|
||||
_,
|
||||
dispatch_ids,
|
||||
dispatch_weights,
|
||||
) = adapter.prepare(
|
||||
hidden_states,
|
||||
topk_weights,
|
||||
router_topk_ids,
|
||||
num_experts=4,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
# Read the shapes before the identity asserts below, which narrow these
|
||||
# names to the fake op's Optional attribute type.
|
||||
router_shape = tuple(router_topk_ids.shape)
|
||||
dispatch_shape = tuple(dispatch_ids.shape)
|
||||
expected_rows = router_shape[0] * 16
|
||||
|
||||
assert dispatch_shape[0] == expected_rows, (
|
||||
"prepare() must return the ids dispatch() produced: EP=16 expands "
|
||||
f"{router_shape[0]} tokens to {expected_rows} rows, but prepare() "
|
||||
f"returned {dispatch_shape[0]} rows"
|
||||
)
|
||||
assert mori_op.dispatch_indices is router_topk_ids, (
|
||||
"dispatch() must be handed this rank's own pre-dispatch routing of "
|
||||
f"shape {router_shape}, but it received "
|
||||
f"{_describe(mori_op.dispatch_indices)}"
|
||||
)
|
||||
|
||||
output = torch.empty_like(hidden_states)
|
||||
adapter.finalize(
|
||||
output,
|
||||
dispatch_a1,
|
||||
dispatch_weights,
|
||||
dispatch_ids,
|
||||
apply_router_weight_on_input=False,
|
||||
weight_and_reduce_impl=SimpleNamespace(),
|
||||
)
|
||||
|
||||
assert mori_op.combine_indices is router_topk_ids, (
|
||||
"combine() reduces over this rank's own routing, so it must be handed "
|
||||
f"the pre-dispatch tensor of shape {router_shape}, but it received "
|
||||
f"{_describe(mori_op.combine_indices)}. finalize() most likely "
|
||||
"forwarded its own topk_ids, which the modular kernel has already "
|
||||
"rebound to the ids dispatch() returned (ROCm/mori#475)"
|
||||
)
|
||||
assert mori_op.combine_indices is not dispatch_ids, (
|
||||
f"combine() was handed the dispatched ids of shape {dispatch_shape} "
|
||||
"instead of this rank's pre-dispatch routing, which silently corrupts "
|
||||
"the reduction (ROCm/mori#475)"
|
||||
)
|
||||
@@ -1724,7 +1724,7 @@ def test_rocm_mxfp4_moe_oracle(
|
||||
w13_conv, w2_conv, w13_scale_conv, w2_scale_conv, w13_bias_conv, w2_bias_conv = (
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
mxfp4_backend=backend,
|
||||
layer=layer, # type: ignore[arg-type]
|
||||
layer=layer,
|
||||
w13_weight=w13_quant,
|
||||
w2_weight=w2_quant,
|
||||
w13_weight_scale=w13_scale,
|
||||
|
||||
@@ -30,7 +30,7 @@ import torch.nn.functional as F
|
||||
|
||||
from tests.kernels.utils import _assert_deterministic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_gfx942, on_gfx950
|
||||
from vllm.platforms.rocm import on_cdna, on_gfx942, on_gfx950
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
@@ -638,6 +638,115 @@ def test_aiter_moe_padding_env_var(
|
||||
assert envs.VLLM_ROCM_MOE_PADDING is moe_padding
|
||||
|
||||
|
||||
# a4w4 (FP4 activation) opt-in gating test ---------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("env_value", "model_type", "expected"),
|
||||
[
|
||||
# Unset/"0" is opt-in-only: a4w4 never turns on by itself, even on
|
||||
# a validated model type.
|
||||
(None, "deepseek_v41", False),
|
||||
("0", "deepseek_v41", False),
|
||||
("1", "deepseek_v41", True),
|
||||
("1", "deepseek_v41_text", True),
|
||||
],
|
||||
)
|
||||
def test_aiter_moe_a4w4_dsv4_is_opt_in(
|
||||
env_value: str | None,
|
||||
model_type: str,
|
||||
expected: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""vllm-project/vllm#58819's a4w4 MoE activation path must stay a plain
|
||||
explicit opt-in (VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4=1), with no
|
||||
default-on behavior, but is still only permitted on the model types
|
||||
this flag has been validated for."""
|
||||
import types
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
fake_vllm_config = types.SimpleNamespace(
|
||||
model_config=types.SimpleNamespace(
|
||||
hf_config=types.SimpleNamespace(model_type=model_type)
|
||||
)
|
||||
)
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
if env_value is None:
|
||||
mp.delenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", raising=False)
|
||||
else:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", env_value)
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
with mock.patch(
|
||||
"vllm.config.get_current_vllm_config_or_none",
|
||||
return_value=fake_vllm_config,
|
||||
):
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=4,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=512,
|
||||
in_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
assert moe_config.use_mxfp4_w4a4_dsv4 is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_type", ["deepseek_v4", "gpt_oss", None])
|
||||
def test_aiter_moe_a4w4_dsv4_rejects_unvalidated_model_type(
|
||||
model_type: str | None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Forcing a4w4 on a model type this flag hasn't been GSM8K/AgentX
|
||||
validated for must raise: it also flips the MXFP4 weight-shuffle layout
|
||||
(ATOM SEPARATED vs INTERLEAVE), which silently produces garbled output
|
||||
on the wrong model rather than just being a slower path — the same bug
|
||||
class vllm-project/vllm#58819's own second commit fixed for DeepSeek
|
||||
V4.1 itself."""
|
||||
import types
|
||||
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
fake_vllm_config = types.SimpleNamespace(
|
||||
model_config=types.SimpleNamespace(
|
||||
hf_config=types.SimpleNamespace(model_type=model_type)
|
||||
)
|
||||
)
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE", "1")
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4", "1")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
with (
|
||||
mock.patch(
|
||||
"vllm.config.get_current_vllm_config_or_none",
|
||||
return_value=fake_vllm_config,
|
||||
),
|
||||
pytest.raises(ValueError, match="VLLM_ROCM_USE_AITER_MOE_A4W4_DSV4"),
|
||||
):
|
||||
make_dummy_moe_config(
|
||||
num_experts=4,
|
||||
experts_per_token=2,
|
||||
hidden_dim=256,
|
||||
intermediate_size=512,
|
||||
in_dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
|
||||
# Dispatch-policy forwarding test ------------------------------------------
|
||||
|
||||
|
||||
@@ -1208,3 +1317,252 @@ def test_aiter_fused_moe_mi3xx_fp8_accuracy():
|
||||
pass_rate=1.0,
|
||||
max_violation_factor=1.5,
|
||||
)
|
||||
|
||||
|
||||
# Weight alignment tests --------------------------------------------------
|
||||
#
|
||||
# AITER's CK 2stages MoE kernel rejects an intermediate size not divisible by
|
||||
# its tile width (64 at or below inter_dim 192, 128 above). Some model + TP
|
||||
# splits land on an unaligned size (e.g. 1792 / TP=8 = 224), so the AITER path
|
||||
# rounds the intermediate dim up in ``maybe_roundup_sizes`` and allocates the
|
||||
# weights at the padded size.
|
||||
|
||||
ALIGNMENT_HIDDEN = 64
|
||||
ALIGNMENT_NUM_EXPERTS = 2
|
||||
|
||||
|
||||
def _make_alignment_moe_config(intermediate: int):
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
|
||||
return make_dummy_moe_config(
|
||||
num_experts=ALIGNMENT_NUM_EXPERTS,
|
||||
hidden_dim=ALIGNMENT_HIDDEN,
|
||||
intermediate_size=intermediate,
|
||||
)
|
||||
|
||||
|
||||
def _make_aiter_method(moe_config):
|
||||
"""Build the unquantized method with the backend pinned to AITER.
|
||||
|
||||
``select_unquantized_moe_backend`` needs a real ROCm + AITER runtime, so
|
||||
stub it out and set the backend directly.
|
||||
"""
|
||||
from unittest.mock import patch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method import (
|
||||
UnquantizedFusedMoEMethod,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"vllm.model_executor.layers.fused_moe.unquantized_fused_moe_method"
|
||||
".select_unquantized_moe_backend",
|
||||
return_value=(UnquantizedMoeBackend.AITER, None),
|
||||
):
|
||||
return UnquantizedFusedMoEMethod(moe_config)
|
||||
|
||||
|
||||
def _roundup(method, moe_config, intermediate):
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||
|
||||
return method.maybe_roundup_sizes(
|
||||
hidden_size=ALIGNMENT_HIDDEN,
|
||||
intermediate_size_per_partition=intermediate,
|
||||
act_dtype=moe_config.in_dtype,
|
||||
moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("intermediate", "expected_padded"),
|
||||
[
|
||||
(64, 64), # already aligned, untouched
|
||||
(96, 128),
|
||||
(160, 192),
|
||||
(192, 192), # must stay 192: it is valid and has tuned configs
|
||||
(224, 256), # K2-Horizon-375B at TP=8
|
||||
(256, 256),
|
||||
(448, 512), # K2-Horizon-375B at TP=4
|
||||
(4096, 4096),
|
||||
],
|
||||
)
|
||||
def test_aiter_moe_roundup_pads_intermediate(
|
||||
intermediate,
|
||||
expected_padded,
|
||||
default_vllm_config,
|
||||
):
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
aiter_moe_intermediate_alignment,
|
||||
)
|
||||
|
||||
moe_config = _make_alignment_moe_config(intermediate)
|
||||
method = _make_aiter_method(moe_config)
|
||||
|
||||
hidden, padded = _roundup(method, moe_config, intermediate)
|
||||
|
||||
assert padded == expected_padded
|
||||
assert hidden == ALIGNMENT_HIDDEN
|
||||
# Padding must not move a size across the <= 192 threshold, or the
|
||||
# alignment we picked is not the one the kernel dispatches the padded
|
||||
# shape to.
|
||||
assert aiter_moe_intermediate_alignment(padded) == (
|
||||
aiter_moe_intermediate_alignment(intermediate)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend_name", ["TRITON", "FLASHINFER_CUTLASS"])
|
||||
def test_aiter_moe_roundup_is_not_applied_to_other_backends(
|
||||
backend_name,
|
||||
default_vllm_config,
|
||||
):
|
||||
"""Backends with no alignment requirement keep the unaligned size."""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
UnquantizedMoeBackend,
|
||||
)
|
||||
|
||||
intermediate = 224
|
||||
moe_config = _make_alignment_moe_config(intermediate)
|
||||
method = _make_aiter_method(moe_config)
|
||||
method.unquantized_backend = UnquantizedMoeBackend[backend_name]
|
||||
|
||||
_, padded = _roundup(method, moe_config, intermediate)
|
||||
|
||||
assert padded == intermediate
|
||||
|
||||
|
||||
@pytest.mark.parametrize("intermediate", [224, 448])
|
||||
def test_aiter_moe_reload_zeroes_intermediate_padding(
|
||||
intermediate,
|
||||
monkeypatch,
|
||||
default_vllm_config,
|
||||
):
|
||||
"""A reload must re-zero pad lanes left dirty in the weight storage.
|
||||
|
||||
The reload writes back only the logical slices, so the pad lanes hold
|
||||
garbage (NaN here) that the AITER conversion must zero. Two passes also
|
||||
cover reloading into storage that already holds a previous conversion.
|
||||
"""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
aiter_moe_intermediate_alignment,
|
||||
)
|
||||
|
||||
alignment = aiter_moe_intermediate_alignment(intermediate)
|
||||
padded = -(-intermediate // alignment) * alignment
|
||||
assert padded != intermediate
|
||||
|
||||
moe_config = _make_alignment_moe_config(intermediate)
|
||||
assert moe_config.is_act_and_mul
|
||||
method = _make_aiter_method(moe_config)
|
||||
# RoutedExperts records the rounded-up size after maybe_roundup_sizes.
|
||||
moe_config.intermediate_size_per_partition = padded
|
||||
layer = torch.nn.Module()
|
||||
layer.moe_config = moe_config
|
||||
|
||||
method.create_weights(
|
||||
layer=layer,
|
||||
num_experts=ALIGNMENT_NUM_EXPERTS,
|
||||
hidden_size=ALIGNMENT_HIDDEN,
|
||||
intermediate_size_per_partition=padded,
|
||||
params_dtype=torch.float32,
|
||||
)
|
||||
|
||||
# Identity shuffle keeps pad lanes sliceable; both stubs need real ROCm.
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized"
|
||||
".rocm_aiter_ops.shuffle_weights",
|
||||
lambda w13, w2: (w13, w2),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
method,
|
||||
"_init_moe_kernel",
|
||||
lambda _layer: setattr(method, "moe_kernel", MagicMock()),
|
||||
)
|
||||
|
||||
up = padded # up projection starts one padded block into the fused rows
|
||||
|
||||
for _ in range(2):
|
||||
gate = torch.randn(ALIGNMENT_NUM_EXPERTS, intermediate, ALIGNMENT_HIDDEN)
|
||||
up_proj = torch.randn(ALIGNMENT_NUM_EXPERTS, intermediate, ALIGNMENT_HIDDEN)
|
||||
down = torch.randn(ALIGNMENT_NUM_EXPERTS, ALIGNMENT_HIDDEN, intermediate)
|
||||
|
||||
# NaN the pads, then write only the logical slices like the loader.
|
||||
layer.w13_weight.data.fill_(float("nan"))
|
||||
layer.w2_weight.data.fill_(float("nan"))
|
||||
layer.w13_weight.data[:, :intermediate].copy_(gate)
|
||||
layer.w13_weight.data[:, up : up + intermediate].copy_(up_proj)
|
||||
layer.w2_weight.data[:, :, :intermediate].copy_(down)
|
||||
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
assert torch.equal(w13[:, :intermediate], gate)
|
||||
assert torch.equal(w13[:, up : up + intermediate], up_proj)
|
||||
assert torch.equal(w2[:, :, :intermediate], down)
|
||||
assert torch.all(w13[:, intermediate:up] == 0)
|
||||
assert torch.all(w13[:, up + intermediate :] == 0)
|
||||
assert torch.all(w2[:, :, intermediate:] == 0)
|
||||
|
||||
|
||||
def _aiter_accepts_intermediate(intermediate: int, num_tokens: int) -> bool:
|
||||
"""Run the real AITER CK MoE kernel and report whether it dispatched."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
ActivationMethod,
|
||||
QuantMethod,
|
||||
)
|
||||
|
||||
case = _make_moe_case(
|
||||
num_tokens=num_tokens,
|
||||
hidden_dim=1024,
|
||||
intermediate_dim=intermediate,
|
||||
num_experts=8,
|
||||
topk=2,
|
||||
seed=0,
|
||||
)
|
||||
try:
|
||||
_run_fused_moe(
|
||||
case["hidden_states"],
|
||||
case["w1"],
|
||||
case["w2"],
|
||||
case["topk_weights"],
|
||||
case["topk_ids"],
|
||||
activation_method=int(ActivationMethod.SILU),
|
||||
quant_method=int(QuantMethod.NO),
|
||||
)
|
||||
except RuntimeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not on_cdna(),
|
||||
reason="CDNA ROCm only",
|
||||
)
|
||||
@pytest.mark.parametrize("intermediate", [192, 224, 256])
|
||||
# With topk=2 and 8 experts, these token counts select block_m 32 / 64 / 128
|
||||
# respectively (AITER's get_block_size_M, on both 256- and 304-CU parts).
|
||||
@pytest.mark.parametrize("num_tokens", [1024, 2560, 5120])
|
||||
def test_aiter_moe_alignment_rule_holds_across_block_m(intermediate, num_tokens):
|
||||
"""The alignment rule must hold at every reachable ``block_m``.
|
||||
|
||||
Stage 1 checks ``NPerBlock`` and stage 2 ``KPerBlock``; both vary with
|
||||
``block_m``, so a size validated at one tile size says nothing about the
|
||||
others.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.unquantized import (
|
||||
aiter_moe_intermediate_alignment,
|
||||
)
|
||||
|
||||
_assert_aiter_supported()
|
||||
|
||||
alignment = aiter_moe_intermediate_alignment(intermediate)
|
||||
padded = -(-intermediate // alignment) * alignment
|
||||
|
||||
assert _aiter_accepts_intermediate(intermediate, num_tokens) == (
|
||||
intermediate % alignment == 0
|
||||
)
|
||||
assert _aiter_accepts_intermediate(padded, num_tokens)
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the AITER biased_grouped_topk expert-group count.
|
||||
|
||||
The AITER kernel is only instantiated for ``NUM_GRP`` in {1, 2, 4, 8}, so the
|
||||
group count derived from ``num_experts`` has to be rounded to a supported
|
||||
value that still divides ``num_experts``. Grouping is a no-op on this path
|
||||
(``topk_group == num_expert_group``), so any such value routes identically;
|
||||
the constraint is purely about which kernel exists.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
AITER_MAX_EXPERTS_PER_GROUP as MAX_EXPERTS_PER_GROUP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
AITER_SUPPORTED_NUM_GRP as SUPPORTED_NUM_GRP,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.fused_topk_bias_router import (
|
||||
_aiter_can_use_biased_grouped_topk,
|
||||
_aiter_get_num_expert_group,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.router.grouped_topk_router import (
|
||||
grouped_topk,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.platforms.rocm import on_cdna
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm-specific tests"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_experts", "expected"),
|
||||
[
|
||||
(8, 1), # fits in a single group
|
||||
(32, 1), # exactly at the per-group limit
|
||||
(64, 2),
|
||||
(128, 4),
|
||||
(256, 8),
|
||||
(72, 8), # naive ceil gives 3 -> unsupported, rounds up to 8
|
||||
(96, 8), # naive ceil gives 3 -> unsupported, rounds up to 8
|
||||
(160, 8), # naive ceil gives 5 -> unsupported, rounds up to 8
|
||||
(192, 8), # naive ceil gives 6 -> unsupported, rounds up to 8
|
||||
# No supported NUM_GRP divides these within the group-size limit, so
|
||||
# the naive value is kept and the call site declines the kernel.
|
||||
(33, 3),
|
||||
(129, 43),
|
||||
(257, 257),
|
||||
(320, 10),
|
||||
(384, 12),
|
||||
],
|
||||
)
|
||||
def test_group_count_and_guard(num_experts, expected):
|
||||
g = _aiter_get_num_expert_group(num_experts)
|
||||
assert g == expected
|
||||
# The router asserts both unconditionally; rounding must never break them.
|
||||
assert num_experts % g == 0
|
||||
assert num_experts // g <= MAX_EXPERTS_PER_GROUP
|
||||
|
||||
supported = g in SUPPORTED_NUM_GRP
|
||||
# Ample topk, so only the membership check can decline.
|
||||
assert _aiter_can_use_biased_grouped_topk(num_experts, topk=64) == supported
|
||||
if supported:
|
||||
assert _aiter_can_use_biased_grouped_topk(num_experts, topk=g)
|
||||
assert not _aiter_can_use_biased_grouped_topk(num_experts, topk=g - 1)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_cdna(), reason="CDNA ROCm only")
|
||||
@pytest.mark.parametrize(
|
||||
"num_experts",
|
||||
[
|
||||
64, # g == 2, not rounded -- covers the no-op claim itself
|
||||
128, # g == 4, not rounded
|
||||
256, # g == 8, not rounded
|
||||
96, # naive 3 -> rounded to 8
|
||||
192, # naive 6 -> rounded to 8
|
||||
],
|
||||
)
|
||||
def test_rounded_group_count_routes_like_the_reference(num_experts):
|
||||
"""The rounded group count must route identically to ``grouped_topk``.
|
||||
|
||||
Gating is continuous random, so exact score ties -- the one place the two
|
||||
may legitimately differ -- have measure zero.
|
||||
"""
|
||||
torch.manual_seed(num_experts)
|
||||
device = "cuda"
|
||||
topk = 8
|
||||
num_tokens = 83 # not a multiple of any warp/tile size
|
||||
g = _aiter_get_num_expert_group(num_experts)
|
||||
assert _aiter_can_use_biased_grouped_topk(num_experts, topk)
|
||||
|
||||
gating = torch.randn(num_tokens, num_experts, dtype=torch.float32, device=device)
|
||||
bias = torch.randn(num_experts, dtype=torch.float32, device=device)
|
||||
|
||||
ref_weights, ref_ids = grouped_topk(
|
||||
hidden_states=torch.empty(num_tokens, 1, device=device),
|
||||
gating_output=gating,
|
||||
topk=topk,
|
||||
renormalize=True,
|
||||
num_expert_group=g,
|
||||
topk_group=g,
|
||||
scoring_func="sigmoid",
|
||||
e_score_correction_bias=bias,
|
||||
)
|
||||
|
||||
weights = torch.empty(num_tokens, topk, dtype=torch.float32, device=device)
|
||||
ids = torch.empty(num_tokens, topk, dtype=torch.int32, device=device)
|
||||
rocm_aiter_ops.biased_grouped_topk(
|
||||
gating,
|
||||
bias,
|
||||
weights,
|
||||
ids,
|
||||
num_expert_group=g,
|
||||
topk_group=g,
|
||||
need_renorm=True,
|
||||
)
|
||||
|
||||
# Neither side promises an order within a token's top-k, so sort by expert
|
||||
# id and carry the weights through the same permutation -- sorting the two
|
||||
# independently would not catch a weight attached to the wrong expert.
|
||||
order = ids.argsort(dim=-1)
|
||||
ref_order = ref_ids.argsort(dim=-1)
|
||||
torch.testing.assert_close(
|
||||
ids.gather(1, order), ref_ids.to(torch.int32).gather(1, ref_order)
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
weights.gather(1, order),
|
||||
ref_weights.gather(1, ref_order),
|
||||
atol=1e-4,
|
||||
rtol=1e-4,
|
||||
)
|
||||
@@ -224,8 +224,9 @@ def test_zen_int8_dispatch_contract(
|
||||
call = mock_zentorch_fused_moe["fused_moe"]
|
||||
assert call["w13_scales"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE)
|
||||
assert call["w2_scales"].shape == (EXPERT_NUM, HIDDEN_SIZE)
|
||||
assert call["w13_scales"].dtype == torch.bfloat16
|
||||
assert call["w2_scales"].dtype == torch.bfloat16
|
||||
# f32 scales keep the layer on zentorch's fast routed MoE executor.
|
||||
assert call["w13_scales"].dtype == torch.float32
|
||||
assert call["w2_scales"].dtype == torch.float32
|
||||
assert call["act"] == act.value.lower()
|
||||
assert call["skip_weighted"] is False
|
||||
assert call["topk_weights"].dtype == torch.float32
|
||||
|
||||
@@ -7,7 +7,6 @@ import torch
|
||||
import vllm._custom_ops as ops
|
||||
from tests.kernels.quant_utils import per_block_cast_to_int8
|
||||
from tests.kernels.quantization.nvfp4_utils import FLOAT4_E2M1_MAX, FLOAT8_E4M3_MAX
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
maybe_make_prepare_finalize,
|
||||
@@ -552,97 +551,6 @@ class TestMLP(torch.nn.Module):
|
||||
return x
|
||||
|
||||
|
||||
class RealMLP(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
intermediate_size: int,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
hidden_act: str = "silu",
|
||||
quant_config=None,
|
||||
reduce_results: bool = True,
|
||||
prefix: str = "",
|
||||
w1_s: torch.Tensor | None = None,
|
||||
w2_s: torch.Tensor | None = None,
|
||||
) -> None:
|
||||
from vllm.model_executor.layers.linear import (
|
||||
MergedColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
|
||||
super().__init__()
|
||||
self.gate_up_proj = MergedColumnParallelLinear(
|
||||
hidden_size,
|
||||
[intermediate_size] * 2,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.gate_up_proj",
|
||||
)
|
||||
self.gate_up_proj.register_parameter(
|
||||
"weight", torch.nn.Parameter(w1, requires_grad=False)
|
||||
)
|
||||
self.gate_up_proj.register_parameter(
|
||||
"weight_scale", torch.nn.Parameter(w1_s, requires_grad=False)
|
||||
)
|
||||
self.gate_up_proj.register_parameter(
|
||||
"input_scale", None
|
||||
) # torch.nn.Parameter(None, requires_grad=False))
|
||||
self.down_proj = RowParallelLinear(
|
||||
intermediate_size,
|
||||
hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
reduce_results=reduce_results,
|
||||
prefix=f"{prefix}.down_proj",
|
||||
)
|
||||
self.down_proj.register_parameter(
|
||||
"weight", torch.nn.Parameter(w2, requires_grad=False)
|
||||
)
|
||||
self.down_proj.register_parameter(
|
||||
"weight_scale", torch.nn.Parameter(w2_s, requires_grad=False)
|
||||
)
|
||||
self.down_proj.register_parameter(
|
||||
"input_scale", None
|
||||
) # torch.nn.Parameter(None, requires_grad=False))
|
||||
if hidden_act != "silu":
|
||||
raise ValueError(
|
||||
f"Unsupported activation: {hidden_act}. Only silu is supported for now."
|
||||
)
|
||||
self.act_fn = SiluAndMul()
|
||||
|
||||
def forward(self, x):
|
||||
gate_up, _ = self.gate_up_proj(x)
|
||||
x = self.act_fn(gate_up)
|
||||
x, _ = self.down_proj(x)
|
||||
return x
|
||||
|
||||
|
||||
def make_shared_experts_with_weights(
|
||||
N: int,
|
||||
K: int,
|
||||
in_dtype: torch.dtype,
|
||||
w1: torch.Tensor,
|
||||
w2: torch.Tensor,
|
||||
w1_s: torch.Tensor | None = None,
|
||||
w2_s: torch.Tensor | None = None,
|
||||
quant_dtype: torch.dtype | str | None = None,
|
||||
) -> torch.nn.Module:
|
||||
old_dtype = torch.get_default_dtype()
|
||||
try:
|
||||
torch.set_default_dtype(in_dtype)
|
||||
if quant_dtype == torch.float8_e4m3fn:
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
|
||||
quant_config = Fp8Config()
|
||||
else:
|
||||
quant_config = None
|
||||
|
||||
return RealMLP(K, N, w1, w2, "silu", quant_config, w1_s=w1_s, w2_s=w2_s)
|
||||
finally:
|
||||
torch.set_default_dtype(old_dtype)
|
||||
|
||||
|
||||
def modular_triton_fused_moe(
|
||||
moe_config: FusedMoEConfig,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
|
||||
@@ -37,7 +37,8 @@ from vllm.utils.flashinfer import (
|
||||
)
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
|
||||
if current_platform.get_device_capability() < (9, 0):
|
||||
capability = current_platform.get_device_capability()
|
||||
if capability is None or capability < (9, 0):
|
||||
pytest.skip("FP8 Triton requires CUDA 9.0 or higher", allow_module_level=True)
|
||||
|
||||
vllm_config = VllmConfig()
|
||||
|
||||
@@ -62,7 +62,8 @@ PER_TOKEN_GROUP_SHAPE = (1, -1)
|
||||
PER_OUT_CH_GROUP_SHAPE = (-1, 1)
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
capability = capability[0] * 10 + capability[1]
|
||||
# Fall back to 0 so import doesn't fail on unknown capability
|
||||
capability_int = (capability[0] * 10 + capability[1]) if capability is not None else 0
|
||||
|
||||
|
||||
def rand_int8(shape: tuple, device: str = "cuda"):
|
||||
@@ -639,7 +640,7 @@ def test_cutlass_cuda_graph(per_act_token: bool, per_out_ch: bool):
|
||||
|
||||
|
||||
def test_cutlass_support_opcheck():
|
||||
opcheck(torch.ops._C.cutlass_scaled_mm_supports_fp8, (capability,))
|
||||
opcheck(torch.ops._C.cutlass_scaled_mm_supports_fp8, (capability_int,))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_experts", [8, 64])
|
||||
|
||||
@@ -29,7 +29,8 @@ if not current_platform.is_cuda():
|
||||
# `is_quant_method_supported` conflates kernels with quantization methods
|
||||
# an assumption which is breaking down as quantizations methods can have
|
||||
# have kernels and some kernels support multiple quantization methods.
|
||||
IS_SUPPORTED_BY_GPU = current_platform.get_device_capability()[0] >= 9
|
||||
capability = current_platform.get_device_capability()
|
||||
IS_SUPPORTED_BY_GPU = capability is not None and capability[0] >= 9
|
||||
|
||||
MNK_SHAPES = [
|
||||
(1, 128, 128),
|
||||
|
||||
@@ -17,8 +17,9 @@ from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import ScalarType, scalar_types
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
capability = current_platform.get_device_capability()
|
||||
IS_SUPPORTED_BY_GPU = (
|
||||
current_platform.is_cuda() and current_platform.get_device_capability()[0] >= 9
|
||||
current_platform.is_cuda() and capability is not None and capability[0] >= 9
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ from tests.kernels.quant_utils import (
|
||||
)
|
||||
from tests.kernels.utils import opcheck
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
scaled_quantize,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -208,7 +209,10 @@ def test_static_fp8_quant_group_2d(
|
||||
|
||||
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
|
||||
ref_out, scale = scaled_quantize(
|
||||
x, group_shape, current_platform.fp8_dtype(), compute_dtype=torch.float32
|
||||
x,
|
||||
GroupShape(*group_shape),
|
||||
current_platform.fp8_dtype(),
|
||||
compute_dtype=torch.float32,
|
||||
)
|
||||
ops_out, ops_scale = ops.scaled_fp8_quant(x, scale=scale, group_shape=group_shape)
|
||||
|
||||
@@ -236,7 +240,7 @@ def test_static_fp8_quant_1d_scale(
|
||||
|
||||
x = torch.rand(num_tokens, hidden_size, dtype=dtype, device="cuda")
|
||||
ref_out, scale_2d = scaled_quantize(
|
||||
x, group_shape, FP8_DTYPE, compute_dtype=torch.float32
|
||||
x, GroupShape(*group_shape), FP8_DTYPE, compute_dtype=torch.float32
|
||||
)
|
||||
|
||||
# Flatten scale to 1D for testing 1D scale path
|
||||
|
||||
@@ -38,7 +38,8 @@ CUDA_DEVICES = [
|
||||
# `is_quant_method_supported` conflates kernels with quantization methods
|
||||
# an assumption which is breaking down as quantizations methods can have
|
||||
# have kernels and some kernels support multiple quantization methods.
|
||||
IS_SUPPORTED_BY_GPU = current_platform.get_device_capability()[0] >= 9
|
||||
capability = current_platform.get_device_capability()
|
||||
IS_SUPPORTED_BY_GPU = capability is not None and capability[0] >= 9
|
||||
|
||||
MNK_SHAPES = [
|
||||
(1, 128, 128),
|
||||
@@ -305,7 +306,7 @@ def test_machete_all_schedules(shape, types: TypeConfig):
|
||||
if types.group_scale_type is None:
|
||||
group_sizes = [None]
|
||||
else:
|
||||
group_sizes = query_machete_supported_group_sizes(types.act_type)
|
||||
group_sizes = list(query_machete_supported_group_sizes(types.act_type))
|
||||
|
||||
for group_size in group_sizes:
|
||||
if not group_size_valid(shape, group_size):
|
||||
@@ -334,7 +335,7 @@ def test_machete_heuristic(shape, types: TypeConfig):
|
||||
if types.group_scale_type is None:
|
||||
group_sizes = [None]
|
||||
else:
|
||||
group_sizes = query_machete_supported_group_sizes(types.act_type)
|
||||
group_sizes = list(query_machete_supported_group_sizes(types.act_type))
|
||||
|
||||
for group_size in group_sizes:
|
||||
if not group_size_valid(shape, group_size):
|
||||
|
||||
@@ -342,7 +342,7 @@ def marlin_generate_valid_test_cases():
|
||||
[quant_test_config["b_type"]],
|
||||
quant_test_config.get("c_type", f16_types),
|
||||
quant_test_config["group_blocks"],
|
||||
)
|
||||
) # type: ignore[call-overload]
|
||||
|
||||
for sub_case in inner_combinations:
|
||||
if (
|
||||
|
||||
@@ -11,6 +11,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
GPTQ_MARLIN_TILE,
|
||||
apply_gptq_marlin_linear,
|
||||
@@ -524,7 +525,7 @@ def test_check_marlin_supports_layer_allow_tile_padding():
|
||||
)
|
||||
|
||||
# Tile-misaligned but group-aligned: rejected strictly, allowed w/ padding
|
||||
layer = _FakeLinear(4640, 512, input_size=2048)
|
||||
layer: LinearBase = _FakeLinear(4640, 512, input_size=2048)
|
||||
assert not check_marlin_supports_layer(layer, 128)
|
||||
assert check_marlin_supports_layer(layer, 128, allow_tile_padding=True)
|
||||
assert check_marlin_supports_layer(layer, -1, allow_tile_padding=True)
|
||||
|
||||
@@ -35,7 +35,7 @@ pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def test_auto_linear_backend_is_unchanged_without_overrides():
|
||||
kernels = [object]
|
||||
kernels: list[type] = [object]
|
||||
quantizations = (
|
||||
"fp8_block_w8a8",
|
||||
"fp8_w8a8",
|
||||
|
||||
@@ -0,0 +1,661 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the zentorch DA8W4 (W4A8) int4 linear and MoE paths on Zen CPUs.
|
||||
|
||||
The zentorch ops are mocked with reference implementations when zentorch is not
|
||||
installed, so the layout and dispatch contracts are covered in CI.
|
||||
"""
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from compressed_tensors.compressors.pack_quantized.helpers import pack_to_int32
|
||||
|
||||
from tests.kernels.quant_utils import ref_dynamic_per_token_quant
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.zentorch import (
|
||||
ZentorchWNA16LinearKernel,
|
||||
_import_unpack_from_int32,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle import int_wna16
|
||||
from vllm.model_executor.layers.fused_moe.oracle.int_wna16 import WNA16MoEBackend
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
GROUP_SIZE = 128
|
||||
IN_FEATURES = 512
|
||||
OUT_FEATURES = 128
|
||||
|
||||
|
||||
def _unpack_s4(packed: torch.Tensor, in_features: int) -> torch.Tensor:
|
||||
"""int8 [N, K/2] (or int32 [N, K/8]) packed s4 -> float32 [N, K]."""
|
||||
words = packed.view(torch.int32)
|
||||
out = torch.zeros(words.shape[0], in_features, dtype=torch.float32)
|
||||
for i in range(8):
|
||||
nibble = (words >> (4 * i)) & 0xF
|
||||
out[:, i::8] = torch.where(nibble > 7, nibble - 16, nibble).float()
|
||||
return out
|
||||
|
||||
|
||||
def _repack_s4(unpacked: torch.Tensor) -> torch.Tensor:
|
||||
"""Reference zentorch_woq_repack_weight: int8 [N, K] -> int32 [N, K/8]."""
|
||||
n, k = unpacked.shape
|
||||
values = unpacked.to(torch.int32).reshape(n, k // 8, 8)
|
||||
out = torch.zeros(n, k // 8, dtype=torch.int32)
|
||||
for i in range(8):
|
||||
out |= (values[:, :, i] & 0xF) << (4 * i)
|
||||
return out
|
||||
|
||||
|
||||
def _quantize_per_group(
|
||||
weight: torch.Tensor, group_size: int
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Symmetric int4 per-group quantization -> (int8 in [-8, 7], scale [N, G])."""
|
||||
out_features, in_features = weight.shape
|
||||
grouped = weight.float().reshape(
|
||||
out_features, in_features // group_size, group_size
|
||||
)
|
||||
scale = grouped.abs().amax(dim=-1) / 8.0
|
||||
quantized = (grouped / scale.unsqueeze(-1)).round().clamp(-8, 7).to(torch.int8)
|
||||
return quantized.reshape(out_features, in_features), scale.to(torch.bfloat16)
|
||||
|
||||
|
||||
def _dequantize_per_group(
|
||||
quantized: torch.Tensor, scale: torch.Tensor, group_size: int
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize with [N, G] scales into a float32 [N, K] weight."""
|
||||
return quantized.float() * scale.float().repeat_interleave(group_size, dim=1)
|
||||
|
||||
|
||||
def _dynamic_quant_matmul(
|
||||
x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None
|
||||
) -> torch.Tensor:
|
||||
"""DA8W4 reference: per-token int8 activation quant against a bf16 weight."""
|
||||
x_q, scale = ref_dynamic_per_token_quant(x.float(), torch.int8)
|
||||
out = (x_q.float() @ weight.t()) * scale
|
||||
if bias is not None:
|
||||
out = out + bias.float()
|
||||
return out
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_zentorch_ops():
|
||||
"""Register reference zentorch ops when zentorch is not installed."""
|
||||
if hasattr(torch.ops.zentorch, "zentorch_dynamic_qlinear"):
|
||||
yield None
|
||||
return
|
||||
|
||||
calls: dict[str, tuple] = {}
|
||||
|
||||
def _dynamic_qlinear(input, weight, weight_scales, bias=None, zentorch_op_name=""):
|
||||
calls["dynamic_qlinear"] = (input, weight, weight_scales, bias)
|
||||
in_features = weight.view(torch.int32).shape[1] * 8
|
||||
group_size = in_features // weight_scales.shape[0]
|
||||
# weight_scales is [G, N]; expand to a [N, K] dequantized weight.
|
||||
scales = weight_scales.float().t().repeat_interleave(group_size, dim=1)
|
||||
weight_deq = _unpack_s4(weight, in_features) * scales
|
||||
out = _dynamic_quant_matmul(input, weight_deq, bias)
|
||||
return out.to(input.dtype)
|
||||
|
||||
def _fused_moe(
|
||||
output,
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
w13_bias,
|
||||
w2_bias,
|
||||
topk_weights,
|
||||
topk_id,
|
||||
skip_weighted,
|
||||
act,
|
||||
w13_scales=None,
|
||||
w2_scales=None,
|
||||
zentorch_op_name="",
|
||||
):
|
||||
calls["fused_moe"] = (
|
||||
input,
|
||||
w13,
|
||||
w2,
|
||||
topk_weights,
|
||||
topk_id,
|
||||
skip_weighted,
|
||||
act,
|
||||
w13_scales,
|
||||
w2_scales,
|
||||
)
|
||||
output.zero_()
|
||||
|
||||
lib_def = torch.library.Library("zentorch", "DEF")
|
||||
lib_def.define("zentorch_woq_repack_weight(Tensor unpacked_weight) -> Tensor")
|
||||
lib_def.define(
|
||||
"zentorch_dynamic_qlinear(Tensor input, Tensor weight, "
|
||||
"Tensor weight_scales, Tensor? bias=None, *, "
|
||||
"str zentorch_op_name='zentorch::zentorch_dynamic_qlinear') -> Tensor"
|
||||
)
|
||||
lib_def.define(
|
||||
"zentorch_fused_moe(Tensor(a!) output, Tensor input, Tensor w13, "
|
||||
"Tensor w2, Tensor? w13_bias, Tensor? w2_bias, Tensor topk_weights, "
|
||||
"Tensor topk_id, bool skip_weighted, str act, "
|
||||
"Tensor? w13_scales=None, Tensor? w2_scales=None, *, "
|
||||
"str zentorch_op_name='zentorch::zentorch_fused_moe') -> ()"
|
||||
)
|
||||
|
||||
lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
|
||||
lib_impl.impl("zentorch_woq_repack_weight", _repack_s4)
|
||||
lib_impl.impl("zentorch_dynamic_qlinear", _dynamic_qlinear)
|
||||
lib_impl.impl("zentorch_fused_moe", _fused_moe)
|
||||
|
||||
yield calls
|
||||
|
||||
lib_impl._destroy()
|
||||
lib_def._destroy()
|
||||
|
||||
|
||||
def _make_config(
|
||||
act_type: torch.dtype = torch.bfloat16,
|
||||
weight_type=scalar_types.uint4b8,
|
||||
zero_points: bool = False,
|
||||
group_size: int = GROUP_SIZE,
|
||||
) -> MPLinearLayerConfig:
|
||||
return MPLinearLayerConfig(
|
||||
full_weight_shape=(IN_FEATURES, OUT_FEATURES),
|
||||
partition_weight_shape=(IN_FEATURES, OUT_FEATURES),
|
||||
weight_type=weight_type,
|
||||
act_type=act_type,
|
||||
group_size=group_size,
|
||||
zero_points=zero_points,
|
||||
)
|
||||
|
||||
|
||||
def _make_layer(group_size: int = GROUP_SIZE) -> tuple[torch.nn.Module, torch.Tensor]:
|
||||
"""Build a layer holding a CT-packed int4 weight, plus its dequantized form."""
|
||||
torch.manual_seed(0)
|
||||
weight = torch.randn(OUT_FEATURES, IN_FEATURES, dtype=torch.bfloat16)
|
||||
quantized, scale = _quantize_per_group(weight, group_size)
|
||||
|
||||
# compressed-tensors stores [N, K//8] int32, packed along the input dim.
|
||||
packed = pack_to_int32(quantized, 4, packed_dim=1)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight_packed = torch.nn.Parameter(packed, requires_grad=False)
|
||||
layer.weight_packed.input_dim = 1
|
||||
layer.weight_packed.packed_dim = 1
|
||||
layer.weight_scale = torch.nn.Parameter(scale, requires_grad=False)
|
||||
layer.weight_zero_point = None
|
||||
return layer, _dequantize_per_group(quantized, scale, group_size)
|
||||
|
||||
|
||||
def _make_kernel(config: MPLinearLayerConfig) -> ZentorchWNA16LinearKernel:
|
||||
return ZentorchWNA16LinearKernel(
|
||||
config, "weight_packed", "weight_scale", "weight_zero_point"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dense DA8W4
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_da8w4_eligible_for_symmetric_bf16_layer(mock_zentorch_ops, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
assert kernel._maybe_process_da8w4_weights(layer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"config_kwargs,reason",
|
||||
[
|
||||
({"act_type": torch.float32}, "f32 activations are rejected"),
|
||||
(
|
||||
{"weight_type": scalar_types.uint4, "zero_points": True},
|
||||
"asymmetric int4 is unsupported",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_da8w4_not_eligible(mock_zentorch_ops, monkeypatch, config_kwargs, reason):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config(**config_kwargs))
|
||||
assert not kernel._maybe_process_da8w4_weights(layer), reason
|
||||
|
||||
|
||||
def test_da8w4_not_eligible_when_env_disabled(mock_zentorch_ops, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"envs.VLLM_CPU_INT4_W4A8",
|
||||
False,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
assert not kernel._maybe_process_da8w4_weights(layer)
|
||||
|
||||
|
||||
def test_da8w4_process_weights_layout(mock_zentorch_ops, monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
assert layer._zentorch_da8w4
|
||||
assert layer._zentorch_kind == "compressed_tensors_w4a8_da8w4"
|
||||
# Packed s4 holds two nibbles per byte, and scales transpose to {G, N}.
|
||||
assert layer._zentorch_da8w4_packed.dtype == torch.int8
|
||||
assert layer._zentorch_da8w4_packed.shape == (OUT_FEATURES, IN_FEATURES // 2)
|
||||
assert layer._zentorch_da8w4_scale.dtype == torch.bfloat16
|
||||
assert layer._zentorch_da8w4_scale.shape == (
|
||||
IN_FEATURES // GROUP_SIZE,
|
||||
OUT_FEATURES,
|
||||
)
|
||||
assert layer._zentorch_da8w4_packed.is_contiguous()
|
||||
assert layer._zentorch_da8w4_scale.is_contiguous()
|
||||
# The checkpoint parameters are released once repacked.
|
||||
assert layer.weight_packed.numel() == 0
|
||||
assert layer.weight_scale.numel() == 0
|
||||
|
||||
|
||||
def test_da8w4_apply_weights_matches_dequantized_reference(
|
||||
mock_zentorch_ops, monkeypatch
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
layer, weight_deq = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
x = torch.randn(4, IN_FEATURES, dtype=torch.bfloat16)
|
||||
bias = torch.randn(OUT_FEATURES, dtype=torch.bfloat16)
|
||||
out = kernel.apply_weights(layer, x, bias)
|
||||
|
||||
expected = _dynamic_quant_matmul(x, weight_deq, bias)
|
||||
assert out.dtype == torch.bfloat16
|
||||
torch.testing.assert_close(out.float(), expected, rtol=2e-2, atol=2e-2)
|
||||
|
||||
|
||||
def test_da8w4_falls_back_to_w4a16_when_op_missing(monkeypatch):
|
||||
"""Without zentorch_dynamic_qlinear the layer must not take the DA8W4 path."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
|
||||
lambda ops: "zentorch_dynamic_qlinear" not in ops,
|
||||
)
|
||||
layer, _ = _make_layer()
|
||||
kernel = _make_kernel(_make_config())
|
||||
assert not kernel._maybe_process_da8w4_weights(layer)
|
||||
|
||||
|
||||
def test_can_implement_requires_zen_cpu(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: False,
|
||||
)
|
||||
ok, reason = ZentorchWNA16LinearKernel.can_implement(_make_config())
|
||||
assert not ok
|
||||
assert reason is not None
|
||||
|
||||
|
||||
def test_can_implement_accepts_shapes_cpuwna16_rejects(monkeypatch):
|
||||
"""CPUWNA16's N/K % 32 rule is a oneDNN packing constraint zentorch lacks."""
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.cpu import (
|
||||
CPUWNA16LinearKernel,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.cpu."
|
||||
"current_platform.is_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
|
||||
lambda ops: True,
|
||||
)
|
||||
|
||||
unaligned = dataclasses.replace(
|
||||
_make_config(), partition_weight_shape=(IN_FEATURES, OUT_FEATURES - 16)
|
||||
)
|
||||
assert unaligned.partition_weight_shape[1] % 32 != 0
|
||||
|
||||
ok, _ = CPUWNA16LinearKernel.can_implement(unaligned)
|
||||
assert not ok, "the parent is expected to turn this shape away"
|
||||
|
||||
ok, reason = ZentorchWNA16LinearKernel.can_implement(unaligned)
|
||||
assert ok, reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"group_size,expected_ok",
|
||||
[
|
||||
(-1, True), # per-channel, one group spanning K
|
||||
(GROUP_SIZE, True),
|
||||
(IN_FEATURES, True),
|
||||
(6, False), # even, so the parent's multiple-of-2 rule would allow it
|
||||
(0, False),
|
||||
],
|
||||
)
|
||||
def test_can_implement_group_size_must_divide_k(monkeypatch, group_size, expected_ok):
|
||||
"""ZenDNN WOQ groups along K, so a group that straddles the end is unusable."""
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch."
|
||||
"current_platform.is_zen_cpu",
|
||||
lambda: True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.mixed_precision.zentorch.has_zentorch_op",
|
||||
lambda ops: True,
|
||||
)
|
||||
|
||||
ok, reason = ZentorchWNA16LinearKernel.can_implement(
|
||||
_make_config(group_size=group_size)
|
||||
)
|
||||
assert ok is expected_ok, reason
|
||||
if not expected_ok:
|
||||
assert reason is not None
|
||||
assert "must divide input size" in reason
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# MoE DA8W4
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
NUM_EXPERTS = 4
|
||||
HIDDEN = 256
|
||||
INTERMEDIATE = 128
|
||||
|
||||
|
||||
def _make_moe_weights(group_size: int = 32):
|
||||
"""Build CT-layout MoE weights: w13 [E, 2I, H//8], w2 [E, H, I//8]."""
|
||||
torch.manual_seed(0)
|
||||
w13_q, w13_s, w2_q, w2_s = [], [], [], []
|
||||
for _ in range(NUM_EXPERTS):
|
||||
w13 = torch.randn(2 * INTERMEDIATE, HIDDEN, dtype=torch.bfloat16)
|
||||
q13, s13 = _quantize_per_group(w13, group_size)
|
||||
# CT is N-first and packs along the input dim: [N, K//8] per expert.
|
||||
w13_q.append(pack_to_int32(q13, 4, packed_dim=1))
|
||||
w13_s.append(s13)
|
||||
|
||||
w2 = torch.randn(HIDDEN, INTERMEDIATE, dtype=torch.bfloat16)
|
||||
q2, s2 = _quantize_per_group(w2, group_size)
|
||||
w2_q.append(pack_to_int32(q2, 4, packed_dim=1))
|
||||
w2_s.append(s2)
|
||||
return (
|
||||
torch.stack(w13_q),
|
||||
torch.stack(w2_q),
|
||||
torch.stack(w13_s),
|
||||
torch.stack(w2_s),
|
||||
)
|
||||
|
||||
|
||||
def test_zen_cpu_first_in_cpu_backend_priority(monkeypatch):
|
||||
monkeypatch.setattr(int_wna16.current_platform, "is_cpu", lambda: True)
|
||||
backends = int_wna16._get_priority_backends()
|
||||
assert backends[0] == WNA16MoEBackend.ZEN_CPU
|
||||
# The generic CPU backend stays available as a fallback.
|
||||
assert WNA16MoEBackend.CPU in backends
|
||||
|
||||
|
||||
def test_zen_cpu_process_weights_layout(mock_zentorch_ops):
|
||||
group_size = 32
|
||||
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
|
||||
assert w13.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8)
|
||||
assert w2.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8)
|
||||
|
||||
converted = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)
|
||||
w13_out, w2_out, w13_s_out, w2_s_out = converted[:4]
|
||||
|
||||
# zentorch consumes [E, N, K/8] packed s4 with per-group [E, G, N] scales.
|
||||
assert w13_out.shape == (NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN // 8)
|
||||
assert w2_out.shape == (NUM_EXPERTS, HIDDEN, INTERMEDIATE // 8)
|
||||
assert w13_out.dtype == w2_out.dtype == torch.int32
|
||||
assert w13_s_out.shape == (NUM_EXPERTS, HIDDEN // group_size, 2 * INTERMEDIATE)
|
||||
assert w2_s_out.shape == (NUM_EXPERTS, INTERMEDIATE // group_size, HIDDEN)
|
||||
assert w13_s_out.dtype == w2_s_out.dtype == torch.bfloat16
|
||||
# Symmetric checkpoints carry no zero points into the kernel.
|
||||
assert converted[8] is None and converted[9] is None
|
||||
|
||||
|
||||
def test_zen_cpu_repack_is_value_exact(mock_zentorch_ops):
|
||||
"""The repacked weight must dequantize back to the checkpoint values."""
|
||||
group_size = 32
|
||||
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
|
||||
w13_out = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0]
|
||||
|
||||
expected = _import_unpack_from_int32()(
|
||||
w13,
|
||||
4,
|
||||
torch.Size([NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN]),
|
||||
packed_dim=1,
|
||||
)
|
||||
for expert in range(NUM_EXPERTS):
|
||||
torch.testing.assert_close(
|
||||
_unpack_s4(w13_out[expert], HIDDEN),
|
||||
expected[expert].float(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"group_size,expected_ok",
|
||||
[(32, True), (128, True), (-1, False), (2, False)],
|
||||
)
|
||||
def test_zen_cpu_group_size_gating(group_size, expected_ok):
|
||||
quant_config = type("Args", (), {"group_size": group_size})()
|
||||
reason = int_wna16._backend_incompatibility_reason(
|
||||
WNA16MoEBackend.ZEN_CPU,
|
||||
moe_config=None,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=False,
|
||||
may_have_bias=False,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
assert (reason is None) == expected_ok
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"may_have_zp,may_have_bias,expected_ok",
|
||||
[(True, False, False), (False, True, True), (True, True, False)],
|
||||
)
|
||||
def test_zen_cpu_rejects_zero_points_but_takes_bias(
|
||||
may_have_zp, may_have_bias, expected_ok
|
||||
):
|
||||
quant_config = type("Args", (), {"group_size": 128})()
|
||||
reason = int_wna16._backend_incompatibility_reason(
|
||||
WNA16MoEBackend.ZEN_CPU,
|
||||
moe_config=None,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=may_have_zp,
|
||||
may_have_bias=may_have_bias,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
assert (reason is None) == expected_ok
|
||||
|
||||
|
||||
def test_zen_cpu_rejects_moe_wna16_layout():
|
||||
from vllm.model_executor.layers.quantization.moe_wna16 import MoeWNA16Config
|
||||
|
||||
quant_config = MoeWNA16Config(
|
||||
linear_quant_method="gptq",
|
||||
weight_bits=4,
|
||||
group_size=128,
|
||||
has_zp=False,
|
||||
lm_head_quantized=False,
|
||||
modules_to_not_convert=None,
|
||||
full_config={},
|
||||
)
|
||||
reason = int_wna16._backend_incompatibility_reason(
|
||||
WNA16MoEBackend.ZEN_CPU,
|
||||
moe_config=None,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=False,
|
||||
may_have_bias=False,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
assert reason is not None
|
||||
|
||||
|
||||
def test_zen_cpu_disabled_by_env(monkeypatch):
|
||||
monkeypatch.setattr(int_wna16.envs, "VLLM_CPU_INT4_W4A8", False)
|
||||
quant_config = type("Args", (), {"group_size": 128})()
|
||||
reason = int_wna16._backend_incompatibility_reason(
|
||||
WNA16MoEBackend.ZEN_CPU,
|
||||
moe_config=None,
|
||||
quant_config=quant_config,
|
||||
may_have_zp=False,
|
||||
may_have_bias=False,
|
||||
allow_tile_padding=True,
|
||||
)
|
||||
assert reason is not None
|
||||
|
||||
|
||||
def test_zen_cpu_backend_maps_to_experts_class():
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
|
||||
assert int_wna16.backend_to_kernel_cls(WNA16MoEBackend.ZEN_CPU) == [
|
||||
ZentorchExpertsInt4
|
||||
]
|
||||
|
||||
|
||||
def test_zen_experts_support_predicates():
|
||||
from vllm.model_executor.kernels.linear.zentorch_utils import (
|
||||
_ZENTORCH_MOE_ACTIVATIONS,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kInt4Static,
|
||||
kInt4Static32,
|
||||
kInt8StaticChannelSym,
|
||||
)
|
||||
|
||||
assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static, None)
|
||||
assert ZentorchExpertsInt4._supports_quant_scheme(kInt4Static32, None)
|
||||
assert not ZentorchExpertsInt4._supports_quant_scheme(kInt8StaticChannelSym, None)
|
||||
for act in MoEActivation:
|
||||
expected = act.value in _ZENTORCH_MOE_ACTIVATIONS
|
||||
assert ZentorchExpertsInt4._supports_activation(act) == expected
|
||||
assert ZentorchExpertsInt4.requires_interleaved_w13
|
||||
assert not ZentorchExpertsInt4._supports_no_act_and_mul()
|
||||
assert ZentorchExpertsInt4._supports_parallel_config(
|
||||
type("Par", (), {"use_ep": False})()
|
||||
)
|
||||
assert not ZentorchExpertsInt4._supports_parallel_config(
|
||||
type("Par", (), {"use_ep": True})()
|
||||
)
|
||||
|
||||
|
||||
def test_zen_experts_take_custom_routing():
|
||||
"""Models with their own router (gemma-4) reach select_experts through the
|
||||
config captured off the layer, since apply() cannot be handed the callable."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
from tests.kernels.moe.test_cpu_fused_moe import _StubMoELayer
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
|
||||
assert ZentorchExpertsInt4._supports_routing_method(
|
||||
RoutingMethodType.Custom, None, None
|
||||
)
|
||||
|
||||
def routing_fn(**kwargs):
|
||||
raise AssertionError("not called")
|
||||
|
||||
layer = _StubMoELayer(
|
||||
torch.zeros(NUM_EXPERTS, 2 * INTERMEDIATE, HIDDEN),
|
||||
torch.zeros(NUM_EXPERTS, HIDDEN, INTERMEDIATE),
|
||||
MoEActivation.SILU,
|
||||
)
|
||||
layer.renormalize = True
|
||||
layer.custom_routing_function = routing_fn
|
||||
|
||||
experts = SimpleNamespace(renormalize=False, custom_routing_function=None)
|
||||
ZentorchExpertsInt4.process_weights_after_loading(experts, layer)
|
||||
|
||||
assert experts.custom_routing_function is routing_fn
|
||||
assert experts.renormalize is True
|
||||
|
||||
|
||||
def _swigluoai_perm(activation, experts_cls=None):
|
||||
from vllm.model_executor.layers.fused_moe.experts.cpu_moe import (
|
||||
swigluoai_w13_interleave_perm,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.zentorch_moe import (
|
||||
ZentorchExpertsInt4,
|
||||
)
|
||||
|
||||
return swigluoai_w13_interleave_perm(
|
||||
experts_cls or ZentorchExpertsInt4,
|
||||
activation,
|
||||
2 * INTERMEDIATE,
|
||||
torch.device("cpu"),
|
||||
)
|
||||
|
||||
|
||||
def test_swigluoai_perm_only_for_zen_swigluoai():
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
assert _swigluoai_perm(MoEActivation.SWIGLUOAI) is not None
|
||||
# Other activations keep the half-split layout the loader leaves.
|
||||
assert _swigluoai_perm(MoEActivation.SILU) is None
|
||||
# object stands in for any kernel that never set requires_interleaved_w13.
|
||||
assert _swigluoai_perm(MoEActivation.SWIGLUOAI, experts_cls=object) is None
|
||||
|
||||
|
||||
def test_swigluoai_perm_interleaves_weights_scales_and_bias(mock_zentorch_ops):
|
||||
"""The permutation is a gather on w13's output-channel axis, which is dim 1
|
||||
for the N-first packed weights and their group scales."""
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
|
||||
group_size = 32
|
||||
w13, w2, w13_scale, w2_scale = _make_moe_weights(group_size)
|
||||
bias = torch.randn(NUM_EXPERTS, 2 * INTERMEDIATE, dtype=torch.bfloat16)
|
||||
perm = _swigluoai_perm(MoEActivation.SWIGLUOAI)
|
||||
|
||||
baseline = int_wna16._process_weights_zen_cpu(w13, w2, w13_scale, w2_scale)[0]
|
||||
permuted = int_wna16._process_weights_zen_cpu(
|
||||
w13[:, perm].contiguous(), w2, w13_scale[:, perm].contiguous(), w2_scale
|
||||
)[0]
|
||||
|
||||
for expert in range(NUM_EXPERTS):
|
||||
before = _unpack_s4(baseline[expert], HIDDEN)
|
||||
after = _unpack_s4(permuted[expert], HIDDEN)
|
||||
torch.testing.assert_close(after[0::2], before[:INTERMEDIATE])
|
||||
torch.testing.assert_close(after[1::2], before[INTERMEDIATE:])
|
||||
|
||||
torch.testing.assert_close(bias[:, perm][:, 0::2], bias[:, :INTERMEDIATE])
|
||||
torch.testing.assert_close(bias[:, perm][:, 1::2], bias[:, INTERMEDIATE:])
|
||||
@@ -203,7 +203,7 @@ class TestInt4ScaledMmCpu:
|
||||
print(f" [PASS] INT4 GEMM with bias: M={M}")
|
||||
|
||||
def test_gemm_3d_input(self):
|
||||
"""apply() reshapes 3D input [B, S, K] -> [B*S, K] -> back to 3D."""
|
||||
"""int4_scaled_mm_cpu flattens 3D [B, S, K] to 2D and reshapes back."""
|
||||
K, N, group_size = 256, 128, 128
|
||||
(packed_qweight, packed_qzeros, scales, float_ref, _, _) = (
|
||||
make_awq_checkpoint_data(K, N, group_size)
|
||||
@@ -220,12 +220,13 @@ class TestInt4ScaledMmCpu:
|
||||
x_3d = torch.randn(B, S, K, dtype=torch.bfloat16)
|
||||
x_2d = x_3d.reshape(-1, K)
|
||||
|
||||
out_3d = int4_scaled_mm_cpu(x_3d, blocked_w, blocked_zp, blocked_s, None)
|
||||
out_2d = int4_scaled_mm_cpu(x_2d, blocked_w, blocked_zp, blocked_s, None)
|
||||
out_3d = out_2d.reshape(B, S, N)
|
||||
|
||||
ref_out = torch.mm(x_2d.float(), float_ref).reshape(B, S, N)
|
||||
|
||||
assert out_3d.shape == (B, S, N)
|
||||
torch.testing.assert_close(out_3d.reshape(-1, N), out_2d)
|
||||
|
||||
ref_out = torch.mm(x_2d.float(), float_ref).reshape(B, S, N)
|
||||
abs_diff = (out_3d.float() - ref_out).abs()
|
||||
mean_abs = abs_diff.mean().item()
|
||||
ref_mag = ref_out.abs().mean().item() + 1e-6
|
||||
|
||||
@@ -85,6 +85,12 @@ QWEN4_EXP_SM90_CASES = [
|
||||
for num_tokens, config in plans.items()
|
||||
]
|
||||
|
||||
QWEN4_EXP_SM100_CASES = [
|
||||
(n, k, num_tokens, config)
|
||||
for (n, k), plans in qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS.items()
|
||||
for num_tokens, config in plans.items()
|
||||
]
|
||||
|
||||
EXPECTED_CUTE_CONFIGS = {
|
||||
(3072, 7168, 1): (224, 3, 4, 8),
|
||||
(3072, 7168, 2): (128, 3, 2, 8),
|
||||
@@ -269,6 +275,9 @@ def test_kda_overlap_configs_match_measured_table() -> None:
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="KDA CuTeDSL helper requires CUDA"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"compute_capability,supported",
|
||||
[((9, 0), False), ((10, 0), True), ((10, 3), True), ((12, 0), True)],
|
||||
@@ -714,6 +723,7 @@ def test_qwen4_exp_hopper_plans_are_valid() -> None:
|
||||
"capability,expected_plans",
|
||||
[
|
||||
((10, 3), qwen4_exp_gemm.QWEN4_EXP_GEMM_PLANS),
|
||||
((10, 0), qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS),
|
||||
((9, 0), qwen4_exp_gemm.QWEN4_EXP_SM90_GEMM_PLANS),
|
||||
((8, 0), {}),
|
||||
],
|
||||
@@ -906,6 +916,29 @@ def test_qwen4_exp_sm90_selected_shapes(
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n,k,num_tokens,config", QWEN4_EXP_SM100_CASES)
|
||||
def test_qwen4_exp_sm100_selected_shapes(
|
||||
n: int,
|
||||
k: int,
|
||||
num_tokens: int,
|
||||
config: SkinnyGemmConfig,
|
||||
) -> None:
|
||||
_require_capability_and_cute((10, 0))
|
||||
torch.manual_seed(42 + num_tokens)
|
||||
x = torch.randn(num_tokens, k, dtype=torch.bfloat16, device="cuda")
|
||||
weight = torch.randn(n, k, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
selected = qwen4_exp_gemm.QWEN4_EXP_SM100_GEMM_PLANS[(n, k)][num_tokens]
|
||||
assert selected == config
|
||||
output = qwen4_exp_gemm._qwen4_exp_low_latency_gemm(x, weight)
|
||||
|
||||
reference = torch.nn.functional.linear(x, weight)
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), reference.float().flatten(), dim=0
|
||||
).item()
|
||||
assert cosine > 0.999
|
||||
|
||||
|
||||
def test_glm52_q_b_nonpacked_single_row_falls_back() -> None:
|
||||
_require_sm103_and_cute()
|
||||
spec = glm52_gemm.GLM52_Q_B_PROJECTION
|
||||
|
||||
@@ -2182,7 +2182,9 @@ def test_v41_rocm_csa2_full_pipeline():
|
||||
)
|
||||
|
||||
q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
|
||||
assert torch.equal(q_out, q)
|
||||
assert torch.equal(
|
||||
q_out, q.view(num_tokens, attention.n_local_heads, attention.head_dim)
|
||||
)
|
||||
assert torch.equal(kv_out, kv)
|
||||
assert torch.equal(index_q_out, index_q)
|
||||
assert torch.equal(index_q_scale_out, index_q_scale)
|
||||
@@ -2273,7 +2275,9 @@ def test_v41_rocm_csa2_reindex_pipeline():
|
||||
)
|
||||
|
||||
q_out, kv_out, index_q_out, index_q_scale_out, index_weights_out_out = result
|
||||
assert torch.equal(q_out, q)
|
||||
assert torch.equal(
|
||||
q_out, q.view(num_tokens, attention.n_local_heads, attention.head_dim)
|
||||
)
|
||||
assert torch.equal(kv_out, kv)
|
||||
assert torch.equal(index_q_out, index_q)
|
||||
assert torch.equal(index_q_scale_out, index_q_scale)
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import tracemalloc
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_cpu():
|
||||
pytest.skip("skipping CPU-only tests", allow_module_level=True)
|
||||
|
||||
import vllm._C # noqa: F401, E402
|
||||
|
||||
VOCAB_SIZES = [32000, 49152, 128256]
|
||||
BATCH_SIZES = [1, 4, 16]
|
||||
|
||||
|
||||
def reference_greedy_sample(logits: torch.Tensor) -> torch.Tensor:
|
||||
return logits.argmax(dim=-1).view(-1)
|
||||
|
||||
|
||||
class TestGreedyArgmax:
|
||||
@pytest.mark.parametrize("vocab_size", VOCAB_SIZES)
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
|
||||
def test_exact_match(self, vocab_size: int, batch_size: int):
|
||||
logits = torch.randn(batch_size, vocab_size, dtype=torch.float32)
|
||||
expected = reference_greedy_sample(logits)
|
||||
result = torch.ops._C.greedy_argmax(logits)
|
||||
torch.testing.assert_close(result, expected)
|
||||
|
||||
def test_single_dominant(self):
|
||||
logits = torch.full((1, 50000), -1e9, dtype=torch.float32)
|
||||
logits[0, 42] = 100.0
|
||||
assert torch.ops._C.greedy_argmax(logits).item() == 42
|
||||
|
||||
def test_negative_logits(self):
|
||||
logits = torch.randn(8, 32000, dtype=torch.float32) - 10.0
|
||||
expected = reference_greedy_sample(logits)
|
||||
result = torch.ops._C.greedy_argmax(logits)
|
||||
torch.testing.assert_close(result, expected)
|
||||
|
||||
|
||||
class TestFusedGumbelArgmax:
|
||||
@pytest.mark.parametrize("vocab_size", VOCAB_SIZES)
|
||||
def test_distribution_chi_squared(self, vocab_size: int):
|
||||
"""Verify sampling distribution via chi-squared goodness of fit."""
|
||||
small_vocab = min(vocab_size, 100)
|
||||
logits = torch.randn(1, small_vocab, dtype=torch.float32)
|
||||
probs = logits.softmax(dim=-1).squeeze(0)
|
||||
|
||||
n_samples = 100_000
|
||||
counts = torch.zeros(small_vocab)
|
||||
for trial in range(n_samples):
|
||||
seed = torch.tensor([trial * 7 + 13], dtype=torch.long)
|
||||
tile = logits.expand(1, -1).contiguous()
|
||||
idx = torch.ops._C.fused_gumbel_argmax(tile, seed)
|
||||
counts[idx.item()] += 1
|
||||
|
||||
expected = probs * n_samples
|
||||
mask = expected > 5
|
||||
chi2 = ((counts[mask] - expected[mask]) ** 2 / expected[mask]).sum()
|
||||
dof = mask.sum().item() - 1
|
||||
from scipy.stats import chi2 as chi2_dist
|
||||
|
||||
p_value = 1.0 - chi2_dist.cdf(chi2.item(), dof)
|
||||
assert p_value > 0.001, (
|
||||
f"Chi-squared test failed: chi2={chi2.item():.1f}, "
|
||||
f"dof={dof}, p={p_value:.6f}"
|
||||
)
|
||||
|
||||
def test_deterministic_same_seed(self):
|
||||
"""Same seed produces same result."""
|
||||
logits = torch.randn(4, 32000, dtype=torch.float32)
|
||||
seeds = torch.tensor([42, 123, 456, 789], dtype=torch.long)
|
||||
r1 = torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
r2 = torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
torch.testing.assert_close(r1, r2)
|
||||
|
||||
def test_different_seeds_differ(self):
|
||||
logits = torch.zeros(16, 50000, dtype=torch.float32)
|
||||
seeds_a = torch.arange(16, dtype=torch.long)
|
||||
seeds_b = torch.arange(16, dtype=torch.long) + 1_000_000
|
||||
r_a = torch.ops._C.fused_gumbel_argmax(logits, seeds_a)
|
||||
r_b = torch.ops._C.fused_gumbel_argmax(logits, seeds_b)
|
||||
assert not torch.equal(r_a, r_b)
|
||||
|
||||
@pytest.mark.parametrize("vocab_size", VOCAB_SIZES)
|
||||
@pytest.mark.parametrize("batch_size", BATCH_SIZES)
|
||||
def test_output_in_range(self, vocab_size: int, batch_size: int):
|
||||
logits = torch.randn(batch_size, vocab_size, dtype=torch.float32)
|
||||
seeds = torch.arange(batch_size, dtype=torch.long)
|
||||
result = torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
assert result.min() >= 0
|
||||
assert result.max() < vocab_size
|
||||
|
||||
|
||||
class TestMemory:
|
||||
def test_fused_no_intermediate_allocs(self):
|
||||
"""Verify that the fused kernel does not allocate large intermediates."""
|
||||
logits = torch.randn(16, 128256, dtype=torch.float32)
|
||||
seeds = torch.arange(16, dtype=torch.long)
|
||||
|
||||
torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
|
||||
tracemalloc.start()
|
||||
snap_before = tracemalloc.take_snapshot()
|
||||
for _ in range(50):
|
||||
torch.ops._C.fused_gumbel_argmax(logits, seeds)
|
||||
snap_after = tracemalloc.take_snapshot()
|
||||
tracemalloc.stop()
|
||||
|
||||
diff = snap_after.compare_to(snap_before, "lineno")
|
||||
total_new_bytes = sum(s.size_diff for s in diff if s.size_diff > 0)
|
||||
vocab_bytes = 16 * 128256 * 4
|
||||
assert total_new_bytes < vocab_bytes, (
|
||||
f"Fused kernel allocated {total_new_bytes} bytes, "
|
||||
f"expected < {vocab_bytes} (one intermediate tensor)"
|
||||
)
|
||||
@@ -9,13 +9,10 @@ import torch
|
||||
|
||||
from vllm.models.deepseek_v41.common import engram as engram_ops
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram as CommonEngram,
|
||||
)
|
||||
from vllm.models.deepseek_v41.common.engram import (
|
||||
Engram,
|
||||
NgramHashState,
|
||||
ParallelEngramEmbedding,
|
||||
)
|
||||
from vllm.models.deepseek_v41.nvidia import engram as nvidia_engram_ops
|
||||
from vllm.models.deepseek_v41.nvidia.engram import Engram, ParallelEngramEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
@@ -111,7 +108,7 @@ def test_fused_engram_post_wkv_matches_reference(
|
||||
)
|
||||
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: tp_rank)
|
||||
|
||||
module = CommonEngram.__new__(CommonEngram)
|
||||
module = Engram.__new__(Engram)
|
||||
torch.nn.Module.__init__(module)
|
||||
module.dim = dim
|
||||
module.hc_mult = hc_mult
|
||||
@@ -121,6 +118,7 @@ def test_fused_engram_post_wkv_matches_reference(
|
||||
module.embed_tokens = torch.nn.Identity()
|
||||
# `forward` reads rows staged by `prepare_embeddings`, so inject kv there.
|
||||
module.embed_tokens.tp_size = module.embed_tokens.n_hash_cols = 1
|
||||
module.embed_tokens.dp_size = 1
|
||||
module.staged_rows = kv.unsqueeze(1)
|
||||
if use_sequence_parallel:
|
||||
padded = torch.nn.functional.pad(kv, (0, 0, 0, (-num_kv_tokens) % tp_size))
|
||||
@@ -592,9 +590,8 @@ def test_engram_rejects_empty_head_shards(tp_size, dp_size, n_heads, monkeypatch
|
||||
monkeypatch.setattr(
|
||||
engram_ops, "get_tensor_model_parallel_world_size", lambda: tp_size
|
||||
)
|
||||
monkeypatch.setattr(nvidia_engram_ops, "get_engram_dp_size", lambda: dp_size)
|
||||
monkeypatch.setattr(engram_ops, "get_engram_dp_size", lambda: dp_size)
|
||||
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(nvidia_engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
with pytest.raises(AssertionError, match="ranks without hash heads"):
|
||||
ParallelEngramEmbedding(n_heads * 17, 64, (17,) * n_heads)
|
||||
|
||||
@@ -602,10 +599,19 @@ def test_engram_rejects_empty_head_shards(tp_size, dp_size, n_heads, monkeypatch
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="CUDA or ROCm required"
|
||||
)
|
||||
@pytest.mark.parametrize("cpu_offload", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"cpu_offload,sort_rows", [(False, False), (True, False), (True, True)]
|
||||
)
|
||||
@pytest.mark.parametrize("tp_size", [1, 2, 4, 8])
|
||||
def test_engram_head_shards_reconstruct_checkpoint(cpu_offload, tp_size, monkeypatch):
|
||||
"""Keep complete buckets and reconstruct head order, including TP padding."""
|
||||
def test_engram_head_shards_reconstruct_checkpoint(
|
||||
cpu_offload, sort_rows, tp_size, monkeypatch
|
||||
):
|
||||
"""Keep complete buckets and reconstruct head order, including TP padding,
|
||||
whether or not the host lookup sorts its rows by table offset."""
|
||||
if sort_rows:
|
||||
monkeypatch.setattr(
|
||||
engram_ops, "_engram_lookup_thresholds", lambda _: (0, None)
|
||||
)
|
||||
head_sizes = (17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73)
|
||||
num_rows, dim = sum(head_sizes), 64
|
||||
torch.manual_seed(0)
|
||||
@@ -796,10 +802,8 @@ def test_engram_shared_prefetch_stream_waits_per_layer():
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA required")
|
||||
@pytest.mark.parametrize(
|
||||
"backend,cpu_offload", [("nvidia", None), ("nvidia", False), ("common", False)]
|
||||
)
|
||||
def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload):
|
||||
@pytest.mark.parametrize("cpu_offload", [None, False])
|
||||
def test_engram_constructor_honors_offload(monkeypatch, cpu_offload):
|
||||
"""Default offload uses pinned storage and the given stream; False uses HBM."""
|
||||
from vllm.config import EngramConfig
|
||||
|
||||
@@ -817,23 +821,18 @@ def test_engram_constructor_honors_offload(monkeypatch, backend, cpu_offload):
|
||||
else EngramConfig(cpu_offload=cpu_offload),
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
|
||||
)
|
||||
offloaded = backend == "nvidia" and vllm_config.engram_config.cpu_offload
|
||||
if backend == "common":
|
||||
vllm_config.engram_config = None
|
||||
offloaded = vllm_config.engram_config.cpu_offload
|
||||
monkeypatch.setattr(engram_ops, "get_current_vllm_config", lambda: vllm_config)
|
||||
monkeypatch.setattr(
|
||||
nvidia_engram_ops, "get_current_vllm_config", lambda: vllm_config
|
||||
)
|
||||
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(engram_ops, "get_tensor_model_parallel_rank", lambda: 0)
|
||||
monkeypatch.setattr(
|
||||
engram_ops, "ColumnParallelLinear", lambda *a, **k: torch.nn.Identity()
|
||||
)
|
||||
stream = torch.cuda.Stream()
|
||||
kwargs = {"prefetch_stream": stream} if backend == "nvidia" else {}
|
||||
with torch.device("cuda"):
|
||||
cls = Engram if backend == "nvidia" else CommonEngram
|
||||
module = cls(config, None, layout, 0, False, "engram", **kwargs)
|
||||
module = Engram(
|
||||
config, None, layout, 0, False, "engram", prefetch_stream=stream
|
||||
)
|
||||
layer = module.embed_tokens
|
||||
if offloaded:
|
||||
assert layer.weight.device.type == "cpu" and layer.weight.is_pinned()
|
||||
@@ -897,7 +896,8 @@ def test_engram_prefetch_detects_missing_dependency(monkeypatch, missing_depende
|
||||
"The allocator reuse negative control requires the native allocator"
|
||||
)
|
||||
|
||||
def start(self, hash_ids, rows, stream):
|
||||
def start(self, hash_ids, rows):
|
||||
stream = self._prefetch_stream
|
||||
if missing_dependency != "producer_wait":
|
||||
stream.wait_stream(torch.cuda.current_stream())
|
||||
if missing_dependency != "record_stream":
|
||||
|
||||
@@ -24,6 +24,7 @@ outputs are checked within 1 representable-step (ULP); bf16 norm/RoPE outputs us
|
||||
rtol/atol=1e-2 (the tolerance the sibling deepseek_v4 fused-kernel test uses).
|
||||
"""
|
||||
|
||||
from functools import partial
|
||||
from typing import NamedTuple
|
||||
|
||||
import pytest
|
||||
@@ -880,22 +881,36 @@ def test_fused_q_no_indexer(num_tokens: int, cfg: ModelConfig):
|
||||
@pytest.mark.parametrize("cfg", MODEL_CONFIGS, ids=MODEL_IDS)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 17, 512])
|
||||
@pytest.mark.parametrize("has_indexer", [True, False])
|
||||
def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig):
|
||||
@pytest.mark.parametrize("use_mha", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"index_interleave,cache_dtype", [(False, torch.float32), (True, torch.bfloat16)]
|
||||
)
|
||||
@pytest.mark.parametrize("capture", [False, True])
|
||||
def test_fused_q_bf16_query(
|
||||
num_tokens: int,
|
||||
has_indexer: bool,
|
||||
use_mha: bool,
|
||||
index_interleave: bool,
|
||||
cache_dtype: torch.dtype,
|
||||
capture: bool,
|
||||
cfg: ModelConfig,
|
||||
):
|
||||
"""bf16-query path (FlashMLA sparse, SM90/SM100): only the RoPE'd q_pe is
|
||||
produced (bf16, unquantized); ql_nope is consumed directly by the caller."""
|
||||
produced; MHA does not need an absorbed ql_nope projection."""
|
||||
torch.manual_seed(6)
|
||||
dev = "cuda"
|
||||
max_pos = 8192
|
||||
pos = torch.arange(num_tokens, device=dev, dtype=torch.int64) % max_pos
|
||||
|
||||
# Attention takes q_pe as a strided slice of the full Q projection.
|
||||
q_pe = torch.randn(
|
||||
num_tokens, cfg.num_heads, cfg.rope_dim, device=dev, dtype=torch.bfloat16
|
||||
)
|
||||
num_tokens, cfg.num_heads, 4 * cfg.rope_dim, device=dev, dtype=torch.bfloat16
|
||||
)[..., -cfg.rope_dim :]
|
||||
ql_nope = torch.randn(
|
||||
num_tokens, cfg.num_heads, cfg.kv_lora, device=dev, dtype=torch.bfloat16
|
||||
)
|
||||
q_scale = torch.tensor([0.37], device=dev, dtype=torch.float32)
|
||||
q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
|
||||
q_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev).to(cache_dtype)
|
||||
|
||||
index_q = index_w = idx_cos_sin = None
|
||||
if has_indexer:
|
||||
@@ -909,24 +924,35 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig
|
||||
index_w = torch.randn(
|
||||
num_tokens, cfg.index_heads, device=dev, dtype=torch.float32
|
||||
)
|
||||
idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev)
|
||||
idx_cos_sin = make_cos_sin(max_pos, cfg.rope_dim, dev).to(cache_dtype)
|
||||
|
||||
iq_fp8, iw_out, q_pe_out = K.fused_q(
|
||||
run = partial(
|
||||
K.fused_q,
|
||||
pos,
|
||||
q_pe,
|
||||
q_cos_sin,
|
||||
index_q,
|
||||
idx_cos_sin,
|
||||
ql_nope,
|
||||
q_pe if use_mha else ql_nope,
|
||||
q_scale,
|
||||
index_w,
|
||||
cfg.index_head_dim**-0.5,
|
||||
cfg.index_heads**-0.5,
|
||||
has_indexer=has_indexer,
|
||||
index_rope_interleave=False,
|
||||
index_rope_interleave=index_interleave,
|
||||
quantize_mqa=False,
|
||||
)
|
||||
|
||||
iq_fp8, iw_out, q_pe_out = run()
|
||||
if capture:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
iq_fp8, iw_out, q_pe_out = run()
|
||||
q_pe.mul_(0.5)
|
||||
if index_q is not None:
|
||||
index_q.mul_(1.5)
|
||||
graph.replay()
|
||||
|
||||
# MQA query: only the RoPE'd q_pe, bf16, unquantized.
|
||||
assert q_pe_out.dtype == torch.bfloat16
|
||||
assert q_pe_out.shape == (num_tokens, cfg.num_heads, cfg.rope_dim)
|
||||
@@ -945,7 +971,7 @@ def test_fused_q_bf16_query(num_tokens: int, has_indexer: bool, cfg: ModelConfig
|
||||
index_q.float(),
|
||||
pos.unsqueeze(-1).expand(num_tokens, cfg.index_heads),
|
||||
idx_cos_sin,
|
||||
interleave=False,
|
||||
interleave=index_interleave,
|
||||
)
|
||||
q_ref, scale_ref = ue8m0_quant(iq_ref)
|
||||
assert_fp8(iq_fp8, q_ref, "indexer-Q fp8 (bf16-query path)")
|
||||
|
||||
@@ -320,6 +320,38 @@ def test_sparse_full(num_tokens, block_size, kv_cache_dtype):
|
||||
index_cache.view(-1, HEAD_DIM), expected_index_cache, rtol=0, atol=0
|
||||
)
|
||||
|
||||
# Without q_out, q is emitted only in fp8: the same bytes, while qkv keeps
|
||||
# its raw q and every other output matches the run above.
|
||||
outputs = [qkv_orig.clone(), torch.empty_like(q_fp8), torch.empty_like(index_q)]
|
||||
caches = [torch.zeros_like(kv_cache), torch.zeros_like(index_cache)]
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
outputs[0],
|
||||
q_w,
|
||||
k_w,
|
||||
cos_sin,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
ROTARY_DIM,
|
||||
eps,
|
||||
iq_w,
|
||||
ik_w,
|
||||
num_idx_heads,
|
||||
slot_mapping,
|
||||
index_slot_mapping,
|
||||
*caches,
|
||||
block_size,
|
||||
None,
|
||||
outputs[2],
|
||||
kv_cache_dtype,
|
||||
q_fp8_out=outputs[1],
|
||||
q_fp8_scale=0.5,
|
||||
)
|
||||
for actual, expected in zip(
|
||||
outputs + caches, [qkv, q_fp8, index_q, kv_cache, index_cache]
|
||||
):
|
||||
assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 64, 513])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
@@ -441,8 +473,8 @@ def test_sparse_skip_index_branch(num_tokens, block_size, kv_cache_dtype):
|
||||
# ── Test 4: fp8 (e4m3) index outputs ─────────────────────────────────────────
|
||||
# The fp8 score path stores index_q and the index-K cache as e4m3 while q/k/v +
|
||||
# q_out stay bf16. Asserts: (1) q/k/v/q_out are bit-identical to the bf16 run
|
||||
# (the index dtype must not perturb the main branch), and (2) the e4m3 index
|
||||
# outputs dequantize close to the bf16 reference.
|
||||
# (the index dtype must not perturb the main branch), and (2) index_q is the
|
||||
# e4m3 cast of the bf16 run's index_q, and the index-K cache is within fp8 ulp.
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
@@ -528,6 +560,196 @@ def test_sparse_full_fp8_index(num_tokens, block_size):
|
||||
torch.testing.assert_close(qkv_fp, qkv_bf, rtol=0, atol=0)
|
||||
torch.testing.assert_close(kvc_fp, kvc_bf, rtol=0, atol=0)
|
||||
|
||||
# (2) Dequantized e4m3 index outputs match the bf16 reference within fp8 ulp.
|
||||
torch.testing.assert_close(iq_fp.float(), iq_bf.float(), rtol=0.13, atol=0.05)
|
||||
# (2) index_q is e4m3(bf16(q)); the index-K cache is within fp8 ulp.
|
||||
assert torch.equal(
|
||||
iq_fp.view(torch.uint8), iq_bf.to(torch.float8_e4m3fn).view(torch.uint8)
|
||||
)
|
||||
torch.testing.assert_close(idxc_fp.float(), idxc_bf.float(), rtol=0.13, atol=0.05)
|
||||
|
||||
|
||||
# ── Test 5: sparse mode with an NVFP4 main KV cache ──────────────────────────
|
||||
# The NVFP4 inserts quantize x * sf_scale, where sf_scale = 1 / the global
|
||||
# scale: per 16-value block,
|
||||
# scale = min(448, max(E4M3(max(amax, 1e-12) / 6), 1/512)) and
|
||||
# code = sign | RNE E2M1 of min(|x / scale|, 6), with zero stored as +0.
|
||||
|
||||
E2M1_MIDPOINTS = (0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5.0)
|
||||
|
||||
|
||||
def _div(a, b):
|
||||
# Division by a Python scalar would become a reciprocal multiply.
|
||||
return a / torch.as_tensor(b, dtype=a.dtype, device=a.device)
|
||||
|
||||
|
||||
def nvfp4_ref(x, sf_scale):
|
||||
"""Returns the E2M1 codes [..., 128] and E4M3 scale bytes [..., 8] of x."""
|
||||
blocks = x.float().unflatten(-1, (-1, 16))
|
||||
amax = blocks.abs().amax(-1, keepdim=True).clamp_min(1e-12)
|
||||
scale = (sf_scale * _div(amax, 6.0)).clamp(max=448.0)
|
||||
scale = scale.to(torch.float8_e4m3fn).float().clamp(min=2.0**-9)
|
||||
q = _div(blocks * sf_scale, scale)
|
||||
# Alternating > / >= rounds exact midpoints to the even code.
|
||||
mag = sum(
|
||||
(q.abs() >= m) if i % 2 else (q.abs() > m) for i, m in enumerate(E2M1_MIDPOINTS)
|
||||
).to(torch.uint8)
|
||||
codes = torch.where(mag > 0, mag | (torch.signbit(q).to(torch.uint8) << 3), 0)
|
||||
scale_bytes = scale.squeeze(-1).to(torch.float8_e4m3fn).view(torch.uint8)
|
||||
return codes.flatten(-2), scale_bytes
|
||||
|
||||
|
||||
def nvfp4_pages(sides, slot_mapping, num_blocks, block_size):
|
||||
"""Expected NVFP4 pages [nb, len(sides) * nkv, bs, 72] for the K (then V)
|
||||
side given as (x [N, nkv, 128], sf_scale). Slot h * len(sides) + side is
|
||||
head h's packed E2M1 data [bs, 64] (low nibble = even element), then its
|
||||
E4M3 scales [bs, 8]: linear for K, 4x4 token-quad swizzled for V."""
|
||||
nkv, ns, device = sides[0][0].shape[1], HEAD_DIM // 16, slot_mapping.device
|
||||
row_bytes, data_bytes = HEAD_DIM // 2 + ns, block_size * HEAD_DIM // 2
|
||||
pages = torch.zeros(
|
||||
num_blocks,
|
||||
nkv,
|
||||
len(sides),
|
||||
block_size * row_bytes,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
valid = slot_mapping >= 0
|
||||
blk, tok = slot_mapping[valid] // block_size, slot_mapping[valid] % block_size
|
||||
g, t = torch.arange(ns, device=device), tok[:, None]
|
||||
scale_offsets = (t * ns + g, ((t // 4) * 4 + g // 2) * ns + (g % 2) * 4 + t % 4)
|
||||
for side, (x, sf_scale) in enumerate(sides):
|
||||
codes, scales = nvfp4_ref(x[valid], sf_scale)
|
||||
data = pages[:, :, side, :data_bytes].view(num_blocks, nkv, block_size, -1)
|
||||
data[blk, :, tok] = codes[..., 0::2] | (codes[..., 1::2] << 4)
|
||||
scale_region = pages[:, :, side, data_bytes:]
|
||||
for h in range(nkv):
|
||||
scale_region[blk[:, None], h, scale_offsets[side]] = scales[:, h]
|
||||
return pages.view(num_blocks, -1, block_size, row_bytes)
|
||||
|
||||
|
||||
def nvfp4_edge_values(num_rows, global_scale):
|
||||
"""[num_rows, 128] bf16 whose 16-value blocks, divided by global_scale, hit
|
||||
the quantizer's corner cases: exact E2M1 midpoints under non-power-of-2 scales
|
||||
(blocks 0-1), amax under the 1/512 scale floor (2), all -0 (3), and
|
||||
saturation with |x| > 4096 up to bf16 max (4) or with inf (5)."""
|
||||
g = torch.Generator().manual_seed(0)
|
||||
x = torch.randn(num_rows, 8, 16, generator=g)
|
||||
sign = torch.where(torch.rand(x.shape, generator=g) < 0.5, -1.0, 1.0)
|
||||
steps = torch.tensor(E2M1_MIDPOINTS + (0.5, 1.0, 3.0))
|
||||
mant = 1 + torch.randint(1, 8, (num_rows, 2, 1), generator=g) / 8
|
||||
scale = mant * 2.0 ** torch.randint(-6, 3, (num_rows, 2, 1), generator=g)
|
||||
x[:, :2] = steps[torch.randint(len(steps), (num_rows, 2, 16), generator=g)] * scale
|
||||
x[:, :2, :1] = 6 * scale # amax = 6 * scale, so the block scale is `scale`
|
||||
x[:, 2] = torch.rand(num_rows, 16, generator=g) * 2.0**-8
|
||||
x[:, 4] *= 5000
|
||||
x = x * sign * global_scale
|
||||
x[:, 3] = -0.0
|
||||
x[:, 4, 0] = sign[:, 4, 0] * torch.finfo(torch.bfloat16).max
|
||||
x[:, 5, 0] = sign[:, 5, 0] * float("inf")
|
||||
return x.flatten(1).to(torch.bfloat16)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available() or torch.cuda.get_device_capability()[0] != 10,
|
||||
reason="NVFP4 KV caches need SM100",
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [1, 7, 64, 513])
|
||||
@pytest.mark.parametrize("num_kv_heads", [1, 4])
|
||||
@pytest.mark.parametrize("block_size", [16, 128])
|
||||
@pytest.mark.parametrize("kv_scales", [(0.5, 2.0), (1.0, 1.0)])
|
||||
def test_sparse_full_nvfp4(num_tokens, num_kv_heads, block_size, kv_scales):
|
||||
"""The fused NVFP4 insert writes the reference quantizer's bytes (packed
|
||||
E2M1, linear K / swizzled V E4M3 block scales) for the kernel's own
|
||||
post-RoPE k and for a raw v that hits the quantizer's corner cases, and
|
||||
skips padded slots. Unit global scales take the kernel's separate fast
|
||||
path."""
|
||||
kv_cache_dtype = "nvfp4"
|
||||
torch.manual_seed(5)
|
||||
device, dtype, eps = "cuda", torch.bfloat16, 1e-6
|
||||
base, max_pos = 5_000_000.0, 4096
|
||||
num_heads, num_idx_heads = 4 * num_kv_heads, 1
|
||||
|
||||
weights = [
|
||||
torch.randn(HEAD_DIM, dtype=dtype, device=device) * 0.1 for _ in range(4)
|
||||
]
|
||||
cos_sin = make_cos_sin_cache(max_pos, ROTARY_DIM, base, dtype, device)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
qsz, kvsz = num_heads * HEAD_DIM, num_kv_heads * HEAD_DIM
|
||||
iqsz = num_idx_heads * HEAD_DIM
|
||||
qkv = torch.randn(
|
||||
num_tokens, qsz + 2 * kvsz + iqsz + HEAD_DIM, dtype=dtype, device=device
|
||||
)
|
||||
k_scale, v_scale = (
|
||||
torch.tensor(s, dtype=torch.float32, device=device) for s in kv_scales
|
||||
)
|
||||
qkv[:, qsz + kvsz : qsz + 2 * kvsz] = nvfp4_edge_values(
|
||||
num_tokens * num_kv_heads, v_scale.item()
|
||||
).view(num_tokens, kvsz)
|
||||
qkv_orig = qkv.clone()
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks,
|
||||
2 * num_kv_heads,
|
||||
block_size,
|
||||
HEAD_DIM // 2 + HEAD_DIM // 16,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
index_cache = torch.zeros(
|
||||
num_blocks, block_size, HEAD_DIM, dtype=dtype, device=device
|
||||
)
|
||||
slot_mapping = torch.randperm(
|
||||
num_blocks * block_size, dtype=torch.int64, device=device
|
||||
)[:num_tokens]
|
||||
slot_mapping[::5] = -1 # padded / unscheduled tokens are not written
|
||||
index_slot_mapping = torch.randperm(
|
||||
num_blocks * block_size, dtype=torch.int64, device=device
|
||||
)[:num_tokens]
|
||||
|
||||
ops.fused_minimax_m3_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
*weights[:2],
|
||||
cos_sin,
|
||||
positions,
|
||||
num_heads,
|
||||
num_kv_heads,
|
||||
ROTARY_DIM,
|
||||
eps,
|
||||
*weights[2:],
|
||||
num_idx_heads,
|
||||
slot_mapping,
|
||||
index_slot_mapping,
|
||||
kv_cache,
|
||||
index_cache,
|
||||
block_size,
|
||||
torch.empty(num_tokens, qsz, dtype=dtype, device=device),
|
||||
torch.empty(num_tokens, iqsz, dtype=dtype, device=device),
|
||||
kv_cache_dtype,
|
||||
kv_k_scale=k_scale,
|
||||
kv_v_scale=v_scale,
|
||||
)
|
||||
|
||||
_, k_out, v_out, _, index_k = qkv.split([qsz, kvsz, kvsz, iqsz, HEAD_DIM], dim=-1)
|
||||
torch.testing.assert_close(
|
||||
v_out, qkv_orig.split([qsz, kvsz, kvsz, iqsz, HEAD_DIM], dim=-1)[2]
|
||||
)
|
||||
expected_kv_cache = nvfp4_pages(
|
||||
[
|
||||
(k_out.reshape(num_tokens, num_kv_heads, HEAD_DIM), 1 / k_scale),
|
||||
(v_out.reshape(num_tokens, num_kv_heads, HEAD_DIM), 1 / v_scale),
|
||||
],
|
||||
slot_mapping,
|
||||
num_blocks,
|
||||
block_size,
|
||||
)
|
||||
assert torch.equal(kv_cache, expected_kv_cache), (
|
||||
f"{int((kv_cache != expected_kv_cache).sum())} NVFP4 cache bytes differ"
|
||||
)
|
||||
|
||||
expected_index_cache = torch.zeros_like(index_cache).view(-1, HEAD_DIM)
|
||||
expected_index_cache[index_slot_mapping] = index_k
|
||||
torch.testing.assert_close(
|
||||
index_cache.view(-1, HEAD_DIM), expected_index_cache, rtol=0, atol=0
|
||||
)
|
||||
|
||||
@@ -233,6 +233,26 @@ def _assert_eq(r_ref, r_kern):
|
||||
)
|
||||
|
||||
|
||||
def _cache_pool_bytes(kv_cache: torch.Tensor, pool_slot: int) -> torch.Tensor:
|
||||
"""Read a logical pool's K and scale bytes from the platform cache layout."""
|
||||
page_size = kv_cache.shape[1]
|
||||
head_dim = kv_cache.shape[2] - 4
|
||||
page_idx, token_offset = divmod(pool_slot, page_size)
|
||||
flat = kv_cache[page_idx].reshape(-1)
|
||||
dims = torch.arange(head_dim, device=kv_cache.device)
|
||||
if current_platform.is_rocm() and page_size > 1:
|
||||
k_offsets = (
|
||||
(token_offset // 16) * 16 * head_dim
|
||||
+ (dims // 16) * 16 * 16
|
||||
+ (token_offset % 16) * 16
|
||||
+ dims % 16
|
||||
)
|
||||
else:
|
||||
k_offsets = token_offset * head_dim + dims
|
||||
scale_offset = page_size * head_dim + 4 * token_offset
|
||||
return torch.cat((flat[k_offsets], flat[scale_offset : scale_offset + 4]))
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
|
||||
def test_amd_prefill_writer_uses_preshuffled_cache_layout():
|
||||
from vllm.models.glm5next.amd.ops.kpool_compress import (
|
||||
@@ -343,7 +363,7 @@ def test_decode_writer_matches_prefill_writer(pool_size, ring_pools):
|
||||
p
|
||||
for p in range(n_pools)
|
||||
if not torch.equal(
|
||||
kv_prefill[p // page, p % page], kv_decode[p // page, p % page]
|
||||
_cache_pool_bytes(kv_prefill, p), _cache_pool_bytes(kv_decode, p)
|
||||
)
|
||||
]
|
||||
assert not differing, (
|
||||
@@ -379,13 +399,6 @@ def test_rejected_draft_redo_needs_ring_slots(ring_pools):
|
||||
kv = torch.zeros_like(kv_ref)
|
||||
tail = torch.zeros(nblk, 2, ring, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
|
||||
def pool_bytes(cache, p):
|
||||
# Page layout: [page * HEAD_DIM bytes of K rows | page * 4 bytes of scales].
|
||||
flat = cache[0].reshape(-1)
|
||||
k_bytes = flat[p * HEAD_DIM : (p + 1) * HEAD_DIM]
|
||||
s_bytes = flat[page * HEAD_DIM + 4 * p : page * HEAD_DIM + 4 * (p + 1)]
|
||||
return torch.cat([k_bytes, s_bytes])
|
||||
|
||||
def step(positions, keys, scores):
|
||||
pos = torch.tensor([positions], dtype=torch.int32, device=dev)
|
||||
slots = [(p // pool) if p % pool == pool - 1 else -1 for p in positions]
|
||||
@@ -415,7 +428,7 @@ def test_rejected_draft_redo_needs_ring_slots(ring_pools):
|
||||
)
|
||||
step([8, 9, 10, 11], k[8:12], score[8:12]) # all drafts rejected
|
||||
for p in (1, 2):
|
||||
assert torch.equal(pool_bytes(kv, p), pool_bytes(kv_ref, p)), p
|
||||
assert torch.equal(_cache_pool_bytes(kv, p), _cache_pool_bytes(kv_ref, p)), p
|
||||
|
||||
# Draft 7 completes pool 1 and is rejected. With a one-pool ring, drafts
|
||||
# 8 and 9 overwrite the slots of positions 4 and 5, which are read by the
|
||||
@@ -430,7 +443,7 @@ def test_rejected_draft_redo_needs_ring_slots(ring_pools):
|
||||
torch.cat([score[6:7], draft_scores]),
|
||||
)
|
||||
step([7, 8, 9, 10], k[7:11], score[7:11])
|
||||
pool1_ok = torch.equal(pool_bytes(kv, 1), pool_bytes(kv_ref, 1))
|
||||
pool1_ok = torch.equal(_cache_pool_bytes(kv, 1), _cache_pool_bytes(kv_ref, 1))
|
||||
if ring_pools == 1:
|
||||
assert not pool1_ok, "expected a one-pool ring to corrupt pool 1"
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""ROCm projection/collective contract corresponding to the SM100 GEMM-RS/AR.
|
||||
|
||||
Exercise the production RowParallelLinear and sequence-parallel reduction
|
||||
paths: native BF16/MXFP8 GEMM, AITER all-reduce, and RCCL reduce-scatter. This
|
||||
checks the same observable contract, not NVIDIA's fused GEMM/communication
|
||||
implementation or its performance.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from tests.kernels.test_gemm_rs_ar import _N, _SHAPES
|
||||
from vllm.config import ModelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.quantization import QuantizationConfigArgs
|
||||
from vllm.distributed import cleanup_dist_env_and_memory, get_tp_group
|
||||
from vllm.distributed.parallel_state import (
|
||||
graph_capture,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.layers.linear import RowParallelLinear
|
||||
from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
_mxfp8_e4m3_quantize_torch,
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.models.common.ops.sequence_parallel import sp_reduce_scatter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
# Retain the expanded parent's shape transitions, plus fully padded RS ranks.
|
||||
_ROCM_SHAPES = ((1, 512), (7, 768), *_SHAPES)
|
||||
|
||||
|
||||
def _reference(
|
||||
x: torch.Tensor,
|
||||
layer: RowParallelLinear,
|
||||
all_reduce: bool,
|
||||
) -> torch.Tensor:
|
||||
weight = layer.weight
|
||||
if hasattr(layer, "weight_scale"):
|
||||
# The production ROCm path uses the Triton quantizer and dot_scaled;
|
||||
# use the independent torch quantizer and FP32 matmul as the oracle.
|
||||
x_q, x_scale = _mxfp8_e4m3_quantize_torch(x)
|
||||
x = dequant_mxfp8_to_bf16(x_q, x_scale)
|
||||
weight = dequant_mxfp8_to_bf16(weight, layer.weight_scale)
|
||||
partial = (x.float() @ weight.float().T).to(torch.bfloat16).float()
|
||||
group = get_tp_group()
|
||||
# Always use FP32 RCCL AR for the oracle, including the RS candidate.
|
||||
dist.all_reduce(partial, group=group.device_group)
|
||||
expected = partial.to(torch.bfloat16)
|
||||
if all_reduce:
|
||||
return expected
|
||||
padding = -x.shape[0] % group.world_size
|
||||
expected = torch.nn.functional.pad(expected, (0, 0, 0, padding))
|
||||
return expected.chunk(group.world_size)[group.rank_in_group]
|
||||
|
||||
|
||||
def _apply(x: torch.Tensor, layer: RowParallelLinear) -> torch.Tensor:
|
||||
output, _ = layer(x)
|
||||
return output if layer.reduce_results else sp_reduce_scatter(output)
|
||||
|
||||
|
||||
def _assert_output(
|
||||
actual: torch.Tensor, expected: torch.Tensor, M: int, all_reduce: bool
|
||||
) -> None:
|
||||
assert actual.dtype == torch.bfloat16
|
||||
assert actual.shape == expected.shape
|
||||
assert torch.isfinite(actual).all()
|
||||
# Match the parent's tolerance at 0.1 weight scale, then bound total error.
|
||||
# BF16 partial/reduction rounding can dominate near a cancelled output.
|
||||
torch.testing.assert_close(actual, expected, rtol=5e-2, atol=0.4)
|
||||
error = (actual.float() - expected.float()).norm()
|
||||
assert error <= 5e-3 * expected.float().norm() + 1e-6
|
||||
if not all_reduce:
|
||||
rank_start = get_tp_group().rank_in_group * actual.shape[0]
|
||||
valid_rows = min(max(M - rank_start, 0), actual.shape[0])
|
||||
assert not actual[valid_rows:].any(), "RS padding must remain zero"
|
||||
|
||||
|
||||
def _run_mode(
|
||||
device: torch.device, backend: str, all_reduce: bool, world_size: int
|
||||
) -> None:
|
||||
group = get_tp_group()
|
||||
generator = torch.Generator(device=device)
|
||||
quant_config = (
|
||||
OnlineQuantizationConfig(QuantizationConfigArgs(linear="mxfp8"))
|
||||
if backend == "mxfp8"
|
||||
else None
|
||||
)
|
||||
layers = {}
|
||||
for K in sorted({K for _, K in _ROCM_SHAPES}):
|
||||
with torch.device(device):
|
||||
layer = RowParallelLinear(
|
||||
K * world_size,
|
||||
_N,
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
reduce_results=all_reduce,
|
||||
)
|
||||
generator.manual_seed(1000 + group.rank_in_group * 100 + K)
|
||||
weight = (
|
||||
torch.randn(_N, K, dtype=torch.bfloat16, device=device, generator=generator)
|
||||
* 0.1
|
||||
)
|
||||
layer.weight = torch.nn.Parameter(weight, requires_grad=False)
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
if backend == "mxfp8":
|
||||
from vllm.model_executor.kernels.linear.mxfp8.rocm_native import (
|
||||
RocmDotScaledMxfp8LinearKernel,
|
||||
)
|
||||
|
||||
assert isinstance(layer.quant_method.kernel, RocmDotScaledMxfp8LinearKernel)
|
||||
assert layer.weight.dtype == torch.float8_e4m3fn
|
||||
layers[K] = layer
|
||||
|
||||
def make_input(M: int, K: int, seed: int) -> torch.Tensor:
|
||||
generator.manual_seed(seed + group.rank_in_group * 100)
|
||||
return torch.randn(
|
||||
M, K, dtype=torch.bfloat16, device=device, generator=generator
|
||||
)
|
||||
|
||||
comm = group.device_communicator
|
||||
assert comm is not None
|
||||
assert comm.use_aiter_allreduce
|
||||
assert comm.aiter_ar_comm is not None and not comm.aiter_ar_comm.disabled
|
||||
assert comm.qr_comm is None or comm.qr_comm.disabled
|
||||
assert comm.pynccl_comm is not None and not comm.pynccl_comm.disabled
|
||||
for M, K in (*_ROCM_SHAPES, *_ROCM_SHAPES[::-1]):
|
||||
x = make_input(M, K, 2000 + M + K)
|
||||
if all_reduce:
|
||||
probe = torch.empty(M, _N, dtype=x.dtype, device=device)
|
||||
assert comm.aiter_ar_comm.should_custom_ar(probe)
|
||||
expected = _reference(x, layers[K], all_reduce)
|
||||
actual = _apply(x, layers[K])
|
||||
_assert_output(actual, expected, M, all_reduce)
|
||||
|
||||
# Retained outputs must survive the next call's collective buffer reuse.
|
||||
x = make_input(257, 768, 2500)
|
||||
first = _apply(x, layers[768])
|
||||
snapshot = first.clone()
|
||||
second = _apply(-x, layers[768])
|
||||
torch.accelerator.synchronize(device)
|
||||
assert first.data_ptr() != second.data_ptr()
|
||||
torch.testing.assert_close(first, snapshot, rtol=0, atol=0)
|
||||
|
||||
M, K = 1025, 4224
|
||||
graph_x = make_input(M, K, 3000)
|
||||
for _ in range(3):
|
||||
_apply(graph_x, layers[K])
|
||||
torch.accelerator.synchronize(device)
|
||||
dist.barrier(group=group.device_group)
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
# vLLM's context registers the AITER IPC addresses used by graph replay.
|
||||
with (
|
||||
graph_capture(device=device) as context,
|
||||
torch.cuda.graph(graph, stream=context.stream),
|
||||
):
|
||||
graph_output = _apply(graph_x, layers[K])
|
||||
torch.cuda.current_stream().wait_stream(context.stream)
|
||||
dist.barrier(group=group.device_group)
|
||||
for replay in range(3):
|
||||
graph_x.copy_(make_input(M, K, 3001 + replay))
|
||||
expected = _reference(graph_x, layers[K], all_reduce)
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize(device)
|
||||
_assert_output(graph_output, expected, M, all_reduce)
|
||||
dist.barrier(group=group.device_group)
|
||||
|
||||
|
||||
def _worker(rank: int, world_size: int, port: int, backend: str) -> None:
|
||||
device = torch.device("cuda", rank)
|
||||
torch.accelerator.set_device_index(device)
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(rank),
|
||||
"LOCAL_RANK": str(rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": str(port),
|
||||
}
|
||||
)
|
||||
config = VllmConfig()
|
||||
config.model_config = ModelConfig(dtype="bfloat16")
|
||||
try:
|
||||
with set_current_vllm_config(config), torch.inference_mode():
|
||||
init_distributed_environment()
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
for all_reduce in (False, True):
|
||||
_run_mode(device, backend, all_reduce, world_size)
|
||||
finally:
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm GEMM-RS/AR contract")
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
@pytest.mark.parametrize(
|
||||
"backend",
|
||||
[
|
||||
"bf16",
|
||||
pytest.param(
|
||||
"mxfp8",
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.supports_mx(), reason="native MXFP8 requires CDNA4"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.distributed(num_gpus=4)
|
||||
def test_rocm_gemm_rs_ar(
|
||||
monkeypatch: pytest.MonkeyPatch, world_size: int, backend: str
|
||||
) -> None:
|
||||
"""Preserve rank reduction, tails, shape reuse, lifetime, and graph replay."""
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"GEMM-RS/AR requires {world_size} GPUs")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_CUSTOM_AR", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_QUICK_REDUCE_QUANTIZATION", "NONE")
|
||||
try:
|
||||
mp.spawn(
|
||||
_worker, args=(world_size, get_open_port(), backend), nprocs=world_size
|
||||
)
|
||||
finally:
|
||||
cleanup_dist_env_and_memory()
|
||||
Reference in New Issue
Block a user