Merge branch 'main' into patches
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,187 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.backends.flash_attn import cascade_attention, merge_attn_states
|
||||
|
||||
try:
|
||||
from vllm.vllm_flash_attn import (
|
||||
fa_version_unsupported_reason,
|
||||
flash_attn_varlen_func,
|
||||
is_fa_version_supported,
|
||||
)
|
||||
except ImportError:
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"vllm_flash_attn is not supported for vLLM on ROCm.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
NUM_HEADS = [(4, 4), (8, 2), (16, 2)]
|
||||
HEAD_SIZES = [128, 192, 256]
|
||||
BLOCK_SIZES = [16]
|
||||
DTYPES = [torch.float16, torch.bfloat16]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 39, 16912])
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@torch.inference_mode()
|
||||
def test_merge_kernel(
|
||||
num_tokens: int,
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
torch.set_default_device("cuda")
|
||||
set_random_seed(0)
|
||||
num_query_heads = num_heads[0]
|
||||
num_kv_heads = num_heads[1]
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
|
||||
# Prepare inputs.
|
||||
prefix_output = torch.randn(num_tokens, num_query_heads, head_size, dtype=dtype)
|
||||
suffix_output = torch.randn(num_tokens, num_query_heads, head_size, dtype=dtype)
|
||||
prefix_lse = torch.randn(num_query_heads, num_tokens, dtype=torch.float32)
|
||||
suffix_lse = torch.randn(num_query_heads, num_tokens, dtype=torch.float32)
|
||||
|
||||
# Run the kernel.
|
||||
output = torch.empty(num_tokens, num_query_heads, head_size, dtype=dtype)
|
||||
merge_attn_states(output, prefix_output, prefix_lse, suffix_output, suffix_lse)
|
||||
|
||||
# Reference implementation.
|
||||
max_lse = torch.maximum(prefix_lse, suffix_lse)
|
||||
p_lse = torch.exp(prefix_lse - max_lse)
|
||||
s_lse = torch.exp(suffix_lse - max_lse)
|
||||
p_scale = p_lse / (p_lse + s_lse)
|
||||
s_scale = s_lse / (p_lse + s_lse)
|
||||
p_scale = p_scale.transpose(0, 1).unsqueeze(2)
|
||||
s_scale = s_scale.transpose(0, 1).unsqueeze(2)
|
||||
ref_output = p_scale * prefix_output + s_scale * suffix_output
|
||||
ref_output = ref_output.to(dtype)
|
||||
|
||||
# Compare the results.
|
||||
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
CASES = [
|
||||
# Case 1. A general case.
|
||||
([(129, 871), (18, 280), (37, 988), (1023, 2304), (1, 257)], 256),
|
||||
# Case 2. Flash-decoding case.
|
||||
([(1, 1023), (1, 879), (1, 778), (1, 1777)] * 100, 512),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seq_lens_and_common_prefix", CASES)
|
||||
@pytest.mark.parametrize("num_heads", NUM_HEADS)
|
||||
@pytest.mark.parametrize("head_size", HEAD_SIZES)
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("block_size", BLOCK_SIZES)
|
||||
@pytest.mark.parametrize("soft_cap", [None, 50])
|
||||
@pytest.mark.parametrize("num_blocks", [2048])
|
||||
@pytest.mark.parametrize("fa_version", [2, 3])
|
||||
@torch.inference_mode()
|
||||
def test_cascade(
|
||||
seq_lens_and_common_prefix: tuple[list[tuple[int, int]], int],
|
||||
num_heads: tuple[int, int],
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
block_size: int,
|
||||
soft_cap: float | None,
|
||||
num_blocks: int,
|
||||
fa_version: int,
|
||||
) -> None:
|
||||
torch.set_default_device("cuda")
|
||||
if not is_fa_version_supported(fa_version):
|
||||
pytest.skip(
|
||||
f"Flash attention version {fa_version} not supported due "
|
||||
f'to: "{fa_version_unsupported_reason(fa_version)}"'
|
||||
)
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
window_size = (-1, -1)
|
||||
scale = head_size**-0.5
|
||||
num_query_heads = num_heads[0]
|
||||
num_kv_heads = num_heads[1]
|
||||
assert num_query_heads % num_kv_heads == 0
|
||||
key_cache = torch.randn(
|
||||
num_blocks, block_size, num_kv_heads, head_size, dtype=dtype
|
||||
)
|
||||
value_cache = torch.randn_like(key_cache)
|
||||
|
||||
seq_lens, common_prefix_len = seq_lens_and_common_prefix
|
||||
num_seqs = len(seq_lens)
|
||||
query_lens = [x[0] for x in seq_lens]
|
||||
kv_lens = [x[1] for x in seq_lens]
|
||||
max_query_len = max(query_lens)
|
||||
max_kv_len = max(kv_lens)
|
||||
|
||||
total_num_query_tokens = sum(query_lens)
|
||||
query = torch.randn(total_num_query_tokens, num_query_heads, head_size, dtype=dtype)
|
||||
cu_query_lens = torch.tensor([0] + query_lens, dtype=torch.int32).cumsum(
|
||||
dim=0, dtype=torch.int32
|
||||
)
|
||||
kv_lens_tensor = torch.tensor(kv_lens, dtype=torch.int32)
|
||||
max_num_blocks_per_seq = (max_kv_len + block_size - 1) // block_size
|
||||
block_tables = torch.randint(
|
||||
0, num_blocks, (num_seqs, max_num_blocks_per_seq), dtype=torch.int32
|
||||
)
|
||||
|
||||
assert common_prefix_len > 0
|
||||
assert common_prefix_len % block_size == 0
|
||||
num_common_kv_blocks = common_prefix_len // block_size
|
||||
# Make sure the first `num_common_kv_blocks` blocks are the same.
|
||||
block_tables[:, :num_common_kv_blocks] = block_tables[0, :num_common_kv_blocks]
|
||||
|
||||
# Run the regular attention.
|
||||
ref_output = flash_attn_varlen_func(
|
||||
q=query,
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
cu_seqlens_q=cu_query_lens,
|
||||
seqused_k=kv_lens_tensor,
|
||||
max_seqlen_q=max_query_len,
|
||||
max_seqlen_k=max_kv_len,
|
||||
softmax_scale=scale,
|
||||
causal=True,
|
||||
window_size=list(window_size),
|
||||
block_table=block_tables,
|
||||
softcap=soft_cap if soft_cap is not None else 0,
|
||||
)
|
||||
|
||||
# Run cascade attention.
|
||||
assert all(common_prefix_len < kv_len for kv_len in kv_lens)
|
||||
cu_prefix_query_lens = torch.tensor([0, total_num_query_tokens], dtype=torch.int32)
|
||||
prefix_kv_lens = torch.tensor([common_prefix_len], dtype=torch.int32)
|
||||
suffix_kv_lens = kv_lens_tensor - common_prefix_len
|
||||
output = torch.empty_like(query)
|
||||
cascade_attention(
|
||||
output=output,
|
||||
query=query,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
cu_query_lens=cu_query_lens,
|
||||
max_query_len=max_query_len,
|
||||
cu_prefix_query_lens=cu_prefix_query_lens,
|
||||
prefix_kv_lens=prefix_kv_lens,
|
||||
suffix_kv_lens=suffix_kv_lens,
|
||||
max_kv_len=max_kv_len,
|
||||
softmax_scale=scale,
|
||||
alibi_slopes=None,
|
||||
sliding_window=window_size,
|
||||
logits_soft_cap=soft_cap if soft_cap is not None else 0,
|
||||
block_table=block_tables,
|
||||
common_prefix_len=common_prefix_len,
|
||||
max_num_splits=0, # no max
|
||||
fa_version=fa_version,
|
||||
)
|
||||
|
||||
# Compare the results.
|
||||
torch.testing.assert_close(output, ref_output, atol=1e-2, rtol=1e-2)
|
||||
@@ -5,6 +5,7 @@ import random
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.sparse_attn_indexer import as_fp32_logits
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
_ceil_to_ue8m0,
|
||||
@@ -13,18 +14,29 @@ from vllm.utils.deep_gemm import (
|
||||
fp8_fp4_paged_mqa_logits,
|
||||
get_num_sms,
|
||||
get_paged_mqa_logits_metadata,
|
||||
mqa_logits_mx_inputs,
|
||||
mqa_logits_use_mx_scales,
|
||||
native_next_n_supported,
|
||||
)
|
||||
from vllm.utils.import_utils import has_deep_gemm
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
|
||||
def encode_indexer_scales(sf: torch.Tensor) -> torch.Tensor:
|
||||
"""Per-token power-of-two fp32 scales as the indexer K cache stores them:
|
||||
four equal packed UE8M0 bytes under MX scales, else the fp32 itself."""
|
||||
if not mqa_logits_use_mx_scales():
|
||||
return sf
|
||||
ue8m0 = (sf.view(torch.int32) >> 23).to(torch.uint8)
|
||||
return ue8m0[..., None].repeat_interleave(4, dim=-1).view(torch.int32)[..., 0]
|
||||
|
||||
|
||||
def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
|
||||
# x: (num_blocks, block_size, 1, head_dim)
|
||||
num_blocks, block_size, num_heads, head_dim = x.shape
|
||||
assert num_heads == 1
|
||||
x_amax = x.abs().float().amax(dim=3, keepdim=True).clamp(1e-4)
|
||||
sf = x_amax / 448.0
|
||||
sf = _ceil_to_ue8m0(x_amax / 448.0)
|
||||
x_scaled = (x * (1.0 / sf)).to(torch.float8_e4m3fn)
|
||||
x_fp8 = torch.empty(
|
||||
(num_blocks, block_size * (head_dim + 4)),
|
||||
@@ -34,9 +46,9 @@ def kv_cache_cast_to_fp8(x: torch.Tensor) -> torch.Tensor:
|
||||
x_fp8[:, : block_size * head_dim] = x_scaled.view(
|
||||
num_blocks, block_size * head_dim
|
||||
).view(dtype=torch.uint8)
|
||||
x_fp8[:, block_size * head_dim :] = sf.view(num_blocks, block_size).view(
|
||||
dtype=torch.uint8
|
||||
)
|
||||
x_fp8[:, block_size * head_dim :] = encode_indexer_scales(
|
||||
sf.view(num_blocks, block_size)
|
||||
).view(dtype=torch.uint8)
|
||||
return x_fp8.view(num_blocks, block_size, num_heads, head_dim + 4)
|
||||
|
||||
|
||||
@@ -91,64 +103,72 @@ def _ref_fp8_mqa_logits(
|
||||
return logits
|
||||
|
||||
|
||||
def _packed_requests_test_data(seq_len: int, seq_len_kv: int):
|
||||
# Two causal prefill requests packed into one KV buffer, so the second
|
||||
# request's rows start at a nonzero K position.
|
||||
half, half_kv = seq_len // 2, seq_len_kv // 2
|
||||
rows = torch.arange(seq_len, dtype=torch.int, device="cuda")
|
||||
ks = torch.where(rows < half, 0, half_kv).to(torch.int)
|
||||
ke = ks + half_kv - half + 1 + rows % half
|
||||
return ks, ke
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
|
||||
)
|
||||
@pytest.mark.parametrize("clean_logits", [True, False])
|
||||
def test_deepgemm_fp8_mqa_logits(clean_logits: bool):
|
||||
@pytest.mark.parametrize("layout", ["cp", "no_cp", "packed"])
|
||||
def test_deepgemm_fp8_mqa_logits(layout: str):
|
||||
"""FP8 indexer prefill logits, compressed to each row's valid K span."""
|
||||
torch.manual_seed(0)
|
||||
random.seed(0)
|
||||
num_heads, head_dim = 32, 128
|
||||
for seq_len in (512,):
|
||||
for seq_len_kv in (1024,):
|
||||
for disable_cp in (False, True):
|
||||
q = torch.randn(
|
||||
seq_len,
|
||||
num_heads,
|
||||
head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv = torch.randn(
|
||||
seq_len_kv, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
)
|
||||
weights = torch.randn(
|
||||
seq_len, num_heads, device="cuda", dtype=torch.float32
|
||||
)
|
||||
seq_len, seq_len_kv = 512, 1024
|
||||
q = torch.randn(seq_len, num_heads, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
kv = torch.randn(seq_len_kv, head_dim, device="cuda", dtype=torch.bfloat16)
|
||||
weights = torch.randn(seq_len, num_heads, device="cuda", dtype=torch.float32)
|
||||
|
||||
if disable_cp:
|
||||
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
|
||||
ke = torch.arange(seq_len, dtype=torch.int, device="cuda") + (
|
||||
seq_len_kv - seq_len
|
||||
)
|
||||
else:
|
||||
ks, ke = _generate_cp_test_data(seq_len, seq_len_kv)
|
||||
if layout == "cp":
|
||||
ks, ke = _generate_cp_test_data(seq_len, seq_len_kv)
|
||||
elif layout == "no_cp":
|
||||
ks = torch.zeros(seq_len, dtype=torch.int, device="cuda")
|
||||
ke = torch.arange(seq_len, dtype=torch.int, device="cuda") + (
|
||||
seq_len_kv - seq_len
|
||||
)
|
||||
else:
|
||||
ks, ke = _packed_requests_test_data(seq_len, seq_len_kv)
|
||||
|
||||
q_fp8 = q.to(torch.float8_e4m3fn)
|
||||
kv_fp8 = per_custom_dims_cast_to_fp8(kv, (0,), False)
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_fp8, None), kv_fp8, weights, ks, ke, clean_logits=clean_logits
|
||||
)
|
||||
q_fp8 = q.to(torch.float8_e4m3fn)
|
||||
kv_fp8, kv_sf = per_custom_dims_cast_to_fp8(kv, (0,), True)
|
||||
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
|
||||
max_seqlen_k = int((ke - ks).max())
|
||||
logits = fp8_fp4_mqa_logits(
|
||||
(q_fp8, q_scale),
|
||||
(kv_fp8, encode_indexer_scales(kv_sf)),
|
||||
kernel_weights,
|
||||
ks,
|
||||
ke,
|
||||
max_seqlen_k,
|
||||
)
|
||||
assert logits.shape == (seq_len, max_seqlen_k)
|
||||
logits = as_fp32_logits(logits, ke - ks)
|
||||
|
||||
ref_logits = _ref_fp8_mqa_logits(
|
||||
q=q,
|
||||
kv=kv,
|
||||
weights=weights,
|
||||
cu_seqlen_ks=ks,
|
||||
cu_seqlen_ke=ke,
|
||||
)
|
||||
ref_neginf_mask = ref_logits == float("-inf")
|
||||
|
||||
if clean_logits:
|
||||
neginf_mask = logits == float("-inf")
|
||||
assert torch.equal(neginf_mask, ref_neginf_mask)
|
||||
|
||||
ref_logits = ref_logits.masked_fill(ref_neginf_mask, 0)
|
||||
logits = logits.masked_fill(ref_neginf_mask, 0)
|
||||
diff = calc_diff(logits, ref_logits)
|
||||
assert diff < 1e-3, f"{diff=}"
|
||||
ref_logits = _ref_fp8_mqa_logits(
|
||||
q=q,
|
||||
kv=kv,
|
||||
weights=kernel_weights.float(),
|
||||
cu_seqlen_ks=ks,
|
||||
cu_seqlen_ke=ke,
|
||||
)
|
||||
# Compressed row i holds K positions [ks[i], ke[i]) from column zero.
|
||||
cols = ks[:, None] + torch.arange(max_seqlen_k, device="cuda")[None, :]
|
||||
valid = cols < ke[:, None]
|
||||
ref_logits = ref_logits.gather(1, cols.clamp(max=seq_len_kv - 1).long())
|
||||
ref_logits = ref_logits.masked_fill(~valid, 0)
|
||||
logits = logits.masked_fill(~valid, 0)
|
||||
diff = calc_diff(logits, ref_logits)
|
||||
assert diff < 1e-3, f"{diff=}"
|
||||
|
||||
|
||||
def _ref_fp8_fp4_paged_mqa_logits(
|
||||
@@ -212,9 +232,6 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
|
||||
if not native_next_n_supported(next_n):
|
||||
pytest.skip(f"next_n={next_n} has no native kernel on this architecture")
|
||||
|
||||
# NOTE: clean_logits=True is incompatible with the 2D context_lens
|
||||
# required by csrc/apis/attention.hpp; only the False path is exercised.
|
||||
clean_logits = False
|
||||
torch.manual_seed(0)
|
||||
random.seed(0)
|
||||
|
||||
@@ -273,26 +290,42 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
|
||||
context_lens_2d = (
|
||||
context_lens.unsqueeze(-1) - next_n + 1 + next_n_arange
|
||||
).contiguous()
|
||||
q_scale, kernel_weights = mqa_logits_mx_inputs(q_fp8, None, weights)
|
||||
kernel_q = (q_fp8, q_scale)
|
||||
kernel_context_lens, kernel_block_tables = context_lens_2d, block_tables
|
||||
indices = None
|
||||
if mqa_logits_use_mx_scales():
|
||||
# SM100 only takes varlen rows: one Q token each, tied to its
|
||||
# request by `indices`.
|
||||
assert q_scale is not None
|
||||
kernel_q = (
|
||||
q_fp8.flatten(0, 1)[:, None],
|
||||
q_scale.flatten(0, 1)[:, None],
|
||||
)
|
||||
kernel_context_lens = context_lens_2d.view(-1, 1)
|
||||
kernel_block_tables = block_tables.repeat_interleave(next_n, dim=0)
|
||||
indices = torch.arange(batch_size, device="cuda", dtype=torch.int32)
|
||||
indices = indices.repeat_interleave(next_n)
|
||||
schedule_metadata = get_paged_mqa_logits_metadata(
|
||||
context_lens_2d,
|
||||
blocksize,
|
||||
get_num_sms(),
|
||||
kernel_context_lens, blocksize, get_num_sms(), indices=indices
|
||||
)
|
||||
logits = fp8_fp4_paged_mqa_logits(
|
||||
(q_fp8, None),
|
||||
kernel_q,
|
||||
kv_cache_fp8,
|
||||
weights,
|
||||
context_lens_2d,
|
||||
block_tables,
|
||||
kernel_weights,
|
||||
kernel_context_lens,
|
||||
kernel_block_tables,
|
||||
schedule_metadata,
|
||||
max_model_len,
|
||||
clean_logits=clean_logits,
|
||||
indices=indices,
|
||||
)
|
||||
if indices is not None:
|
||||
logits = as_fp32_logits(logits, kernel_context_lens.view(-1))
|
||||
|
||||
ref_logits = _ref_fp8_fp4_paged_mqa_logits(
|
||||
q,
|
||||
kv_cache,
|
||||
weights,
|
||||
kernel_weights.float(),
|
||||
context_lens,
|
||||
block_tables,
|
||||
max_model_len,
|
||||
@@ -309,7 +342,69 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
|
||||
context_lens[row_indices] - next_n + next_n_offset
|
||||
).unsqueeze(1)
|
||||
|
||||
logits = logits.masked_fill(~mask, 0)
|
||||
logits = logits.float().masked_fill(~mask, 0)
|
||||
ref_logits = ref_logits.masked_fill(~mask, 0)
|
||||
diff = calc_diff(logits, ref_logits)
|
||||
assert diff < 1e-3, f"{diff=}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
|
||||
)
|
||||
def test_deepgemm_paged_mqa_packed_manager_block_stride():
|
||||
torch.manual_seed(0)
|
||||
num_blocks, block_size, packed_rows = 4, 256, 320
|
||||
num_heads, head_dim, row_bytes = 64, 128, 132
|
||||
page_size, pages_per_block = 64, block_size // 64
|
||||
dense_stride, packed_stride = pages_per_block, packed_rows // page_size
|
||||
compact_pages = kv_cache_cast_to_fp8(
|
||||
torch.randn(
|
||||
num_blocks, block_size, 1, head_dim, device="cuda", dtype=torch.bfloat16
|
||||
).view(num_blocks * pages_per_block, page_size, 1, head_dim)
|
||||
)
|
||||
packed_backing = torch.zeros(
|
||||
num_blocks,
|
||||
packed_rows,
|
||||
row_bytes,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
packed_manager = packed_backing[:, :block_size]
|
||||
packed_manager.copy_(compact_pages.squeeze(2).view(num_blocks, block_size, -1))
|
||||
packed_pages = packed_backing.view(-1, page_size, row_bytes).unsqueeze(2)
|
||||
offsets = torch.arange(pages_per_block, device="cuda", dtype=torch.int32)
|
||||
tables = [
|
||||
(stride + offsets).unsqueeze(0) for stride in (dense_stride, packed_stride)
|
||||
]
|
||||
|
||||
q = torch.randn(1, 1, num_heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
|
||||
torch.float8_e4m3fn
|
||||
)
|
||||
q_scale, weights = mqa_logits_mx_inputs(
|
||||
q, None, torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
|
||||
)
|
||||
context_lens = torch.full((1, 1), block_size, device="cuda", dtype=torch.int32)
|
||||
indices = (
|
||||
torch.zeros(1, device="cuda", dtype=torch.int32)
|
||||
if mqa_logits_use_mx_scales()
|
||||
else None
|
||||
)
|
||||
schedule = get_paged_mqa_logits_metadata(
|
||||
context_lens, page_size, get_num_sms(), indices=indices
|
||||
)
|
||||
logits = [
|
||||
fp8_fp4_paged_mqa_logits(
|
||||
(q, q_scale),
|
||||
cache,
|
||||
weights,
|
||||
context_lens,
|
||||
table,
|
||||
schedule,
|
||||
block_size,
|
||||
indices=indices,
|
||||
)
|
||||
for cache, table in zip((compact_pages, packed_pages), tables)
|
||||
]
|
||||
torch.testing.assert_close(*logits, rtol=0, atol=0)
|
||||
|
||||
@@ -175,7 +175,15 @@ def test_flashinfer_trtllm_sparse_mla_decode_without_rope(dtype, mode, monkeypat
|
||||
backend, "prepare_sparse_mla_safe_lengths", unexpected_helper
|
||||
)
|
||||
|
||||
out, lse = impl._run_mqa_kernel(query.squeeze(1), kv_cache, indices, counts)
|
||||
out, lse = impl._forward_mqa_kernel(
|
||||
query.squeeze(1),
|
||||
kv_cache,
|
||||
indices,
|
||||
counts,
|
||||
layer=None,
|
||||
block_size=block_size,
|
||||
is_decode=True,
|
||||
)
|
||||
reference = trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=query,
|
||||
kv_cache=kv_cache.unsqueeze(1),
|
||||
|
||||
@@ -1833,6 +1833,35 @@ def test_aiter_indexer_requires_the_aiter_attend(monkeypatch):
|
||||
assert reason is None or "AITER sparse PA attend" not in reason
|
||||
|
||||
|
||||
def test_aiter_indexer_rejects_unbuilt_index_heads(monkeypatch):
|
||||
"""AITER builds the MSA score kernels for 1 or 2 index heads only.
|
||||
|
||||
MiniMax-M3 has 4, so at attention TP=1 the indexer has to fall back at load
|
||||
time rather than fail at the first forward.
|
||||
"""
|
||||
import vllm.models.minimax_m3.amd.indexer_aiter as indexer_aiter_mod
|
||||
|
||||
monkeypatch.setattr(indexer_aiter_mod.current_platform, "is_rocm", lambda: True)
|
||||
monkeypatch.setattr(
|
||||
indexer_aiter_mod, "_minimax_m3_aiter_sparse_pa_requested", lambda: True
|
||||
)
|
||||
|
||||
def unsupported_reason(num_index_heads: int) -> str | None:
|
||||
return indexer_aiter_mod.aiter_indexer_unsupported_reason(
|
||||
topk_blocks=TOPK,
|
||||
sparse_block_size=BLOCK_SIZE,
|
||||
num_index_heads=num_index_heads,
|
||||
index_head_dim=HEAD_DIM,
|
||||
indexer_kv_dtype="fp8",
|
||||
max_model_len=8192,
|
||||
)
|
||||
|
||||
if reason := unsupported_reason(2):
|
||||
pytest.skip(f"AITER MSA indexer unavailable: {reason}")
|
||||
reason = unsupported_reason(4)
|
||||
assert reason is not None and "num_index_heads" in reason
|
||||
|
||||
|
||||
def test_aiter_consolidated_qknorm_enabled(monkeypatch):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
|
||||
@@ -65,30 +65,40 @@ SM_SCALE = HEAD_DIM**-0.5
|
||||
"decode_query_len",
|
||||
"num_q_heads",
|
||||
"num_kv_heads",
|
||||
"is_sm103",
|
||||
"expected",
|
||||
),
|
||||
[
|
||||
pytest.param(8, 4, 64, 4, False, id="tp1-below-min-batch"),
|
||||
pytest.param(16, 4, 64, 4, True, id="tp1-supported"),
|
||||
pytest.param(16, 4, 16, 1, True, id="tp4-min-batch"),
|
||||
pytest.param(24, 4, 16, 1, True, id="tp4-intermediate-batch"),
|
||||
pytest.param(32, 4, 16, 1, True, id="tp4-supported"),
|
||||
pytest.param(16, 1, 64, 4, True, id="tp1-query-len-1"),
|
||||
pytest.param(16, 1, 16, 1, True, id="tp4-query-len-1"),
|
||||
pytest.param(16, 2, 64, 4, True, id="tp1-query-len-2"),
|
||||
pytest.param(16, 2, 16, 1, True, id="tp4-query-len-2"),
|
||||
pytest.param(16, 32, 64, 4, True, id="query-len-upper-bound"),
|
||||
pytest.param(16, 0, 64, 4, False, id="query-len-zero"),
|
||||
pytest.param(16, 33, 64, 4, False, id="query-len-above-bound"),
|
||||
pytest.param(15, 1, 64, 4, True, False, id="qlen1-below-row-floor"),
|
||||
pytest.param(16, 1, 64, 4, True, True, id="qlen1-at-row-floor"),
|
||||
pytest.param(10, 3, 32, 2, True, False, id="qlen3-below-row-floor"),
|
||||
pytest.param(11, 3, 32, 2, True, True, id="qlen3-at-row-floor"),
|
||||
pytest.param(3, 4, 64, 4, True, False, id="qlen4-below-row-floor"),
|
||||
pytest.param(4, 4, 64, 4, True, True, id="qlen4-at-row-floor"),
|
||||
pytest.param(7, 8, 16, 1, True, False, id="qlen8-below-row-floor"),
|
||||
pytest.param(8, 8, 16, 1, True, True, id="qlen8-at-row-floor"),
|
||||
pytest.param(15, 9, 64, 4, True, False, id="unmeasured-query-length"),
|
||||
pytest.param(15, 4, 8, 1, True, False, id="unmeasured-heads"),
|
||||
pytest.param(15, 4, 64, 4, False, False, id="sm100-keeps-old-floor"),
|
||||
pytest.param(16, 32, 64, 4, True, True, id="query-len-upper-bound"),
|
||||
pytest.param(16, 0, 64, 4, True, False, id="query-len-zero"),
|
||||
pytest.param(16, 33, 64, 4, True, False, id="query-len-above-bound"),
|
||||
],
|
||||
)
|
||||
def test_msa_cutlass_decode_static_dispatch(
|
||||
batch_size: int,
|
||||
decode_query_len: int,
|
||||
expected: bool,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
is_sm103: bool,
|
||||
expected: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(
|
||||
current_platform,
|
||||
"is_device_capability",
|
||||
lambda capability: is_sm103 and capability == (10, 3),
|
||||
)
|
||||
assert (
|
||||
should_prepare_decode_metadata(
|
||||
batch_size,
|
||||
@@ -291,6 +301,7 @@ def test_msa_metadata_builder_prepares_cutlass_for_regular_decode(
|
||||
builder.kv_cache_dtype = "fp8_e4m3"
|
||||
builder.decode_backend = "cutlass"
|
||||
builder.msa_cutlass_plan_cache = object()
|
||||
builder.msa_cutlass_min_batch_size = None
|
||||
|
||||
metadata = builder.build(
|
||||
0,
|
||||
@@ -486,6 +497,10 @@ def _make_topk(
|
||||
pytest.param(64, 4, 8, 8, False, id="tp1-query-len-8"),
|
||||
pytest.param(16, 1, 8, 1, True, id="tp4-query-len-1"),
|
||||
pytest.param(16, 1, 16, 4, True, id="tp4-query-len-4"),
|
||||
# Measured B300 EAGLE3 dispatch boundaries.
|
||||
pytest.param(64, 4, 2, 4, True, id="tp1-batch-4-query-len-4"),
|
||||
pytest.param(32, 2, 4, 4, True, id="tp2-batch-8-query-len-4"),
|
||||
pytest.param(16, 1, 8, 4, True, id="tp4-batch-16-query-len-4"),
|
||||
],
|
||||
)
|
||||
def test_msa_cutlass_decode_matches_triton_with_interleaved_cache(
|
||||
@@ -782,6 +797,46 @@ def _make_nvfp4_kv_cache(
|
||||
return kv_cache, torch.cat(dequantized, dim=-1).to(torch.bfloat16)
|
||||
|
||||
|
||||
def _e4m3_staged_kv_reference(
|
||||
kv_reference: torch.Tensor, k_scale: torch.Tensor, v_scale: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
"""The dequantized NVFP4 cache as an E4M3-query kernel reads it: each
|
||||
code * block_scale rounded to E4M3, then the global scale."""
|
||||
sides = []
|
||||
for side, global_scale in zip(
|
||||
kv_reference.float().split(HEAD_DIM, dim=-1), (k_scale, v_scale)
|
||||
):
|
||||
sides.append(
|
||||
(side / global_scale).to(torch.float8_e4m3fn).float() * global_scale
|
||||
)
|
||||
return torch.cat(sides, dim=-1).to(torch.bfloat16)
|
||||
|
||||
|
||||
def _assert_close_fp8_prefill(actual: torch.Tensor, expected: torch.Tensor) -> None:
|
||||
"""The fp8-q prefill rounds each split's probabilities to E4M3, and the splits
|
||||
follow the SM count, so a rare element moves past the BF16 tolerance."""
|
||||
close = torch.isclose(actual, expected, atol=0.05, rtol=0.05)
|
||||
assert close.float().mean().item() > 1 - 1e-4
|
||||
torch.testing.assert_close(actual, expected, atol=0.25, rtol=0.05)
|
||||
|
||||
|
||||
def _record_query_dtypes(
|
||||
monkeypatch: pytest.MonkeyPatch, kernel_name: str
|
||||
) -> list[torch.dtype]:
|
||||
"""Record the query dtype of every ``fmha_sm100.sparse.<kernel_name>`` call."""
|
||||
from vllm.third_party.fmha_sm100 import sparse as msa_sparse
|
||||
|
||||
kernel = getattr(msa_sparse, kernel_name)
|
||||
query_dtypes: list[torch.dtype] = []
|
||||
|
||||
def record(q, *args, **kwargs):
|
||||
query_dtypes.append(q.dtype)
|
||||
return kernel(q, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(msa_sparse, kernel_name, record)
|
||||
return query_dtypes
|
||||
|
||||
|
||||
def test_nvfp4_kv_cache_views_address_head_slots() -> None:
|
||||
"""Slot 2h (K) and 2h + 1 (V) hold head h's data then scales, so each head
|
||||
is one contiguous run of the page: a TP rank's page is a byte slice."""
|
||||
@@ -898,27 +953,14 @@ def test_msa_cutlass_decode_nvfp4_matches_triton_on_dequantized_cache(
|
||||
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_q_heads", "num_kv_heads"),
|
||||
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
|
||||
)
|
||||
def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
|
||||
num_q_heads: int,
|
||||
def _make_prefill_metadata(
|
||||
q_lens: list[int],
|
||||
seq_lens_list: list[int],
|
||||
block_table: torch.Tensor,
|
||||
num_kv_heads: int,
|
||||
) -> None:
|
||||
"""MSA prefill reads NVFP4 pages as their exact dequantized values,
|
||||
including chunked prefill over cached context, and takes q from its fp8
|
||||
copy (the only one the fused insert writes on this path)."""
|
||||
torch.manual_seed(0)
|
||||
q_lens = [300, 128, 77]
|
||||
seq_lens_list = [300, 640, 1100]
|
||||
block_table, num_pages = _make_paged_layout(seq_lens_list)
|
||||
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
|
||||
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
|
||||
kv_cache, kv_reference = _make_nvfp4_kv_cache(
|
||||
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
|
||||
)
|
||||
|
||||
) -> tuple[MiniMaxM3SparseMetadata, torch.Tensor]:
|
||||
"""Prefill-only metadata (chunks over cached context) and a top-k selecting
|
||||
every causally visible page (<= TOPK per query), ending with the local one."""
|
||||
total_q = sum(q_lens)
|
||||
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32, device="cuda")
|
||||
context_lens = seq_lens - torch.tensor(q_lens, dtype=torch.int32, device="cuda")
|
||||
@@ -948,7 +990,6 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
|
||||
prefill=prefill,
|
||||
)
|
||||
|
||||
# Every causally visible page (<= TOPK per query), ending with the local one.
|
||||
topk = torch.full(
|
||||
(total_q, num_kv_heads, TOPK), -1, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
@@ -959,6 +1000,35 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
|
||||
assert visible <= TOPK
|
||||
topk[token, :, :visible] = torch.arange(visible, device="cuda")
|
||||
token += 1
|
||||
return metadata, topk
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_q_heads", "num_kv_heads"),
|
||||
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
|
||||
)
|
||||
def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
) -> None:
|
||||
"""MSA prefill reads NVFP4 pages as their dequantized values, staged to E4M3
|
||||
for the E4M3 q, including chunked prefill over cached context; it attends
|
||||
with the fp8 q (the only one the fused insert writes on this path)."""
|
||||
torch.manual_seed(0)
|
||||
q_lens = [300, 128, 77]
|
||||
seq_lens_list = [300, 640, 1100]
|
||||
block_table, num_pages = _make_paged_layout(seq_lens_list)
|
||||
k_scale = torch.tensor(0.5, dtype=torch.float32, device="cuda")
|
||||
v_scale = torch.tensor(2.0, dtype=torch.float32, device="cuda")
|
||||
kv_cache, kv_reference = _make_nvfp4_kv_cache(
|
||||
num_pages, num_kv_heads, "nvfp4", k_scale, v_scale
|
||||
)
|
||||
|
||||
metadata, topk = _make_prefill_metadata(
|
||||
q_lens, seq_lens_list, block_table, num_kv_heads
|
||||
)
|
||||
total_q = sum(q_lens)
|
||||
|
||||
layer_name = "model.layers.0.self_attn.attn"
|
||||
q_scale = 0.5
|
||||
@@ -976,6 +1046,7 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
|
||||
/ q_scale
|
||||
).to(torch.float8_e4m3fn)
|
||||
query = query_fp8.to(torch.bfloat16) * q_scale
|
||||
kv_reference = _e4m3_staged_kv_reference(kv_reference, k_scale, v_scale)
|
||||
common = dict(
|
||||
num_heads=num_q_heads,
|
||||
head_size=HEAD_DIM,
|
||||
@@ -984,6 +1055,7 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
|
||||
topk_blocks=TOPK,
|
||||
sparse_block_size=BLOCK_SIZE,
|
||||
)
|
||||
query_dtypes = _record_query_dtypes(monkeypatch, "sparse_atten_nvfp4_kv_func")
|
||||
forward_context = ForwardContext(
|
||||
no_compile_layers={},
|
||||
attn_metadata={layer_name: metadata},
|
||||
@@ -1001,7 +1073,8 @@ def test_msa_prefill_nvfp4_matches_triton_on_dequantized_cache(
|
||||
query_fp8=query_fp8,
|
||||
)
|
||||
current_platform.synchronize()
|
||||
torch.testing.assert_close(actual, expected, atol=0.05, rtol=0.05)
|
||||
assert query_dtypes == [torch.float8_e4m3fn]
|
||||
_assert_close_fp8_prefill(actual, expected)
|
||||
|
||||
|
||||
def test_msa_triton_decode_fallback_reads_dequantized_query_fp8() -> None:
|
||||
@@ -1079,3 +1152,80 @@ def test_msa_triton_decode_fallback_reads_dequantized_query_fp8() -> None:
|
||||
)
|
||||
current_platform.synchronize()
|
||||
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_q_heads", "num_kv_heads"),
|
||||
[pytest.param(64, 4, id="tp1"), pytest.param(16, 1, id="tp4")],
|
||||
)
|
||||
def test_msa_prefill_reads_query_fp8_over_fp8_cache(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
num_q_heads: int,
|
||||
num_kv_heads: int,
|
||||
) -> None:
|
||||
"""With CUTLASS decode on, FP8 prefill (incl. chunks over cached context)
|
||||
attends with the fp8 q the fused insert writes, not a BF16 copy, and
|
||||
matches Triton on the dequantized q."""
|
||||
torch.manual_seed(0)
|
||||
q_lens = [300, 128, 77]
|
||||
seq_lens_list = [300, 640, 1100]
|
||||
block_table, num_pages = _make_paged_layout(seq_lens_list)
|
||||
kv_cache = (
|
||||
torch.randn(
|
||||
num_pages,
|
||||
num_kv_heads,
|
||||
BLOCK_SIZE,
|
||||
2 * HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
* 0.5
|
||||
).to(torch.float8_e4m3fn)
|
||||
metadata, topk = _make_prefill_metadata(
|
||||
q_lens, seq_lens_list, block_table, num_kv_heads
|
||||
)
|
||||
layer_name = "model.layers.0.self_attn.attn"
|
||||
q_scale = 0.5
|
||||
layer = SimpleNamespace(
|
||||
layer_name=layer_name,
|
||||
topk_indices_buffer=topk,
|
||||
_q_scale_float=q_scale,
|
||||
)
|
||||
query_fp8 = (
|
||||
torch.randn(
|
||||
sum(q_lens), num_q_heads * HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
/ q_scale
|
||||
).to(torch.float8_e4m3fn)
|
||||
query = query_fp8.to(torch.bfloat16) * q_scale
|
||||
common = dict(
|
||||
num_heads=num_q_heads,
|
||||
head_size=HEAD_DIM,
|
||||
scale=SM_SCALE,
|
||||
num_kv_heads=num_kv_heads,
|
||||
kv_cache_dtype="fp8",
|
||||
topk_blocks=TOPK,
|
||||
sparse_block_size=BLOCK_SIZE,
|
||||
)
|
||||
impl = MiniMaxM3SparseMSAImpl(**common, msa_decode_backend="cutlass")
|
||||
assert impl.use_cutlass_decode
|
||||
query_dtypes = _record_query_dtypes(monkeypatch, "sparse_atten_func")
|
||||
forward_context = ForwardContext(
|
||||
no_compile_layers={},
|
||||
attn_metadata={layer_name: metadata},
|
||||
slot_mapping={},
|
||||
)
|
||||
with override_forward_context(forward_context):
|
||||
expected = MiniMaxM3SparseTritonImpl(**common).forward(
|
||||
layer, query, kv_cache, torch.empty_like(query)
|
||||
)
|
||||
actual = impl.forward(
|
||||
layer,
|
||||
torch.full_like(query, float("nan")),
|
||||
kv_cache,
|
||||
torch.zeros_like(query),
|
||||
query_fp8=query_fp8,
|
||||
)
|
||||
current_platform.synchronize()
|
||||
assert query_dtypes == [torch.float8_e4m3fn]
|
||||
_assert_close_fp8_prefill(actual, expected)
|
||||
|
||||
@@ -346,7 +346,6 @@ def test_rocm_backend_sliding_window_includes_boundary_token(
|
||||
value_cache[..., :seq_len] = value.permute(1, 2, 0)
|
||||
scale = torch.tensor(1.0, device=device)
|
||||
metadata = SimpleNamespace(
|
||||
use_cascade=False,
|
||||
num_actual_tokens=query_len,
|
||||
query_start_loc=torch.tensor([0, query_len], dtype=torch.int32, device=device),
|
||||
seq_lens=torch.tensor([seq_len], dtype=torch.int32, device=device),
|
||||
|
||||
@@ -47,16 +47,18 @@ pytestmark = pytest.mark.skipif(
|
||||
#
|
||||
# The kernel requires 16-aligned heads, so _mla_fp8_prefill_attn replicate-pads
|
||||
# q/k/v up to get_fp8_prefill_num_heads(num_heads) and slices the output back.
|
||||
# Cover all three regimes:
|
||||
# Cover all four regimes:
|
||||
# 8 -> padded to 16 (divisor, repeat_interleave), so the real heads sit at
|
||||
# stride 2 in the padded output. Live case: Kimi-K2.x at TP8.
|
||||
# 12 -> padded to 16 (non-divisor, tile+slice). Live case: a K3 rank at TP8.
|
||||
# 16 -> no padding, output aliases the caller's buffer.
|
||||
# 24 -> padded to 32. No current model and TP reaches this band (96 heads
|
||||
# would need TP4, which does not fit in 288 GiB; 128-head models give
|
||||
# 128/64/32/16/8), so this case exists to keep the general path honest
|
||||
# for future architectures: it is the only one of the three where
|
||||
# padding above 16 actually happens, so it is what proves the
|
||||
# replicate-pad/slice-back argument holds for a target other than 16.
|
||||
HEAD_COUNTS = [12, 16, 24]
|
||||
# for future architectures: it is the only one where padding above
|
||||
# 16 actually happens, so it is what proves the replicate-pad/slice-back
|
||||
# argument holds for a target other than 16.
|
||||
HEAD_COUNTS = [8, 12, 16, 24]
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
QK_HEAD_DIM = QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM # 192
|
||||
|
||||
@@ -33,6 +33,26 @@ def _require_aiter():
|
||||
pytest.skip("aiter is required on supported ROCm hardware for this test")
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_gather_kv_b_proj_op_schema() -> None:
|
||||
"""Validate fake tensors and mutation of both expanded K/V outputs."""
|
||||
_require_aiter()
|
||||
from vllm._aiter_ops import rocm_aiter_ops # noqa: F401
|
||||
|
||||
rows, num_heads = 32, 12
|
||||
cache = torch.randn(rows, 1, 576, device="cuda").to(current_platform.fp8_dtype())
|
||||
scale = torch.tensor([0.07], device="cuda")
|
||||
cu_seq_lens = torch.tensor([0, 13, rows], dtype=torch.int32, device="cuda")
|
||||
indices = torch.arange(rows, dtype=torch.int32, device="cuda").flip(0)
|
||||
weight = torch.randn(num_heads * 256, 512, dtype=torch.bfloat16, device="cuda")
|
||||
k = torch.empty(rows, num_heads, 192, dtype=torch.bfloat16, device="cuda")
|
||||
v = torch.empty(rows, num_heads, 128, dtype=torch.bfloat16, device="cuda")
|
||||
opcheck(
|
||||
torch.ops.vllm.rocm_aiter_gather_kv_b_proj,
|
||||
(cache, scale, cu_seq_lens, indices, cu_seq_lens, weight, k, v),
|
||||
)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_mla_decode_fwd_op_schema() -> None:
|
||||
"""Opcheck validates registration, schema, fake-tensor, and ``o`` aliasing.
|
||||
|
||||
@@ -463,7 +463,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
|
||||
set_random_seed(412)
|
||||
num_tokens, num_heads, block_size = 2, 8, 16
|
||||
q = torch.randn(num_tokens, num_heads, Q_HEAD_DIM, device="cuda").to(dtype)
|
||||
kv = torch.randn(2 * block_size, 1, Q_HEAD_DIM, device="cuda").to(dtype)
|
||||
kv = torch.randn(2, 2, block_size, Q_HEAD_DIM, device="cuda").to(dtype)[:, 1]
|
||||
selected = torch.tensor([[1, 7, 18], [0, 3, 20]], dtype=torch.int32, device="cuda")
|
||||
sinks = torch.linspace(-3.0, 5.0, num_heads, device="cuda")
|
||||
impl = object.__new__(ROCMAiterMLASparseImpl)
|
||||
@@ -476,6 +476,8 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
|
||||
(num_tokens, 128), -1, dtype=torch.int32, device="cuda"
|
||||
)
|
||||
impl.topk_indices_buffer[:, : selected.shape[1]] = selected
|
||||
impl.owns_indexer = True
|
||||
impl.index_epoch = 0
|
||||
impl.q_concat_buffer = torch.empty_like(q)
|
||||
metadata = SimpleNamespace(
|
||||
attn_out_dtype=dtype,
|
||||
@@ -504,7 +506,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
|
||||
SimpleNamespace(_q_scale=None, _k_scale=None),
|
||||
)
|
||||
references = [
|
||||
_sink_reference(q[i], kv[:, 0][selected[i].long()], sinks, SM_SCALE)
|
||||
_sink_reference(q[i], kv.flatten(0, 1)[selected[i].long()], sinks, SM_SCALE)
|
||||
for i in range(num_tokens)
|
||||
]
|
||||
expected, _, value_scale, score_error = (
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"ROCm AITER sparse MLA index remap test requires ROCm.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
if not is_aiter_found_and_supported():
|
||||
pytest.skip(
|
||||
"ROCm AITER sparse MLA index remap test requires a supported AITER "
|
||||
"installation.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.attention import sparse_mla_attention as sparse_mla_mod
|
||||
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
|
||||
|
||||
NUM_HEADS = 16
|
||||
KV_LORA_RANK = 512
|
||||
QK_NOPE_HEAD_DIM = 128
|
||||
QK_ROPE_HEAD_DIM = 64
|
||||
HEAD_SIZE = 576
|
||||
NUM_TOKENS = 2
|
||||
TOPK_TOKENS = 4
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_index_epochs():
|
||||
# The counter is keyed by id(), and a freed test buffer can hand its id to
|
||||
# the next one, so a leftover count would leak across tests.
|
||||
sparse_mod._INDEX_EPOCHS.clear()
|
||||
yield
|
||||
sparse_mod._INDEX_EPOCHS.clear()
|
||||
|
||||
|
||||
def _make_metadata():
|
||||
return SimpleNamespace(
|
||||
num_actual_tokens=NUM_TOKENS,
|
||||
req_id_per_token=torch.zeros(NUM_TOKENS, dtype=torch.int32, device="cpu"),
|
||||
block_table=torch.arange(16, dtype=torch.int32, device="cpu").view(2, 8),
|
||||
paged_kv_indptr=torch.zeros(NUM_TOKENS + 1, dtype=torch.int32, device="cpu"),
|
||||
paged_kv_indices=torch.zeros(
|
||||
NUM_TOKENS * TOPK_TOKENS, dtype=torch.int32, device="cpu"
|
||||
),
|
||||
block_size=1,
|
||||
topk_tokens=TOPK_TOKENS,
|
||||
remapped_buf=None,
|
||||
remapped_epoch=-1,
|
||||
)
|
||||
|
||||
|
||||
def _patch_init_deps(monkeypatch, *, speculative: bool = False):
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.AiterMLAHelper,
|
||||
"check_num_heads_validity",
|
||||
staticmethod(lambda num_heads: None),
|
||||
)
|
||||
vllm_config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(max_num_batched_tokens=NUM_TOKENS),
|
||||
model_config=SimpleNamespace(dtype=torch.bfloat16),
|
||||
speculative_config=SimpleNamespace() if speculative else None,
|
||||
attention_config=SimpleNamespace(hisparse_config=None),
|
||||
)
|
||||
monkeypatch.setattr(sparse_mod, "get_current_vllm_config", lambda: vllm_config)
|
||||
monkeypatch.setattr(sparse_mla_mod, "get_current_vllm_config", lambda: vllm_config)
|
||||
monkeypatch.setattr(
|
||||
sparse_mla_mod, "get_tensor_model_parallel_world_size", lambda: 1
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod,
|
||||
"current_workspace_manager",
|
||||
lambda: SimpleNamespace(
|
||||
get_simultaneous=lambda *specs: tuple(
|
||||
torch.zeros(shape, dtype=dtype, device="cpu") for shape, dtype in specs
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _build_impl(indexer, topk_indices_buffer):
|
||||
return sparse_mod.ROCMAiterMLASparseImpl(
|
||||
num_heads=NUM_HEADS,
|
||||
head_size=HEAD_SIZE,
|
||||
scale=1.0,
|
||||
num_kv_heads=1,
|
||||
alibi_slopes=None,
|
||||
sliding_window=None,
|
||||
kv_cache_dtype="auto",
|
||||
logits_soft_cap=None,
|
||||
attn_type="decoder",
|
||||
kv_sharing_target_layer_name=None,
|
||||
topk_indices_buffer=topk_indices_buffer,
|
||||
indexer=indexer,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
qk_rope_head_dim=QK_ROPE_HEAD_DIM,
|
||||
qk_head_dim=QK_NOPE_HEAD_DIM + QK_ROPE_HEAD_DIM,
|
||||
v_head_dim=QK_NOPE_HEAD_DIM,
|
||||
kv_b_proj=None,
|
||||
)
|
||||
|
||||
|
||||
def _new_buffer():
|
||||
return torch.zeros(NUM_TOKENS, TOPK_TOKENS, dtype=torch.int32, device="cpu")
|
||||
|
||||
|
||||
def _patch_forward_deps(monkeypatch, impl, remap_calls):
|
||||
monkeypatch.setattr(
|
||||
sparse_mod,
|
||||
"triton_convert_req_index_to_global_index",
|
||||
lambda *args, **kwargs: remap_calls.append(args),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.AiterMLAHelper,
|
||||
"get_mla_padded_q",
|
||||
staticmethod(lambda num_heads, q: q),
|
||||
)
|
||||
attn_out = torch.zeros(NUM_TOKENS, NUM_HEADS, KV_LORA_RANK, device="cpu")
|
||||
monkeypatch.setattr(
|
||||
type(impl),
|
||||
"_forward_mla",
|
||||
lambda self, layer, q, kv_cache, attn_metadata: (attn_out, None),
|
||||
)
|
||||
return attn_out
|
||||
|
||||
|
||||
def _run_forward(impl, metadata):
|
||||
return impl.forward_mqa(
|
||||
torch.zeros(NUM_TOKENS, NUM_HEADS, HEAD_SIZE, device="cpu"),
|
||||
torch.zeros(4, 1, HEAD_SIZE, device="cpu"),
|
||||
metadata,
|
||||
SimpleNamespace(_q_scale=None, _k_scale=None),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_indexer", [True, False])
|
||||
def test_owns_indexer_is_structural(monkeypatch, has_indexer):
|
||||
"""owns_indexer reflects the layer, not the serving config."""
|
||||
_patch_init_deps(monkeypatch)
|
||||
buffer = _new_buffer()
|
||||
indexer = SimpleNamespace(topk_indices_buffer=buffer) if has_indexer else None
|
||||
|
||||
impl = _build_impl(indexer, buffer)
|
||||
|
||||
assert impl.owns_indexer is has_indexer
|
||||
assert impl.topk_indices_buffer is buffer
|
||||
|
||||
|
||||
def test_forward_mqa_remaps_once_per_metadata(monkeypatch):
|
||||
"""Skip layers sharing a metadata share its remap; the first one pays.
|
||||
|
||||
All layers read one top-k buffer, so the reuse has to key off the metadata
|
||||
that owns ``paged_kv_indices``, not off the buffer being distinct.
|
||||
"""
|
||||
_patch_init_deps(monkeypatch)
|
||||
buffer = _new_buffer()
|
||||
first = _build_impl(None, buffer)
|
||||
second = _build_impl(None, buffer)
|
||||
|
||||
remap_calls: list[tuple] = []
|
||||
attn_out = _patch_forward_deps(monkeypatch, first, remap_calls)
|
||||
metadata = _make_metadata()
|
||||
|
||||
out, lse = _run_forward(first, metadata)
|
||||
assert out is attn_out
|
||||
assert lse is None
|
||||
assert len(remap_calls) == 1
|
||||
assert metadata.remapped_buf is buffer
|
||||
|
||||
_run_forward(second, metadata)
|
||||
assert len(remap_calls) == 1
|
||||
|
||||
|
||||
def test_forward_mqa_remaps_for_each_metadata(monkeypatch):
|
||||
"""Skip layers on a second metadata must remap its own paged_kv_indices.
|
||||
|
||||
Regression: gating on ``indexer is not None`` skipped every layer in the
|
||||
non-indexer metadata group, leaving its ``paged_kv_indices`` unwritten.
|
||||
"""
|
||||
_patch_init_deps(monkeypatch)
|
||||
buffer = _new_buffer()
|
||||
indexer_layer = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
|
||||
skip_layer = _build_impl(None, buffer)
|
||||
|
||||
remap_calls: list[tuple] = []
|
||||
_patch_forward_deps(monkeypatch, indexer_layer, remap_calls)
|
||||
|
||||
indexer_metadata = _make_metadata()
|
||||
_run_forward(indexer_layer, indexer_metadata)
|
||||
assert len(remap_calls) == 1
|
||||
assert indexer_metadata.remapped_buf is buffer
|
||||
|
||||
# A distinct metadata instance carries its own, still-unwritten indices.
|
||||
skip_metadata = _make_metadata()
|
||||
_run_forward(skip_layer, skip_metadata)
|
||||
assert len(remap_calls) == 2
|
||||
assert skip_metadata.remapped_buf is buffer
|
||||
|
||||
|
||||
def test_forward_mqa_remaps_after_a_later_indexer_write(monkeypatch):
|
||||
"""A skip metadata must be refreshed once per indexer write, not once ever.
|
||||
|
||||
Regression for cross-layer index sharing (``index_topk_freq``). With
|
||||
freq=4 on a 78-layer model the 57 skip layers share one metadata while the
|
||||
21 indexer layers share another, so the skip metadata has to pick up every
|
||||
new selection. Recording only the buffer left layers 4 onwards attending
|
||||
to the first selection of the pass, which collapsed RULER niah_single_2 to
|
||||
0.026 on GLM-5.3 and took GPQA from 0.884 to 0.672.
|
||||
"""
|
||||
_patch_init_deps(monkeypatch)
|
||||
buffer = _new_buffer()
|
||||
# Construction order mirrors the layer stack: indexer, three skips, indexer.
|
||||
_build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
|
||||
first_skip = _build_impl(None, buffer)
|
||||
reuse_skip = _build_impl(None, buffer)
|
||||
_build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
|
||||
next_skip = _build_impl(None, buffer)
|
||||
|
||||
remap_calls: list[tuple] = []
|
||||
_patch_forward_deps(monkeypatch, first_skip, remap_calls)
|
||||
skip_metadata = _make_metadata()
|
||||
|
||||
_run_forward(first_skip, skip_metadata)
|
||||
assert len(remap_calls) == 1
|
||||
|
||||
# Same selection, same metadata: reuse.
|
||||
_run_forward(reuse_skip, skip_metadata)
|
||||
assert len(remap_calls) == 1
|
||||
|
||||
# A later indexer layer has since rewritten the buffer in place. The object
|
||||
# is unchanged, so only the epoch can reveal it.
|
||||
_run_forward(next_skip, skip_metadata)
|
||||
assert len(remap_calls) == 2
|
||||
assert skip_metadata.remapped_epoch == next_skip.index_epoch
|
||||
|
||||
|
||||
def test_index_epoch_tracks_the_preceding_indexer_write(monkeypatch):
|
||||
"""Layers between two indexer layers share the earlier one's epoch."""
|
||||
_patch_init_deps(monkeypatch)
|
||||
buffer = _new_buffer()
|
||||
first_indexer = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
|
||||
skip_a = _build_impl(None, buffer)
|
||||
skip_b = _build_impl(None, buffer)
|
||||
second_indexer = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
|
||||
skip_c = _build_impl(None, buffer)
|
||||
|
||||
assert skip_a.index_epoch == first_indexer.index_epoch
|
||||
assert skip_b.index_epoch == first_indexer.index_epoch
|
||||
assert second_indexer.index_epoch == first_indexer.index_epoch + 1
|
||||
assert skip_c.index_epoch == second_indexer.index_epoch
|
||||
|
||||
|
||||
def test_forward_mqa_remaps_when_buffer_differs(monkeypatch):
|
||||
"""A layer reading a different buffer cannot reuse the metadata's remap."""
|
||||
_patch_init_deps(monkeypatch)
|
||||
remapped_buffer = _new_buffer()
|
||||
other_buffer = _new_buffer()
|
||||
layer = _build_impl(None, other_buffer)
|
||||
|
||||
remap_calls: list[tuple] = []
|
||||
_patch_forward_deps(monkeypatch, layer, remap_calls)
|
||||
metadata = _make_metadata()
|
||||
metadata.remapped_buf = remapped_buffer
|
||||
|
||||
_run_forward(layer, metadata)
|
||||
assert len(remap_calls) == 1
|
||||
assert metadata.remapped_buf is other_buffer
|
||||
|
||||
|
||||
def test_forward_mqa_indexer_layer_always_remaps(monkeypatch):
|
||||
"""The indexer rewrites its buffer in place, so its remap is never stale."""
|
||||
_patch_init_deps(monkeypatch)
|
||||
buffer = _new_buffer()
|
||||
impl = _build_impl(SimpleNamespace(topk_indices_buffer=buffer), buffer)
|
||||
|
||||
remap_calls: list[tuple] = []
|
||||
_patch_forward_deps(monkeypatch, impl, remap_calls)
|
||||
metadata = _make_metadata()
|
||||
metadata.remapped_buf = buffer
|
||||
metadata.remapped_epoch = impl.index_epoch
|
||||
|
||||
_run_forward(impl, metadata)
|
||||
assert len(remap_calls) == 1
|
||||
@@ -33,6 +33,101 @@ class _FakeAiter(ModuleType):
|
||||
get_mla_metadata_v1: Mock
|
||||
|
||||
|
||||
@pytest.mark.parametrize("block_size", [16, 64, 128])
|
||||
@pytest.mark.parametrize("topk_tokens", [128, 256])
|
||||
def test_rocm_index_conversion_matrix(block_size, topk_tokens):
|
||||
"""Convert valid, masked, and out-of-range sparse indices exactly."""
|
||||
device = torch.device("cuda")
|
||||
num_tokens = 4
|
||||
max_blocks = 4
|
||||
req_ids = torch.tensor([0, 1, 0, 1], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor(
|
||||
[[3, 5, 7, 9], [2, 4, 6, 8]], dtype=torch.int32, device=device
|
||||
)
|
||||
token_indices = torch.arange(
|
||||
num_tokens * topk_tokens, dtype=torch.int32, device=device
|
||||
).view(num_tokens, topk_tokens)
|
||||
token_indices %= max_blocks * block_size
|
||||
token_indices[0, 0] = -1
|
||||
token_indices[1, 1] = max_blocks * block_size
|
||||
cu_seqlens = torch.arange(
|
||||
0,
|
||||
(num_tokens + 1) * topk_tokens,
|
||||
topk_tokens,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
output = torch.full(
|
||||
(num_tokens * topk_tokens,), -99, dtype=torch.int32, device=device
|
||||
)
|
||||
|
||||
sparse_mod.triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
token_indices,
|
||||
cu_seqlens,
|
||||
output,
|
||||
BLOCK_STRIDE_ROWS=block_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=topk_tokens,
|
||||
)
|
||||
|
||||
block_ids = token_indices // block_size
|
||||
valid = (token_indices >= 0) & (block_ids < max_blocks)
|
||||
safe_blocks = block_ids.clamp(0, max_blocks - 1)
|
||||
bases = block_table[req_ids[:, None].long(), safe_blocks.long()]
|
||||
expected = torch.where(valid, bases * block_size + token_indices % block_size, 0)
|
||||
torch.testing.assert_close(output.view_as(expected), expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_rocm_index_conversion_rejects_mismatched_token_extents():
|
||||
"""Reject full-batch request IDs paired with decode-subset indices."""
|
||||
with pytest.raises(AssertionError, match="must cover the same tokens"):
|
||||
sparse_mod.triton_convert_req_index_to_global_index(
|
||||
torch.zeros(3, dtype=torch.int32),
|
||||
torch.zeros(1, 4, dtype=torch.int32),
|
||||
torch.zeros(2, 128, dtype=torch.int32),
|
||||
torch.arange(3, dtype=torch.int32),
|
||||
torch.zeros(256, dtype=torch.int32),
|
||||
BLOCK_STRIDE_ROWS=64,
|
||||
NUM_TOPK_TOKENS=128,
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_index_conversion_respects_packed_valid_counts():
|
||||
"""ROCm's indptr contract packs only each token's valid top-k prefix."""
|
||||
device = torch.device("cuda")
|
||||
topk_tokens = 128
|
||||
valid_counts = torch.tensor([1, 17, 128], dtype=torch.int32, device=device)
|
||||
cu_seqlens = torch.zeros(4, dtype=torch.int32, device=device)
|
||||
torch.cumsum(valid_counts, dim=0, out=cu_seqlens[1:])
|
||||
req_ids = torch.tensor([0, 0, 0], dtype=torch.int32, device=device)
|
||||
block_table = torch.tensor([[2, 3, 4]], dtype=torch.int32, device=device)
|
||||
token_indices = torch.arange(
|
||||
3 * topk_tokens, dtype=torch.int32, device=device
|
||||
).view(3, topk_tokens)
|
||||
token_indices %= 3 * 64
|
||||
output = torch.full((int(cu_seqlens[-1]),), -99, dtype=torch.int32, device=device)
|
||||
|
||||
sparse_mod.triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
token_indices,
|
||||
cu_seqlens,
|
||||
output,
|
||||
BLOCK_STRIDE_ROWS=64,
|
||||
BLOCK_SIZE=64,
|
||||
NUM_TOPK_TOKENS=topk_tokens,
|
||||
)
|
||||
|
||||
expected_parts = []
|
||||
for token_id, count in enumerate(valid_counts.tolist()):
|
||||
local = token_indices[token_id, :count]
|
||||
expected_parts.append(block_table[0, (local // 64).long()] * 64 + local % 64)
|
||||
expected = torch.cat(expected_parts)
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def _make_builder():
|
||||
builder = object.__new__(sparse_mod.ROCMAiterMLASparseMetadataBuilder)
|
||||
max_num_batched_tokens = 8
|
||||
@@ -44,6 +139,18 @@ def _make_builder():
|
||||
builder.kv_cache_dtype = "fp8"
|
||||
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64)
|
||||
builder.topk_tokens = topk_tokens
|
||||
# SparseMLACommonMetadataBuilder state consumed by its build() method.
|
||||
builder.cp_kv_cache_interleave_size = 1
|
||||
builder.use_pcp = False
|
||||
builder._prefill_backend = None
|
||||
|
||||
def cpu_token_to_req(metadata):
|
||||
return torch.repeat_interleave(
|
||||
torch.arange(metadata.num_reqs, dtype=torch.int32),
|
||||
metadata.query_start_loc_cpu.diff(),
|
||||
)
|
||||
|
||||
builder._build_req_id_per_token = cpu_token_to_req # type: ignore[method-assign]
|
||||
builder.req_id_per_token_buffer = torch.zeros(
|
||||
max_num_batched_tokens, dtype=torch.int32, device="cpu"
|
||||
)
|
||||
@@ -62,6 +169,8 @@ def _make_builder():
|
||||
builder._use_persistent_metadata = True
|
||||
builder._num_attention_heads = 16
|
||||
builder._num_compute_units = current_platform.num_compute_units()
|
||||
builder._mla_q_dtype = torch.bfloat16
|
||||
builder._mla_kv_dtype = torch.bfloat16
|
||||
builder._mla_work_meta_data = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
builder._mla_work_indptr = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
builder._mla_work_info_set = torch.empty(1, dtype=torch.int32, device="cpu")
|
||||
@@ -114,7 +223,7 @@ def _patch_build_deps(monkeypatch, events=None):
|
||||
on CPU."""
|
||||
|
||||
def fake_generate_sparse_seqlen_triton(
|
||||
query_lens, seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
|
||||
seq_lens, cu_query_lens, topk_token, num_tokens, max_query_len
|
||||
):
|
||||
return torch.zeros(num_tokens, dtype=torch.int32, device="cpu")
|
||||
|
||||
@@ -167,6 +276,25 @@ def test_build_populates_mixed_split_fields(monkeypatch):
|
||||
assert md.prefill is None
|
||||
|
||||
|
||||
def test_build_fills_paged_kv_indptr_tail(monkeypatch):
|
||||
"""Rows past num_tokens must read as empty: the tail repeats the total."""
|
||||
builder = _make_builder()
|
||||
_patch_build_deps(monkeypatch)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod,
|
||||
"generate_sparse_seqlen_triton",
|
||||
lambda *args, **kwargs: torch.tensor([3, 5], dtype=torch.int32),
|
||||
)
|
||||
builder.paged_kv_indptr[1:].fill_(-1)
|
||||
|
||||
md = builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=_make_common_metadata()
|
||||
)
|
||||
|
||||
assert md.paged_kv_indptr.tolist() == [0, 3, 8]
|
||||
assert builder.paged_kv_indptr.tolist() == [0, 3, 8, 8, 8, 8, 8, 8, 8]
|
||||
|
||||
|
||||
def test_sink_build_skips_persistent_metadata(monkeypatch):
|
||||
builder = _make_builder()
|
||||
builder._use_persistent_metadata = False
|
||||
@@ -216,6 +344,8 @@ def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
|
||||
assert events == ["metadata", "sync"]
|
||||
assert fake_get_mla_metadata_v1_mock.call_count == 1
|
||||
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["max_split_per_batch"] == 1
|
||||
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_q"] == torch.bfloat16
|
||||
assert fake_get_mla_metadata_v1_mock.call_args.kwargs["dtype_kv"] == torch.bfloat16
|
||||
|
||||
events.clear()
|
||||
|
||||
@@ -223,3 +353,399 @@ def test_sparse_persistent_metadata_syncs_only_after_recompute(monkeypatch):
|
||||
|
||||
assert events == []
|
||||
assert fake_get_mla_metadata_v1_mock.call_count == 1
|
||||
|
||||
|
||||
def test_dense_prefill_builds_sparse_metadata_for_decode_subset(monkeypatch):
|
||||
"""Dense prefill must exclude prefill tokens from AITER sparse metadata.
|
||||
|
||||
A mixed batch is routed as sparse MQA for its leading decode requests and
|
||||
dense MHA for its trailing short prefill requests. The AITER metadata
|
||||
buffers therefore must describe only the decode subset; including prefill
|
||||
tokens would mismatch the query passed to ``forward_mqa``.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
builder.vllm_config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
|
||||
)
|
||||
|
||||
query_start_loc = torch.tensor([0, 1, 5], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([16, 4], dtype=torch.int32)
|
||||
common_metadata = SimpleNamespace(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
max_query_len=4,
|
||||
max_seq_len=16,
|
||||
)
|
||||
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
|
||||
num_reqs=2,
|
||||
max_query_len=4,
|
||||
max_seq_len=16,
|
||||
num_actual_tokens=5,
|
||||
query_start_loc=query_start_loc,
|
||||
slot_mapping=torch.arange(5),
|
||||
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
|
||||
req_id_per_token=torch.tensor([0, 1, 1, 1, 1], dtype=torch.int32),
|
||||
topk_tokens=4,
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
seq_lens=seq_lens,
|
||||
prefill_max_seq_len=4,
|
||||
prefill=SimpleNamespace(use_dense_mha=True),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.SparseMLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args, **kwargs: metadata,
|
||||
)
|
||||
sparse_call: dict[str, torch.Tensor] = {}
|
||||
|
||||
def fake_generate_sparse_seqlen(
|
||||
sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
|
||||
):
|
||||
sparse_call.update(
|
||||
query_lens=torch.diff(sparse_query_start_loc),
|
||||
seq_lens=sparse_seq_lens.clone(),
|
||||
query_start_loc=sparse_query_start_loc.clone(),
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
return torch.ones(num_tokens, dtype=torch.int32)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
|
||||
)
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
fake_aiter.get_mla_metadata_v1 = Mock()
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: None),
|
||||
)
|
||||
|
||||
result = builder.build(0, common_metadata)
|
||||
|
||||
assert result.prefill is metadata.prefill
|
||||
assert result.num_prefills == 1
|
||||
assert result.req_id_per_token.tolist() == [0]
|
||||
assert result.qo_indptr.numel() == 2
|
||||
assert result.paged_kv_indptr.numel() == 2
|
||||
assert sparse_call["num_tokens"] == 1
|
||||
assert sparse_call["query_lens"].tolist() == [1]
|
||||
assert sparse_call["seq_lens"].tolist() == [16]
|
||||
assert sparse_call["query_start_loc"].tolist() == [0, 1]
|
||||
|
||||
|
||||
def test_long_prefill_shares_full_sparse_metadata_with_decode(monkeypatch):
|
||||
"""A prefill longer than top-k must remain in shared sparse metadata.
|
||||
|
||||
Dense MHA is only selected when the prefill sequence fits within the
|
||||
sparse top-k window. Once it exceeds that window, both decode and prefill
|
||||
are handled by sparse MQA, so every query token must be represented by the
|
||||
same sparse paging metadata.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
builder.vllm_config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
|
||||
)
|
||||
|
||||
# One decode token followed by a five-token prefill. topk_tokens is four,
|
||||
# so the prefill must not be split onto the dense-MHA path.
|
||||
query_start_loc = torch.tensor([0, 1, 6], dtype=torch.int32)
|
||||
seq_lens = torch.tensor([16, 5], dtype=torch.int32)
|
||||
common_metadata = SimpleNamespace(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
max_query_len=5,
|
||||
max_seq_len=16,
|
||||
)
|
||||
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
|
||||
num_reqs=2,
|
||||
max_query_len=5,
|
||||
max_seq_len=16,
|
||||
num_actual_tokens=6,
|
||||
query_start_loc=query_start_loc,
|
||||
slot_mapping=torch.arange(6),
|
||||
block_table=torch.arange(16, dtype=torch.int32).view(2, 8),
|
||||
req_id_per_token=torch.tensor([0, 1, 1, 1, 1, 1], dtype=torch.int32),
|
||||
topk_tokens=4,
|
||||
num_decodes=1,
|
||||
num_prefills=1,
|
||||
num_decode_tokens=1,
|
||||
seq_lens=seq_lens,
|
||||
prefill_max_seq_len=5,
|
||||
prefill=SimpleNamespace(use_dense_mha=False),
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.SparseMLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args, **kwargs: metadata,
|
||||
)
|
||||
sparse_call: dict[str, torch.Tensor] = {}
|
||||
|
||||
def fake_generate_sparse_seqlen(
|
||||
sparse_seq_lens, sparse_query_start_loc, topk, num_tokens, max_q
|
||||
):
|
||||
sparse_call.update(
|
||||
query_lens=torch.diff(sparse_query_start_loc),
|
||||
seq_lens=sparse_seq_lens.clone(),
|
||||
query_start_loc=sparse_query_start_loc.clone(),
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
return torch.ones(num_tokens, dtype=torch.int32)
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "generate_sparse_seqlen_triton", fake_generate_sparse_seqlen
|
||||
)
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
get_persistent_metadata = Mock()
|
||||
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: None),
|
||||
)
|
||||
|
||||
result = builder.build(0, common_metadata)
|
||||
|
||||
assert result.num_decodes == 1
|
||||
assert result.num_prefills == 1
|
||||
assert result.req_id_per_token.tolist() == [0, 1, 1, 1, 1, 1]
|
||||
assert result.qo_indptr.numel() == 7
|
||||
assert result.paged_kv_indptr.numel() == 7
|
||||
assert result.paged_kv_indices.numel() == 6 * result.topk_tokens
|
||||
assert sparse_call["num_tokens"] == 6
|
||||
assert sparse_call["query_lens"].tolist() == [1, 5]
|
||||
assert sparse_call["seq_lens"].tolist() == [16, 5]
|
||||
assert sparse_call["query_start_loc"].tolist() == [0, 1, 6]
|
||||
get_persistent_metadata.assert_called_once()
|
||||
assert result.work_meta_data is builder._mla_work_meta_data
|
||||
assert result.reduce_indptr is builder._mla_reduce_indptr
|
||||
|
||||
|
||||
def test_forward_mqa_slices_full_batch_metadata_to_query_tokens(monkeypatch):
|
||||
"""The sparse kernel inputs must have the same token extent as its query.
|
||||
|
||||
In a mixed dense-prefill batch, ``forward_mqa`` receives only decode query
|
||||
tokens while some metadata buffers may represent the full batch. This
|
||||
guards the slicing that prevents the index conversion from writing beyond
|
||||
the decode-sized output.
|
||||
"""
|
||||
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.num_heads = 4
|
||||
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
|
||||
impl.owns_indexer = True
|
||||
impl.index_epoch = 0
|
||||
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
|
||||
forward_mla = Mock(return_value=(expected_output, None))
|
||||
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
|
||||
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
|
||||
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
|
||||
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
|
||||
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
|
||||
block_size=1,
|
||||
topk_tokens=4,
|
||||
)
|
||||
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
|
||||
captured = {}
|
||||
|
||||
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
|
||||
captured["req_ids"] = req_ids.clone()
|
||||
captured["token_indices"] = token_indices.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
|
||||
)
|
||||
|
||||
output, lse = impl.forward_mqa(
|
||||
query,
|
||||
torch.zeros(1, 1, 576, dtype=torch.bfloat16),
|
||||
metadata,
|
||||
SimpleNamespace(),
|
||||
)
|
||||
|
||||
assert output is expected_output
|
||||
assert lse is None
|
||||
assert captured["req_ids"].tolist() == [0, 1]
|
||||
assert captured["token_indices"].shape == (2, 4)
|
||||
forward_mla.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
def test_forward_mqa_slices_decode_subset_for_bf16_and_fp8(monkeypatch, kv_cache_dtype):
|
||||
"""Both cache dtypes must slice mixed-batch metadata to decode queries."""
|
||||
impl = object.__new__(sparse_mod.ROCMAiterMLASparseImpl)
|
||||
impl.kv_cache_dtype = kv_cache_dtype
|
||||
impl.num_heads = 4
|
||||
impl.topk_indices_buffer = torch.arange(48, dtype=torch.int32).view(12, 4)
|
||||
impl.owns_indexer = True
|
||||
impl.index_epoch = 0
|
||||
expected_output = torch.zeros(2, 4, 512, dtype=torch.bfloat16)
|
||||
forward_mla = Mock(return_value=(expected_output, None))
|
||||
monkeypatch.setattr(impl, "_forward_mla", forward_mla)
|
||||
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.tensor([0, 1] + [2] * 5 + [3] * 5, dtype=torch.int32),
|
||||
block_table=torch.arange(40, dtype=torch.int32).view(4, 10),
|
||||
paged_kv_indptr=torch.arange(3, dtype=torch.int32),
|
||||
paged_kv_indices=torch.zeros(8, dtype=torch.int32),
|
||||
block_size=1,
|
||||
topk_tokens=4,
|
||||
)
|
||||
query = torch.zeros(2, 4, 576, dtype=torch.bfloat16)
|
||||
kv_cache = torch.zeros(1, 1, 576, dtype=torch.uint8)
|
||||
captured = {}
|
||||
|
||||
def fake_convert(req_ids, block_table, token_indices, *args, **kwargs):
|
||||
captured["req_ids"] = req_ids.clone()
|
||||
captured["token_indices"] = token_indices.clone()
|
||||
|
||||
monkeypatch.setattr(
|
||||
sparse_mod, "triton_convert_req_index_to_global_index", fake_convert
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.AiterMLAHelper, "get_mla_padded_q", lambda num_heads, q: q
|
||||
)
|
||||
if kv_cache_dtype == "fp8":
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.ops,
|
||||
"scaled_fp8_quant",
|
||||
lambda q, scale: (q.to(fp8_dtype), None),
|
||||
)
|
||||
|
||||
output, lse = impl.forward_mqa(
|
||||
query,
|
||||
kv_cache,
|
||||
metadata,
|
||||
SimpleNamespace(_q_scale=torch.tensor(1.0)),
|
||||
)
|
||||
|
||||
assert output is expected_output
|
||||
assert lse is None
|
||||
assert captured["req_ids"].tolist() == [0, 1]
|
||||
assert captured["token_indices"].shape == (2, 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("length_relation", ["short", "long"])
|
||||
@pytest.mark.parametrize("workload", ["prefill", "mixed", "decode"])
|
||||
def test_sparse_metadata_workload_length_dtype_matrix(
|
||||
monkeypatch, workload, length_relation, kv_cache_dtype
|
||||
):
|
||||
"""Cover routing and persistent metadata across the ROCm support matrix.
|
||||
|
||||
The matrix crosses pure prefill, mixed prefill/decode, and decode-only
|
||||
workloads with sequence lengths below/above top-k and BF16/FP8 KV cache.
|
||||
"""
|
||||
builder = _make_builder()
|
||||
builder.vllm_config = SimpleNamespace(
|
||||
attention_config=SimpleNamespace(sparse_mla_force_mqa=False)
|
||||
)
|
||||
if kv_cache_dtype == "fp8":
|
||||
builder._mla_q_dtype = current_platform.fp8_dtype()
|
||||
builder._mla_kv_dtype = current_platform.fp8_dtype()
|
||||
|
||||
is_long = length_relation == "long"
|
||||
prefill_len = 5 if is_long else 3
|
||||
decode_seq_len = 8 if is_long else 3
|
||||
if workload == "prefill":
|
||||
query_lens = [prefill_len]
|
||||
seq_lens_list = [prefill_len]
|
||||
num_decodes, num_prefills = 0, 1
|
||||
elif workload == "mixed":
|
||||
query_lens = [1, prefill_len]
|
||||
seq_lens_list = [decode_seq_len, prefill_len]
|
||||
num_decodes, num_prefills = 1, 1
|
||||
else:
|
||||
query_lens = [1, 1]
|
||||
seq_lens_list = [decode_seq_len, decode_seq_len + 1]
|
||||
num_decodes, num_prefills = 2, 0
|
||||
|
||||
query_start_loc = torch.tensor(
|
||||
[0, *torch.tensor(query_lens).cumsum(0).tolist()], dtype=torch.int32
|
||||
)
|
||||
seq_lens = torch.tensor(seq_lens_list, dtype=torch.int32)
|
||||
num_tokens = sum(query_lens)
|
||||
common_metadata = SimpleNamespace(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
max_query_len=max(query_lens),
|
||||
max_seq_len=max(seq_lens_list),
|
||||
)
|
||||
metadata = sparse_mod.ROCMAiterMLASparseMetadata(
|
||||
num_reqs=len(query_lens),
|
||||
max_query_len=max(query_lens),
|
||||
max_seq_len=max(seq_lens_list),
|
||||
num_actual_tokens=num_tokens,
|
||||
query_start_loc=query_start_loc,
|
||||
slot_mapping=torch.arange(num_tokens),
|
||||
block_table=torch.arange(len(query_lens) * 8, dtype=torch.int32).view(
|
||||
len(query_lens), 8
|
||||
),
|
||||
req_id_per_token=torch.repeat_interleave(
|
||||
torch.arange(len(query_lens), dtype=torch.int32),
|
||||
torch.tensor(query_lens),
|
||||
),
|
||||
topk_tokens=4,
|
||||
num_decodes=num_decodes,
|
||||
num_prefills=num_prefills,
|
||||
num_decode_tokens=num_decodes,
|
||||
seq_lens=seq_lens,
|
||||
prefill_max_seq_len=prefill_len if num_prefills else 0,
|
||||
prefill=SimpleNamespace(use_dense_mha=not is_long) if num_prefills else None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.SparseMLACommonMetadataBuilder,
|
||||
"build",
|
||||
lambda *args, **kwargs: metadata,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod,
|
||||
"generate_sparse_seqlen_triton",
|
||||
lambda *args, **kwargs: torch.ones(args[3], dtype=torch.int32),
|
||||
)
|
||||
get_persistent_metadata = Mock()
|
||||
fake_aiter = _FakeAiter("aiter")
|
||||
fake_aiter.get_mla_metadata_v1 = get_persistent_metadata
|
||||
monkeypatch.setitem(sys.modules, "aiter", fake_aiter)
|
||||
monkeypatch.setattr(
|
||||
sparse_mod.torch.cuda,
|
||||
"current_stream",
|
||||
lambda device=None: SimpleNamespace(synchronize=lambda: None),
|
||||
)
|
||||
|
||||
result = builder.build(0, common_metadata)
|
||||
|
||||
uses_dense_prefill = workload in ("prefill", "mixed") and not is_long
|
||||
expected_sparse_tokens = num_decodes if uses_dense_prefill else num_tokens
|
||||
assert result.req_id_per_token.numel() == expected_sparse_tokens
|
||||
assert result.qo_indptr.numel() == expected_sparse_tokens + 1
|
||||
assert result.paged_kv_indptr.numel() == expected_sparse_tokens + 1
|
||||
|
||||
uses_fallback = workload == "prefill" and is_long and kv_cache_dtype == "auto"
|
||||
if expected_sparse_tokens == 0 or uses_fallback:
|
||||
get_persistent_metadata.assert_not_called()
|
||||
else:
|
||||
get_persistent_metadata.assert_called_once()
|
||||
if uses_fallback or expected_sparse_tokens == 0:
|
||||
assert result.work_meta_data is None
|
||||
assert result.reduce_indptr is None
|
||||
else:
|
||||
assert result.work_meta_data is builder._mla_work_meta_data
|
||||
assert result.reduce_indptr is builder._mla_reduce_indptr
|
||||
|
||||
@@ -253,6 +253,7 @@ def _prefill_metadata(
|
||||
cu_seq_lens=torch.zeros(1, dtype=torch.int32, device=DEVICE),
|
||||
token_to_seq=torch.zeros(1, dtype=torch.int32, device=DEVICE),
|
||||
total_seq_lens=1,
|
||||
max_seqlen_k=int(ends.max()),
|
||||
token_start=t0,
|
||||
token_end=t1,
|
||||
num_reqs=req_hi - req_lo,
|
||||
|
||||
@@ -70,7 +70,7 @@ def enable_aiter_mqa(monkeypatch):
|
||||
def test_unpaged_mqa_logits_preserves_intervals_and_scale_layout(
|
||||
column_scales, monkeypatch, enable_aiter_mqa
|
||||
) -> None:
|
||||
"""The vLLM AITER dispatch preserves ragged masking and per-token scales."""
|
||||
"""The vLLM AITER dispatch honors ragged intervals and per-token scales."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
|
||||
@@ -108,7 +108,7 @@ def test_unpaged_mqa_logits_preserves_intervals_and_scale_layout(
|
||||
valid = (positions >= starts[:, None]) & (positions < ends[:, None])
|
||||
assert actual.shape == expected.shape
|
||||
assert actual.dtype == torch.float32
|
||||
assert torch.equal(torch.isneginf(actual), ~valid)
|
||||
assert not torch.isneginf(actual[valid]).any()
|
||||
torch.testing.assert_close(actual[valid], expected[valid], rtol=1e-3, atol=1e-3)
|
||||
|
||||
|
||||
@@ -1983,6 +1983,272 @@ def test_dsv4_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
|
||||
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_gfx950
|
||||
@torch.inference_mode()
|
||||
def test_dsv41_adaptive_mla_swa_metadata_graph_replay(monkeypatch) -> None:
|
||||
"""Replay V4.1 ROCm decode after device-only query reallocation."""
|
||||
from tests.v1.attention.utils import create_vllm_config
|
||||
from vllm.models.deepseek_v41.amd.rocm import (
|
||||
DeepseekV4ROCMAiterSparseSWAMetadataBuilder,
|
||||
DeepseekV41ROCMAiterMLASparseMetadataBuilder,
|
||||
compute_global_topk_ragged_indices_and_indptr,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionCGSupport, CommonAttentionMetadata
|
||||
from vllm.v1.attention.ops import rocm_aiter_mla_sparse as mod
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec, SlidingWindowMLASpec
|
||||
|
||||
device = torch.device("cuda")
|
||||
num_reqs = 3
|
||||
upper_query_len = 6
|
||||
graph_tokens = num_reqs * upper_query_len
|
||||
block_size = 128
|
||||
compress_ratio = 2
|
||||
compressed_block_size = block_size // compress_ratio
|
||||
|
||||
vllm_config = create_vllm_config(
|
||||
model_name="facebook/opt-125m",
|
||||
max_model_len=1024,
|
||||
block_size=block_size,
|
||||
max_num_seqs=num_reqs,
|
||||
max_num_batched_tokens=graph_tokens,
|
||||
hf_config_override={
|
||||
"compress_ratios": [0, 1, 2],
|
||||
"index_topk": 8,
|
||||
"sliding_window": 32,
|
||||
},
|
||||
)
|
||||
vllm_config.speculative_config = SimpleNamespace(
|
||||
num_speculative_tokens=upper_query_len - 1,
|
||||
parallel_drafting=False,
|
||||
enable_adaptive_verification=True,
|
||||
use_dspark=lambda: True,
|
||||
)
|
||||
mla_spec = MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=compress_ratio,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
swa_spec = SlidingWindowMLASpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=HEAD_DIM,
|
||||
dtype=torch.uint8,
|
||||
sliding_window=32,
|
||||
cache_dtype_str="fp8_ds_mla",
|
||||
model_version="deepseek_v4",
|
||||
)
|
||||
mla_builder = DeepseekV41ROCMAiterMLASparseMetadataBuilder(
|
||||
mla_spec, ["c2a"], vllm_config, device
|
||||
)
|
||||
swa_builder = DeepseekV4ROCMAiterSparseSWAMetadataBuilder(
|
||||
swa_spec, ["c2a"], vllm_config, device
|
||||
)
|
||||
assert (
|
||||
mla_builder.get_cudagraph_support(vllm_config, mla_spec)
|
||||
== AttentionCGSupport.ALWAYS
|
||||
)
|
||||
assert (
|
||||
swa_builder.get_cudagraph_support(vllm_config, swa_spec)
|
||||
== AttentionCGSupport.ALWAYS
|
||||
)
|
||||
|
||||
seq_lens = torch.tensor([100, 110, 120], dtype=torch.int32, device=device)
|
||||
seq_lens_cpu = seq_lens.cpu()
|
||||
query_start_loc_cpu = torch.arange(
|
||||
0, graph_tokens + 1, upper_query_len, dtype=torch.int32
|
||||
)
|
||||
block_table = torch.arange(num_reqs, dtype=torch.int32, device=device).view(
|
||||
num_reqs, 1
|
||||
)
|
||||
|
||||
def build_metadata(query_lens: list[int]):
|
||||
query_lens_tensor = torch.tensor(query_lens, dtype=torch.int32, device=device)
|
||||
query_start_loc = torch.cat(
|
||||
[
|
||||
torch.zeros(1, dtype=torch.int32, device=device),
|
||||
query_lens_tensor.cumsum(0),
|
||||
]
|
||||
)
|
||||
active_tokens = sum(query_lens)
|
||||
slot_mapping = torch.full((graph_tokens,), -1, dtype=torch.int64, device=device)
|
||||
slot_mapping[:active_tokens] = torch.arange(
|
||||
active_tokens, dtype=torch.int64, device=device
|
||||
)
|
||||
common = CommonAttentionMetadata(
|
||||
query_start_loc=query_start_loc,
|
||||
query_start_loc_cpu=query_start_loc_cpu,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu_upper_bound=seq_lens_cpu,
|
||||
num_reqs=num_reqs,
|
||||
num_actual_tokens=graph_tokens,
|
||||
max_query_len=upper_query_len,
|
||||
max_seq_len=int(seq_lens_cpu.max()),
|
||||
block_table_tensor=block_table,
|
||||
slot_mapping=slot_mapping,
|
||||
causal=True,
|
||||
)
|
||||
return (
|
||||
mla_builder.build_for_cudagraph_capture(common),
|
||||
swa_builder.build_for_cudagraph_capture(common),
|
||||
)
|
||||
|
||||
mla_metadata, swa_metadata = build_metadata([6, 6, 6])
|
||||
metadata_ptrs = (
|
||||
mla_metadata.req_id_per_token.data_ptr(),
|
||||
mla_metadata.slot_mapping.data_ptr(),
|
||||
swa_metadata.token_to_req_indices.data_ptr(),
|
||||
swa_metadata.decode_swa_ragged_indices.data_ptr(),
|
||||
swa_metadata.decode_swa_ragged_indptr.data_ptr(),
|
||||
)
|
||||
|
||||
topk_indices = (
|
||||
torch.arange(8, dtype=torch.int32, device=device)
|
||||
.expand(graph_tokens, -1)
|
||||
.contiguous()
|
||||
)
|
||||
torch.manual_seed(23)
|
||||
num_heads = 16
|
||||
q = (
|
||||
torch.randn(
|
||||
graph_tokens,
|
||||
num_heads,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125
|
||||
)
|
||||
swa_cache = _pack_fp8_ds_mla_cache(
|
||||
torch.randn(
|
||||
num_reqs * block_size,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125,
|
||||
block_size,
|
||||
use_fnuz=False,
|
||||
)
|
||||
compressed_cache = _pack_fp8_ds_mla_cache(
|
||||
torch.randn(
|
||||
num_reqs * compressed_block_size,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
* 0.125,
|
||||
compressed_block_size,
|
||||
use_fnuz=False,
|
||||
)
|
||||
attn_sink = torch.linspace(-0.1, 0.1, num_heads, dtype=torch.float32, device=device)
|
||||
out = torch.empty_like(q)
|
||||
monkeypatch.setattr(mod, "_decode_gfx950_num_splits", lambda *args: 1)
|
||||
|
||||
def build_topk_ragged(mla_md, swa_md):
|
||||
return compute_global_topk_ragged_indices_and_indptr(
|
||||
topk_indices,
|
||||
swa_md.token_to_req_indices,
|
||||
mla_md.block_table,
|
||||
compressed_block_size,
|
||||
swa_md.is_valid_token,
|
||||
)
|
||||
|
||||
def run_decode(mla_md, swa_md) -> None:
|
||||
topk_ragged_indices, topk_ragged_indptr, topk_lens = build_topk_ragged(
|
||||
mla_md, swa_md
|
||||
)
|
||||
mod.rocm_sparse_attn_decode(
|
||||
q=q,
|
||||
kv_cache=compressed_cache,
|
||||
swa_k_cache=swa_cache,
|
||||
swa_only=False,
|
||||
topk_indices=None,
|
||||
topk_lens=topk_lens,
|
||||
swa_indices=swa_md.decode_swa_indices,
|
||||
swa_lens=swa_md.decode_swa_lens,
|
||||
swa_ragged_indices=swa_md.decode_swa_ragged_indices,
|
||||
swa_ragged_indptr=swa_md.decode_swa_ragged_indptr,
|
||||
topk_ragged_indices=topk_ragged_indices,
|
||||
topk_ragged_indptr=topk_ragged_indptr,
|
||||
attn_sink=attn_sink,
|
||||
scale=HEAD_DIM**-0.5,
|
||||
head_dim=HEAD_DIM,
|
||||
nope_head_dim=NOPE_HEAD_DIM,
|
||||
rope_head_dim=ROPE_HEAD_DIM,
|
||||
output=out,
|
||||
extra_cache_nan_free=True,
|
||||
)
|
||||
|
||||
def reference(mla_md, swa_md) -> torch.Tensor:
|
||||
topk_ragged_indices, topk_ragged_indptr, _ = build_topk_ragged(mla_md, swa_md)
|
||||
return _ref_sparse_decode_ragged(
|
||||
q=q,
|
||||
main_cache=swa_cache,
|
||||
main_rows=_rows_from_ragged(
|
||||
swa_md.decode_swa_ragged_indices,
|
||||
swa_md.decode_swa_ragged_indptr,
|
||||
),
|
||||
scale=HEAD_DIM**-0.5,
|
||||
attn_sink=attn_sink,
|
||||
block_size=block_size,
|
||||
extra_cache=compressed_cache,
|
||||
extra_rows=_rows_from_ragged(
|
||||
topk_ragged_indices,
|
||||
topk_ragged_indptr,
|
||||
),
|
||||
extra_block_size=compressed_block_size,
|
||||
)
|
||||
|
||||
run_decode(mla_metadata, swa_metadata)
|
||||
torch.accelerator.synchronize()
|
||||
expected_full = reference(mla_metadata, swa_metadata)
|
||||
torch.testing.assert_close(out, expected_full, atol=2e-2, rtol=2e-2)
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run_decode(mla_metadata, swa_metadata)
|
||||
torch.accelerator.synchronize()
|
||||
captured_full = out.clone()
|
||||
|
||||
reallocated_mla, reallocated_swa = build_metadata([2, 6, 4])
|
||||
assert metadata_ptrs == (
|
||||
reallocated_mla.req_id_per_token.data_ptr(),
|
||||
reallocated_mla.slot_mapping.data_ptr(),
|
||||
reallocated_swa.token_to_req_indices.data_ptr(),
|
||||
reallocated_swa.decode_swa_ragged_indices.data_ptr(),
|
||||
reallocated_swa.decode_swa_ragged_indptr.data_ptr(),
|
||||
)
|
||||
expected_owners = torch.tensor(
|
||||
[0, 0, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2],
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
torch.testing.assert_close(reallocated_mla.req_id_per_token[:12], expected_owners)
|
||||
torch.testing.assert_close(
|
||||
reallocated_swa.token_to_req_indices[:12], expected_owners
|
||||
)
|
||||
assert reallocated_swa.is_valid_token[:12].all()
|
||||
assert not reallocated_swa.is_valid_token[12:].any()
|
||||
torch.testing.assert_close(
|
||||
reallocated_swa.decode_swa_lens[12:],
|
||||
torch.zeros(6, dtype=torch.int32, device=device),
|
||||
)
|
||||
|
||||
expected_reallocated = reference(reallocated_mla, reallocated_swa)
|
||||
run_decode(reallocated_mla, reallocated_swa)
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
|
||||
assert not torch.equal(captured_full, expected_reallocated)
|
||||
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
torch.testing.assert_close(out, expected_reallocated, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# o-projection: fused inverse-RoPE + cached bf16 wo_a (rocm_inv_rope_einsum)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -9,11 +11,17 @@ from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.attention.ops import triton_unified_attention as attention
|
||||
from vllm.v1.attention.ops.triton_attention_helpers import (
|
||||
apply_softcap,
|
||||
compute_tile_loop_bounds,
|
||||
softmax_step,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_unified_attention import (
|
||||
MAX_3D_QUERY_LEN,
|
||||
reduce_segments,
|
||||
unified_attention,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_unified_attention import unified_attention
|
||||
from vllm.v1.kv_cache_interface import KVQuantMode
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
@@ -37,6 +45,259 @@ NUM_BLOCKS = [32768, 2048]
|
||||
SEQ_THRESHOLD_3D_VALUES = [0, 8]
|
||||
|
||||
|
||||
def _scratch(rows):
|
||||
shapes = [(rows, 8, 16, 128), (rows, 8, 16), (rows, 8, 16)]
|
||||
return [torch.empty(shape, dtype=torch.float32, device="cpu") for shape in shapes]
|
||||
|
||||
|
||||
def _uses_splitk(
|
||||
monkeypatch,
|
||||
query_len,
|
||||
scratch,
|
||||
*,
|
||||
causal=True,
|
||||
threshold=8,
|
||||
mm_prefix=None,
|
||||
num_seqs=2,
|
||||
):
|
||||
query = torch.empty(
|
||||
(num_seqs * query_len, 8, 128), dtype=torch.bfloat16, device="cpu"
|
||||
)
|
||||
cache = torch.empty((1, 128, 1, 128), dtype=torch.bfloat16, device="cpu")
|
||||
kernel = MagicMock()
|
||||
monkeypatch.setattr(attention, "kernel_unified_attention", kernel)
|
||||
monkeypatch.setattr(attention, "reduce_segments", MagicMock())
|
||||
unified_attention(
|
||||
query,
|
||||
cache,
|
||||
cache,
|
||||
torch.empty_like(query),
|
||||
torch.arange(num_seqs + 1, dtype=torch.int32, device="cpu") * query_len,
|
||||
query_len,
|
||||
torch.full((num_seqs,), 128, dtype=torch.int32, device="cpu"),
|
||||
128,
|
||||
128**-0.5,
|
||||
causal,
|
||||
(-1, -1),
|
||||
torch.zeros((num_seqs, 1), dtype=torch.int32, device="cpu"),
|
||||
0.0,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
seq_threshold_3D=threshold,
|
||||
num_par_softmax_segments=16,
|
||||
softmax_segm_output=scratch[0],
|
||||
softmax_segm_max=scratch[1],
|
||||
softmax_segm_expsum=scratch[2],
|
||||
mm_prefix_range=mm_prefix,
|
||||
)
|
||||
return kernel.__getitem__.return_value.call_args.kwargs["IS_3D"]
|
||||
|
||||
|
||||
def _spy_reduce_segments(monkeypatch):
|
||||
original = attention.reduce_segments
|
||||
grids = []
|
||||
|
||||
def launch(grid):
|
||||
grids.append(grid)
|
||||
return original[grid]
|
||||
|
||||
spy = MagicMock()
|
||||
spy.__getitem__.side_effect = launch
|
||||
monkeypatch.setattr(attention, "reduce_segments", spy)
|
||||
return grids
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query_len,options,expected",
|
||||
[
|
||||
(1, {}, True),
|
||||
(1, {"causal": False}, True),
|
||||
(5, {}, True),
|
||||
(6, {"num_seqs": 1}, True),
|
||||
(8, {}, True),
|
||||
(9, {"num_seqs": 1}, False),
|
||||
(5, {"causal": False}, False),
|
||||
(5, {"causal": "per_sequence"}, False),
|
||||
(5, {"threshold": 1}, False),
|
||||
(5, {"batch_invariant": True}, False),
|
||||
(5, {"mm_prefix": True}, False),
|
||||
# BLOCK_Q is 2 for 8 query heads per KV head: 4 * ceil(4 / 2) <= 8.
|
||||
(4, {"num_seqs": 4}, True),
|
||||
(5, {"num_seqs": 4}, False),
|
||||
(5, {"scratch_rows": 9}, False),
|
||||
],
|
||||
)
|
||||
def test_splitk_admission_preserves_decode_and_bounds_verification(
|
||||
monkeypatch, query_len, options, expected
|
||||
):
|
||||
options = options.copy()
|
||||
monkeypatch.setattr(
|
||||
attention, "is_batch_invariant", options.pop("batch_invariant", False)
|
||||
)
|
||||
num_seqs = options.setdefault("num_seqs", 2)
|
||||
if options.get("causal") == "per_sequence":
|
||||
options["causal"] = torch.ones(num_seqs, dtype=torch.bool, device="cpu")
|
||||
if options.get("mm_prefix"):
|
||||
options["mm_prefix"] = torch.zeros(
|
||||
(num_seqs, 1, 2), dtype=torch.int32, device="cpu"
|
||||
)
|
||||
scratch = _scratch(options.pop("scratch_rows", num_seqs * query_len))
|
||||
assert _uses_splitk(monkeypatch, query_len, scratch, **options) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("empty_state", ["masked", "unvisited", "all_empty"])
|
||||
def test_reduce_segments_ignores_empty_softmax_states(empty_state: str) -> None:
|
||||
"""Empty segments must not suppress strongly negative, nonempty segments."""
|
||||
output = torch.full((1, 1, 32), float("nan"), device=DEVICE_TYPE)
|
||||
partial = torch.zeros((1, 1, 16, 32), device=DEVICE_TYPE)
|
||||
maxima = torch.full((1, 1, 16), -float("inf"), device=DEVICE_TYPE)
|
||||
masses = torch.ones_like(maxima)
|
||||
if empty_state != "all_empty":
|
||||
partial[:, :, 0] = 3.0
|
||||
maxima[:, :, 0] = -1000.0
|
||||
if empty_state in ("masked", "all_empty"):
|
||||
maxima[:, :, 1:] = 0.0
|
||||
masses[:, :, 1:] = 0.0
|
||||
reduce_segments[(1, 1)](
|
||||
output,
|
||||
partial,
|
||||
maxima,
|
||||
masses,
|
||||
torch.tensor([32], dtype=torch.int32, device=DEVICE_TYPE),
|
||||
1,
|
||||
1,
|
||||
1.0,
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
1,
|
||||
16,
|
||||
32,
|
||||
32,
|
||||
torch.tensor([0, 1], dtype=torch.int32, device=DEVICE_TYPE),
|
||||
16,
|
||||
16,
|
||||
False,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
output, torch.full_like(output, 0.0 if empty_state == "all_empty" else 3.0)
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _softmax_after_empty_tile(output_ptr):
|
||||
maximum = tl.full((1,), float("-inf"), tl.float32)
|
||||
mass = tl.full((1,), 1.0, tl.float32)
|
||||
empty_scores = tl.full((1, 16), float("-inf"), tl.float32)
|
||||
maximum, mass, _, _ = softmax_step(empty_scores, maximum, mass)
|
||||
scores = tl.full((1, 16), -1000.0, tl.float32)
|
||||
_, mass, probabilities, _ = softmax_step(scores, maximum, mass)
|
||||
tl.store(output_ptr + tl.arange(0, 16), tl.sum(probabilities, axis=0) / mass)
|
||||
|
||||
|
||||
def test_splitk_softmax_recovers_after_fully_masked_tile() -> None:
|
||||
"""A masked first tile must not suppress later, strongly negative logits."""
|
||||
probabilities = torch.empty(16, device=DEVICE_TYPE)
|
||||
_softmax_after_empty_tile[(1,)](probabilities)
|
||||
torch.testing.assert_close(probabilities, torch.full_like(probabilities, 1 / 16))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"query_lens,seq_lens", [([5, 0], [5, 0]), ([0], [0]), ([5], [0]), ([5], [5])]
|
||||
)
|
||||
def test_reduce_segments_leaves_graph_padding_untouched(query_lens, seq_lens) -> None:
|
||||
"""A padded reducer row must return before reading sequence or scratch data."""
|
||||
output = torch.full((10, 1, 32), 123.0, device=DEVICE_TYPE)
|
||||
partial = torch.zeros((10, 1, 16, 32), device=DEVICE_TYPE)
|
||||
maxima = torch.zeros((10, 1, 16), device=DEVICE_TYPE)
|
||||
masses = torch.ones_like(maxima)
|
||||
query_starts = torch.tensor(
|
||||
[0] + query_lens, dtype=torch.int32, device=DEVICE_TYPE
|
||||
).cumsum(0, dtype=torch.int32)
|
||||
reduce_segments[(10, 1)](
|
||||
output,
|
||||
partial,
|
||||
maxima,
|
||||
masses,
|
||||
torch.tensor(seq_lens, dtype=torch.int32, device=DEVICE_TYPE),
|
||||
len(seq_lens),
|
||||
1,
|
||||
1.0,
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
1,
|
||||
16,
|
||||
32,
|
||||
32,
|
||||
query_starts,
|
||||
16,
|
||||
16,
|
||||
False,
|
||||
)
|
||||
active_rows = sum(q for q, kv in zip(query_lens, seq_lens) if kv > 0)
|
||||
torch.testing.assert_close(
|
||||
output[:active_rows], torch.zeros_like(output[:active_rows])
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
output[active_rows:], torch.full_like(output[active_rows:], 123.0)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("seq_lens", "num_heads", "q_dtype"),
|
||||
[
|
||||
*[
|
||||
pytest.param([(w, 128), (w, 129), (w, 257)], (8, 1), None, id=f"w{w}")
|
||||
for w in range(2, MAX_3D_QUERY_LEN + 1, 2)
|
||||
],
|
||||
# Prefill chunk, verification, decode, padding, short query.
|
||||
pytest.param(
|
||||
[(8, 8), (6, 129), (1, 257), (0, 0), (3, 64)], (8, 1), None, id="mixed"
|
||||
),
|
||||
*[
|
||||
pytest.param(
|
||||
[(6, 128), (6, 129), (6, 257)],
|
||||
heads,
|
||||
FP8_DTYPE,
|
||||
id=f"w6-fp8q-kv{heads[1]}",
|
||||
)
|
||||
for heads in [(8, 1), (8, 8)]
|
||||
],
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("sliding_window", [None, 32])
|
||||
def test_speculative_splitk_matches_reference(
|
||||
monkeypatch, seq_lens, num_heads, q_dtype, sliding_window
|
||||
):
|
||||
reduce_grids = _spy_reduce_segments(monkeypatch)
|
||||
test_triton_unified_attn(
|
||||
seq_lens=seq_lens,
|
||||
num_heads=num_heads,
|
||||
head_size=128,
|
||||
sliding_window=sliding_window,
|
||||
dtype=torch.bfloat16,
|
||||
block_size=128,
|
||||
soft_cap=None,
|
||||
num_blocks=8,
|
||||
q_dtype=q_dtype,
|
||||
seq_threshold_3D=32,
|
||||
)
|
||||
assert len(reduce_grids) == 1
|
||||
|
||||
|
||||
def test_speculative_splitk_bf16_query_fp8_kv(monkeypatch):
|
||||
reduce_grids = _spy_reduce_segments(monkeypatch)
|
||||
test_triton_unified_attn_bf16_query_fp8_kv(
|
||||
seq_lens=[(6, 128), (6, 129), (6, 4097)],
|
||||
num_heads=(8, 1),
|
||||
head_size=128,
|
||||
block_size=128,
|
||||
num_blocks=64,
|
||||
seq_threshold_3D=16,
|
||||
)
|
||||
assert len(reduce_grids) == 1
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _compute_clamped_mm_tile_bounds(
|
||||
output_ptr,
|
||||
@@ -471,16 +732,19 @@ def test_triton_unified_attn(
|
||||
|
||||
num_par_softmax_segments = 16
|
||||
head_size_padded = next_power_of_2(head_size)
|
||||
scratch_rows = seq_threshold_3D * (
|
||||
max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1
|
||||
)
|
||||
softmax_segm_output = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_max = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_expsum = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
@@ -591,16 +855,19 @@ def test_triton_unified_attn_bf16_query_fp8_kv(
|
||||
|
||||
num_par_softmax_segments = 16
|
||||
head_size_padded = next_power_of_2(head_size)
|
||||
scratch_rows = seq_threshold_3D * (
|
||||
max_query_len if max_query_len <= MAX_3D_QUERY_LEN else 1
|
||||
)
|
||||
softmax_segm_output = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments, head_size_padded),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_max = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
softmax_segm_expsum = torch.empty(
|
||||
(seq_threshold_3D, num_query_heads, num_par_softmax_segments),
|
||||
(scratch_rows, num_query_heads, num_par_softmax_segments),
|
||||
dtype=torch.float32,
|
||||
)
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the Triton DiffKV unified-attention kernel."""
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -16,6 +18,7 @@ from vllm.v1.attention.backends.fa_utils import (
|
||||
get_flash_attn_version,
|
||||
is_flash_attn_varlen_func_available,
|
||||
)
|
||||
from vllm.v1.attention.ops import triton_unified_attention_diffkv as diffkv_module
|
||||
from vllm.v1.attention.ops.triton_unified_attention_diffkv import (
|
||||
unified_attention_diffkv,
|
||||
)
|
||||
@@ -258,3 +261,95 @@ def test_triton_unified_attn_diffkv_fp8_query(seq_lens, seq_threshold_3D):
|
||||
seq_threshold_3D=seq_threshold_3D,
|
||||
quant_query=True,
|
||||
)
|
||||
|
||||
|
||||
def _capture_diffkv_launch(
|
||||
query_lens: list[int], max_seqlen_q: int, segm_first_dim: int
|
||||
):
|
||||
"""Run the launcher with both Triton kernels mocked; return grid/kwargs."""
|
||||
num_query_heads, num_kv_heads = 8, 2
|
||||
head_size_qk, head_size_v = 128, 128
|
||||
num_seqs = len(query_lens)
|
||||
num_tokens = sum(query_lens)
|
||||
|
||||
query = torch.zeros(
|
||||
num_tokens, num_query_heads, head_size_qk, dtype=torch.bfloat16, device="cpu"
|
||||
)
|
||||
kv_cache = torch.zeros(
|
||||
NUM_BLOCKS,
|
||||
BLOCK_SIZES[0],
|
||||
num_kv_heads,
|
||||
head_size_qk + head_size_v,
|
||||
dtype=torch.bfloat16,
|
||||
device="cpu",
|
||||
)
|
||||
key_cache = kv_cache[..., :head_size_qk]
|
||||
value_cache = kv_cache[..., head_size_qk:]
|
||||
out = torch.zeros(
|
||||
num_tokens, num_query_heads, head_size_v, dtype=torch.bfloat16, device="cpu"
|
||||
)
|
||||
cu_seqlens_q = torch.tensor(
|
||||
[0] + query_lens, dtype=torch.int32, device="cpu"
|
||||
).cumsum(dim=0, dtype=torch.int32)
|
||||
seqused_k = torch.full((num_seqs,), 512, dtype=torch.int32, device="cpu")
|
||||
block_tables = torch.zeros(num_seqs, 32, dtype=torch.int32, device="cpu")
|
||||
segm_output, segm_max, segm_expsum = _alloc_segm_buffers(
|
||||
segm_first_dim, num_query_heads, head_size_v
|
||||
)
|
||||
|
||||
with (
|
||||
patch.object(diffkv_module, "kernel_unified_attention_diffkv") as mock_kernel,
|
||||
patch.object(diffkv_module, "kernel_reduce_segments_diffkv") as mock_reduce,
|
||||
):
|
||||
unified_attention_diffkv(
|
||||
q=query,
|
||||
k=key_cache,
|
||||
v=value_cache,
|
||||
out=out,
|
||||
cu_seqlens_q=cu_seqlens_q,
|
||||
seqused_k=seqused_k,
|
||||
softmax_scale=head_size_qk**-0.5,
|
||||
causal=True,
|
||||
window_size=(-1, -1),
|
||||
block_table=block_tables,
|
||||
softcap=0,
|
||||
max_seqlen_q=max_seqlen_q,
|
||||
seq_threshold_3D=8,
|
||||
num_par_softmax_segments=NUM_PAR_SOFTMAX_SEGMENTS,
|
||||
softmax_segm_output=segm_output,
|
||||
softmax_segm_max=segm_max,
|
||||
softmax_segm_expsum=segm_expsum,
|
||||
)
|
||||
|
||||
grid = mock_kernel.__getitem__.call_args.args[0]
|
||||
kwargs = mock_kernel.__getitem__.return_value.call_args.kwargs
|
||||
return grid, kwargs, mock_reduce.__getitem__.called
|
||||
|
||||
|
||||
def test_spec_verify_block_m_grouping_and_launch_kwargs() -> None:
|
||||
"""Pin program granularity for verify/decode without launching Triton."""
|
||||
verify_grid, verify_kwargs, verify_reduce = _capture_diffkv_launch(
|
||||
query_lens=[8, 8], max_seqlen_q=8, segm_first_dim=16
|
||||
)
|
||||
num_queries_per_kv = 4
|
||||
expected_block_m = min(128, next_power_of_2(8 * num_queries_per_kv))
|
||||
assert verify_grid == (4, 2, NUM_PAR_SOFTMAX_SEGMENTS)
|
||||
assert verify_kwargs["IS_3D"] is True
|
||||
assert verify_kwargs["BLOCK_M"] == expected_block_m
|
||||
assert verify_kwargs["BLOCK_Q"] == expected_block_m // num_queries_per_kv
|
||||
assert verify_kwargs["TILE_SIZE"] == 16
|
||||
assert verify_kwargs["num_warps"] == 4
|
||||
assert "num_stages" not in verify_kwargs
|
||||
assert verify_reduce
|
||||
|
||||
decode_grid, decode_kwargs, decode_reduce = _capture_diffkv_launch(
|
||||
query_lens=[1], max_seqlen_q=1, segm_first_dim=8
|
||||
)
|
||||
assert decode_grid == (1, 2, NUM_PAR_SOFTMAX_SEGMENTS)
|
||||
assert decode_kwargs["IS_3D"] is True
|
||||
assert decode_kwargs["BLOCK_M"] == 16
|
||||
assert decode_kwargs["BLOCK_Q"] == 4
|
||||
assert decode_kwargs["TILE_SIZE"] == 16
|
||||
assert "num_warps" not in decode_kwargs
|
||||
assert "num_stages" not in decode_kwargs
|
||||
assert decode_reduce
|
||||
|
||||
@@ -21,6 +21,9 @@ DEVICES = [
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_cpu_write(device):
|
||||
torch.set_default_device(device)
|
||||
# Initializes the device; otherwise torch.accelerator.synchronize() is a no-op,
|
||||
# as the UVA view's ops never trigger lazy init.
|
||||
torch.accelerator.set_device_index(device)
|
||||
cpu_tensor = torch.zeros(10, 10, device="cpu", pin_memory=True, dtype=torch.int32)
|
||||
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
|
||||
assert gpu_view.device.type == DEVICE_TYPE
|
||||
@@ -44,6 +47,9 @@ def test_cpu_write(device):
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_gpu_write(device):
|
||||
torch.set_default_device(device)
|
||||
# Initializes the device; otherwise torch.accelerator.synchronize() is a no-op,
|
||||
# as the UVA view's ops never trigger lazy init.
|
||||
torch.accelerator.set_device_index(device)
|
||||
cpu_tensor = torch.zeros(10, 10, device="cpu", pin_memory=True, dtype=torch.int32)
|
||||
gpu_view = get_accelerator_view_from_cpu_tensor(cpu_tensor)
|
||||
assert gpu_view.device.type == DEVICE_TYPE
|
||||
|
||||
@@ -9,6 +9,8 @@ each rank, and summing the per-rank outputs (what the all-reduce does)
|
||||
reconstructs a full-table lookup.
|
||||
"""
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -124,3 +126,43 @@ def test_rank_sum_is_full_lookup(tp_size):
|
||||
total += _fused(ids, _shard(full_table, si, rows, dtype), si)
|
||||
|
||||
torch.testing.assert_close(total, full_table[ids.long()], atol=0.0, rtol=0.0)
|
||||
|
||||
|
||||
@requires_cuda
|
||||
@torch.inference_mode()
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float8_e4m3fn])
|
||||
@pytest.mark.parametrize("use_fused", [False, True])
|
||||
def test_embedding_optionally_reduces_output(dtype, use_fused, default_vllm_config):
|
||||
"""SP's deferred reduction preserves masked contributions and their dtype."""
|
||||
with torch.device("cuda"):
|
||||
values = torch.arange(128 * 16).reshape(128, 16)
|
||||
table = ((values % 29 - 14) / 16).to(dtype)
|
||||
ids = torch.tensor([0, 63, 64, 127, 12])
|
||||
expected = table[ids].float()
|
||||
for rank in range(2):
|
||||
local = expected.clone()
|
||||
local[ids // 64 != rank] = 0
|
||||
remote = (expected - local).to(dtype)
|
||||
group = Mock(world_size=2, rank_in_group=rank)
|
||||
# Model the other vocabulary owner's contribution, including FP8 bytes.
|
||||
group.all_reduce.side_effect = lambda output, remote=remote: (
|
||||
output + remote.view(output.dtype)
|
||||
)
|
||||
for reduce_results in (True, False):
|
||||
embedding = VocabParallelEmbedding(
|
||||
128,
|
||||
16,
|
||||
params_dtype=dtype,
|
||||
parallel_group=group,
|
||||
reduce_results=reduce_results,
|
||||
)
|
||||
embedding.use_fused_embedding = use_fused
|
||||
embedding.weight_loader(embedding.get_parameter("weight"), table)
|
||||
actual = embedding(ids)
|
||||
assert actual.dtype == dtype
|
||||
torch.testing.assert_close(
|
||||
actual.float(),
|
||||
expected if reduce_results else local,
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
|
||||
@@ -18,17 +18,32 @@ pytestmark = pytest.mark.skipif(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("strided_mixed_qkv", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"batch,num_key_heads,num_value_heads,dtype,state_dtype",
|
||||
[
|
||||
(32, 4, 8, torch.float16, torch.float16),
|
||||
(32, 4, 8, torch.bfloat16, torch.bfloat16),
|
||||
(32, 4, 8, torch.float32, torch.float32),
|
||||
(1, 16, 64, torch.bfloat16, torch.float32),
|
||||
(9, 4, 12, torch.bfloat16, torch.float32),
|
||||
(1, 4, 12, torch.bfloat16, torch.bfloat16),
|
||||
],
|
||||
)
|
||||
def test_fused_recurrent_packed_decode_matches_reference(
|
||||
dtype: torch.dtype, strided_mixed_qkv: bool
|
||||
dtype: torch.dtype,
|
||||
strided_mixed_qkv: bool,
|
||||
batch: int,
|
||||
num_key_heads: int,
|
||||
num_value_heads: int,
|
||||
state_dtype: torch.dtype,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
generator = torch.Generator(device=DEVICE).manual_seed(0)
|
||||
|
||||
# Small but representative GDN config (Qwen3Next defaults are K=128, V=128).
|
||||
B = 32
|
||||
H = 4
|
||||
HV = 8 # grouped value attention: HV must be divisible by H
|
||||
B = batch
|
||||
H = num_key_heads
|
||||
HV = num_value_heads
|
||||
K = 128
|
||||
V = 128
|
||||
qkv_dim = 2 * (H * K) + (HV * V)
|
||||
@@ -38,22 +53,33 @@ def test_fused_recurrent_packed_decode_matches_reference(
|
||||
if strided_mixed_qkv:
|
||||
# Simulate a packed view into a larger projection buffer:
|
||||
# mixed_qkv.stride(0) > mixed_qkv.shape[1]
|
||||
proj = torch.randn((B, qkv_dim + 64), device=device, dtype=dtype)
|
||||
proj = torch.randn(
|
||||
(B, qkv_dim + 64), device=device, dtype=dtype, generator=generator
|
||||
)
|
||||
mixed_qkv = proj[:, :qkv_dim]
|
||||
else:
|
||||
mixed_qkv = torch.randn((B, qkv_dim), device=device, dtype=dtype)
|
||||
mixed_qkv = torch.randn(
|
||||
(B, qkv_dim), device=device, dtype=dtype, generator=generator
|
||||
)
|
||||
|
||||
a = torch.randn((B, HV), device=device, dtype=dtype)
|
||||
b = torch.randn((B, HV), device=device, dtype=dtype)
|
||||
A_log = torch.randn((HV,), device=device, dtype=dtype)
|
||||
dt_bias = torch.randn((HV,), device=device, dtype=dtype)
|
||||
a = torch.randn((B, HV), device=device, dtype=dtype, generator=generator)
|
||||
b = torch.randn((B, HV), device=device, dtype=dtype, generator=generator)
|
||||
A_log = torch.randn((HV,), device=device, dtype=dtype, generator=generator)
|
||||
dt_bias = torch.randn((HV,), device=device, dtype=dtype, generator=generator)
|
||||
|
||||
# Continuous batching indices (include PAD_SLOT_ID=-1 cases). Index 0 is
|
||||
# reserved as NULL_BLOCK_ID (CUDA graph padding), so valid slots start at 1.
|
||||
ssm_state_indices = torch.arange(1, B + 1, device=device, dtype=torch.int32)
|
||||
ssm_state_indices[-3:] = -1
|
||||
if B > 3:
|
||||
ssm_state_indices[-3:] = -1
|
||||
ssm_state_indices[-1] = 0
|
||||
|
||||
state0 = torch.randn((B + 1, HV, V, K), device=device, dtype=dtype)
|
||||
state0 = torch.randn(
|
||||
(B + 1, HV, V, K),
|
||||
device=device,
|
||||
dtype=state_dtype,
|
||||
generator=generator,
|
||||
)
|
||||
state_ref = state0.clone()
|
||||
state_packed = state0.clone()
|
||||
|
||||
|
||||
@@ -13,10 +13,19 @@ if current_platform.is_rocm():
|
||||
)
|
||||
|
||||
import flashinfer.gdn_prefill # noqa: E402
|
||||
import torch.nn.functional as F # noqa: E402
|
||||
|
||||
from vllm.model_executor.layers.mamba.checkpoint import ( # noqa: E402
|
||||
MambaPrefillCheckpointExporter,
|
||||
MambaPrefillCheckpointMetadata,
|
||||
)
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
fi_chunk_gated_delta_rule,
|
||||
) # noqa: E402
|
||||
from vllm.v1.attention.backends.gdn_attn import ( # noqa: E402
|
||||
plan_flashinfer_checkpoints,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID # noqa: E402
|
||||
|
||||
|
||||
def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
|
||||
@@ -51,3 +60,130 @@ def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
|
||||
assert captured_cu_seqlens.dtype == torch.int64
|
||||
assert output.shape == q.shape
|
||||
assert final_state is None
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_device_capability(90)
|
||||
or current_platform.is_device_capability_family(100)
|
||||
or current_platform.is_device_capability_family(120)
|
||||
),
|
||||
reason="FlashInfer GDN prefill checkpoints need SM90, SM100 or SM120.",
|
||||
)
|
||||
@pytest.mark.parametrize("use_cp", [True, False])
|
||||
@pytest.mark.parametrize("ssm_dtype", [torch.float32, torch.bfloat16])
|
||||
def test_flashinfer_checkpoints_export_the_state_at_each_offset(use_cp, ssm_dtype):
|
||||
"""Each exported checkpoint is the state of the prefill stopped there.
|
||||
|
||||
The leading decode row is outside the prefill kernel. The two checkpointed
|
||||
prefills put FlashInfer's stride (their offsets' gcd) below both offsets,
|
||||
so the exporter must pick each state out of the shared emitted buffer;
|
||||
the third prefill takes no checkpoint. Both FlashInfer routes are covered
|
||||
because ``use_cp="auto"`` may take either. The conv cache has MTP's spec
|
||||
slots after the window, which the conv kernels leave alone.
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
device = "cuda"
|
||||
num_qk_heads, num_v_heads, head_dim, conv_dim, state_len = 2, 4, 128, 16, 3
|
||||
num_spec = 2
|
||||
offsets = [0, 128, 0, 192]
|
||||
query_lens = [200, 50, 300]
|
||||
state_indices = [NULL_BLOCK_ID, 3, NULL_BLOCK_ID, 5]
|
||||
num_blocks = 8
|
||||
|
||||
num_tokens = sum(query_lens)
|
||||
q = F.normalize(
|
||||
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
|
||||
).bfloat16()
|
||||
k = F.normalize(
|
||||
torch.randn(1, num_tokens, num_qk_heads, head_dim, device=device), dim=-1
|
||||
).bfloat16()
|
||||
v = torch.randn(1, num_tokens, num_v_heads, head_dim, device=device).bfloat16()
|
||||
g = F.logsigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
|
||||
beta = torch.sigmoid(torch.randn(1, num_tokens, num_v_heads, device=device))
|
||||
initial_state = 0.1 * torch.randn(
|
||||
len(query_lens), num_v_heads, head_dim, head_dim, device=device
|
||||
)
|
||||
prefill_cu = [0]
|
||||
for query_len in query_lens:
|
||||
prefill_cu.append(prefill_cu[-1] + query_len)
|
||||
|
||||
every_n_tokens, cu_starts, rows = plan_flashinfer_checkpoints(offsets, query_lens)
|
||||
state_checkpoints = torch.empty(
|
||||
cu_starts[-1], num_v_heads, head_dim, head_dim, device=device
|
||||
)
|
||||
fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
k=k,
|
||||
v=v,
|
||||
g=g,
|
||||
beta=beta,
|
||||
initial_state=initial_state,
|
||||
output_final_state=True,
|
||||
cu_seqlens=torch.tensor(prefill_cu, device=device),
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
state_checkpoints=state_checkpoints,
|
||||
checkpoint_cu_starts=torch.tensor(cu_starts, device=device),
|
||||
checkpoint_every_n_tokens=every_n_tokens,
|
||||
use_cp=use_cp,
|
||||
)
|
||||
|
||||
# The exporter sees the decode row's token in front of the prefills.
|
||||
raw_qkv = torch.randn(1 + num_tokens, conv_dim, device=device).bfloat16()
|
||||
cu_seqlens = torch.tensor([0] + [1 + n for n in prefill_cu], device=device)
|
||||
conv_state = torch.zeros(
|
||||
num_blocks,
|
||||
state_len + num_spec,
|
||||
conv_dim,
|
||||
device=device,
|
||||
dtype=torch.bfloat16,
|
||||
).transpose(-1, -2)
|
||||
ssm_state = torch.zeros(
|
||||
num_blocks, num_v_heads, head_dim, head_dim, device=device, dtype=ssm_dtype
|
||||
)
|
||||
MambaPrefillCheckpointExporter(state_len=state_len).export(
|
||||
MambaPrefillCheckpointMetadata(
|
||||
checkpoint_offsets=torch.tensor(offsets, device=device, dtype=torch.int32),
|
||||
state_indices=torch.tensor(state_indices, device=device),
|
||||
),
|
||||
raw_qkv=raw_qkv,
|
||||
conv_state=conv_state,
|
||||
recurrent_checkpoint=state_checkpoints,
|
||||
recurrent_state=ssm_state,
|
||||
cu_seqlens=cu_seqlens,
|
||||
recurrent_rows=torch.tensor(rows, device=device),
|
||||
)
|
||||
|
||||
written = []
|
||||
for row, (offset, block) in enumerate(zip(offsets, state_indices)):
|
||||
if not offset:
|
||||
continue
|
||||
written.append(block)
|
||||
seq = row - 1
|
||||
start = prefill_cu[seq]
|
||||
_, expected_state = fi_chunk_gated_delta_rule(
|
||||
q=q[:, start : start + offset],
|
||||
k=k[:, start : start + offset],
|
||||
v=v[:, start : start + offset],
|
||||
g=g[:, start : start + offset],
|
||||
beta=beta[:, start : start + offset],
|
||||
initial_state=initial_state[seq : seq + 1],
|
||||
output_final_state=True,
|
||||
cu_seqlens=torch.tensor([0, offset], device=device),
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
ssm_state[block].float(), expected_state[0], atol=2e-2, rtol=2e-2
|
||||
)
|
||||
end = cu_seqlens[row].item() + offset
|
||||
torch.testing.assert_close(
|
||||
conv_state[block, :, :state_len],
|
||||
raw_qkv[end - state_len : end].T,
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
assert not conv_state[block, :, state_len:].any()
|
||||
|
||||
untouched = [i for i in range(num_blocks) if i not in written]
|
||||
assert not ssm_state[untouched].any()
|
||||
assert not conv_state[untouched].any()
|
||||
|
||||
@@ -178,7 +178,9 @@ def test_rocm_prefill_numerics_and_output_layout(state_dtype, correlated):
|
||||
model="Qwen/Qwen3.5-0.8B",
|
||||
revision="2fc06364715b967f1860aea9cf38778875588b17",
|
||||
max_model_len=1024,
|
||||
)
|
||||
),
|
||||
# auto picks AITER FlyDSL on gfx942/gfx950; this pins the Triton path.
|
||||
additional_config={"gdn_prefill_backend": "triton"},
|
||||
)
|
||||
with set_current_vllm_config(config):
|
||||
prefill = ChunkGatedDeltaRule()
|
||||
|
||||
@@ -114,23 +114,27 @@ def test_checkpoint_does_not_perturb_the_final_state():
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("spec_slots", [0, 3])
|
||||
@pytest.mark.parametrize("dim_first", [True, False])
|
||||
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first):
|
||||
def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first, spec_slots):
|
||||
"""The store reads the checkpoint row out of every chunk state.
|
||||
|
||||
Row 0 checkpoints 32 tokens in, its state being chunk 3 rather than the
|
||||
program's own row, so the store only passes if ``chunk_idx`` is used.
|
||||
Row 1 declines and must write nothing. The conv input is a column slice of
|
||||
a wider projection, the SD conv layout is a transposed view, and SSM rows
|
||||
are padded pages, matching the strides the mixer passes.
|
||||
are padded pages, matching the strides the mixer passes. Spec decoding
|
||||
widens the conv state, and the store must still fill only the first
|
||||
``state_len`` slots with the newest inputs.
|
||||
"""
|
||||
dim, state_len, heads, head_dim, d_state = 16, 4, 2, 4, 8
|
||||
projection = torch.randn(64, dim + 8, device=DEVICE)
|
||||
conv_input = projection[:, :dim]
|
||||
num_slots = state_len + spec_slots
|
||||
if dim_first:
|
||||
conv_state = torch.zeros(3, dim, state_len, device=DEVICE)
|
||||
conv_state = torch.zeros(3, dim, num_slots, device=DEVICE)
|
||||
else:
|
||||
conv_state = torch.zeros(3, state_len, dim, device=DEVICE).transpose(-1, -2)
|
||||
conv_state = torch.zeros(3, num_slots, dim, device=DEVICE).transpose(-1, -2)
|
||||
varlen_states = torch.randn(5, heads, head_dim, d_state, device=DEVICE)
|
||||
# Blocks are pages holding more than the SSM state, so rows are padded.
|
||||
row = heads * head_dim * d_state
|
||||
@@ -152,12 +156,14 @@ def test_store_selects_the_checkpoint_from_all_chunk_states(dim_first):
|
||||
varlen_states,
|
||||
ssm_state,
|
||||
torch.tensor([0, 40, 64], dtype=torch.int32, device=DEVICE),
|
||||
state_len=state_len,
|
||||
)
|
||||
|
||||
exact = dict(rtol=0, atol=0)
|
||||
torch.testing.assert_close(
|
||||
conv_state[2], conv_input[32 - state_len : 32].T, **exact
|
||||
conv_state[2, :, :state_len], conv_input[32 - state_len : 32].T, **exact
|
||||
)
|
||||
assert not conv_state[2, :, state_len:].any(), "wrote into the spec slots"
|
||||
torch.testing.assert_close(ssm_state[2], varlen_states[3], **exact)
|
||||
assert not conv_state[:2].any() and not ssm_storage[:2].any()
|
||||
assert not ssm_storage[:, row:].any(), "wrote into the page padding"
|
||||
|
||||
@@ -96,15 +96,17 @@ def mixer2_gated_norm_tensor_parallel(
|
||||
gate_states = torch.randn(batch_size, seq_len, hidden_size)
|
||||
|
||||
# create gated-norm with TP
|
||||
mixer = Mixer2RMSNormGated(
|
||||
full_hidden_size=hidden_size,
|
||||
full_n_groups=n_groups,
|
||||
)
|
||||
with ensure_current_vllm_config():
|
||||
mixer = Mixer2RMSNormGated(
|
||||
full_hidden_size=hidden_size,
|
||||
full_n_groups=n_groups,
|
||||
)
|
||||
mixer.weight.weight_loader(mixer.weight, weight) # load
|
||||
|
||||
# create gated-norm without TP to compute reference
|
||||
# - utilize mock patching to disable TP when
|
||||
with (
|
||||
ensure_current_vllm_config(),
|
||||
unittest.mock.patch(
|
||||
"vllm.model_executor.layers.mamba.mamba_mixer2."
|
||||
"get_tensor_model_parallel_world_size",
|
||||
|
||||
@@ -637,3 +637,27 @@ def test_replayssm_physical_ring_shape(
|
||||
(8, expected_ring_len),
|
||||
(2, expected_ring_len, 16),
|
||||
)
|
||||
|
||||
|
||||
def test_replayssm_ring_shape_extends_groups_for_head_shards():
|
||||
"""A single group under TP is replicated to every head shard, as in
|
||||
``mamba2_state_shape``, so each rank's B_cache holds one group."""
|
||||
base_shapes = MambaStateShapeCalculator.mamba2_state_shape(
|
||||
tp_world_size=2,
|
||||
intermediate_size=256,
|
||||
n_groups=1,
|
||||
num_heads=4,
|
||||
head_dim=64,
|
||||
state_size=128,
|
||||
conv_kernel=4,
|
||||
)
|
||||
|
||||
shapes = MambaStateShapeCalculator.append_replayssm_ring(
|
||||
base_shapes,
|
||||
n_groups=1,
|
||||
tp_world_size=2,
|
||||
logical_window=16,
|
||||
backend=MambaBackendEnum.TRITON,
|
||||
)
|
||||
|
||||
assert shapes[4] == (1, 16, 128)
|
||||
|
||||
@@ -439,13 +439,16 @@ def test_deepseek_v41_mhc_fused_post_pre_delayed(num_tokens, hidden_size, carrie
|
||||
return
|
||||
|
||||
# The post mapping and the collapse are what the unfused kernels produce.
|
||||
# The two kernels may round a near-midpoint residual differently (FMA
|
||||
# contraction is up to the compiler), so allow one bf16 ulp there and
|
||||
# check the collapse against the fused kernel's own residual.
|
||||
residual_ref = torch.ops.vllm.mhc_post_tilelang(
|
||||
x, residual, post_layer_mix, comb_res_mix
|
||||
)
|
||||
layer_input_ref = mhc_pre_delayed_tilelang(
|
||||
residual_ref, *mix_args, pre_mix=pre_mix, norm_weight=weight, norm_eps=1e-6
|
||||
residual_cur, *mix_args, pre_mix=pre_mix, norm_weight=weight, norm_eps=1e-6
|
||||
)[2]
|
||||
torch.testing.assert_close(residual_cur, residual_ref, atol=0, rtol=0)
|
||||
torch.testing.assert_close(residual_cur, residual_ref, atol=0, rtol=2**-7)
|
||||
torch.testing.assert_close(layer_input, layer_input_ref, atol=0, rtol=0)
|
||||
|
||||
post_ref, comb_ref, _, next_pre_ref = mhc_fused_post_pre_delayed_ref(
|
||||
|
||||
@@ -167,6 +167,9 @@ class Config:
|
||||
vllm_config.model_config = SimpleNamespace(
|
||||
enforce_eager=True,
|
||||
is_moe=True,
|
||||
enable_sleep_mode=False,
|
||||
sleep_mode_backend="cumem",
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
)
|
||||
vllm_config.parallel_config.data_parallel_size = self.world_size
|
||||
vllm_config.parallel_config.enable_expert_parallel = True
|
||||
@@ -722,6 +725,9 @@ def _maybe_convert_weights_for_experts(
|
||||
rank_weights: WeightTensors,
|
||||
) -> WeightTensors:
|
||||
"""Convert weights to expert-specific format (e.g., TrtLLM BlockMajorK)."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
convert_to_fp8_moe_kernel_format,
|
||||
@@ -734,7 +740,7 @@ def _maybe_convert_weights_for_experts(
|
||||
# intermediate sizes that are not a multiple of 256, but only for the fp8 per-tensor
|
||||
# and per-token schemes. Those keep JIT-compiling until
|
||||
# https://github.com/ROCm/aiter/issues/5766 is fixed.
|
||||
if fe_name == "AiterExperts" and (
|
||||
if issubclass(fe_type, AiterExperts) and (
|
||||
config.quant_dtype is None or config.quant_block_shape is not None
|
||||
):
|
||||
return _shuffle_weights_for_aiter(rank_weights)
|
||||
|
||||
@@ -93,6 +93,7 @@ common_float_types: list[torch.dtype | str] = [
|
||||
]
|
||||
common_float_and_int_types = common_float_types + [torch.int8]
|
||||
nvfp4_types = ["nvfp4"]
|
||||
mxfp4_types = ["mxfp4"]
|
||||
fp8_types = [current_platform.fp8_dtype()]
|
||||
|
||||
|
||||
@@ -318,20 +319,37 @@ if has_flashinfer_trtllm_fused_moe() and current_platform.has_device_capability(
|
||||
|
||||
if has_aiter():
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterFp8BlockExperts,
|
||||
AiterFp8ChannelExperts,
|
||||
AiterFp8TensorExperts,
|
||||
AiterMxfp4Experts,
|
||||
AiterUnquantizedExperts,
|
||||
)
|
||||
|
||||
register_experts(
|
||||
AiterExperts,
|
||||
standard_format,
|
||||
# AiterExperts also supports the fully-unquantized (None, None)
|
||||
# scheme (see SUPPORTED_W_A in rocm_aiter_moe.py), not just fp8.
|
||||
common_float_types,
|
||||
blocked_quantization_support=True,
|
||||
needs_aiter=True,
|
||||
)
|
||||
# Registration dtypes gate which quant configs the modular-kernel test
|
||||
# harness will ever pair with a given class (see Config.is_valid()):
|
||||
# they must match what each subclass's _supports_quant_scheme() actually
|
||||
# declares, or the harness will generate a case the class rejects.
|
||||
for aiter_experts_cls, aiter_supported_dtypes in (
|
||||
(AiterUnquantizedExperts, common_float_types),
|
||||
(AiterFp8BlockExperts, fp8_types),
|
||||
(AiterFp8TensorExperts, fp8_types),
|
||||
(AiterFp8ChannelExperts, fp8_types),
|
||||
(AiterMxfp4Experts, mxfp4_types),
|
||||
):
|
||||
register_experts(
|
||||
aiter_experts_cls,
|
||||
standard_format,
|
||||
aiter_supported_dtypes,
|
||||
blocked_quantization_support=True,
|
||||
needs_aiter=True,
|
||||
)
|
||||
else:
|
||||
AiterExperts = None
|
||||
AiterUnquantizedExperts = None
|
||||
AiterFp8BlockExperts = None
|
||||
AiterFp8TensorExperts = None
|
||||
AiterFp8ChannelExperts = None
|
||||
AiterMxfp4Experts = None
|
||||
|
||||
if has_deep_gemm() and is_deep_gemm_supported():
|
||||
register_experts(
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
"""DeepEP test utilities."""
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
import traceback
|
||||
from collections.abc import Callable
|
||||
from typing import Concatenate
|
||||
@@ -14,7 +13,7 @@ from torch.multiprocessing import spawn # pyright: ignore[reportPrivateImportUs
|
||||
from typing_extensions import ParamSpec
|
||||
|
||||
from vllm.utils.import_utils import has_deep_ep, has_deep_ep_v2
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.network_utils import get_file_store_init_method
|
||||
|
||||
if has_deep_ep():
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_ht import (
|
||||
@@ -105,7 +104,7 @@ def parallel_launch(
|
||||
world_size,
|
||||
world_size,
|
||||
0,
|
||||
f"tcp://{os.getenv('LOCALHOST', 'localhost')}:{get_open_port()}",
|
||||
get_file_store_init_method(),
|
||||
worker,
|
||||
)
|
||||
+ args,
|
||||
|
||||
@@ -179,7 +179,14 @@ def _make_b12x_moe_kernel(
|
||||
num_experts=num_experts,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=hidden_states.shape[1],
|
||||
intermediate_size=w2.shape[2] * 2,
|
||||
intermediate_size=w2.shape[2]
|
||||
* (
|
||||
32
|
||||
if quant_config.weight_quant_dtype == "q8_0"
|
||||
else 256
|
||||
if quant_config.weight_quant_dtype in ("iq2_xs", "iq2_xxs")
|
||||
else 2
|
||||
),
|
||||
in_dtype=hidden_states.dtype,
|
||||
activation=activation,
|
||||
)
|
||||
@@ -1272,3 +1279,78 @@ def test_b12x_moe_cuda_graph_replay(
|
||||
assert torch.isfinite(expected).all()
|
||||
assert torch.isfinite(actual).all()
|
||||
torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _has_b12x_moe(), reason="requires b12x MoE on SM120")
|
||||
@pytest.mark.parametrize("codec", ["iq2_xxs", "q8_0"])
|
||||
@pytest.mark.parametrize("tokens", [1, 2, 4])
|
||||
@torch.inference_mode()
|
||||
def test_b12x_block_quant_relu2_moe_graph(codec, tokens, workspace_init):
|
||||
from b12x.testing.iq2_xs_reference import moe_reference_iq2_xs
|
||||
|
||||
from tests.model_executor.kernels.test_b12x_linear import _block_weights
|
||||
|
||||
with set_current_vllm_config(VllmConfig()):
|
||||
torch.manual_seed(71)
|
||||
e, h, i, topk = 4, 256, 256, 2
|
||||
raw1, raw2 = _block_weights(e * i, h, codec), _block_weights(e * h, i, codec)
|
||||
w1 = raw1.view(e, i, *raw1.shape[1:]).cuda()
|
||||
w2 = raw2.view(e, h, *raw2.shape[1:]).cuda()
|
||||
x = torch.randn(tokens, h, device="cuda", dtype=torch.bfloat16) * 0.125
|
||||
scores = torch.randn(tokens, e, device="cuda", dtype=torch.bfloat16)
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEQuantDesc
|
||||
|
||||
quant = FusedMoEQuantConfig(
|
||||
_a1=FusedMoEQuantDesc(),
|
||||
_a2=FusedMoEQuantDesc(),
|
||||
_w1=FusedMoEQuantDesc(dtype=codec),
|
||||
_w2=FusedMoEQuantDesc(dtype=codec),
|
||||
)
|
||||
activation = MoEActivation.RELU2_NO_MUL
|
||||
kernel = _make_b12x_moe_kernel(x, w1, w2, topk, activation, quant)
|
||||
weights, ids, _ = fused_topk(x, scores, topk, renormalize=False)
|
||||
|
||||
def apply():
|
||||
return kernel.apply(
|
||||
hidden_states=x,
|
||||
w1=w1,
|
||||
w2=w2,
|
||||
topk_weights=weights,
|
||||
topk_ids=ids,
|
||||
activation=activation,
|
||||
global_num_experts=e,
|
||||
expert_map=None,
|
||||
apply_router_weight_on_input=False,
|
||||
)
|
||||
|
||||
apply()
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
output = apply()
|
||||
for _ in range(2):
|
||||
x.normal_(std=0.125)
|
||||
scores.normal_()
|
||||
new_weights, new_ids, _ = fused_topk(x, scores, topk, renormalize=False)
|
||||
weights.copy_(new_weights)
|
||||
ids.copy_(new_ids)
|
||||
allocated = torch.accelerator.memory_allocated()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert torch.accelerator.memory_allocated() == allocated
|
||||
expected = moe_reference_iq2_xs(
|
||||
x,
|
||||
raw1.view(e, i, *raw1.shape[1:]),
|
||||
raw2.view(e, h, *raw2.shape[1:]),
|
||||
ids,
|
||||
weights,
|
||||
activation="relu2",
|
||||
)
|
||||
assert torch.isfinite(output).all()
|
||||
relative_l2 = (
|
||||
output.float() - expected.float()
|
||||
).norm() / expected.float().norm()
|
||||
cosine = torch.nn.functional.cosine_similarity(
|
||||
output.float().flatten(), expected.float().flatten(), dim=0
|
||||
)
|
||||
assert cosine >= 0.999 and relative_l2 <= 0.01
|
||||
graph.reset()
|
||||
|
||||
@@ -119,6 +119,8 @@ class BatchedMMTensors:
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("block_shape", [None, [128, 128]])
|
||||
@pytest.mark.parametrize("per_act_token_quant", [False, True])
|
||||
# Tuned configs can pick a BLOCK_SIZE_K wider than the quantization block.
|
||||
@pytest.mark.parametrize("fp8_block_k", [32, 256])
|
||||
def test_batched_mm(
|
||||
num_experts: int,
|
||||
max_tokens_per_expert: int,
|
||||
@@ -127,6 +129,7 @@ def test_batched_mm(
|
||||
dtype: torch.dtype,
|
||||
block_shape: list[int] | None,
|
||||
per_act_token_quant: bool,
|
||||
fp8_block_k: int,
|
||||
):
|
||||
"""Note: float8_e4m3fn is not supported on CUDA architecture < 89,
|
||||
and those tests will be skipped on unsupported hardware."""
|
||||
@@ -149,6 +152,9 @@ def test_batched_mm(
|
||||
if per_act_token_quant and block_shape is not None:
|
||||
pytest.skip("Skip illegal quantization test.")
|
||||
|
||||
if fp8_block_k != 32 and not use_fp8_w8a8:
|
||||
pytest.skip("BLOCK_SIZE_K is only varied for fp8.")
|
||||
|
||||
if dtype.itemsize == 1:
|
||||
act_dtype = torch.bfloat16
|
||||
quant_dtype = dtype
|
||||
@@ -214,7 +220,7 @@ def test_batched_mm(
|
||||
config={
|
||||
"BLOCK_SIZE_M": 16,
|
||||
"BLOCK_SIZE_N": 16,
|
||||
"BLOCK_SIZE_K": 16 if dtype.itemsize > 1 else 32,
|
||||
"BLOCK_SIZE_K": 16 if dtype.itemsize > 1 else fp8_block_k,
|
||||
},
|
||||
per_act_token_quant=per_act_token_quant,
|
||||
block_shape=block_shape,
|
||||
|
||||
@@ -9,7 +9,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config, make_test_quant_config
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
@@ -26,8 +26,10 @@ from vllm.model_executor.layers.fused_moe.all2all_utils import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FUSED_MOE_UNQUANTIZED_CONFIG,
|
||||
FusedMoEParallelConfig,
|
||||
FusedMoEQuantConfig,
|
||||
fp8_w8a8_moe_quant_config,
|
||||
mxfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsFp4,
|
||||
@@ -41,7 +43,10 @@ from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle import mxfp4 as mxfp4_oracle
|
||||
from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle
|
||||
from vllm.model_executor.layers.fused_moe.utils import moe_kernel_quantize_input
|
||||
from vllm.model_executor.layers.fused_moe.utils import (
|
||||
moe_kernel_quantize_input,
|
||||
remap_topk_to_local,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp4Dynamic
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
@@ -183,6 +188,189 @@ def test_cutlass_moe_permutation_maps_padding_to_zero():
|
||||
torch.testing.assert_close(restored[3], torch.zeros_like(restored[3]))
|
||||
|
||||
|
||||
def test_remap_topk_ids_to_local():
|
||||
device = "cuda"
|
||||
expert_map = torch.tensor([-1, 0, -1, 1, -1, 2], dtype=torch.int32, device=device)
|
||||
topk_ids = torch.tensor([[1, 3], [0, 5], [-1, 4]], dtype=torch.int32, device=device)
|
||||
expected = torch.tensor(
|
||||
[[0, 1], [-1, 2], [-1, -1]], dtype=torch.int32, device=device
|
||||
)
|
||||
actual = remap_topk_to_local(topk_ids, expert_map, out_dtype=torch.int32)
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
def test_remap_topk_ids_to_local_requires_contiguous_topk_ids():
|
||||
expert_map = torch.tensor([0, 1], dtype=torch.int32)
|
||||
topk_ids = torch.tensor([[0, 1], [1, 0]], dtype=torch.int32).t()
|
||||
assert not topk_ids.is_contiguous()
|
||||
with pytest.raises(AssertionError):
|
||||
remap_topk_to_local(topk_ids, expert_map)
|
||||
|
||||
|
||||
def test_cutlass_fp4_moe_supports_expert_parallel_config():
|
||||
ep_config = FusedMoEParallelConfig(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
pcp_size=1,
|
||||
pcp_rank=0,
|
||||
dp_size=8,
|
||||
dp_rank=0,
|
||||
ep_size=8,
|
||||
ep_rank=0,
|
||||
sp_size=1,
|
||||
use_ep=True,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
enable_eplb=False,
|
||||
)
|
||||
assert CutlassExpertsMxfp4._supports_parallel_config(ep_config)
|
||||
assert CutlassExpertsFp4._supports_parallel_config(ep_config)
|
||||
|
||||
|
||||
def _slice_moe_quant_config(
|
||||
quant_config: FusedMoEQuantConfig, start: int, end: int
|
||||
) -> FusedMoEQuantConfig:
|
||||
num_experts = quant_config.w1_scale.shape[0]
|
||||
sliced_config = copy.deepcopy(quant_config)
|
||||
for desc_name in ("_a1", "_a2", "_w1", "_w2"):
|
||||
desc = getattr(sliced_config, desc_name)
|
||||
for field_name in ("scale", "alpha_or_gscale", "bias", "zp"):
|
||||
tensor = getattr(desc, field_name)
|
||||
if (
|
||||
isinstance(tensor, torch.Tensor)
|
||||
and tensor.ndim > 0
|
||||
and tensor.shape[0] == num_experts
|
||||
):
|
||||
setattr(desc, field_name, tensor[start:end])
|
||||
return sliced_config
|
||||
|
||||
|
||||
def _swizzle_mxfp4_moe_scales(
|
||||
w1_scale: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
e, w1_n, w1_scale_k = w1_scale.shape
|
||||
w1_k = w1_scale_k * 32
|
||||
w2_m, w2_scale_n = w2_scale.shape[1], w2_scale.shape[2]
|
||||
w2_n = w2_scale_n * 32
|
||||
|
||||
swizzled_w1 = []
|
||||
swizzled_w2 = []
|
||||
for expert in range(e):
|
||||
swizzled_w1.append(
|
||||
swizzle_mxfp4_scales(w1_scale[expert], w1_n, w1_k).reshape(w1_n, w1_scale_k)
|
||||
)
|
||||
swizzled_w2.append(
|
||||
swizzle_mxfp4_scales(w2_scale[expert], w2_m, w2_n).reshape(w2_m, w2_scale_n)
|
||||
)
|
||||
return torch.stack(swizzled_w1), torch.stack(swizzled_w2)
|
||||
|
||||
|
||||
def _make_cutlass_fp4_moe_weights(
|
||||
quantization: str,
|
||||
num_experts: int,
|
||||
n: int,
|
||||
k: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, FusedMoEQuantConfig]:
|
||||
if quantization == "nvfp4":
|
||||
return make_test_quant_config(
|
||||
num_experts,
|
||||
n,
|
||||
k,
|
||||
in_dtype=dtype,
|
||||
quant_dtype="nvfp4",
|
||||
block_shape=None,
|
||||
per_act_token_quant=False,
|
||||
make_gate=True,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp4_utils import (
|
||||
mxfp4_quantize,
|
||||
)
|
||||
|
||||
w1, w1_scale = mxfp4_quantize(
|
||||
torch.randn((num_experts, 2 * n, k), device=device, dtype=dtype) / 8
|
||||
)
|
||||
w2, w2_scale = mxfp4_quantize(
|
||||
torch.randn((num_experts, k, n), device=device, dtype=dtype) / 8
|
||||
)
|
||||
w1_scale, w2_scale = _swizzle_mxfp4_moe_scales(w1_scale, w2_scale)
|
||||
return (
|
||||
w1.contiguous(),
|
||||
w2.contiguous(),
|
||||
mxfp4_moe_quant_config(w1_scale=w1_scale, w2_scale=w2_scale),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quantization", ["nvfp4", "mxfp4"])
|
||||
@torch.inference_mode()
|
||||
def test_cutlass_fp4_moe_EP(
|
||||
quantization: str,
|
||||
workspace_init,
|
||||
):
|
||||
"""Sharded CUTLASS FP4/MXFP4 MoE with expert_map must match full weights."""
|
||||
experts_cls = CutlassExpertsFp4 if quantization == "nvfp4" else CutlassExpertsMxfp4
|
||||
if not experts_cls._supports_current_device():
|
||||
pytest.skip(f"CUTLASS {quantization} MoE is not supported on this GPU")
|
||||
|
||||
set_random_seed(7)
|
||||
m, n, k = 8, 128, 128
|
||||
num_experts, num_local_experts, topk = 8, 2, 2
|
||||
dtype = torch.bfloat16
|
||||
device = torch.device("cuda")
|
||||
|
||||
w1, w2, quant_config = _make_cutlass_fp4_moe_weights(
|
||||
quantization, num_experts, n, k, dtype, device
|
||||
)
|
||||
hidden_states = torch.randn((m, k), device=device, dtype=dtype) / 10
|
||||
score = torch.randn((m, num_experts), device=device, dtype=dtype)
|
||||
topk_weights, topk_ids, _ = fused_topk(
|
||||
hidden_states, score, topk, renormalize=False
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config(
|
||||
num_experts=num_experts,
|
||||
num_local_experts=num_experts,
|
||||
experts_per_token=topk,
|
||||
hidden_dim=k,
|
||||
intermediate_size=n,
|
||||
in_dtype=dtype,
|
||||
max_num_tokens=m,
|
||||
)
|
||||
kernel = mk.FusedMoEKernel(
|
||||
maybe_make_prepare_finalize(
|
||||
moe=moe_config,
|
||||
quant_config=quant_config,
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
experts_cls(moe_config=moe_config, quant_config=quant_config),
|
||||
)
|
||||
apply_kwargs = {
|
||||
"hidden_states": hidden_states,
|
||||
"w1": w1,
|
||||
"w2": w2,
|
||||
"topk_weights": topk_weights,
|
||||
"topk_ids": topk_ids,
|
||||
"global_num_experts": num_experts,
|
||||
"activation": MoEActivation.SILU,
|
||||
"expert_map": None,
|
||||
"apply_router_weight_on_input": False,
|
||||
}
|
||||
reference = kernel.apply(**apply_kwargs)
|
||||
sharded = run_with_expert_maps(
|
||||
num_experts,
|
||||
num_local_experts,
|
||||
quant_config,
|
||||
experts_cls,
|
||||
**apply_kwargs,
|
||||
)
|
||||
|
||||
assert reference.abs().max() > 0
|
||||
torch.testing.assert_close(sharded, reference, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("quantization", ["nvfp4", "mxfp4"])
|
||||
@torch.inference_mode()
|
||||
def test_cutlass_fp4_moe_padded_routes_do_not_change_valid_output(quantization: str):
|
||||
@@ -486,6 +674,7 @@ def run_with_expert_maps(
|
||||
num_experts: int,
|
||||
num_local_experts: int,
|
||||
quant_config: FusedMoEQuantConfig,
|
||||
experts_cls: type[mk.FusedMoEExperts],
|
||||
**cutlass_moe_kwargs,
|
||||
):
|
||||
def slice_experts():
|
||||
@@ -513,9 +702,7 @@ def run_with_expert_maps(
|
||||
for k, t in full_tensors.items():
|
||||
cutlass_moe_kwargs[k] = t[s:e]
|
||||
|
||||
new_quant_config = copy.deepcopy(quant_config)
|
||||
new_quant_config._w1.scale = quant_config.w1_scale[s:e]
|
||||
new_quant_config._w2.scale = quant_config.w2_scale[s:e]
|
||||
new_quant_config = _slice_moe_quant_config(quant_config, s, e)
|
||||
|
||||
yield cutlass_moe_kwargs, new_quant_config
|
||||
|
||||
@@ -523,13 +710,14 @@ def run_with_expert_maps(
|
||||
for kwargs, new_quant_config in slice_experts():
|
||||
w2 = kwargs["w2"]
|
||||
a = kwargs["hidden_states"]
|
||||
w1 = kwargs["w1"]
|
||||
moe_config = make_dummy_moe_config(
|
||||
max_num_tokens=kwargs.get("hidden_states").shape[0],
|
||||
experts_per_token=kwargs.get("topk_ids").shape[1],
|
||||
num_experts=num_experts,
|
||||
num_local_experts=num_local_experts,
|
||||
hidden_dim=w2.shape[1],
|
||||
intermediate_size=w2.shape[2],
|
||||
intermediate_size=w1.shape[1] // 2,
|
||||
in_dtype=a.dtype,
|
||||
)
|
||||
kernel = mk.FusedMoEKernel(
|
||||
@@ -539,7 +727,7 @@ def run_with_expert_maps(
|
||||
allow_new_interface=True,
|
||||
use_monolithic=False,
|
||||
),
|
||||
CutlassExpertsFp8(
|
||||
experts_cls(
|
||||
moe_config=moe_config,
|
||||
quant_config=new_quant_config,
|
||||
),
|
||||
@@ -624,6 +812,7 @@ def run_8_bit(
|
||||
num_experts,
|
||||
num_local_experts,
|
||||
quant_config,
|
||||
CutlassExpertsFp8,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,422 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.kernels.moe.modular_kernel_tools.parallel_utils import (
|
||||
ProcessGroupInfo,
|
||||
parallel_launch_with_config,
|
||||
)
|
||||
from vllm.config import ParallelConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoEFactory
|
||||
from vllm.model_executor.layers.fused_moe.expert_substitution import (
|
||||
ConstantExpertSubstitution,
|
||||
SubstitutedRoutedExperts,
|
||||
get_expert_substitution_spec,
|
||||
make_expert_substitution,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe import (
|
||||
FlashInferExperts,
|
||||
)
|
||||
from vllm.model_executor.models.mixtral import MixtralMoE
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
MOE_PREFIX = "model.layers.1.mlp.experts"
|
||||
|
||||
|
||||
def _model_config(approximate_experts, **kwargs) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
hf_config=SimpleNamespace(approximate_experts=approximate_experts),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _expert_substitution(targets: dict[str, dict[int, str]]) -> dict:
|
||||
"""The ``compression_config.transform_config.expert_substitution`` format."""
|
||||
return {
|
||||
"version": 1,
|
||||
"router_semantics": {
|
||||
"preserve_logical_expert_ids": True,
|
||||
"preserve_router_weights": True,
|
||||
"renormalize_after_substitution": False,
|
||||
},
|
||||
"targets": {
|
||||
path: {
|
||||
"num_logical_experts": 5,
|
||||
"weight_layout": "compact_retained_experts",
|
||||
"replacements": {
|
||||
str(expert_id): {
|
||||
"format": "constant-v1",
|
||||
"tensors": {"value": name},
|
||||
}
|
||||
for expert_id, name in replacements.items()
|
||||
},
|
||||
}
|
||||
for path, replacements in targets.items()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _compressed_model_config(expert_substitution: dict, **kwargs) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
hf_config=SimpleNamespace(
|
||||
compression_config={
|
||||
"transform_config": {"expert_substitution": expert_substitution}
|
||||
}
|
||||
),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _substitution(hidden_size: int, dtype=torch.float32):
|
||||
# Experts 1 and 3 of 5 are substituted; 0, 2, 4 map to physical 0, 1, 2.
|
||||
return ConstantExpertSubstitution(1, 5, (1, 3), hidden_size, dtype)
|
||||
|
||||
|
||||
def test_parse_approximate_experts():
|
||||
assert get_expert_substitution_spec(None) is None
|
||||
assert get_expert_substitution_spec(_model_config(None)) is None
|
||||
spec = get_expert_substitution_spec(_model_config({"1": [3, 1], "2": [], 5: (0,)}))
|
||||
assert spec.experts == {1: (1, 3), 5: (0,)}
|
||||
assert spec.value_names == {}
|
||||
text_config = SimpleNamespace(
|
||||
hf_text_config=SimpleNamespace(approximate_experts={"0": [2]}),
|
||||
hf_config=SimpleNamespace(),
|
||||
)
|
||||
assert get_expert_substitution_spec(text_config).experts == {0: (2,)}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"approximate_experts",
|
||||
[[1, 2], {"layer": [1]}, {"1": [1, 1]}, {"1": [-1]}, {"1": ["x"]}],
|
||||
)
|
||||
def test_parse_approximate_experts_rejects_invalid_entries(approximate_experts):
|
||||
with pytest.raises(ValueError, match="approximate_experts"):
|
||||
get_expert_substitution_spec(_model_config(approximate_experts))
|
||||
|
||||
|
||||
def test_parse_expert_substitution_with_explicit_value_names():
|
||||
spec = get_expert_substitution_spec(
|
||||
_compressed_model_config(
|
||||
_expert_substitution(
|
||||
{
|
||||
MOE_PREFIX: {3: "shared", 1: "layer_1.expert_1"},
|
||||
"model.layers.2.mlp.experts": {0: "shared"},
|
||||
}
|
||||
)
|
||||
)
|
||||
)
|
||||
assert spec.experts == {1: (1, 3), 2: (0,)}
|
||||
assert spec.value_names == {
|
||||
"shared": ((1, 3), (2, 0)),
|
||||
"layer_1.expert_1": ((1, 1),),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mutate",
|
||||
[
|
||||
lambda raw: raw.update(version=2),
|
||||
lambda raw: raw["router_semantics"].update(renormalize_after_substitution=True),
|
||||
lambda raw: raw["targets"].update({"mlp.experts": raw["targets"][MOE_PREFIX]}),
|
||||
lambda raw: raw["targets"][MOE_PREFIX].update(weight_layout="dense"),
|
||||
lambda raw: raw["targets"][MOE_PREFIX]["replacements"]["1"].update(
|
||||
format="linear-v1"
|
||||
),
|
||||
lambda raw: raw["targets"][MOE_PREFIX]["replacements"]["1"].pop("tensors"),
|
||||
],
|
||||
)
|
||||
def test_parse_expert_substitution_rejects_unsupported_metadata(mutate):
|
||||
raw = _expert_substitution({MOE_PREFIX: {1: "value"}})
|
||||
mutate(raw)
|
||||
with pytest.raises(ValueError, match="invalid expert_substitution metadata"):
|
||||
get_expert_substitution_spec(_compressed_model_config(raw))
|
||||
|
||||
|
||||
def test_parse_rejects_both_formats():
|
||||
config = _compressed_model_config(_expert_substitution({MOE_PREFIX: {1: "v"}}))
|
||||
config.hf_config.approximate_experts = {"1": [1]}
|
||||
with pytest.raises(ValueError, match="both"):
|
||||
get_expert_substitution_spec(config)
|
||||
|
||||
|
||||
def test_make_expert_substitution_matches_layer_index():
|
||||
config = _model_config({"1": [3, 1]})
|
||||
substitution = make_expert_substitution(config, MOE_PREFIX, 5, 16, torch.bfloat16)
|
||||
|
||||
assert substitution is not None
|
||||
assert substitution.layer_idx == 1
|
||||
assert substitution.substituted_expert_ids == (1, 3)
|
||||
assert substitution.num_compute_experts == 3
|
||||
assert substitution.values.dtype == torch.bfloat16
|
||||
assert [substitution.physical_expert_id(i) for i in range(5)] == [0, -1, 1, -1, 2]
|
||||
assert (
|
||||
make_expert_substitution(config, "model.layers.2.mlp.experts", 5, 16, None)
|
||||
is None
|
||||
)
|
||||
assert make_expert_substitution(None, MOE_PREFIX, 5, 16, None) is None
|
||||
explicit = _compressed_model_config(_expert_substitution({MOE_PREFIX: {1: "v"}}))
|
||||
substitution = make_expert_substitution(explicit, MOE_PREFIX, 5, 16, None)
|
||||
assert substitution is not None
|
||||
assert substitution.substituted_expert_ids == (1,)
|
||||
with pytest.raises(ValueError, match="exactly one layer index"):
|
||||
make_expert_substitution(config, "mlp.experts", 5, 16, None)
|
||||
with pytest.raises(ValueError, match="within"):
|
||||
make_expert_substitution(config, MOE_PREFIX, 3, 16, None)
|
||||
with pytest.raises(ValueError, match="retain at least one expert"):
|
||||
make_expert_substitution(_model_config({"1": [0, 1]}), MOE_PREFIX, 2, 16, None)
|
||||
|
||||
|
||||
def test_constant_substitution_loads_values():
|
||||
substitution = _substitution(2)
|
||||
substitution.weight_loader(substitution.values, torch.tensor([1.0, 2.0]), 3)
|
||||
with pytest.raises(ValueError, match="missing approx_value .* \\[1\\]"):
|
||||
substitution.validate_loaded_values("layer")
|
||||
substitution.weight_loader(substitution.values, torch.tensor([3.0, 4.0]), 1)
|
||||
substitution.validate_loaded_values("layer")
|
||||
torch.testing.assert_close(
|
||||
substitution.values, torch.tensor([[3.0, 4.0], [1.0, 2.0]])
|
||||
)
|
||||
with pytest.raises(ValueError, match="not listed"):
|
||||
substitution.weight_loader(substitution.values, torch.ones(2), 2)
|
||||
with pytest.raises(ValueError, match="has shape"):
|
||||
substitution.weight_loader(substitution.values, torch.ones(3), 1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [0, 1, 3, 7])
|
||||
@pytest.mark.parametrize("hidden_size", [2, 4])
|
||||
def test_constant_substitution_transforms_routes_and_computes_side_output(
|
||||
num_tokens, hidden_size
|
||||
):
|
||||
substitution = _substitution(2)
|
||||
substitution.values.data.copy_(torch.tensor([[10.0, 20.0], [30.0, 40.0]]))
|
||||
|
||||
rows = torch.arange(num_tokens) % 3
|
||||
topk_ids = torch.tensor([[1, 2], [4, 3], [99, -1]])[rows]
|
||||
topk_weights = torch.tensor([[0.25, 0.75], [0.6, 0.4], [0.5, 0.5]])[rows]
|
||||
constant_output = substitution.transform_routes(
|
||||
torch.zeros(num_tokens, hidden_size), topk_weights, topk_ids
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
topk_weights, torch.tensor([[0.0, 0.75], [0.6, 0.0], [0.0, 0.0]])[rows]
|
||||
)
|
||||
torch.testing.assert_close(topk_ids, torch.tensor([[0, 1], [2, 0], [0, 0]])[rows])
|
||||
expected = torch.zeros(num_tokens, hidden_size)
|
||||
expected[:, :2] = torch.tensor([[2.5, 5.0], [12.0, 16.0], [0.0, 0.0]])[rows]
|
||||
torch.testing.assert_close(constant_output, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"device",
|
||||
[
|
||||
"cpu",
|
||||
pytest.param(
|
||||
"cuda",
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="requires a CUDA-like platform",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_constant_substitution_preserves_router_weight_precision(dtype, device):
|
||||
substitution = _substitution(1, dtype).to(device)
|
||||
substitution.values.data.copy_(
|
||||
torch.tensor([[60000.0], [-60000.0]], dtype=dtype, device=device)
|
||||
)
|
||||
topk_weights = torch.tensor([[0.5001, 0.4999]], dtype=torch.float32, device=device)
|
||||
topk_ids = torch.tensor([[1, 3]], device=device)
|
||||
# Nearly cancelling constants must retain the difference between FP32 weights.
|
||||
expected = (topk_weights.double() @ substitution.values.double()).to(dtype)
|
||||
|
||||
actual = substitution.transform_routes(
|
||||
torch.zeros(1, 1, dtype=dtype, device=device), topk_weights, topk_ids
|
||||
)
|
||||
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
def _bare_substituted_routed_experts(**attrs) -> SubstitutedRoutedExperts:
|
||||
layer = SubstitutedRoutedExperts.__new__(SubstitutedRoutedExperts)
|
||||
torch.nn.Module.__init__(layer)
|
||||
layer.expert_substitution = _substitution(2)
|
||||
for name, value in attrs.items():
|
||||
setattr(layer, name, value)
|
||||
return layer
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tp_rank", [0, 1])
|
||||
@pytest.mark.parametrize("padded_input", [False, True])
|
||||
def test_substituted_routed_experts_adds_constant_on_one_tp_rank(tp_rank, padded_input):
|
||||
def apply(*, layer, x, topk_weights, topk_ids, **kwargs):
|
||||
torch.testing.assert_close(topk_weights, torch.tensor([[0.0, 0.75]]))
|
||||
torch.testing.assert_close(topk_ids, torch.tensor([[0, 1]]))
|
||||
return torch.ones(x.shape[0], 2)
|
||||
|
||||
layer = _bare_substituted_routed_experts(
|
||||
moe_config=SimpleNamespace(tp_rank=tp_rank),
|
||||
quant_method=SimpleNamespace(is_monolithic=False, apply=apply),
|
||||
)
|
||||
layer.expert_substitution.values.data.copy_(
|
||||
torch.tensor([[10.0, 20.0], [30.0, 40.0]])
|
||||
)
|
||||
actual = layer.forward_modular(
|
||||
torch.zeros(1, 4 if padded_input else 2),
|
||||
torch.tensor([[0.25, 0.75]]),
|
||||
torch.tensor([[1, 2]]),
|
||||
)
|
||||
expected = torch.ones(1, 2)
|
||||
if tp_rank == 0:
|
||||
expected += torch.tensor([[2.5, 5.0]])
|
||||
torch.testing.assert_close(actual, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like platform"
|
||||
)
|
||||
def test_standard_fused_moe_model_discovers_substitution(dist_init):
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = _model_config(
|
||||
{"0": [1]}, dtype=torch.bfloat16, is_moe=True
|
||||
)
|
||||
vllm_config.kernel_config.moe_backend = "triton"
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
moe = MixtralMoE(
|
||||
num_experts=4,
|
||||
top_k=2,
|
||||
hidden_size=256,
|
||||
intermediate_size=512,
|
||||
params_dtype=torch.bfloat16,
|
||||
prefix="model.layers.0.block_sparse_moe",
|
||||
)
|
||||
|
||||
routed_experts = moe.experts.routed_experts
|
||||
assert isinstance(routed_experts, SubstitutedRoutedExperts)
|
||||
assert routed_experts.expert_substitution.substituted_expert_ids == (1,)
|
||||
assert routed_experts.w13_weight.shape[0] == 3
|
||||
assert moe.experts.moe_config.require_decomposed_backend
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="requires a CUDA-like platform"
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"moe_backend",
|
||||
[
|
||||
pytest.param("triton", id="triton"),
|
||||
pytest.param(
|
||||
"flashinfer_cutlass",
|
||||
id="flashinfer-cutlass",
|
||||
marks=pytest.mark.skipif(
|
||||
not FlashInferExperts._supports_current_device(),
|
||||
reason="FlashInfer CUTLASS is unavailable on this platform",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_fused_moe_combines_compact_experts_and_substitution(dist_init, moe_backend):
|
||||
hidden_size = 256
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.model_config = _model_config(
|
||||
{"0": [1, 3]}, dtype=torch.bfloat16, is_moe=True
|
||||
)
|
||||
vllm_config.compilation_config.static_forward_context = {}
|
||||
vllm_config.kernel_config.moe_backend = moe_backend
|
||||
|
||||
with set_current_vllm_config(vllm_config), set_forward_context(None, vllm_config):
|
||||
init_workspace_manager(torch.accelerator.current_device_index())
|
||||
layer = FusedMoEFactory(
|
||||
num_experts=4,
|
||||
top_k=2,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=512,
|
||||
params_dtype=torch.bfloat16,
|
||||
prefix="model.layers.0.mlp.experts",
|
||||
renormalize=False,
|
||||
).cuda()
|
||||
substitution = layer.routed_experts.expert_substitution
|
||||
assert layer.routed_experts.w13_weight.shape[0] == 2
|
||||
with torch.no_grad():
|
||||
layer.routed_experts.w13_weight.normal_(0, 0.01)
|
||||
layer.routed_experts.w2_weight.normal_(0, 0.01)
|
||||
substitution.values.normal_(0, 0.01)
|
||||
layer._quant_method.process_weights_after_loading(layer.routed_experts)
|
||||
|
||||
hidden_states = torch.randn(8, hidden_size, dtype=torch.bfloat16, device="cuda")
|
||||
router_logits = torch.randn(8, 4, dtype=torch.float32, device="cuda")
|
||||
topk_weights, topk_ids = layer.router.select_experts(
|
||||
hidden_states.clone(), router_logits
|
||||
)
|
||||
expected = substitution.transform_routes(hidden_states, topk_weights, topk_ids)
|
||||
expected += layer._quant_method.apply(
|
||||
layer=layer.routed_experts,
|
||||
x=hidden_states.clone(),
|
||||
topk_weights=topk_weights,
|
||||
topk_ids=topk_ids,
|
||||
shared_experts=None,
|
||||
shared_experts_input=None,
|
||||
)
|
||||
|
||||
get_forward_context().all_moe_layers = None
|
||||
actual = layer(hidden_states.clone(), router_logits)
|
||||
torch.testing.assert_close(actual, expected, atol=0, rtol=0)
|
||||
|
||||
|
||||
def _tp_substitution_worker(
|
||||
process_group: ProcessGroupInfo,
|
||||
vllm_config: VllmConfig,
|
||||
_cpu_group,
|
||||
) -> None:
|
||||
with set_forward_context(None, vllm_config):
|
||||
init_workspace_manager(process_group.local_rank)
|
||||
layer = FusedMoEFactory(
|
||||
num_experts=4,
|
||||
top_k=1,
|
||||
hidden_size=256,
|
||||
intermediate_size=512,
|
||||
params_dtype=torch.bfloat16,
|
||||
prefix="model.layers.0.mlp.experts",
|
||||
renormalize=True,
|
||||
)
|
||||
with torch.no_grad():
|
||||
layer.routed_experts.w13_weight.zero_()
|
||||
layer.routed_experts.w2_weight.zero_()
|
||||
layer.routed_experts.expert_substitution.values.fill_(0.25)
|
||||
layer._quant_method.process_weights_after_loading(layer.routed_experts)
|
||||
|
||||
hidden_states = torch.zeros(2, 256, dtype=torch.bfloat16)
|
||||
router_logits = torch.tensor(
|
||||
[[-100.0, 100.0, -100.0, -100.0]] * 2,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
get_forward_context().all_moe_layers = None
|
||||
actual = layer(hidden_states, router_logits)
|
||||
torch.testing.assert_close(
|
||||
actual, torch.full_like(actual, 0.25), atol=0, rtol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike() or current_platform.device_count() < 2,
|
||||
reason="requires two CUDA-like devices",
|
||||
)
|
||||
def test_constant_substitution_is_added_once_with_tp2():
|
||||
vllm_config = VllmConfig(parallel_config=ParallelConfig(tensor_parallel_size=2))
|
||||
vllm_config.model_config = _model_config(
|
||||
{"0": [1]},
|
||||
dtype=torch.bfloat16,
|
||||
is_moe=True,
|
||||
sleep_mode_offload_cudagraph=False,
|
||||
)
|
||||
vllm_config.compilation_config.static_forward_context = {}
|
||||
vllm_config.kernel_config.moe_backend = "triton"
|
||||
|
||||
parallel_launch_with_config(2, _tp_substitution_worker, vllm_config, None)
|
||||
@@ -10,9 +10,14 @@ CuTeDSL kernel (NVFP4 activations and weights, E4M3 scales per 16) as well as
|
||||
the DeepGEMM kernel (FP8 activations and FP4 weights, power-of-two scales per
|
||||
32). The expected output is then known in closed form and compared bit for
|
||||
bit.
|
||||
|
||||
A test that needs no SM100 also checks the adapter's kwargs against the
|
||||
installed FlashInfer's config types, so config drift fails on any CI runner
|
||||
with FlashInfer installed.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -22,6 +27,7 @@ from tests.distributed.eplb_utils import distributed_run, set_env_vars_and_devic
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.kernel import FLASHINFER_MOE_EP_CUTEDSL, FLASHINFER_MOE_EP_DEEP_GEMM
|
||||
from vllm.distributed.parallel_state import ensure_model_parallel_initialized
|
||||
from vllm.model_executor.layers.fused_moe import flashinfer_moe_ep as fi_ep
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEConfig,
|
||||
@@ -305,3 +311,59 @@ def test_init_warmup_forward_is_exact_on_grid_inputs(
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs")
|
||||
distributed_run(_init_warmup_forward, world_size, geometry, backend)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", BACKENDS, ids=("cutedsl", "deep_gemm"))
|
||||
def test_adapter_kwargs_match_installed_flashinfer(
|
||||
monkeypatch: pytest.MonkeyPatch, backend: str
|
||||
):
|
||||
"""The adapter's kwargs construct the installed FlashInfer's config types.
|
||||
|
||||
Needs no SM100, so config kwarg drift fails in CI rather than at model load.
|
||||
Fakes only the megakernel layer (symmetric memory), the EP group, the device
|
||||
index and the DeepGEMM import.
|
||||
"""
|
||||
moe_ep = pytest.importorskip("flashinfer.moe_ep")
|
||||
layer_args: list[tuple] = []
|
||||
|
||||
class FakeMegaLayer:
|
||||
def __init__(self, *args) -> None:
|
||||
layer_args.append(args)
|
||||
|
||||
def destroy(self) -> None:
|
||||
pass
|
||||
|
||||
monkeypatch.setattr(moe_ep, "MoEEpMegaLayer", FakeMegaLayer)
|
||||
monkeypatch.setattr(fi_ep, "_expose_deep_gemm_to_flashinfer", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
fi_ep,
|
||||
"get_ep_group",
|
||||
lambda: SimpleNamespace(world_size=2, rank_in_group=0, device_group=None),
|
||||
)
|
||||
monkeypatch.setattr(torch.accelerator, "current_device_index", lambda: 0)
|
||||
|
||||
geometry = GEOMETRIES[0]
|
||||
moe = SimpleNamespace(
|
||||
moe_backend=backend,
|
||||
num_experts=geometry.num_experts,
|
||||
experts_per_token=geometry.top_k,
|
||||
hidden_dim=geometry.hidden_size,
|
||||
intermediate_size=geometry.intermediate_size,
|
||||
max_num_tokens=MAX_TOKENS_PER_RANK,
|
||||
swiglu_limit=10.0,
|
||||
routing_method=_routing_method(backend),
|
||||
)
|
||||
weight = torch.zeros(1, dtype=torch.bfloat16)
|
||||
FlashInferMoeEp(
|
||||
moe,
|
||||
FlashInferMoeEpWeights(w13=weight, w2=weight),
|
||||
apply_topk_in_fc1=apply_topk_in_fc1(moe),
|
||||
)
|
||||
|
||||
((_, _, _, mega_config),) = layer_args
|
||||
expected = (
|
||||
moe_ep.Nvfp4CutedslMegaMoeConfig
|
||||
if backend == FLASHINFER_MOE_EP_CUTEDSL
|
||||
else moe_ep.DeepGemmMegaMoeConfig
|
||||
)
|
||||
assert isinstance(mega_config.megakernel, expected)
|
||||
|
||||
@@ -0,0 +1,229 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Correctness of the fused globalize + align + sort decode metadata kernel.
|
||||
|
||||
The fused op replaces the DeepEP-v2 INDEXED cudagraph-decode chain
|
||||
(_globalize_recv_topk_idx + moe_align_block_size + count_and_sort). It is
|
||||
checked against that exact reference chain over a grid of shapes / EP ranks /
|
||||
fill distributions.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.fused_globalize_align_block_size import (
|
||||
fused_globalize_align_block_size,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_align_block_size import (
|
||||
moe_align_block_size,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.import_utils import has_deep_ep_v2
|
||||
|
||||
if current_platform.is_cuda() and has_deep_ep_v2():
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.deepep_v2 import (
|
||||
_globalize_recv_topk_idx,
|
||||
)
|
||||
else:
|
||||
pytest.skip("Requires CUDA and DeepEP v2", allow_module_level=True)
|
||||
|
||||
|
||||
def _expert_map(global_num_experts, rank_expert_offset, local_num_experts, device):
|
||||
expert_map = torch.full((global_num_experts,), -1, device=device, dtype=torch.int32)
|
||||
rng = torch.arange(local_num_experts, device=device, dtype=torch.int32)
|
||||
expert_map[rank_expert_offset : rank_expert_offset + local_num_experts] = rng
|
||||
return expert_map
|
||||
|
||||
|
||||
def _make_case(n, topk, ep_size, local_e, block_size, seed, frac_recv, rank, fill):
|
||||
torch.manual_seed(seed)
|
||||
device = "cuda"
|
||||
global_num_experts = local_e * ep_size
|
||||
rank_idx = {"first": 0, "mid": ep_size // 2, "last": ep_size - 1}[rank]
|
||||
rank_expert_offset = rank_idx * local_e
|
||||
|
||||
if fill == "all_nonlocal":
|
||||
local_idx = torch.full((n, topk), -1, device=device, dtype=torch.int64)
|
||||
elif fill == "single":
|
||||
local_idx = torch.zeros((n, topk), device=device, dtype=torch.int64)
|
||||
elif fill == "all_local":
|
||||
local_idx = torch.randint(
|
||||
0, local_e, (n, topk), device=device, dtype=torch.int64
|
||||
)
|
||||
else:
|
||||
local_idx = torch.randint(
|
||||
-1, local_e, (n, topk), device=device, dtype=torch.int64
|
||||
)
|
||||
|
||||
num_recv = max(0, min(n, int(round(n * frac_recv))))
|
||||
if num_recv < n:
|
||||
local_idx[num_recv:] = torch.randint(
|
||||
-1, local_e, (n - num_recv, topk), device=device, dtype=torch.int64
|
||||
)
|
||||
psum = torch.zeros(4, device=device, dtype=torch.int32)
|
||||
psum[-1] = num_recv
|
||||
return local_idx, psum, rank_expert_offset, global_num_experts
|
||||
|
||||
|
||||
def _group_by_expert(sorted_ids, expert_ids, block_size, valid_len, total):
|
||||
groups: dict[int, list[int]] = {}
|
||||
for b in range(valid_len // block_size):
|
||||
eid = int(expert_ids[b].item())
|
||||
blk = sorted_ids[b * block_size : (b + 1) * block_size]
|
||||
groups.setdefault(eid, []).extend(blk[blk < total].tolist())
|
||||
return groups
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [1, 7, 64, 257])
|
||||
# topk=16 with local_num_experts=28 is the Kimi K3 decode config. topk=6 and
|
||||
# block_size=16, 128 are outside the old template dispatch table: they guard
|
||||
# against a regression to fixed (topk, block_size) specialization.
|
||||
@pytest.mark.parametrize("topk", [1, 6, 8, 16])
|
||||
@pytest.mark.parametrize("block_size", [16, 32, 64, 128])
|
||||
@pytest.mark.parametrize("local_e", [28, 32])
|
||||
@pytest.mark.parametrize("rank", ["first", "mid", "last"])
|
||||
@pytest.mark.parametrize("fill", ["rand", "all_local", "all_nonlocal", "single"])
|
||||
def test_fused_matches_reference(n, topk, block_size, local_e, rank, fill):
|
||||
ep_size = 4
|
||||
local_idx, psum, reo, gne = _make_case(
|
||||
n,
|
||||
topk,
|
||||
ep_size,
|
||||
local_e,
|
||||
block_size,
|
||||
seed=n * 131 + topk,
|
||||
frac_recv=0.75,
|
||||
rank=rank,
|
||||
fill=fill,
|
||||
)
|
||||
|
||||
ref_global = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
|
||||
ref_sorted, ref_expert_ids, ref_num = moe_align_block_size(
|
||||
topk_ids=ref_global,
|
||||
block_size=block_size,
|
||||
num_experts=gne,
|
||||
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
|
||||
ignore_invalid_experts=True,
|
||||
)
|
||||
|
||||
fused_global, fused_sorted, fused_expert_ids, fused_num = (
|
||||
fused_globalize_align_block_size(
|
||||
local_idx.clone(), psum, reo, gne, local_e, block_size
|
||||
)
|
||||
)
|
||||
|
||||
total = n * topk
|
||||
assert torch.equal(fused_global, ref_global)
|
||||
assert torch.equal(fused_num, ref_num)
|
||||
|
||||
valid = int(fused_num.item())
|
||||
nb = valid // block_size
|
||||
assert torch.equal(fused_expert_ids[:nb], ref_expert_ids[:nb])
|
||||
assert bool((fused_expert_ids[nb:] == -1).all())
|
||||
assert bool((fused_sorted[valid:] == total).all())
|
||||
|
||||
ref_groups = _group_by_expert(
|
||||
ref_sorted, ref_expert_ids, block_size, int(ref_num.item()), total
|
||||
)
|
||||
fused_groups = _group_by_expert(
|
||||
fused_sorted, fused_expert_ids, block_size, valid, total
|
||||
)
|
||||
assert set(ref_groups) == set(fused_groups)
|
||||
for eid in ref_groups:
|
||||
assert sorted(ref_groups[eid]) == sorted(fused_groups[eid])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n", [7, 257])
|
||||
def test_fused_scatter_idx_matches_reference(n):
|
||||
"""The optional scatter output masks invalid and padded routes."""
|
||||
local_e = 28
|
||||
local_idx, psum, reo, gne = _make_case(
|
||||
n, 16, 4, local_e, 64, seed=n, frac_recv=0.75, rank="mid", fill="rand"
|
||||
)
|
||||
ref_global = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
|
||||
*_, ref_scatter = moe_align_block_size(
|
||||
topk_ids=ref_global,
|
||||
block_size=64,
|
||||
num_experts=gne,
|
||||
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
|
||||
ignore_invalid_experts=True,
|
||||
return_scatter_idx=True,
|
||||
)
|
||||
|
||||
fused_global, *_, fused_scatter = fused_globalize_align_block_size(
|
||||
local_idx.clone(), psum, reo, gne, local_e, 64, return_scatter_idx=True
|
||||
)
|
||||
assert torch.equal(fused_global, ref_global)
|
||||
assert torch.equal(fused_scatter, ref_scatter)
|
||||
|
||||
|
||||
# These have effective_work 1504 (single block) and 6068 (cooperative).
|
||||
@pytest.mark.parametrize("n", [1, 64])
|
||||
def test_fused_cuda_graph_replay(n):
|
||||
"""Both launch paths remain capturable and consume replay-time inputs."""
|
||||
topk = 16
|
||||
local_e = 28
|
||||
block_size = 64
|
||||
ep_size = 4
|
||||
|
||||
local_idx, psum, reo, gne = _make_case(
|
||||
n,
|
||||
topk,
|
||||
ep_size,
|
||||
local_e,
|
||||
block_size,
|
||||
seed=n,
|
||||
frac_recv=0.75,
|
||||
rank="mid",
|
||||
fill="rand",
|
||||
)
|
||||
graph_idx = torch.empty_like(local_idx)
|
||||
graph_psum = torch.empty_like(psum)
|
||||
|
||||
# Initialize lazy CUDA state before capture.
|
||||
fused_globalize_align_block_size(
|
||||
local_idx.clone(), psum, reo, gne, local_e, block_size
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
graph_idx.copy_(local_idx)
|
||||
graph_psum.copy_(psum)
|
||||
graph_outputs = fused_globalize_align_block_size(
|
||||
graph_idx, graph_psum, reo, gne, local_e, block_size
|
||||
)
|
||||
|
||||
for frac_recv in (0.75, 0.5):
|
||||
psum[-1] = int(round(n * frac_recv))
|
||||
ref_idx = _globalize_recv_topk_idx(local_idx.clone(), psum, reo, gne)
|
||||
ref_sorted, ref_expert_ids, ref_num = moe_align_block_size(
|
||||
topk_ids=ref_idx,
|
||||
block_size=block_size,
|
||||
num_experts=gne,
|
||||
expert_map=_expert_map(gne, reo, local_e, local_idx.device),
|
||||
ignore_invalid_experts=True,
|
||||
)
|
||||
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
fused_idx, fused_sorted, fused_expert_ids, fused_num = graph_outputs
|
||||
|
||||
assert torch.equal(fused_idx, ref_idx)
|
||||
assert torch.equal(fused_num, ref_num)
|
||||
valid = int(fused_num.item())
|
||||
num_blocks = valid // block_size
|
||||
assert torch.equal(fused_expert_ids[:num_blocks], ref_expert_ids[:num_blocks])
|
||||
fused_groups = _group_by_expert(
|
||||
fused_sorted, fused_expert_ids, block_size, valid, n * topk
|
||||
)
|
||||
ref_groups = _group_by_expert(
|
||||
ref_sorted,
|
||||
ref_expert_ids,
|
||||
block_size,
|
||||
int(ref_num.item()),
|
||||
n * topk,
|
||||
)
|
||||
assert set(fused_groups) == set(ref_groups)
|
||||
for eid in ref_groups:
|
||||
assert sorted(fused_groups[eid]) == sorted(ref_groups[eid])
|
||||
@@ -431,11 +431,6 @@ def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
|
||||
|
||||
if gptoss_moe.triton_kernels_version == "3.5.1":
|
||||
pytest.skip("SparseMatrix path requires triton_kernels v3.6.0+")
|
||||
if gptoss_moe.triton_kernels_version == "3.8":
|
||||
pytest.skip(
|
||||
"3.8 rebuilds routing from raw ids, so routing_data_from_sparse_topk "
|
||||
"just forwards to make_routing_data and the comparison is vacuous"
|
||||
)
|
||||
|
||||
from triton_kernels.topk import topk as topk_fn
|
||||
|
||||
@@ -467,11 +462,52 @@ def test_routing_data_from_sparse_topk_parity(n_tokens, n_experts, topk):
|
||||
elif hasattr(ref, "__dict__"):
|
||||
for k, v in vars(ref).items():
|
||||
assert_equivalent(getattr(new, k), v, f"{path}.{k}")
|
||||
elif hasattr(type(ref), "__slots__"):
|
||||
for k in type(ref).__slots__:
|
||||
assert_equivalent(getattr(new, k), getattr(ref, k), f"{path}.{k}")
|
||||
else:
|
||||
assert type(new) is type(ref), f"{path}: {type(new)} vs {type(ref)}"
|
||||
|
||||
assert_equivalent(rd_new, rd_ref)
|
||||
if isinstance(gather_ref, torch.Tensor):
|
||||
torch.testing.assert_close(gather_new, gather_ref)
|
||||
torch.testing.assert_close(scatter_new, scatter_ref)
|
||||
return
|
||||
torch.testing.assert_close(gather_new.src_indx, gather_ref.src_indx)
|
||||
torch.testing.assert_close(gather_new.dst_indx, gather_ref.dst_indx)
|
||||
torch.testing.assert_close(scatter_new.src_indx, scatter_ref.src_indx)
|
||||
torch.testing.assert_close(scatter_new.dst_indx, scatter_ref.dst_indx)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("n_tokens", [1, 33, 4099])
|
||||
@pytest.mark.parametrize("n_experts,topk", [(128, 4), (256, 6)])
|
||||
def test_make_routing_data_tk38_matches_reference(n_tokens, n_experts, topk):
|
||||
from vllm.model_executor.layers.fused_moe.experts import (
|
||||
gpt_oss_triton_kernels_moe as gptoss_moe,
|
||||
)
|
||||
|
||||
if gptoss_moe.triton_kernels_version != "3.8":
|
||||
pytest.skip("requires triton_kernels 3.8")
|
||||
|
||||
torch.manual_seed(0)
|
||||
topk_ids = torch.stack(
|
||||
[torch.randperm(n_experts, device="cuda")[:topk] for _ in range(n_tokens)]
|
||||
).to(torch.int32)
|
||||
topk_ids[torch.rand(topk_ids.shape, device="cuda") < 0.5] = -1
|
||||
topk_weights = torch.rand(topk_ids.shape, device="cuda")
|
||||
|
||||
routing, gather_tok, scatter_tok = gptoss_moe.make_routing_data(
|
||||
topk_ids, topk_weights, n_experts
|
||||
)
|
||||
|
||||
flat = topk_ids.reshape(-1).to(torch.int64)
|
||||
order = torch.argsort(flat.clamp(min=0), stable=True)
|
||||
counts = torch.bincount(flat.clamp(min=0), minlength=n_experts).to(torch.int32)
|
||||
weights = topk_weights.reshape(-1) * (flat >= 0).to(torch.float32)
|
||||
|
||||
assert torch.equal(routing.ragged.slice_sizes, counts)
|
||||
assert torch.equal(gather_tok, (order // topk).to(torch.int32))
|
||||
assert torch.equal(scatter_tok, gather_tok)
|
||||
assert torch.equal(routing.gate_scal, weights[order])
|
||||
assert routing.n_expts_act == topk
|
||||
assert routing.n_valid == flat.numel()
|
||||
|
||||
@@ -16,7 +16,13 @@ import vllm.model_executor.layers.fused_moe.modular_kernel as mk
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterFp8BlockExperts,
|
||||
AiterFp8ChannelExperts,
|
||||
AiterFp8TensorExperts,
|
||||
AiterUnquantizedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8DynamicTensorSym,
|
||||
kFp8StaticTensorSym,
|
||||
@@ -80,10 +86,10 @@ def assert_aiter_quant_scheme_case(config: Config) -> None:
|
||||
indirectly through the general quant-config sweep.
|
||||
See https://github.com/vllm-project/vllm/issues/54966."""
|
||||
fe_cls = config.fused_experts_type
|
||||
if fe_cls is not AiterExperts:
|
||||
if not issubclass(fe_cls, AiterExperts):
|
||||
return
|
||||
|
||||
if config.quant_config is None:
|
||||
if config.quant_config is None or config.quant_dtype is None:
|
||||
w_key, a_key = None, None
|
||||
else:
|
||||
w_key, a_key = config.fp8_quant_key_pair()
|
||||
@@ -101,7 +107,7 @@ def assert_aiter_activation_case(config: Config) -> None:
|
||||
AiterExperts being reached only indirectly through the general
|
||||
activation sweep. See https://github.com/vllm-project/vllm/issues/54966."""
|
||||
fe_cls = config.fused_experts_type
|
||||
if fe_cls is not AiterExperts:
|
||||
if not issubclass(fe_cls, AiterExperts):
|
||||
return
|
||||
|
||||
assert fe_cls._supports_activation(config.activation), (
|
||||
@@ -156,7 +162,7 @@ def rank_worker(
|
||||
if (
|
||||
topk == 1
|
||||
and config.supports_apply_weight_on_input()
|
||||
and config.fused_experts_type is AiterExperts
|
||||
and issubclass(config.fused_experts_type, AiterExperts)
|
||||
and config.quant_block_shape is not None
|
||||
):
|
||||
print(
|
||||
@@ -172,7 +178,7 @@ def rank_worker(
|
||||
# https://github.com/vllm-project/vllm/issues/57029
|
||||
if (
|
||||
config.world_size > 1
|
||||
and config.fused_experts_type is AiterExperts
|
||||
and issubclass(config.fused_experts_type, AiterExperts)
|
||||
and getattr(config.prepare_finalize_type, "__name__", "")
|
||||
in ("DeepEPHTPrepareAndFinalize", "MoriPrepareAndFinalize")
|
||||
):
|
||||
@@ -210,7 +216,7 @@ def rank_worker(
|
||||
|
||||
is_aiter_fp8 = (
|
||||
_cp.is_rocm()
|
||||
and config.fused_experts_type is AiterExperts
|
||||
and issubclass(config.fused_experts_type, AiterExperts)
|
||||
and config.quant_config is not None
|
||||
)
|
||||
if is_aiter_fp8:
|
||||
@@ -270,7 +276,7 @@ def is_nyi_config(config: Config) -> bool:
|
||||
return True
|
||||
|
||||
if config.activation != MoEActivation.SILU:
|
||||
if config.fused_experts_type is not AiterExperts:
|
||||
if not issubclass(config.fused_experts_type, AiterExperts):
|
||||
return True # AITER-only for this axis, for now
|
||||
if config.quant_dtype is not None:
|
||||
return True # unquantized-only for this axis, for now
|
||||
@@ -591,7 +597,7 @@ def test_aiter_moe_sorting_backend_dispatch_env_matrix(
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=None,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
fused_experts_type=AiterUnquantizedExperts,
|
||||
world_size=1,
|
||||
)
|
||||
assert config.is_valid()[0]
|
||||
@@ -671,7 +677,7 @@ def test_aiter_moe_dispatch_policy_forwarded_through_apply(dispatch_policy: int)
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=None,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
fused_experts_type=AiterUnquantizedExperts,
|
||||
world_size=1,
|
||||
)
|
||||
assert config.is_valid()[0]
|
||||
@@ -738,6 +744,20 @@ _PADDING_QUANT_IDS = [
|
||||
]
|
||||
|
||||
|
||||
def _padding_matrix_fe_cls(
|
||||
quant_config: TestMoEQuantConfig | None,
|
||||
) -> type[AiterExperts]:
|
||||
"""Map a _PADDING_QUANT_CONFIGS entry to the AiterExperts leaf class that
|
||||
supports it (mirrors Config.fp8_quant_key_pair's scheme derivation)."""
|
||||
if quant_config is None:
|
||||
return AiterUnquantizedExperts
|
||||
if quant_config.block_shape is not None:
|
||||
return AiterFp8BlockExperts
|
||||
if quant_config.per_out_ch_quant:
|
||||
return AiterFp8ChannelExperts
|
||||
return AiterFp8TensorExperts
|
||||
|
||||
|
||||
def _slice_gate_up_rows(t: torch.Tensor, real_per_half: int) -> torch.Tensor:
|
||||
"""Slice a (E, 2*padded_per_half, ...) tensor's dim=1 down to the real
|
||||
gate/up halves of `real_per_half` size each."""
|
||||
@@ -950,7 +970,7 @@ def test_aiter_moe_padding_matrix(mode: str, quant_config: TestMoEQuantConfig |
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
fused_experts_type=_padding_matrix_fe_cls(quant_config),
|
||||
world_size=1,
|
||||
hidden_dim_unpadded=_PADDING_K_UNPADDED if pad_hidden else None,
|
||||
intermediate_size_per_partition_unpadded=(
|
||||
@@ -1113,7 +1133,7 @@ def test_aiter_moe_token_padding_garbage_rows(
|
||||
dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prepare_finalize_type=MoEPrepareAndFinalizeNoDPEPModular,
|
||||
fused_experts_type=AiterExperts,
|
||||
fused_experts_type=_padding_matrix_fe_cls(quant_config),
|
||||
world_size=1,
|
||||
)
|
||||
assert config.is_valid()[0]
|
||||
|
||||
@@ -40,9 +40,37 @@ MNKS = [
|
||||
]
|
||||
|
||||
|
||||
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
|
||||
def _mxfp4_aiter_weights(w1, w2, start, end):
|
||||
"""MXFP4-quantize experts [start, end) into AITER's W4A4 kernel layout."""
|
||||
from aiter.ops.triton.quant import dynamic_mxfp4_quant
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
convert_gpt_oss_weight_to_mxfp4_moe_kernel_format,
|
||||
make_mxfp4_moe_quant_config,
|
||||
)
|
||||
|
||||
def quantize(w):
|
||||
w = w[start:end]
|
||||
q, s = dynamic_mxfp4_quant(w.flatten(0, 1))
|
||||
return q.view(*w.shape[:2], -1), s.view(*w.shape[:2], -1)
|
||||
|
||||
backend = Mxfp4MoeBackend.AITER_MXFP4_MXFP4
|
||||
(q1, s1), (q2, s2) = quantize(w1), quantize(w2)
|
||||
q1, q2, s1, s2, _, _ = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
|
||||
backend, None, q1, q2, s1, s2
|
||||
)
|
||||
return q1, q2, make_mxfp4_moe_quant_config(backend, s1, s2)
|
||||
|
||||
|
||||
def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, scheme, graph):
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import AiterExperts
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterFp8ChannelExperts,
|
||||
AiterMxfp4Experts,
|
||||
AiterUnquantizedExperts,
|
||||
rocm_aiter_fused_experts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.mori import (
|
||||
MoriPrepareAndFinalize,
|
||||
)
|
||||
@@ -50,6 +78,8 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
init_workspace_manager(pgi.device)
|
||||
set_random_seed(47)
|
||||
fp8 = scheme == "fp8_token_channel"
|
||||
mxfp4 = scheme == "mxfp4"
|
||||
dtype = current_platform.fp8_dtype() if fp8 else None
|
||||
(_, w1, s1, _), (_, w2, s2, _) = make_test_weights(
|
||||
32, n, k, quant_dtype=dtype, per_out_ch_quant=fp8
|
||||
@@ -83,28 +113,44 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
|
||||
moe_parallel_config=parallel,
|
||||
)
|
||||
start, end = pgi.rank * 16, (pgi.rank + 1) * 16
|
||||
quant = FusedMoEQuantConfig.make(
|
||||
dtype,
|
||||
per_act_token_quant=fp8,
|
||||
per_out_ch_quant=fp8,
|
||||
w1_scale=None if s1 is None else s1[start:end].contiguous(),
|
||||
w2_scale=None if s2 is None else s2[start:end].contiguous(),
|
||||
)
|
||||
if mxfp4:
|
||||
local_w1, local_w2, quant = _mxfp4_aiter_weights(w1, w2, start, end)
|
||||
ref_w1, ref_w2, ref_quant = _mxfp4_aiter_weights(w1, w2, 0, 32)
|
||||
else:
|
||||
quant = FusedMoEQuantConfig.make(
|
||||
dtype,
|
||||
per_act_token_quant=fp8,
|
||||
per_out_ch_quant=fp8,
|
||||
w1_scale=None if s1 is None else s1[start:end].contiguous(),
|
||||
w2_scale=None if s2 is None else s2[start:end].contiguous(),
|
||||
)
|
||||
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
|
||||
w1[start:end].contiguous(), w2[start:end].contiguous()
|
||||
)
|
||||
local_w1.is_shuffled = local_w2.is_shuffled = True
|
||||
prepare = maybe_make_prepare_finalize(config, quant)
|
||||
assert isinstance(prepare, MoriPrepareAndFinalize)
|
||||
assert prepare.use_fp8_dispatch == fp8
|
||||
assert prepare.use_fp4_dispatch == mxfp4
|
||||
assert prepare.num_dispatchers() == 2
|
||||
experts = AiterExperts(config, quant)
|
||||
experts_cls = {
|
||||
"bf16": AiterUnquantizedExperts,
|
||||
"fp8_token_channel": AiterFp8ChannelExperts,
|
||||
"mxfp4": AiterMxfp4Experts,
|
||||
}[scheme]
|
||||
experts = experts_cls(config, quant)
|
||||
assert not experts.expects_unquantized_inputs
|
||||
kernel = FusedMoEKernel(prepare, experts)
|
||||
local_w1, local_w2 = rocm_aiter_ops.shuffle_weights(
|
||||
w1[start:end].contiguous(), w2[start:end].contiguous()
|
||||
)
|
||||
local_w1.is_shuffled = local_w2.is_shuffled = True
|
||||
expert_mask = torch.zeros(32, dtype=torch.int32, device=pgi.device)
|
||||
expert_mask[start:end] = 1
|
||||
|
||||
def reference(route_weights):
|
||||
if mxfp4:
|
||||
# AITER's non-EP MXFP4 path quantizes the BF16 input the same way
|
||||
# as FP4 dispatch, so the BF16 tolerances below still apply.
|
||||
return rocm_aiter_fused_experts(
|
||||
x, ref_w1, ref_w2, route_weights, ids, config, quant_config=ref_quant
|
||||
)
|
||||
return torch_experts(
|
||||
x,
|
||||
w1,
|
||||
@@ -177,9 +223,14 @@ def _mori_worker(pgi, vllm_config, cpu_group, m, n, k, fp8, graph):
|
||||
not current_platform.is_rocm() or not has_mori(), reason="Requires ROCm MoRI"
|
||||
)
|
||||
@pytest.mark.parametrize("m,n,k", MNKS)
|
||||
@pytest.mark.parametrize("fp8", [False, True], ids=["bf16", "fp8_token_channel"])
|
||||
@pytest.mark.parametrize("scheme", ["bf16", "fp8_token_channel", "mxfp4"])
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
def test_mori_moe(m, n, k, fp8):
|
||||
def test_mori_moe(m, n, k, scheme):
|
||||
if scheme == "mxfp4":
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if not on_gfx950():
|
||||
pytest.skip("AITER MXFP4 W4A4 MoE requires gfx950")
|
||||
config = VllmConfig(
|
||||
parallel_config=ParallelConfig(
|
||||
data_parallel_size=2,
|
||||
@@ -193,4 +244,4 @@ def test_mori_moe(m, n, k, fp8):
|
||||
"VLLM_ROCM_USE_AITER_MOE": "1",
|
||||
"VLLM_ROCM_USE_AITER_FUSION_SHARED_EXPERTS": "0",
|
||||
}
|
||||
parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, fp8, m == 32)
|
||||
parallel_launch_with_config(2, _mori_worker, config, env, m, n, k, scheme, m == 32)
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize import naive_dp_ep
|
||||
from vllm.model_executor.layers.fused_moe.prepare_finalize.naive_dp_ep import (
|
||||
MoEPrepareAndFinalizeNaiveDPEPModular,
|
||||
make_moe_prepare_and_finalize_naive_dp_ep,
|
||||
)
|
||||
|
||||
|
||||
class _RegionProbe(MoEPrepareAndFinalizeNaiveDPEPModular):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.body_ran = False
|
||||
|
||||
def run_comm(self) -> None:
|
||||
with self._comm_region():
|
||||
self.body_ran = True
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dbo_mocks():
|
||||
yield_switch = MagicMock()
|
||||
switch_back = MagicMock()
|
||||
with (
|
||||
patch.object(
|
||||
naive_dp_ep, "dbo_yield_and_switch_from_compute_to_comm", yield_switch
|
||||
),
|
||||
patch.object(naive_dp_ep, "dbo_switch_to_compute_sync", switch_back),
|
||||
):
|
||||
yield {"yield_switch": yield_switch, "switch_back": switch_back}
|
||||
|
||||
|
||||
def _dbo_state(dbo_active: bool, capturing: bool):
|
||||
return (
|
||||
patch.object(naive_dp_ep, "dbo_enabled", return_value=dbo_active),
|
||||
patch.object(torch.cuda, "is_current_stream_capturing", return_value=capturing),
|
||||
)
|
||||
|
||||
|
||||
def test_comm_region_noop_without_dbo(dbo_mocks):
|
||||
probe = _RegionProbe()
|
||||
dbo, capturing = _dbo_state(dbo_active=False, capturing=False)
|
||||
with dbo, capturing:
|
||||
probe.run_comm()
|
||||
assert probe.body_ran
|
||||
dbo_mocks["yield_switch"].assert_not_called()
|
||||
dbo_mocks["switch_back"].assert_not_called()
|
||||
|
||||
|
||||
def test_comm_region_handoff_when_dbo_active(dbo_mocks):
|
||||
probe = _RegionProbe()
|
||||
dbo, capturing = _dbo_state(dbo_active=True, capturing=False)
|
||||
with dbo, capturing:
|
||||
probe.run_comm()
|
||||
assert probe.body_ran
|
||||
dbo_mocks["yield_switch"].assert_called_once_with()
|
||||
dbo_mocks["switch_back"].assert_called_once_with()
|
||||
|
||||
|
||||
def test_comm_region_skipped_during_graph_capture(dbo_mocks):
|
||||
probe = _RegionProbe()
|
||||
dbo, capturing = _dbo_state(dbo_active=True, capturing=True)
|
||||
with dbo, capturing:
|
||||
probe.run_comm()
|
||||
assert probe.body_ran
|
||||
dbo_mocks["yield_switch"].assert_not_called()
|
||||
dbo_mocks["switch_back"].assert_not_called()
|
||||
|
||||
|
||||
def test_comm_region_restores_compute_stream_on_error(dbo_mocks):
|
||||
probe = _RegionProbe()
|
||||
dbo, capturing = _dbo_state(dbo_active=True, capturing=False)
|
||||
with (
|
||||
dbo,
|
||||
capturing,
|
||||
pytest.raises(RuntimeError, match="collective failed"),
|
||||
probe._comm_region(),
|
||||
):
|
||||
raise RuntimeError("collective failed")
|
||||
dbo_mocks["yield_switch"].assert_called_once_with()
|
||||
dbo_mocks["switch_back"].assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_rocm", [True, False])
|
||||
def test_supports_dbo_only_on_rocm(is_rocm):
|
||||
inst = MoEPrepareAndFinalizeNaiveDPEPModular()
|
||||
platform = SimpleNamespace(is_rocm=lambda: is_rocm)
|
||||
with patch.object(naive_dp_ep, "current_platform", platform):
|
||||
assert inst.supports_dbo() is is_rocm
|
||||
# supports_async() also drives MK-internal shared-expert overlap, so
|
||||
# it must not change with DBO support.
|
||||
assert not inst.supports_async()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_rocm", [True, False])
|
||||
def test_factory_returns_base_classes(is_rocm):
|
||||
platform = SimpleNamespace(is_rocm=lambda: is_rocm)
|
||||
with patch.object(naive_dp_ep, "current_platform", platform):
|
||||
mono = make_moe_prepare_and_finalize_naive_dp_ep(use_monolithic=True)
|
||||
modular = make_moe_prepare_and_finalize_naive_dp_ep(use_monolithic=False)
|
||||
assert type(mono) is naive_dp_ep.MoEPrepareAndFinalizeNaiveDPEPMonolithic
|
||||
assert type(modular) is MoEPrepareAndFinalizeNaiveDPEPModular
|
||||
|
||||
|
||||
def test_default_supports_dbo_follows_supports_async():
|
||||
# Classes that do not override supports_dbo() keep the old contract:
|
||||
# DBO only with async prepare/finalize.
|
||||
mono = naive_dp_ep.MoEPrepareAndFinalizeNaiveDPEPMonolithic()
|
||||
assert not mono.supports_async()
|
||||
assert not mono.supports_dbo()
|
||||
with patch.object(type(mono), "supports_async", return_value=True):
|
||||
assert mono.supports_dbo()
|
||||
@@ -2114,6 +2114,117 @@ def test_aiter_mxfp4_monolithic_rejects_expert_parallel(
|
||||
assert "parallel config" in reason
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"all2all_backend",
|
||||
[
|
||||
"deepep_high_throughput",
|
||||
"flashinfer_nvlink_two_sided",
|
||||
"flashinfer_nvlink_one_sided",
|
||||
],
|
||||
)
|
||||
def test_make_mxfp4_moe_kernel_rejects_cutlass_unsupported_expert_parallel(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
all2all_backend: str,
|
||||
):
|
||||
"""Compressed-tensors CUTLASS W4A4 bypasses the oracle; guard at assembly."""
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe.config import (
|
||||
FusedMoEParallelConfig,
|
||||
mxfp4_moe_quant_config,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsMxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
make_mxfp4_moe_kernel,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
CutlassExpertsMxfp4, "_supports_current_device", staticmethod(lambda: True)
|
||||
)
|
||||
|
||||
moe_config = replace(
|
||||
make_dummy_moe_config(
|
||||
num_experts=256,
|
||||
num_local_experts=32,
|
||||
hidden_dim=4096,
|
||||
intermediate_size=2048,
|
||||
),
|
||||
moe_parallel_config=FusedMoEParallelConfig(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
pcp_size=1,
|
||||
pcp_rank=0,
|
||||
dp_size=8,
|
||||
dp_rank=0,
|
||||
ep_size=8,
|
||||
ep_rank=0,
|
||||
sp_size=1,
|
||||
use_ep=True,
|
||||
all2all_backend=all2all_backend,
|
||||
enable_eplb=False,
|
||||
),
|
||||
)
|
||||
w1_scale = torch.empty(32, 64, 128, dtype=torch.uint8)
|
||||
w2_scale = torch.empty(32, 4096, 64, dtype=torch.uint8)
|
||||
quant_config = mxfp4_moe_quant_config(w1_scale=w1_scale, w2_scale=w2_scale)
|
||||
|
||||
with pytest.raises(ValueError, match="parallel config"):
|
||||
make_mxfp4_moe_kernel(
|
||||
quant_config,
|
||||
moe_config,
|
||||
CutlassExpertsMxfp4,
|
||||
Mxfp4MoeBackend.MARLIN,
|
||||
)
|
||||
|
||||
|
||||
def test_make_mxfp4_moe_kernel_validates_cutlass_expert_parallel(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""CUTLASS MXFP4 supports EP with allgather_reducescatter after expert_map."""
|
||||
from tests.kernels.moe.utils import make_dummy_moe_config
|
||||
from vllm.model_executor.layers.fused_moe.config import FusedMoEParallelConfig
|
||||
from vllm.model_executor.layers.fused_moe.experts.cutlass_moe import (
|
||||
CutlassExpertsMxfp4,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
_validate_mxfp4_experts_config,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
CutlassExpertsMxfp4, "_supports_current_device", staticmethod(lambda: True)
|
||||
)
|
||||
|
||||
moe_config = replace(
|
||||
make_dummy_moe_config(
|
||||
num_experts=256,
|
||||
num_local_experts=32,
|
||||
hidden_dim=4096,
|
||||
intermediate_size=2048,
|
||||
),
|
||||
moe_parallel_config=FusedMoEParallelConfig(
|
||||
tp_size=1,
|
||||
tp_rank=0,
|
||||
pcp_size=1,
|
||||
pcp_rank=0,
|
||||
dp_size=8,
|
||||
dp_rank=0,
|
||||
ep_size=8,
|
||||
ep_rank=0,
|
||||
sp_size=1,
|
||||
use_ep=True,
|
||||
all2all_backend="allgather_reducescatter",
|
||||
enable_eplb=False,
|
||||
),
|
||||
)
|
||||
|
||||
_validate_mxfp4_experts_config(
|
||||
CutlassExpertsMxfp4, moe_config, Mxfp4MoeBackend.MARLIN
|
||||
)
|
||||
|
||||
|
||||
# Every activation-quantizing OCP MX scheme must map to a `quant_dtype` that
|
||||
# `moe_kernel_quantize_input` actually dispatches on. Its final `else` returns
|
||||
# the activation untouched, so a name it does not know (e.g. "mxfp6" instead of
|
||||
@@ -2255,3 +2366,79 @@ def test_emulation_a_mxfp6_moe_forward_quantizes_activations():
|
||||
"w_mxfp4_a_mxfp6_e3m2 output is bit-identical to weight-only w_mxfp4:"
|
||||
" the emulation never fake-quantized the activations"
|
||||
)
|
||||
|
||||
|
||||
# (backend, activation, hidden_in, inter_in, hidden_out, inter_out)
|
||||
#
|
||||
# The AITER_MXFP4_MXFP4 (W4A4) rows are the ones this table exists for: the
|
||||
# per-partition intermediate is aligned to 128, not 256, so a model whose native
|
||||
# shard is already 128-aligned is allocated at its real width instead of being
|
||||
# padded up. `hidden_size` deliberately stays on 256 -- the gfx950 stage-1 mxfp4
|
||||
# kernels are generated with tile_k 256 only, and stage-1 reduces over model_dim.
|
||||
#
|
||||
# The remaining rows pin the neighbouring branches so a future edit to this
|
||||
# function cannot silently move a backend between them.
|
||||
ROCM_MXFP4_ROUNDUP_CASES = [
|
||||
# W4A4: 3072 / TP8 = 384 stays 384. This is the case the alignment change is
|
||||
# for; the generic 256 round-up would allocate 512.
|
||||
("AITER_MXFP4_MXFP4", "SILU", 6144, 384, 6144, 384),
|
||||
# W4A4, already 256-aligned (e.g. moe_intermediate 2048 at TP8): identity in
|
||||
# both the old and the new form, so such models are provably untouched.
|
||||
("AITER_MXFP4_MXFP4", "SILU", 6144, 256, 6144, 256),
|
||||
# W4A4, 1024 / TP8 = 128: the most aggressive case, halved rather than trimmed.
|
||||
("AITER_MXFP4_MXFP4", "SILU", 4096, 128, 4096, 128),
|
||||
# W4A4 ignores the activation -- the branch does not read it.
|
||||
("AITER_MXFP4_MXFP4", None, 6144, 384, 6144, 384),
|
||||
("AITER_MXFP4_MXFP4", "SWIGLUOAI", 6144, 384, 6144, 384),
|
||||
# W4A4 hidden_size is still rounded to 256, unlike the intermediate.
|
||||
("AITER_MXFP4_MXFP4", "SILU", 384, 384, 512, 384),
|
||||
# W4A16 CK + SiTU/SILU keeps its own 128/128 carve-out.
|
||||
("AITER_MXFP4_BF16", "SITU", 384, 384, 384, 384),
|
||||
("AITER_MXFP4_BF16", "SILU", 384, 384, 384, 384),
|
||||
# ... but only for those activations; anything else takes the generic ROCm arm.
|
||||
("AITER_MXFP4_BF16", "SWIGLUOAI", 384, 384, 512, 512),
|
||||
# Backends that stay on the generic ROCm 256/256 round-up.
|
||||
("AITER_MXFP4_FP8", "SILU", 384, 384, 512, 512),
|
||||
("AITER_TRITON_MXFP4_BF16", "SILU", 384, 384, 512, 512),
|
||||
("TRITON", "SWIGLUOAI", 384, 384, 512, 512),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"backend_name,activation_name,hidden_in,inter_in,hidden_out,inter_out",
|
||||
ROCM_MXFP4_ROUNDUP_CASES,
|
||||
)
|
||||
@pytest.mark.skipif(not ROCM_AVAILABLE, reason="ROCm is required for this test")
|
||||
def test_rocm_mxfp4_round_up_hidden_and_intermediate(
|
||||
backend_name: str,
|
||||
activation_name: str | None,
|
||||
hidden_in: int,
|
||||
inter_in: int,
|
||||
hidden_out: int,
|
||||
inter_out: int,
|
||||
):
|
||||
"""Pin the ROCm arm of mxfp4_round_up_hidden_size_and_intermediate_size.
|
||||
|
||||
The allocation this function returns is what every MXFP4 MoE weight buffer,
|
||||
scale buffer and aiter tuned-config lookup is sized from, so a change here is
|
||||
invisible locally and expensive everywhere else. Asserting the table keeps
|
||||
each backend on the alignment its kernels actually require.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.activation import MoEActivation
|
||||
from vllm.model_executor.layers.fused_moe.oracle.mxfp4 import (
|
||||
Mxfp4MoeBackend,
|
||||
mxfp4_round_up_hidden_size_and_intermediate_size,
|
||||
)
|
||||
|
||||
backend = Mxfp4MoeBackend[backend_name]
|
||||
activation = MoEActivation[activation_name] if activation_name else None
|
||||
|
||||
hidden, intermediate = mxfp4_round_up_hidden_size_and_intermediate_size(
|
||||
backend, hidden_in, inter_in, activation
|
||||
)
|
||||
|
||||
assert (hidden, intermediate) == (hidden_out, inter_out), (
|
||||
f"{backend_name} / {activation_name}: "
|
||||
f"hidden {hidden_in}->{hidden} (expected {hidden_out}), "
|
||||
f"intermediate {inter_in}->{intermediate} (expected {inter_out})"
|
||||
)
|
||||
|
||||
@@ -848,13 +848,13 @@ def test_activation_method_enum_values():
|
||||
def test_aiter_mxfp4_quant_scheme_support_matches_gfx950():
|
||||
"""AITER MXFP4 MoE support should stay gfx950-only."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.rocm_aiter_moe import (
|
||||
AiterExperts,
|
||||
AiterMxfp4Experts,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kMxfp4Static,
|
||||
)
|
||||
|
||||
assert AiterExperts._supports_quant_scheme(kMxfp4Static, None) is on_gfx950()
|
||||
assert AiterMxfp4Experts._supports_quant_scheme(kMxfp4Static, None) is on_gfx950()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
|
||||
@@ -87,7 +87,7 @@ def _torch_topk_softplus_sqrt(
|
||||
"backend,expected_experts",
|
||||
[
|
||||
("aiter_triton_mxfp4_bf16", "AiterW4A16ExpertsMonolithic"),
|
||||
("aiter", "AiterExperts"),
|
||||
("aiter", "AiterMxfp4Experts"),
|
||||
("triton_unfused", "UnfusedOAITritonExperts"),
|
||||
],
|
||||
)
|
||||
@@ -302,15 +302,18 @@ def test_fused_topk_softplus_sqrt_hash(
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "num_experts", "indices_type"),
|
||||
[
|
||||
(0, 256, torch.uint32),
|
||||
(17, 256, torch.uint32),
|
||||
(0, 256, torch.int32),
|
||||
(17, 256, torch.int32),
|
||||
(17, 384, torch.int64),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("use_padding", [False, True])
|
||||
def test_dsv4_fast_topk(
|
||||
num_tokens: int,
|
||||
num_experts: int,
|
||||
indices_type: torch.dtype,
|
||||
use_padding: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
torch.manual_seed(0)
|
||||
gating_output = torch.randn(
|
||||
@@ -325,8 +328,25 @@ def test_dsv4_fast_topk(
|
||||
routed_scaling_factor=1.5,
|
||||
e_score_correction_bias=correction_bias,
|
||||
)
|
||||
topk_weights, topk_ids = dsv4_topk(
|
||||
gating_output, correction_bias, indices_type, 1.5
|
||||
is_padding = torch.arange(num_tokens, device="cuda") % 2 == 1
|
||||
if use_padding:
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.fused_moe.router."
|
||||
"fused_topk_bias_router._get_padding_mask",
|
||||
lambda _: is_padding,
|
||||
)
|
||||
gating_output[is_padding] = float("nan")
|
||||
topk_weights_ref[is_padding] = 0
|
||||
topk_ids_ref[is_padding] = -1
|
||||
topk_weights, topk_ids = fused_topk_bias(
|
||||
gating_output,
|
||||
gating_output,
|
||||
"sqrtsoftplus",
|
||||
correction_bias,
|
||||
6,
|
||||
True,
|
||||
indices_type=indices_type,
|
||||
routed_scaling_factor=1.5,
|
||||
)
|
||||
|
||||
assert topk_ids.dtype == indices_type
|
||||
@@ -338,6 +358,17 @@ def test_dsv4_fast_topk(
|
||||
rtol=2e-5,
|
||||
)
|
||||
|
||||
if use_padding and num_tokens:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
weights, ids = dsv4_topk(
|
||||
gating_output, correction_bias, indices_type, 1.5, is_padding=is_padding
|
||||
)
|
||||
is_padding.fill_(True)
|
||||
graph.replay()
|
||||
assert (ids == -1).all()
|
||||
assert (weights == 0).all()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
|
||||
@@ -136,7 +136,13 @@ def test_arm_cpu_unquantized_kernel_selection(
|
||||
],
|
||||
)
|
||||
@patch(
|
||||
"vllm.utils.flashinfer.has_flashinfer",
|
||||
"vllm.model_executor.layers.fused_moe.experts.flashinfer_cutlass_moe."
|
||||
"has_flashinfer_cutlass_fused_moe",
|
||||
return_value=False,
|
||||
)
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe."
|
||||
"has_flashinfer_trtllm_fused_moe",
|
||||
return_value=False,
|
||||
)
|
||||
@patch(
|
||||
@@ -145,13 +151,18 @@ def test_arm_cpu_unquantized_kernel_selection(
|
||||
)
|
||||
def test_select_default_backend_by_platform(
|
||||
mock_aiter_enabled,
|
||||
mock_has_flashinfer,
|
||||
mock_has_flashinfer_trtllm,
|
||||
mock_has_flashinfer_cutlass,
|
||||
monkeypatch,
|
||||
platform_method,
|
||||
expected_backend,
|
||||
):
|
||||
"""Test default backend selection per platform with all optional
|
||||
accelerators (FlashInfer, AITER) disabled."""
|
||||
accelerators (FlashInfer, AITER) disabled.
|
||||
|
||||
Patch the feature probes directly because their cached results can outlive
|
||||
changes to has_flashinfer().
|
||||
"""
|
||||
with patch(
|
||||
"vllm.model_executor.layers.fused_moe.oracle.unquantized.current_platform"
|
||||
) as mock_platform:
|
||||
@@ -407,6 +418,41 @@ def test_select_cuda_flashinfer_trtllm_modular_for_standard_all2all(
|
||||
assert experts_cls.__name__ == "TrtLlmBf16ExpertsModular"
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
|
||||
return_value=True,
|
||||
)
|
||||
@pytest.mark.parametrize("require_decomposed_backend", [False, True])
|
||||
def test_require_decomposed_backend_rejects_monolithic_kernels(
|
||||
mock_supports_current_device,
|
||||
require_decomposed_backend,
|
||||
):
|
||||
"""Expert substitution adjusts routes, so the kernel must not route itself."""
|
||||
from vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe import (
|
||||
TrtLlmBf16ExpertsMonolithic,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.modular_kernel import (
|
||||
FusedMoEActivationFormat,
|
||||
)
|
||||
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.routing_method = RoutingMethodType.Renormalize
|
||||
moe_config.require_decomposed_backend = require_decomposed_backend
|
||||
|
||||
supported, reason = TrtLlmBf16ExpertsMonolithic.is_supported_config(
|
||||
TrtLlmBf16ExpertsMonolithic,
|
||||
moe_config,
|
||||
None,
|
||||
None,
|
||||
FusedMoEActivationFormat.Standard,
|
||||
)
|
||||
|
||||
assert supported != require_decomposed_backend
|
||||
assert (reason is not None and "routes adjusted" in reason) == (
|
||||
require_decomposed_backend
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"vllm.model_executor.layers.fused_moe.experts.trtllm_bf16_moe.TrtLlmBf16ExpertsBase._supports_current_device",
|
||||
return_value=True,
|
||||
@@ -580,3 +626,17 @@ def test_select_explicit_triton_backend(is_lora_enabled):
|
||||
|
||||
assert selected_backend == UnquantizedMoeBackend.TRITON
|
||||
assert experts_cls is not None
|
||||
|
||||
|
||||
@skipif_not_cuda_rocm
|
||||
@pytest.mark.parametrize("moe_backend", ["humming", "marlin"])
|
||||
def test_quantization_only_backend_falls_back_to_auto(moe_backend):
|
||||
"""A quantization-only --moe-backend also reaches the unquantized MoE
|
||||
layers of a quantized model (e.g. a BF16 MTP draft), which must use the
|
||||
auto-selected backend instead of failing."""
|
||||
moe_config = make_dummy_moe_config()
|
||||
moe_config.moe_backend = moe_backend
|
||||
|
||||
assert select_unquantized_moe_backend(
|
||||
moe_config=moe_config
|
||||
) == select_unquantized_moe_backend(moe_config=make_dummy_moe_config())
|
||||
|
||||
@@ -5,7 +5,10 @@
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import (
|
||||
_FP8_MIN_SCALING_FACTOR,
|
||||
QuantFP8,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
@@ -186,3 +189,22 @@ def test_quantfp8_group_edge_cases(default_vllm_config, seed: int) -> None:
|
||||
assert x_quant_large.shape == x_large.shape
|
||||
# FP8 max is typically 448 or 224, so scales should be > 1
|
||||
assert (scales_large > 1.0).all(), "Large values should have scales > 1"
|
||||
|
||||
# Test UE8M0 clamping still returns a correctly rounded value
|
||||
quant_op_ue8m0 = QuantFP8(
|
||||
static=False,
|
||||
group_shape=group_shape,
|
||||
column_major_scales=False,
|
||||
use_ue8m0=True,
|
||||
)
|
||||
x_quant_zero_ue8m0, scales_zero_ue8m0 = quant_op_ue8m0.forward_native(
|
||||
x_zero.clone()
|
||||
)
|
||||
assert x_quant_zero_ue8m0.shape == x_zero.shape
|
||||
# 0x807FFFFF selects only the sign and mantissa bits.
|
||||
assert ((scales_zero_ue8m0.view(torch.int32) & 0x807FFFFF) == 0).all(), (
|
||||
"Clamped scales did not have zero sign and mantissa bits."
|
||||
)
|
||||
assert (scales_zero_ue8m0 >= _FP8_MIN_SCALING_FACTOR).all(), (
|
||||
"Clamped scales were not larger than or equal to _FP8_MIN_SCALING_FACTOR."
|
||||
)
|
||||
|
||||
@@ -19,7 +19,7 @@ if not current_platform.is_rocm():
|
||||
|
||||
pytest.importorskip("triton")
|
||||
|
||||
from vllm.platforms.rocm import on_gfx1x # noqa: E402
|
||||
from vllm.platforms.rocm import on_gfx1x, on_gfx115x # noqa: E402
|
||||
|
||||
device = "cuda"
|
||||
|
||||
@@ -446,22 +446,28 @@ def _hip_skinny_reference(
|
||||
scales_nkg: torch.Tensor,
|
||||
*,
|
||||
group_size: int,
|
||||
zp_bias: int,
|
||||
zp_nkg: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""Reference for symmetric HIP skinny: C = A @ (W - zp_bias) * S."""
|
||||
"""Reference for HIP skinny: C = A @ (W - zp) * S.
|
||||
|
||||
``zp_nkg`` is [N, K//G] int32 raw zero points for the asymmetric case, or
|
||||
None for symmetric uint4b8, where the kernel subtracts a constant 8.
|
||||
"""
|
||||
K = a_mk.shape[1]
|
||||
N = w_int4_nk.shape[0]
|
||||
num_groups = K // group_size
|
||||
|
||||
w_fp = (w_int4_nk.to(torch.float32) - zp_bias).view(N, num_groups, group_size)
|
||||
w_g = w_int4_nk.to(torch.float32).view(N, num_groups, group_size)
|
||||
zp = 8.0 if zp_nkg is None else zp_nkg.to(torch.float32).unsqueeze(-1)
|
||||
s = scales_nkg.to(torch.float32).unsqueeze(-1)
|
||||
w_dequant = (w_fp * s).view(N, K)
|
||||
w_dequant = ((w_g - zp) * s).view(N, K)
|
||||
|
||||
return (a_mk.to(torch.float32) @ w_dequant.t()).to(a_mk.dtype)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("has_zp", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,N,G",
|
||||
[
|
||||
@@ -470,13 +476,50 @@ def _hip_skinny_reference(
|
||||
(1, 512, 256, 128),
|
||||
(2, 512, 256, 64),
|
||||
(3, 256, 512, 64),
|
||||
# M is the kernel's batch dimension; 4 and 5 reach dispatch tuples that
|
||||
# 1..3 never take, and 5 is MAX_SKINNY_BATCH_SIZE.
|
||||
(4, 512, 256, 32),
|
||||
(4, 256, 512, 64),
|
||||
(5, 512, 256, 128),
|
||||
(5, 256, 256, 32),
|
||||
# N == 8 is exactly one packed zero-point word.
|
||||
(1, 256, 8, 32),
|
||||
# K must exceed THRDS * A_CHUNK * UNRL for the K loop to run a whole
|
||||
# unrolled block; every K above is below that bound, so without these
|
||||
# the deep-K path goes untested. 2560 is not a multiple of the 1024 K
|
||||
# step the batch 2 and 4 tuples take, so it also hits the ragged tail.
|
||||
(1, 4096, 256, 128),
|
||||
(2, 2560, 256, 32),
|
||||
(3, 4096, 256, 64),
|
||||
(4, 2560, 256, 128),
|
||||
(5, 4096, 256, 32),
|
||||
# On gfx115x, batch 1 with K == 4096, or symmetric with K a multiple of
|
||||
# 8192, takes a tuple with A_CHUNK = 32 instead of 16. G = 32 is the
|
||||
# tight case there: each lane's A_CHUNK spans exactly one scale group.
|
||||
(1, 4096, 256, 32),
|
||||
(1, 8192, 256, 128),
|
||||
(1, 16384, 256, 64),
|
||||
# K * batch beyond what LDS holds (32768 fp16 elements), so gfx115x
|
||||
# walks K in chunks; elsewhere the op rejects these, as the layer
|
||||
# routes them to Triton. The per-row window shrinks as the batch grows,
|
||||
# so these cover a single reload and several, with and without a short
|
||||
# final chunk. 20992 is not a multiple of the K step either, so it
|
||||
# crosses a chunk boundary *and* has a ragged K tail.
|
||||
(2, 20480, 512, 64),
|
||||
(3, 16384, 512, 32),
|
||||
(4, 16384, 512, 128),
|
||||
(5, 21504, 512, 32),
|
||||
(5, 20992, 512, 128),
|
||||
],
|
||||
)
|
||||
def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G):
|
||||
def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G, has_zp):
|
||||
"""Test HIP wvSplitK_int4_g kernel directly via _custom_ops."""
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
if K * M > LDS_CAPACITY_ELEMENTS and not on_gfx115x():
|
||||
pytest.skip("deep-K HIP path is gfx115x only")
|
||||
|
||||
set_random_seed(0)
|
||||
|
||||
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
@@ -489,10 +532,64 @@ def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G):
|
||||
dtype
|
||||
)
|
||||
|
||||
cu_count = num_compute_units()
|
||||
out = ops.wvSplitK_int4_g(b_packed_i8, a, scales, cu_count, G)
|
||||
zp_nkg = None
|
||||
zp_packed = None
|
||||
if has_zp:
|
||||
zp_nkg = torch.randint(0, 16, (N, K // G), device=device, dtype=torch.int32)
|
||||
zp_packed = _pack_zp_rows_for_kernel(zp_nkg)
|
||||
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_bias=8)
|
||||
cu_count = num_compute_units()
|
||||
out = ops.wvSplitK_int4_g(b_packed_i8, a, scales, cu_count, G, zp_packed)
|
||||
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=zp_nkg)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("G", [32, 128])
|
||||
@pytest.mark.parametrize("has_zp", [False, True])
|
||||
@pytest.mark.parametrize("M", [2, 64], ids=["M=2_decode", "M=64_prefill"])
|
||||
def test_hip_skinny_padded_group_stride(dtype, G, has_zp, M):
|
||||
"""Scale and zero-point rows may be padded past K // group_size.
|
||||
|
||||
Both paths index them by their own row stride, so a padded row is read
|
||||
correctly rather than bleeding into the next row. M picks the path: the
|
||||
HIP skinny kernel at 2, the Triton prefill kernel at 64.
|
||||
"""
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
|
||||
set_random_seed(0)
|
||||
K, N = 2048, 256
|
||||
num_groups = K // G
|
||||
pad = 7
|
||||
|
||||
a = (0.25 * torch.randn((M, K), device=device, dtype=torch.float32)).to(dtype)
|
||||
w_int4_nk = torch.randint(0, 16, (N, K), device=device, dtype=torch.int32)
|
||||
b_packed_i8 = pack_int4_exllama_shuffle(w_int4_nk).view(torch.int8)
|
||||
|
||||
scales = (0.05 * torch.rand((N, num_groups), device=device)).to(dtype)
|
||||
scales_padded = torch.zeros((N, num_groups + pad), device=device, dtype=dtype)
|
||||
scales_padded[:, :num_groups] = scales
|
||||
scales_view = scales_padded[:, :num_groups]
|
||||
assert scales_view.stride(0) == num_groups + pad
|
||||
|
||||
zp_nkg = zp_view = None
|
||||
if has_zp:
|
||||
zp_nkg = torch.randint(0, 16, (N, num_groups), device=device, dtype=torch.int32)
|
||||
zp_packed = _pack_zp_rows_for_kernel(zp_nkg)
|
||||
zp_padded = torch.zeros(
|
||||
(N // 8, num_groups + pad), device=device, dtype=torch.int32
|
||||
)
|
||||
zp_padded[:, :num_groups] = zp_packed
|
||||
zp_view = zp_padded[:, :num_groups]
|
||||
|
||||
cu_count = num_compute_units()
|
||||
out = torch.ops.vllm.rdna_hybrid_w4a16_apply(
|
||||
a, b_packed_i8, scales_view, zp_view, None, cu_count, G
|
||||
)
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=zp_nkg)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
|
||||
|
||||
@@ -510,6 +607,8 @@ def test_hip_skinny_wvSplitK_int4_g(dtype, M, K, N, G):
|
||||
(1, 256, 256, 64),
|
||||
(1, 512, 256, 32),
|
||||
(1, 512, 256, 128),
|
||||
# Past LDS: HIP on gfx115x, Triton elsewhere.
|
||||
(2, 20480, 256, 64),
|
||||
(32, 512, 256, 64),
|
||||
(64, 1024, 256, 128),
|
||||
],
|
||||
@@ -535,7 +634,7 @@ def test_rdna_hybrid_w4a16_dispatch(dtype, M, K, N, G):
|
||||
a, b_packed_i8, scales, None, None, cu_count, G
|
||||
)
|
||||
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_bias=8)
|
||||
ref = _hip_skinny_reference(a, w_int4_nk, scales, group_size=G, zp_nkg=None)
|
||||
|
||||
torch.testing.assert_close(out, ref, rtol=1e-2, atol=5e-2)
|
||||
|
||||
@@ -748,17 +847,30 @@ def test_process_weights_pads_cliff_rows(K, expected_weight_stride, dist_init):
|
||||
|
||||
@pytest.mark.skipif(not on_gfx1x(), reason="Hybrid path is gfx11/gfx12 only")
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,expected_path",
|
||||
"M,K,gfx115x,expected_path",
|
||||
[
|
||||
(4, 8192, "hip"),
|
||||
(4, 9728, "hip"),
|
||||
(4, 9856, "triton"),
|
||||
(MAX_SKINNY_BATCH_SIZE + 1, 1024, "triton"),
|
||||
(4, 8192, False, "hip"),
|
||||
(4, 9728, False, "hip"),
|
||||
(4, 9856, False, "triton"),
|
||||
(4, 9856, True, "hip"),
|
||||
(MAX_SKINNY_BATCH_SIZE + 1, 1024, True, "triton"),
|
||||
],
|
||||
ids=[
|
||||
"regular_limit",
|
||||
"medium_range",
|
||||
"above_medium",
|
||||
"above_medium_gfx115x",
|
||||
"batch_too_large",
|
||||
],
|
||||
ids=["regular_limit", "medium_range", "above_medium", "batch_too_large"],
|
||||
)
|
||||
def test_rdna_hybrid_w4a16_dispatch_boundaries(M, K, expected_path, monkeypatch):
|
||||
"""Route only supported regular and medium skinny shapes to HIP."""
|
||||
def test_rdna_hybrid_w4a16_dispatch_boundaries(
|
||||
M, K, gfx115x, expected_path, monkeypatch
|
||||
):
|
||||
"""Route only supported regular and medium skinny shapes to HIP.
|
||||
|
||||
gfx115x drops the K*M bound, so a shape past the medium limit still goes to
|
||||
HIP there; the batch bound holds on every target.
|
||||
"""
|
||||
import vllm._custom_ops as ops
|
||||
|
||||
N, G = 16, 128
|
||||
@@ -777,6 +889,7 @@ def test_rdna_hybrid_w4a16_dispatch_boundaries(M, K, expected_path, monkeypatch)
|
||||
|
||||
monkeypatch.setattr(ops, "wvSplitK_int4_g", fake_hip)
|
||||
monkeypatch.setattr(hybrid_module, "triton_w4a16_skinny_fmt_gemm", fake_triton)
|
||||
monkeypatch.setattr(hybrid_module, "_on_gfx115x", lambda: gfx115x)
|
||||
|
||||
output = hybrid_module._rdna_hybrid_w4a16_apply_impl(
|
||||
x_mk, w_q, scales, None, None, 40, G
|
||||
|
||||
@@ -30,7 +30,7 @@ fp8_only = pytest.mark.skipif(
|
||||
|
||||
ENV_DEFAULTS = {
|
||||
"VLLM_ROCM_FP8_PADDING": True,
|
||||
"VLLM_ROCM_USE_AITER_FP8BMM": True,
|
||||
"VLLM_ROCM_USE_AITER_FP8BMM": False,
|
||||
"VLLM_ROCM_FP8_MFMA_PAGE_ATTN": False,
|
||||
}
|
||||
|
||||
|
||||
@@ -148,7 +148,7 @@ def test_fp4_env_defaults():
|
||||
import vllm.envs as envs
|
||||
|
||||
assert envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM is False
|
||||
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is True
|
||||
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -32,11 +32,9 @@ def torch_scaled_mm(
|
||||
out = torch.mm(a.to(torch.float32), b.to(torch.float32))
|
||||
out = scale_a * out
|
||||
out = scale_b.T * out
|
||||
out = out.to(out_dtype)
|
||||
if bias is not None:
|
||||
out = out + bias
|
||||
|
||||
return out
|
||||
out = out + bias.float()
|
||||
return out.to(out_dtype)
|
||||
|
||||
|
||||
def get_8bit_types():
|
||||
@@ -125,6 +123,25 @@ def test_scaled_mm(
|
||||
torch.testing.assert_close(c_check, c_actual, rtol=1e-1, atol=1e-1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("in_dtype", get_8bit_types())
|
||||
@pytest.mark.parametrize("use_heuristic", [True, False])
|
||||
def test_scaled_mm_adds_bias_before_rounding(in_dtype, use_heuristic):
|
||||
# The accumulator is 1 + 3 * 2**-10 and the bias 2**-8. Rounding to bf16
|
||||
# first gives 1.0 (1 + 2**-8 is a tie, to even); adding first, 1 + 2**-7.
|
||||
a = torch.ones((1, 32), device=device).to(in_dtype)
|
||||
col = torch.tensor([64.0] * 16 + [2.0, 1.0] + [0.0] * 14, device=device)
|
||||
b = col[:, None].expand(32, 16).to(in_dtype).contiguous()
|
||||
scale_a = torch.ones((1, 1), device=device)
|
||||
scale_b = torch.full((1, 1), 2**-10, device=device)
|
||||
bias = torch.full((16,), 2**-8, device=device, dtype=torch.bfloat16)
|
||||
args = (a, b, scale_a, scale_b, torch.bfloat16, bias)
|
||||
|
||||
out = triton_scaled_mm(*args, use_heuristic=use_heuristic)
|
||||
expected = torch_scaled_mm(*args)
|
||||
assert (expected == 1 + 2**-7).all()
|
||||
torch.testing.assert_close(out, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
# TD operand loads must be bit-exact vs the plain masked-load path.
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
|
||||
@@ -185,10 +185,7 @@ def test_every_dsv3_routed_shape_is_instantiated() -> None:
|
||||
let (3216, 7168) ship without its DISPATCH_DSV3_SHAPE(7168, 3216).
|
||||
"""
|
||||
source = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "csrc"
|
||||
/ "libtorch_stable"
|
||||
/ "dsv3_fused_a_gemm.cu"
|
||||
Path(__file__).resolve().parents[2] / "csrc" / "dsv3_fused_a_gemm.cu"
|
||||
).read_text(encoding="utf-8")
|
||||
# Benchmark-only shapes live behind VLLM_K3_BENCH_SHAPES and are not built
|
||||
# by default, so they must not count as available.
|
||||
|
||||
@@ -1205,6 +1205,84 @@ def test_get_c128_boundary(starts, query_start_loc, expected):
|
||||
assert _get_c128_boundary(metadata) is expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
@torch.inference_mode()
|
||||
def test_c128_skip_is_not_baked_into_cuda_graph(monkeypatch):
|
||||
"""A graph captured on a batch with no C128 boundary must still launch the
|
||||
compressor on replay; only eager steps may skip it."""
|
||||
from vllm.config import CUDAGraphMode
|
||||
from vllm.forward_context import ForwardContext, override_forward_context
|
||||
from vllm.models.deepseek_v4 import compressor as compressor_mod
|
||||
from vllm.models.deepseek_v4.compressor import DeepseekCompressor
|
||||
from vllm.models.deepseek_v4.nvidia.ops import sparse_attn_compress_cutedsl
|
||||
|
||||
launches = torch.zeros(1, dtype=torch.int32, device="cuda")
|
||||
monkeypatch.setattr(
|
||||
compressor_mod, "_SAVE_PARTIAL_STATES_KERNEL", lambda **kw: None
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sparse_attn_compress_cutedsl,
|
||||
"_SPARSE_ATTN_COMPRESSOR_CUTEDSL_KERNEL",
|
||||
lambda **kw: launches.add_(1),
|
||||
)
|
||||
|
||||
compressor = DeepseekCompressor.__new__(DeepseekCompressor)
|
||||
torch.nn.Module.__init__(compressor)
|
||||
compressor.head_dim, compressor.rope_head_dim = 512, 64
|
||||
compressor.compress_ratio, compressor.coff, compressor.overlap = 128, 1, False
|
||||
compressor.use_fp4_cache = False
|
||||
compressor._use_two_stage_fused_compressor = False
|
||||
compressor._quant_block = 64
|
||||
compressor._token_stride = 576
|
||||
compressor._scale_dim = 8
|
||||
compressor.rms_norm_eps = 1e-6
|
||||
compressor.ape = torch.zeros(128, 512, device="cuda")
|
||||
compressor.norm = SimpleNamespace(weight=torch.ones(512, device="cuda"))
|
||||
compressor.state_cache = SimpleNamespace(
|
||||
prefix="state", kv_cache=torch.zeros(2, 8, 1024, device="cuda")
|
||||
)
|
||||
compressor.k_cache_prefix = "main"
|
||||
kv_cache = torch.zeros(1, 64, 584, dtype=torch.uint8, device="cuda")
|
||||
compressor._static_forward_context = {"main": SimpleNamespace(kv_cache=kv_cache)}
|
||||
|
||||
num_tokens = 4
|
||||
slots = torch.arange(num_tokens, device="cuda")
|
||||
metadata = {
|
||||
"state": SimpleNamespace(
|
||||
slot_mapping=slots,
|
||||
block_table=torch.zeros(num_tokens, 1, dtype=torch.int32, device="cuda"),
|
||||
block_size=8,
|
||||
token_to_req_indices=torch.arange(
|
||||
num_tokens, dtype=torch.int32, device="cuda"
|
||||
),
|
||||
c128_boundary=False,
|
||||
),
|
||||
"main": SimpleNamespace(slot_mapping=slots),
|
||||
}
|
||||
kv_score = torch.zeros(num_tokens, 1024, device="cuda")
|
||||
positions = torch.arange(num_tokens, device="cuda")
|
||||
rotary = SimpleNamespace(cos_sin_cache=torch.zeros(1, 64, device="cuda"))
|
||||
context = ForwardContext(
|
||||
{}, metadata, {}, cudagraph_runtime_mode=CUDAGraphMode.PIECEWISE
|
||||
)
|
||||
|
||||
def run():
|
||||
with override_forward_context(context):
|
||||
compressor(kv_score, positions, rotary)
|
||||
|
||||
run()
|
||||
torch.accelerator.synchronize()
|
||||
assert launches.item() == 0, "eager step with no boundary should skip"
|
||||
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run()
|
||||
launches.zero_()
|
||||
graph.replay()
|
||||
torch.accelerator.synchronize()
|
||||
assert launches.item() == 1, "captured graph must not bake in the skip"
|
||||
|
||||
|
||||
# ── Test A: DeepseekV4 Attention path ──────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -1422,7 +1500,10 @@ def test_dequantize_and_gather_k_cache(
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 4, 8, 17])
|
||||
@pytest.mark.parametrize("block_size", [16, 64])
|
||||
def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
|
||||
@pytest.mark.parametrize("scale_fmt", ["ue8m0", "mxfp8"])
|
||||
def test_indexer_quant_cache_roundtrip(
|
||||
num_tokens: int, block_size: int, scale_fmt: str
|
||||
):
|
||||
"""K → indexer_k_quant_and_cache → cp_gather_indexer_k_quant_cache
|
||||
→ manual dequant → compare against original."""
|
||||
HEAD_DIM = 128
|
||||
@@ -1442,7 +1523,9 @@ def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
|
||||
)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
|
||||
ops.indexer_k_quant_and_cache(k, kv_cache, slot_mapping, QUANT_BLOCK_SIZE, "ue8m0")
|
||||
ops.indexer_k_quant_and_cache(
|
||||
k, kv_cache, slot_mapping, QUANT_BLOCK_SIZE, scale_fmt
|
||||
)
|
||||
|
||||
# ── Gather ──────────────────────────────────────────────────────────
|
||||
max_blocks_per_seq = num_blocks
|
||||
@@ -1465,7 +1548,12 @@ def test_indexer_quant_cache_roundtrip(num_tokens: int, block_size: int):
|
||||
|
||||
# ── Manual dequant ──────────────────────────────────────────────────
|
||||
k_fp8 = dst_k.view(torch.float8_e4m3fn).float() # [num_tokens, 128]
|
||||
scale = dst_scale.view(torch.float32) # [num_tokens, 1]
|
||||
if scale_fmt == "mxfp8":
|
||||
# One UE8M0 byte per 32 elements, all equal to the token's scale.
|
||||
assert (dst_scale == dst_scale[:, :1]).all()
|
||||
scale = (dst_scale[:, :1].to(torch.int32) << 23).view(torch.float32)
|
||||
else:
|
||||
scale = dst_scale.view(torch.float32) # [num_tokens, 1]
|
||||
k_recovered = k_fp8 * scale # [num_tokens, 128]
|
||||
|
||||
# ── Compare ─────────────────────────────────────────────────────────
|
||||
@@ -1749,8 +1837,13 @@ def _reference_kv_compress_norm_rope(
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: bool):
|
||||
@pytest.mark.parametrize("mx_scale", [False, True])
|
||||
def test_fused_kv_insert_indexer(
|
||||
num_tokens: int, kv_block_size: int, use_fp4: bool, mx_scale: bool
|
||||
):
|
||||
"""Fused K compress+norm+rope+quant+insert for the indexer KV cache."""
|
||||
if use_fp4 and mx_scale:
|
||||
pytest.skip("MX scales only change the FP8 cache")
|
||||
HEAD_DIM = 128
|
||||
ROPE_DIM = 64
|
||||
BLOCK_SIZE = 16
|
||||
@@ -1833,6 +1926,7 @@ def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: b
|
||||
TOKEN_STRIDE=TOKEN_STRIDE,
|
||||
SCALE_DIM=SCALE_DIM,
|
||||
KV_BLOCK_STRIDE=kv_cache.stride(0),
|
||||
MX_SCALE=mx_scale,
|
||||
num_warps=1,
|
||||
)
|
||||
|
||||
@@ -1876,12 +1970,18 @@ def test_fused_kv_insert_indexer(num_tokens: int, kv_block_size: int, use_fp4: b
|
||||
), f"token {i}: FP8 bytes differ"
|
||||
|
||||
scale_off = kv_block_size * TOKEN_STRIDE + pos * SCALE_DIM
|
||||
actual_scale = kv_cache[blk, scale_off : scale_off + SCALE_DIM].view(
|
||||
torch.float32
|
||||
)
|
||||
assert torch.equal(actual_scale, scale[i : i + 1]), (
|
||||
f"token {i}: scale {actual_scale.item()} != {scale[i].item()}"
|
||||
)
|
||||
actual_scale = kv_cache[blk, scale_off : scale_off + SCALE_DIM]
|
||||
if mx_scale:
|
||||
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
|
||||
ue8m0 = (scale[i : i + 1].view(torch.int32) >> 23).to(torch.uint8)
|
||||
assert torch.equal(actual_scale, ue8m0.repeat(SCALE_DIM)), (
|
||||
f"token {i}: mx scale {actual_scale.tolist()} != {ue8m0.item()}"
|
||||
)
|
||||
else:
|
||||
actual_scale = actual_scale.view(torch.float32)
|
||||
assert torch.equal(actual_scale, scale[i : i + 1]), (
|
||||
f"token {i}: scale {actual_scale.item()} != {scale[i].item()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("compress_ratio", [4, 128])
|
||||
|
||||
@@ -32,6 +32,7 @@ import torch
|
||||
|
||||
from vllm.models.deepseek_v32.common import kernels as K
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
|
||||
FP8 = torch.float8_e4m3fn
|
||||
FP8_MAX = 448.0
|
||||
@@ -116,6 +117,16 @@ def rope(
|
||||
return out
|
||||
|
||||
|
||||
def indexer_scales(scale_bytes: torch.Tensor) -> torch.Tensor:
|
||||
"""Per-token fp32 scales from the indexer cache's 4 scale bytes per token."""
|
||||
if not mqa_logits_use_mx_scales():
|
||||
return scale_bytes.view(torch.float32)
|
||||
# MX scales: the power-of-two scale's exponent as 4 equal UE8M0 bytes.
|
||||
ue8m0 = scale_bytes.view(-1, 4)
|
||||
assert (ue8m0 == ue8m0[:, :1]).all(), "MX scale bytes differ within a token"
|
||||
return (ue8m0[:, 0].to(torch.int32) << 23).view(torch.float32)
|
||||
|
||||
|
||||
def ue8m0_quant(vals: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Per-row (last dim) UE8M0 fp8 quant matching kernels._fp8_ue8m0_quantize."""
|
||||
amax = vals.float().abs().amax(dim=-1, keepdim=True)
|
||||
@@ -243,7 +254,7 @@ def test_fused_norm_rope(
|
||||
q_ref, s_ref = ue8m0_quant(ik_ref)
|
||||
flat = idx_cache[0].reshape(-1)
|
||||
vals = flat[: bs * cfg.index_head_dim].view(FP8).reshape(bs, cfg.index_head_dim)
|
||||
scales = flat[bs * cfg.index_head_dim :].view(torch.float32)
|
||||
scales = indexer_scales(flat[bs * cfg.index_head_dim :])
|
||||
assert_fp8(vals[:num_tokens], q_ref, "indexer-K fp8")
|
||||
torch.testing.assert_close(scales[:num_tokens], s_ref, rtol=0, atol=0)
|
||||
|
||||
@@ -318,7 +329,7 @@ def test_fused_norm_rope_packed_indexer_block_stride(cfg: ModelConfig):
|
||||
.view(FP8)
|
||||
.view(block_size, cfg.index_head_dim)
|
||||
)
|
||||
scales = packed[block_size * cfg.index_head_dim :].view(torch.float32)
|
||||
scales = indexer_scales(packed[block_size * cfg.index_head_dim :])
|
||||
assert_fp8(values, q_ref, "packed indexer-K fp8")
|
||||
torch.testing.assert_close(scales, s_ref, rtol=0, atol=0)
|
||||
assert (backing[block_size * idx_row : packed_block_stride] == 0).all()
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
)
|
||||
from vllm.models.deepseek_v4.common.ops import fused_indexer_q_rope_quant
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
|
||||
HEAD_DIM = 128
|
||||
@@ -347,6 +348,10 @@ def _indexer_k_reference(
|
||||
exponent = k.float().abs().amax(dim=-1, keepdim=True).clamp(min=1e-4) / 448
|
||||
scale = exponent.log2().ceil().exp2()
|
||||
values = (k.float() / scale).clamp(-448, 448).to(torch.float8_e4m3fn)
|
||||
if mqa_logits_use_mx_scales():
|
||||
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
|
||||
ue8m0 = (scale.view(torch.int32) >> 23).to(torch.uint8)
|
||||
return values.view(torch.uint8), ue8m0.repeat(1, 4)
|
||||
return values.view(torch.uint8), scale.view(torch.uint8)
|
||||
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import mqa_logits_use_mx_scales
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.glm5next.amd.ops.kpool_compress import (
|
||||
@@ -206,7 +207,12 @@ def _torch_reference(
|
||||
k_off = page_base + loc_tok * HEAD_DIM + torch.arange(HEAD_DIM)
|
||||
s_off = page_base + k_region + loc_tok * 4
|
||||
kv_flat[k_off] = quantized.view(torch.uint8)
|
||||
kv_flat[s_off : s_off + 4] = scale.detach().reshape(1).view(torch.uint8)
|
||||
scale_bytes = scale.detach().reshape(1).view(torch.uint8)
|
||||
if mqa_logits_use_mx_scales():
|
||||
# The power-of-two scale's exponent as 4 equal UE8M0 bytes.
|
||||
ue8m0 = scale.reshape(1).view(torch.int32) >> 23
|
||||
scale_bytes = ue8m0.to(torch.uint8).repeat(4)
|
||||
kv_flat[s_off : s_off + 4] = scale_bytes
|
||||
|
||||
# stash -- gated on the TOKEN-granular tail slot, not on pos_valid.
|
||||
# pos_valid keys off the pool-granular cache_loc, which is -1 for
|
||||
@@ -220,12 +226,19 @@ def _torch_reference(
|
||||
return kv_out, tail_cpu.to(torch.bfloat16).to(device="cuda")
|
||||
|
||||
|
||||
def _assert_eq(r_ref, r_kern):
|
||||
def _assert_eq(r_ref, r_kern, k_byte_tol=0):
|
||||
kv_ref, tail_ref = r_ref
|
||||
kv_kern, tail_kern = r_kern
|
||||
assert torch.equal(kv_ref, kv_kern), (
|
||||
"kv_cache differs: max diff "
|
||||
f"{(kv_ref.int() - kv_kern.int()).abs().max().item()}"
|
||||
# Each page holds the fp8 K bytes followed by the fp32 scales.
|
||||
k_region = PAGE_SIZE * HEAD_DIM
|
||||
pages_ref = kv_ref.view(kv_ref.shape[0], -1)
|
||||
pages_kern = kv_kern.view(kv_kern.shape[0], -1)
|
||||
k_diff = (pages_ref[:, :k_region].int() - pages_kern[:, :k_region].int()).abs()
|
||||
assert k_diff.max().item() <= k_byte_tol, (
|
||||
f"kv_cache K differs: max diff {k_diff.max().item()}"
|
||||
)
|
||||
assert torch.equal(pages_ref[:, k_region:], pages_kern[:, k_region:]), (
|
||||
"kv_cache scales differ"
|
||||
)
|
||||
assert torch.equal(tail_ref, tail_kern), (
|
||||
"tail_kv_cache differs: max diff "
|
||||
@@ -652,4 +665,6 @@ def test_batched_matches_reference_fuzz(seed):
|
||||
|
||||
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
_assert_eq(r_ref, r_kern)
|
||||
# Random inputs can land the pooled value on a bf16 rounding midpoint,
|
||||
# where the kernel and the CPU reference may round apart by one fp8 step.
|
||||
_assert_eq(r_ref, r_kern, k_byte_tol=1)
|
||||
|
||||
@@ -0,0 +1,544 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for cuteDSL low-latency GEMM with fp32 weights."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not torch.accelerator.is_available(), reason="CUDA required"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True, scope="module")
|
||||
def _require_sm90_and_cutedsl():
|
||||
if not current_platform.has_device_capability(90):
|
||||
pytest.skip("Requires SM90+ (Hopper/Blackwell)")
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
|
||||
is_available,
|
||||
)
|
||||
|
||||
if not is_available():
|
||||
pytest.skip("cuteDSL (CUTLASS Python) not installed")
|
||||
|
||||
|
||||
def _ref(a, b):
|
||||
return torch.mm(a.float(), b.float().T)
|
||||
|
||||
|
||||
def _assert_close(out, ref, *, min_cos_sim=0.99, context=""):
|
||||
assert out.device.type == "cuda", f"{context}: not on CUDA"
|
||||
assert torch.isfinite(out).all(), f"{context}: NaN/Inf"
|
||||
cos = F.cosine_similarity(
|
||||
out.reshape(-1).float(), ref.reshape(-1).float(), dim=0
|
||||
).item()
|
||||
assert cos > min_cos_sim, (
|
||||
f"{context}: cos_sim {cos:.6f} < {min_cos_sim} "
|
||||
f"(abs_err={(out.float() - ref.float()).abs().max().item():.2e})"
|
||||
)
|
||||
|
||||
|
||||
def _gemm(a, b, output_dtype=torch.float32):
|
||||
from vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w import (
|
||||
LL_FP32W_GEMM_KERNEL,
|
||||
)
|
||||
|
||||
return LL_FP32W_GEMM_KERNEL(a, b, output_dtype)
|
||||
|
||||
|
||||
SHAPES = [
|
||||
(256, 3072, "MiniMax-M2"),
|
||||
(256, 7168, "DSV3-sized"),
|
||||
(128, 5120, "DeepSeek-V2"),
|
||||
(64, 2880, "non-aligned-K"),
|
||||
(8, 4096, "Mixtral"),
|
||||
]
|
||||
|
||||
A_DTYPES = [
|
||||
pytest.param(torch.bfloat16, id="bf16"),
|
||||
pytest.param(torch.float16, id="fp16"),
|
||||
pytest.param(torch.float32, id="fp32"),
|
||||
]
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Correctness: all dtype combos x shapes x M values
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("a_dtype", A_DTYPES)
|
||||
@pytest.mark.parametrize("M", [1, 2, 4, 8, 16])
|
||||
@pytest.mark.parametrize("N,K,desc", SHAPES, ids=[s[2] for s in SHAPES])
|
||||
def test_correctness(M, N, K, desc, a_dtype):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, device="cuda").to(a_dtype)
|
||||
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
|
||||
out = _gemm(a, b)
|
||||
assert out.dtype == torch.float32
|
||||
assert out.shape == (M, N)
|
||||
_assert_close(out, _ref(a, b), context=f"{a_dtype} M={M} {desc}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Small K (k_main=0, only tail/scalar paths)
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("K", [16, 32, 64, 128, 256, 512, 1024, 1536])
|
||||
def test_small_K(K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, K, dtype=torch.float32, device="cuda")
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context=f"small_K={K}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# K sweep (covers main loop + various tail combinations)
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("K", [2048, 2304, 2880, 3072, 4096, 5120, 7168])
|
||||
def test_K_sweep(K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(8, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, K, dtype=torch.float32, device="cuda")
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context=f"K={K}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# MiniMax-M2 specific shapes
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 4, 8, 16, 32])
|
||||
def test_minimax_m2(M):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, 3072, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(256, 3072, dtype=torch.float32, device="cuda")
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context=f"MiniMax-M2 M={M}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Single token (M=1, decode path)
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"N,K",
|
||||
[(256, 3072), (256, 7168), (8, 4096)],
|
||||
ids=["MiniMax-M2", "DSV3-sized", "Mixtral"],
|
||||
)
|
||||
def test_single_token(N, K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(1, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
|
||||
out = _gemm(a, b)
|
||||
assert out.shape == (1, N)
|
||||
_assert_close(out, _ref(a, b), context=f"M=1 {N}x{K}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Arbitrary N
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("N", [1, 3, 16, 64, 256])
|
||||
def test_arbitrary_N(N):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 3072, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, 3072, dtype=torch.float32, device="cuda")
|
||||
out = _gemm(a, b)
|
||||
assert out.shape == (4, N)
|
||||
_assert_close(out, _ref(a, b), context=f"N={N}")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Numerical robustness
|
||||
# =================================================================
|
||||
|
||||
|
||||
def test_large_values():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda") * 100
|
||||
b = torch.randn(128, 5120, dtype=torch.float32, device="cuda") * 100
|
||||
out = _gemm(a, b)
|
||||
assert torch.isfinite(out).all()
|
||||
_assert_close(out, _ref(a, b), context="large_values")
|
||||
|
||||
|
||||
def test_near_zero():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda") * 1e-4
|
||||
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda") * 1e-4
|
||||
out = _gemm(a, b)
|
||||
assert torch.isfinite(out).all()
|
||||
assert out.abs().max() < 1.0
|
||||
|
||||
|
||||
def test_zeros():
|
||||
a = torch.zeros(4, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 2048, dtype=torch.float32, device="cuda")
|
||||
assert (_gemm(a, b) == 0).all()
|
||||
|
||||
|
||||
def test_ones():
|
||||
a = torch.ones(1, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(32, 4096, dtype=torch.float32, device="cuda")
|
||||
_assert_close(_gemm(a, b), _ref(a, b), context="ones")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Output dtype
|
||||
# =================================================================
|
||||
|
||||
|
||||
def test_output_fp32():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(128, 5120, dtype=torch.float32, device="cuda")
|
||||
assert _gemm(a, b).dtype == torch.float32
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Determinism
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [1, 4, 8, 16])
|
||||
@pytest.mark.parametrize("K", [1024, 3072, 7168])
|
||||
def test_deterministic(M, K):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, K, dtype=torch.float32, device="cuda")
|
||||
torch.testing.assert_close(_gemm(a, b), _gemm(a, b), atol=0, rtol=0)
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Cross-dtype consistency
|
||||
# =================================================================
|
||||
|
||||
|
||||
def test_bf16_vs_fp32_activations():
|
||||
"""bf16 and fp32 activations on same data should roughly agree."""
|
||||
torch.manual_seed(42)
|
||||
K, N = 4096, 128
|
||||
a_bf16 = torch.randn(4, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
|
||||
out_bf16 = _gemm(a_bf16, b)
|
||||
a_fp32 = a_bf16.float()
|
||||
out_fp32 = _gemm(a_fp32, b)
|
||||
_assert_close(
|
||||
out_bf16, out_fp32, min_cos_sim=0.999, context="bf16_vs_fp32_activations"
|
||||
)
|
||||
|
||||
|
||||
# =================================================================
|
||||
# CUDA graph
|
||||
# =================================================================
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"M,K,N", [(4, 3072, 256), (16, 5120, 128)], ids=["MiniMax", "DeepSeek"]
|
||||
)
|
||||
def test_cudagraph(M, K, N):
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(N, K, dtype=torch.float32, device="cuda")
|
||||
_gemm(a, b)
|
||||
torch.accelerator.synchronize()
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
out = _gemm(a, b)
|
||||
for _ in range(5):
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
_assert_close(out, _ref(a, b), context=f"cudagraph M={M} K={K}")
|
||||
|
||||
|
||||
def test_cudagraph_20x_replay():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
|
||||
_gemm(a, b)
|
||||
torch.accelerator.synchronize()
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
out = _gemm(a, b)
|
||||
results = []
|
||||
for _ in range(20):
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
results.append(out.clone())
|
||||
for i in range(1, len(results)):
|
||||
torch.testing.assert_close(
|
||||
results[0], results[i], atol=0, rtol=0, msg=f"Replay {i} differs"
|
||||
)
|
||||
|
||||
|
||||
def test_cudagraph_input_update():
|
||||
torch.manual_seed(42)
|
||||
a = torch.randn(4, 5120, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(128, 5120, dtype=torch.float32, device="cuda")
|
||||
_gemm(a, b)
|
||||
torch.accelerator.synchronize()
|
||||
g = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(g):
|
||||
out = _gemm(a, b)
|
||||
a.copy_(torch.randn_like(a))
|
||||
g.replay()
|
||||
torch.accelerator.synchronize()
|
||||
_assert_close(out, _ref(a, b), context="cudagraph input update")
|
||||
|
||||
|
||||
# =================================================================
|
||||
# Negative tests
|
||||
# =================================================================
|
||||
|
||||
|
||||
def test_invalid_weight_dtype():
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_device_cpu():
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16)
|
||||
b = torch.randn(64, 4096, dtype=torch.float32)
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_1d_input():
|
||||
a = torch.randn(4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_activation_dtype():
|
||||
a = torch.ones(4, 4096, dtype=torch.int32, device="cuda")
|
||||
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_output_dtype():
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b, torch.bfloat16)
|
||||
|
||||
|
||||
def test_invalid_k_mismatch():
|
||||
a = torch.randn(4, 4096, dtype=torch.bfloat16, device="cuda")
|
||||
b = torch.randn(64, 2048, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
def test_invalid_non_contiguous():
|
||||
base = torch.randn(4, 8192, dtype=torch.bfloat16, device="cuda")
|
||||
a = base[:, ::2]
|
||||
b = torch.randn(64, 4096, dtype=torch.float32, device="cuda")
|
||||
with pytest.raises(ValueError):
|
||||
_gemm(a, b)
|
||||
|
||||
|
||||
# =================================================================
|
||||
# GateLinear dispatch integration
|
||||
# =================================================================
|
||||
|
||||
|
||||
def _make_gate_linear_fp32w(
|
||||
monkeypatch,
|
||||
*,
|
||||
params_dtype,
|
||||
out_dtype=torch.float32,
|
||||
input_size=2048,
|
||||
output_size=64,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.linear.get_tensor_model_parallel_rank",
|
||||
lambda: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.linear.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_rank",
|
||||
lambda: 0,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.parameter.get_tensor_model_parallel_world_size",
|
||||
lambda: 1,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_bf16.is_available",
|
||||
lambda: False,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.is_available",
|
||||
lambda: True,
|
||||
)
|
||||
|
||||
from vllm.model_executor.layers.fused_moe.router.gate_linear import GateLinear
|
||||
|
||||
return GateLinear(
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
bias=False,
|
||||
out_dtype=out_dtype,
|
||||
params_dtype=params_dtype,
|
||||
).cuda()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("x_dtype", [torch.bfloat16, torch.float16, torch.float32])
|
||||
def test_gate_linear_uses_ll_fp32w_for_fp32_weight_fast_path(monkeypatch, x_dtype):
|
||||
gate = _make_gate_linear_fp32w(monkeypatch, params_dtype=torch.float32)
|
||||
x = torch.randn(4, 2048, dtype=x_dtype, device="cuda")
|
||||
calls = []
|
||||
|
||||
def fake_ll_fp32w_gemm(hidden_states, router_weight):
|
||||
calls.append((hidden_states, router_weight))
|
||||
return torch.full(
|
||||
(hidden_states.shape[0], router_weight.shape[0]),
|
||||
1.0,
|
||||
dtype=torch.float32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
|
||||
fake_ll_fp32w_gemm,
|
||||
)
|
||||
out, bias = gate(x)
|
||||
assert bias is None
|
||||
assert out.shape == (4, 64)
|
||||
assert out.dtype == torch.float32
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0] is x
|
||||
assert calls[0][1] is gate.weight
|
||||
|
||||
|
||||
PREFERRED_FP32W_GATE_CASES = [
|
||||
*((6144, 128, M) for M in range(1, 33)),
|
||||
*((6144, 256, M) for M in (1, 8, 12, 14, 16, 20, 22, 25, 27, 32)),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("input_size", "output_size", "M"), PREFERRED_FP32W_GATE_CASES)
|
||||
def test_gate_linear_uses_ll_fp32w_when_cpp_tier_disabled(
|
||||
monkeypatch, input_size, output_size, M
|
||||
):
|
||||
gate = _make_gate_linear_fp32w(
|
||||
monkeypatch,
|
||||
params_dtype=torch.float32,
|
||||
input_size=input_size,
|
||||
output_size=output_size,
|
||||
)
|
||||
gate.allow_fp32_router_gemm = False
|
||||
x = torch.randn(M, input_size, dtype=torch.bfloat16, device="cuda")
|
||||
calls = []
|
||||
|
||||
def fake_ll_fp32w_gemm(hidden_states, router_weight):
|
||||
calls.append((hidden_states, router_weight))
|
||||
return torch.full(
|
||||
(hidden_states.shape[0], router_weight.shape[0]),
|
||||
2.0,
|
||||
dtype=torch.float32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
|
||||
fake_ll_fp32w_gemm,
|
||||
)
|
||||
out, _ = gate(x)
|
||||
assert len(calls) == 1
|
||||
assert calls[0][0] is x
|
||||
assert calls[0][1] is gate.weight
|
||||
assert torch.all(out == 2.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("M", [8, 16, 32])
|
||||
def test_gate_linear_uses_cpp_fp32_router_for_supported_cpp_shape(monkeypatch, M):
|
||||
gate = _make_gate_linear_fp32w(
|
||||
monkeypatch,
|
||||
params_dtype=torch.float32,
|
||||
input_size=3072,
|
||||
output_size=256,
|
||||
)
|
||||
assert gate.allow_fp32_router_gemm
|
||||
x = torch.randn(M, 3072, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
def fail_ll_fp32w_gemm(hidden_states, router_weight):
|
||||
raise AssertionError(
|
||||
"ll_fp32w_gemm should not bypass fp32_router_gemm_dispatch"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
|
||||
fail_ll_fp32w_gemm,
|
||||
)
|
||||
out, _ = gate(x)
|
||||
assert out.shape == (M, 256)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_gate_linear_bf16_weight_skips_ll_fp32w(monkeypatch):
|
||||
gate = _make_gate_linear_fp32w(monkeypatch, params_dtype=torch.bfloat16)
|
||||
assert not gate.allow_ll_fp32w_gemm
|
||||
x = torch.randn(4, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
def fail_ll_fp32w_gemm(hidden_states, router_weight):
|
||||
raise AssertionError("ll_fp32w_gemm should not run for bf16 weights")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
|
||||
fail_ll_fp32w_gemm,
|
||||
)
|
||||
out, _ = gate(x)
|
||||
assert out.shape == (4, 64)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
def test_gate_linear_set_out_dtype_enables_ll_fp32w(monkeypatch):
|
||||
gate = _make_gate_linear_fp32w(
|
||||
monkeypatch, params_dtype=torch.float32, out_dtype=None
|
||||
)
|
||||
assert not gate.allow_ll_fp32w_gemm
|
||||
gate.set_out_dtype(torch.float32)
|
||||
assert gate.allow_ll_fp32w_gemm
|
||||
|
||||
|
||||
def test_gate_linear_non_fp32_out_dtype_disables_ll_fp32w(monkeypatch):
|
||||
gate = _make_gate_linear_fp32w(
|
||||
monkeypatch, params_dtype=torch.float32, out_dtype=None
|
||||
)
|
||||
gate.set_out_dtype(torch.bfloat16)
|
||||
assert not gate.allow_ll_fp32w_gemm
|
||||
|
||||
|
||||
def test_gate_linear_m_gt_32_skips_ll_fp32w(monkeypatch):
|
||||
gate = _make_gate_linear_fp32w(monkeypatch, params_dtype=torch.float32)
|
||||
x = torch.randn(33, 2048, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
def fail_ll_fp32w_gemm(hidden_states, router_weight):
|
||||
raise AssertionError("ll_fp32w_gemm should not run for M > 32")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.kernels.linear.cute_dsl.ll_fp32w.LL_FP32W_GEMM_KERNEL",
|
||||
fail_ll_fp32w_gemm,
|
||||
)
|
||||
out, _ = gate(x)
|
||||
assert out.shape == (33, 64)
|
||||
assert out.dtype == torch.float32
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
Reference in New Issue
Block a user