Merge branch 'main' into patches

This commit is contained in:
2026-10-18 16:04:36 +07:00
299 changed files with 51483 additions and 2195 deletions
@@ -18,13 +18,16 @@ from vllm.distributed import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
## Parallel Processes Utils
P = ParamSpec("P")
LAUNCH_TIMEOUT_S = 900
# On ROCm the first AITER CK MoE launch JIT-compiles two stage modules that the
# AITER wheel does not prebuild, ~10 min each (ROCm/aiter#5766).
LAUNCH_TIMEOUT_S = 1800 if current_platform.is_rocm() else 900
@dataclasses.dataclass
+41 -30
View File
@@ -147,26 +147,15 @@ def test_fp4_env_defaults():
"""ROCm FP4 env defaults should stay stable for the AITER gates."""
import vllm.envs as envs
assert envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM is False
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is False
@pytest.mark.parametrize(
(
"use_aiter",
"use_fp4_asm_gemm",
"use_fp4bmm",
),
[
(True, True, True),
(True, True, False),
(True, False, True),
(False, True, True),
],
("use_aiter", "use_fp4bmm"),
[(True, True), (True, False), (False, True)],
)
def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
use_aiter,
use_fp4_asm_gemm,
use_fp4bmm,
monkeypatch,
):
@@ -182,22 +171,16 @@ def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
_assert_aiter_supported()
on_gfx950_value = on_gfx950()
expected_asm_gemm = use_aiter and use_fp4_asm_gemm and on_gfx950_value
expected_asm_gemm = use_aiter and on_gfx950_value
expected_fp4bmm = use_aiter and use_fp4bmm and on_gfx950_value
with monkeypatch.context() as mp:
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
mp.setenv(
"VLLM_ROCM_USE_AITER_FP4_ASM_GEMM",
"1" if use_fp4_asm_gemm else "0",
)
mp.setenv("VLLM_ROCM_USE_AITER_FP4BMM", "1" if use_fp4bmm else "0")
_reload_envs()
rocm_aiter_ops.refresh_env_variables()
assert (
rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled() is expected_asm_gemm
)
assert rocm_aiter_ops.is_asm_fp4_gemm_supported() is expected_asm_gemm
assert rocm_aiter_ops.is_fp4bmm_enabled() is expected_fp4bmm
@@ -695,24 +678,52 @@ def test_asm_fp4_scale_swizzle_rejects_non_2d():
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
def test_aiter_mxfp4_process_weights_falls_back_to_triton_for_misaligned_scale():
"""A misaligned weight_scale must disable ASM and take the Triton path."""
@pytest.mark.parametrize(
("asm_tuned", "preshuffle_tuned", "weight_shape", "scale_shape", "expected"),
[
(True, True, (1280, 4096), (1280, 256), True),
(True, False, (1280, 4096), (1280, 256), False),
(False, True, (1280, 4096), (1280, 256), False),
# Tuned, but the scale can't be swizzled for ASM (columns % 8 != 0).
(True, True, (2560, 160), (2560, 10), False),
],
ids=["tuned", "preshuffle_untuned", "asm_untuned", "misaligned_scale"],
)
def test_aiter_mxfp4_asm_layout_follows_aiter_tuned_configs(
asm_tuned, preshuffle_tuned, weight_shape, scale_shape, expected, monkeypatch
):
"""A layer takes the ASM layout only when aiter has tuned configs for both
the ASM GEMM and the preshuffled Triton GEMM, and the scale can be swizzled.
Otherwise it stays on the Triton layout."""
from torch.nn.parameter import Parameter
from vllm._aiter_ops import rocm_aiter_ops
from vllm.model_executor.kernels.linear.mxfp4.aiter import AiterMxfp4LinearKernel
monkeypatch.setattr(rocm_aiter_ops, "is_asm_fp4_gemm_tuned", lambda n, k: asm_tuned)
monkeypatch.setattr(
rocm_aiter_ops,
"is_triton_gemm_afp4wfp4_presh_ws_tuned",
lambda n, k: preshuffle_tuned,
)
kernel = object.__new__(AiterMxfp4LinearKernel)
kernel.use_asm_gemm = True
kernel.asm_gemm_supported = True
kernel.use_asm_gemm = False
kernel.out_dtype = torch.bfloat16
layer = torch.nn.Module()
layer.weight = Parameter(
torch.zeros(weight_shape, dtype=torch.uint8, device="cuda"),
requires_grad=False,
)
layer.weight_scale = Parameter(
torch.zeros(2560, 10, dtype=torch.uint8, device="cpu"), requires_grad=False
torch.zeros(scale_shape, dtype=torch.uint8, device="cuda"),
requires_grad=False,
)
kernel.process_weights_after_loading(layer)
assert kernel.use_asm_gemm is False
# Triton path stores the transposed, contiguous scale.
assert tuple(layer.weight_scale.shape) == (10, 2560)
assert layer.weight_scale.is_contiguous()
assert kernel.use_asm_gemm is expected
if not expected:
# Triton path stores the transposed, contiguous scale.
assert tuple(layer.weight_scale.shape) == scale_shape[::-1]
assert layer.weight_scale.is_contiguous()
@@ -0,0 +1,340 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm._custom_ops as ops
from vllm.platforms import current_platform
HEAD_DIM = 64
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike() or not torch.cuda.is_available(),
reason="Muse Spark kernel requires a CUDA or ROCm GPU",
)
def make_cos_sin_cache(max_pos, rotary_dim, base, device, dtype=torch.float32):
"""The table ``RotaryEmbedding`` builds: cos||sin, fp32 for Muse-Spark."""
inv_freq = 1.0 / (
base
** (
torch.arange(0, rotary_dim, 2, dtype=torch.float32, device=device)
/ rotary_dim
)
)
t = torch.arange(max_pos, dtype=torch.float32, device=device)
freqs = torch.einsum("i,j->ij", t, inv_freq)
return torch.cat((freqs.cos(), freqs.sin()), dim=-1).to(dtype)
def qk_norm_rope_ref(x, positions, cos_sin_cache, rotary_dim, eps):
"""fp32 reference for one q or k branch, rounded once; no table means NoPE."""
out = x.float()
out = out * torch.rsqrt(out.pow(2).mean(dim=-1, keepdim=True) + eps)
if cos_sin_cache is not None:
half = rotary_dim // 2
cs = cos_sin_cache[positions].float()
cos = cs[..., :half].unsqueeze(1)
sin = cs[..., half:].unsqueeze(1)
# GPT-J interleaved: the rotated pair is (2i, 2i+1).
x0 = out[..., :rotary_dim:2]
x1 = out[..., 1:rotary_dim:2]
out = out.clone()
out[..., :rotary_dim:2] = x0 * cos - x1 * sin
out[..., 1:rotary_dim:2] = x0 * sin + x1 * cos
return out.to(x.dtype)
def assert_fp8_cache_close(got, expected, exact_fraction=0.999):
"""Compare raw cache codes in the platform's FP8 encoding."""
diff = (got.int() - expected.int()).abs()
within = diff <= 1
assert bool(within.all()), (
f"fp8 cache differs by more than one code in {int((~within).sum())} "
f"of {diff.numel()} elements"
)
exact = float((diff == 0).float().mean())
assert exact >= exact_fraction, (
f"only {exact:.5f} of the fp8 cache is bit-identical to "
f"reshape_and_cache_flash, expected at least {exact_fraction}"
)
def make_case(num_tokens, nq, nkv, dtype, device, with_gate=True, seed=0):
torch.manual_seed(seed)
qsz, kvsz = nq * HEAD_DIM, nkv * HEAD_DIM
gate = qsz if with_gate else 0
qkv = torch.randn(num_tokens, qsz + 2 * kvsz + gate, dtype=dtype, device=device)
return qkv, qsz, kvsz, gate
def split_qkv(qkv, qsz, kvsz, gate):
return qkv.split([qsz, kvsz, kvsz, gate], dim=-1)
def check_qk_norm_rope(
num_tokens,
nq,
nkv,
dtype,
use_rope,
rotary_dim=HEAD_DIM,
with_gate=True,
rope_dtype=torch.float32,
):
device, eps, max_pos = "cuda", 1e-5, 4096
qkv, qsz, kvsz, gate = make_case(num_tokens, nq, nkv, dtype, device, with_gate)
qkv_orig = qkv.clone()
cos_sin = (
make_cos_sin_cache(max_pos, rotary_dim, 500_000.0, device, rope_dtype)
if use_rope
else None
)
positions = torch.randint(
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
)
ops.fused_muse_spark_qknorm_rope_kv_insert(
qkv,
positions,
nq,
nkv,
HEAD_DIM,
eps,
cos_sin_cache=cos_sin,
rotary_dim=rotary_dim if use_rope else 0,
)
q_out, k_out, v_out, og_out = split_qkv(qkv, qsz, kvsz, gate)
q_in, k_in, v_in, og_in = split_qkv(qkv_orig, qsz, kvsz, gate)
q_ref = qk_norm_rope_ref(
q_in.view(num_tokens, nq, HEAD_DIM), positions, cos_sin, rotary_dim, eps
).view(num_tokens, qsz)
k_ref = qk_norm_rope_ref(
k_in.view(num_tokens, nkv, HEAD_DIM), positions, cos_sin, rotary_dim, eps
).view(num_tokens, kvsz)
# rsqrtf and lane summation order cost a couple of fp32 ulp, well inside
# one ulp of the stored dtype.
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
torch.testing.assert_close(og_out, og_in, rtol=0, atol=0)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("use_rope", [True, False])
def test_qk_norm_rope(dtype, use_rope):
check_qk_norm_rope(17, 32, 4, dtype, use_rope)
@pytest.mark.parametrize(
"num_tokens,variant",
[
pytest.param(64, {"rotary_dim": 32}, id="partial_rotary_dim"),
# A [q | k | v] row (no o-gate) is addressed by its own stride.
pytest.param(33, {"with_gate": False}, id="no_gate_row"),
pytest.param(128, {"rope_dtype": torch.bfloat16}, id="bf16_cos_sin_cache"),
],
)
def test_qk_norm_rope_layout_variants(num_tokens, variant):
check_qk_norm_rope(num_tokens, 32, 4, torch.bfloat16, True, **variant)
# Cover narrow/wide head groups for TP4/TP8 and the one-chunk-per-token case.
@pytest.mark.parametrize(
"num_tokens,nq,nkv,kv_cache_dtype,use_rope",
[
pytest.param(1, 32, 4, "auto", True, id="tp4-decode"),
pytest.param(1, 16, 2, "fp8", False, id="tp8-decode-fp8-nope"),
pytest.param(1025, 32, 4, "auto", True, id="tp4-group2"),
pytest.param(1025, 32, 4, "fp8", True, id="tp4-group2-fp8"),
pytest.param(1537, 32, 4, "auto", True, id="tp4-group4"),
pytest.param(1537, 32, 4, "auto", False, id="tp4-nope"),
pytest.param(3073, 16, 2, "auto", True, id="tp8-group4"),
pytest.param(3073, 16, 2, "fp8", False, id="tp8-fp8-nope"),
pytest.param(40000, 2, 1, "auto", True, id="single-chunk-per-token"),
],
)
def test_kv_insert(num_tokens, nq, nkv, kv_cache_dtype, use_rope):
device, dtype, eps = "cuda", torch.bfloat16, 1e-5
rotary_dim, max_pos, block_size = HEAD_DIM, 4096, 16
qkv, qsz, kvsz, gate = make_case(num_tokens, nq, nkv, dtype, device, seed=1)
qkv_orig = qkv.clone()
cos_sin = (
make_cos_sin_cache(max_pos, rotary_dim, 500_000.0, device) if use_rope else None
)
positions = torch.randint(
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
)
num_blocks = (num_tokens + block_size - 1) // block_size + 1
storage = torch.uint8 if kv_cache_dtype == "fp8" else dtype
kv_cache = torch.zeros(
num_blocks, nkv, block_size, 2 * HEAD_DIM, dtype=storage, device=device
)
slot_mapping = torch.randperm(
num_blocks * block_size, dtype=torch.int64, device=device
)[:num_tokens]
if num_tokens > 1:
# -1 marks a token the scheduler did not place; the kernel must skip it
# and leave the cache alone.
slot_mapping[::3] = -1
# Non-unit scales, so a kernel that ignored them would fail loudly.
k_scale = torch.tensor(0.37, dtype=torch.float32, device=device)
v_scale = torch.tensor(1.83, dtype=torch.float32, device=device)
ops.fused_muse_spark_qknorm_rope_kv_insert(
qkv,
positions,
nq,
nkv,
HEAD_DIM,
eps,
cos_sin_cache=cos_sin,
rotary_dim=rotary_dim if use_rope else 0,
slot_mapping=slot_mapping,
kv_cache=kv_cache,
block_size=block_size,
kv_cache_dtype=kv_cache_dtype,
k_scale=k_scale if kv_cache_dtype != "auto" else None,
v_scale=v_scale if kv_cache_dtype != "auto" else None,
)
q_out, k_out, v_out, og_out = split_qkv(qkv, qsz, kvsz, gate)
q_in, k_in, v_in, og_in = split_qkv(qkv_orig, qsz, kvsz, gate)
q_ref = qk_norm_rope_ref(
q_in.view(num_tokens, nq, HEAD_DIM), positions, cos_sin, rotary_dim, eps
).view(num_tokens, qsz)
k_ref = qk_norm_rope_ref(
k_in.view(num_tokens, nkv, HEAD_DIM), positions, cos_sin, rotary_dim, eps
).view(num_tokens, kvsz)
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
torch.testing.assert_close(og_out, og_in, rtol=0, atol=0)
expected = torch.zeros_like(kv_cache)
exp_k, exp_v = expected.transpose(1, 2).split(HEAD_DIM, dim=-1)
ops.reshape_and_cache_flash(
k_out.view(num_tokens, nkv, HEAD_DIM),
v_out.view(num_tokens, nkv, HEAD_DIM),
exp_k,
exp_v,
slot_mapping,
kv_cache_dtype,
k_scale,
v_scale,
)
if kv_cache_dtype == "auto":
torch.testing.assert_close(kv_cache, expected, rtol=0, atol=0)
else:
assert_fp8_cache_close(kv_cache, expected)
# The fused Q output requires E4M3FN; scaled_fp8_quant uses the platform dtype.
@pytest.mark.skipif(
current_platform.is_rocm() and current_platform.fp8_dtype() != torch.float8_e4m3fn,
reason="Fused E4M3FN Q output cannot be compared with ROCm FNUZ quantization",
)
@pytest.mark.parametrize(
"head_dim,num_tokens,rotary_fraction,use_cuda_graph",
[
pytest.param(64, 17, 0, False, id="head64-nope"),
pytest.param(128, 17, 0.5, False, id="head128-partial-rope"),
pytest.param(256, 17, 0, False, id="head256-nope"),
pytest.param(256, 3073, 0.5, False, id="head256-partial-rope-large"),
pytest.param(64, 3073, 0.5, True, id="graph-rope"),
pytest.param(64, 3073, 0, True, id="graph-nope"),
],
)
def test_fused_query_quant_matches_separate_quant(
head_dim, num_tokens, rotary_fraction, use_cuda_graph
):
"""Preserve the rounding boundary, scales, padded Q, and KV/gate writes."""
torch.manual_seed(42)
nq, nkv, block_size = 16, 2, 16
qsz, kvsz = nq * head_dim, nkv * head_dim
qkv = torch.randn(num_tokens, 2 * (qsz + kvsz), device="cuda", dtype=torch.bfloat16)
original = qkv.clone()
reference = qkv.clone()
q_fp8 = torch.empty(num_tokens, qsz, device="cuda", dtype=torch.float8_e4m3fn)
positions = torch.arange(num_tokens, device="cuda", dtype=torch.int64)
rotary_dim = int(head_dim * rotary_fraction)
cos_sin = (
make_cos_sin_cache(num_tokens, rotary_dim, 500_000.0, "cuda")
if rotary_dim
else None
)
cache = torch.zeros(
(num_tokens + block_size - 1) // block_size,
nkv,
block_size,
2 * head_dim,
device="cuda",
dtype=torch.uint8,
)
expected_cache = torch.zeros_like(cache)
slots = positions.clone()
slots[::3] = -1
# Small enough that part of Q saturates the e4m3 range.
scale = torch.tensor(0.001, device="cuda", dtype=torch.float32)
kwargs = dict(
cos_sin_cache=cos_sin,
rotary_dim=rotary_dim,
slot_mapping=slots,
block_size=block_size,
kv_cache_dtype="fp8",
k_scale=torch.tensor(0.37, device="cuda"),
v_scale=torch.tensor(1.83, device="cuda"),
)
ops.fused_muse_spark_qknorm_rope_kv_insert(
reference,
positions,
nq,
nkv,
head_dim,
1e-5,
kv_cache=expected_cache,
**kwargs,
)
expected_q, _ = ops.scaled_fp8_quant(reference[:, :qsz], scale=scale)
def run():
qkv.copy_(original)
ops.fused_muse_spark_qknorm_rope_kv_insert(
qkv,
positions,
nq,
nkv,
head_dim,
1e-5,
kv_cache=cache,
q_fp8_out=q_fp8,
q_scale=scale,
**kwargs,
)
run()
if use_cuda_graph:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
run()
qkv.zero_()
cache.zero_()
q_fp8.zero_()
graph.replay()
torch.testing.assert_close(
q_fp8.view(torch.uint8), expected_q.view(torch.uint8), rtol=0, atol=0
)
torch.testing.assert_close(cache, expected_cache, rtol=0, atol=0)
torch.testing.assert_close(qkv[:, :qsz], original[:, :qsz], rtol=0, atol=0)
torch.testing.assert_close(qkv[:, qsz:], reference[:, qsz:], rtol=0, atol=0)
@@ -0,0 +1,195 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Weighted RMSNorm + partial NeoX RoPE with separate paged K/V caches."""
import pytest
import torch
import vllm._custom_ops as ops
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda() or not torch.cuda.is_available(),
reason="Step-5 kernel is only built for CUDA",
)
def norm_rope_reference(x, weight, cos_sin, positions, rotary_dim, eps, bias):
xf = x.float()
normed = xf * torch.rsqrt(xf.square().mean(-1, keepdim=True) + eps)
out = (normed * (weight.float() + bias)).to(x.dtype)
half = rotary_dim // 2
cos, sin = cos_sin[positions].float().unsqueeze(1).chunk(2, dim=-1)
x0, x1 = out[..., :half].float(), out[..., half:rotary_dim].float()
out[..., :half] = x0 * cos - x1 * sin
out[..., half:rotary_dim] = x0 * sin + x1 * cos
return out
def check_norm_rope_and_cache(
dtype=torch.bfloat16,
kv_cache_dtype="auto",
head_dim=128,
rotary_dim=64,
num_tokens=19,
block_size=16,
fp32_weights=False,
bias=0.0,
strided_cache=False,
use_cuda_graph=False,
):
"""Cover rounding, 192-wide RoPE, padded rows and short/negative slots."""
torch.manual_seed(0)
nq, nkv, max_pos, eps = 5, 2, 64, 1e-5
width = (nq + 2 * nkv) * head_dim
storage = torch.randn(num_tokens, width + 8, device="cuda", dtype=dtype)
qkv = storage[:, :width]
original = storage.clone()
weight_dtype = torch.float32 if fp32_weights else dtype
qw = torch.randn(head_dim, device="cuda", dtype=weight_dtype)
kw = torch.randn_like(qw)
angles = torch.randn(max_pos, rotary_dim // 2, device="cuda")
rope_dtype = torch.float32 if fp32_weights else dtype
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1).to(rope_dtype)
positions = torch.randint(max_pos, (num_tokens,), device="cuda")
num_blocks = (num_tokens + block_size - 1) // block_size + 1
cache_dtype = torch.uint8 if kv_cache_dtype == "fp8" else dtype
shape = (num_blocks, block_size, nkv, head_dim)
if strided_cache:
shape = (num_blocks, nkv, block_size, head_dim)
key_cache = torch.zeros(shape, device="cuda", dtype=cache_dtype)
value_cache = torch.zeros_like(key_cache)
if strided_cache:
key_cache = key_cache.transpose(1, 2)
value_cache = value_cache.transpose(1, 2)
# A short mapping models graph padding; negative slots must stay unwritten.
num_slots = max(0, num_tokens - 3) if num_tokens > 1 else num_tokens
slots = torch.randperm(num_blocks * block_size, device="cuda")[:num_slots]
slots[1::3] = -1
k_scale = torch.tensor(0.37, device="cuda")
v_scale = torch.tensor(1.83, device="cuda")
def run():
return ops.fused_qknorm_rope_cache(
qkv,
qw,
kw,
cos_sin,
positions,
slots,
key_cache,
value_cache,
nq,
nkv,
head_dim,
rotary_dim,
eps,
norm_weight_bias=bias,
kv_cache_dtype=kv_cache_dtype,
k_scale=k_scale,
v_scale=v_scale,
)
q, k, v = run()
if use_cuda_graph:
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
q, k, v = run()
key_cache.zero_()
value_cache.zero_()
graph.replay()
q_in, k_in, v_in = qkv.split(
(nq * head_dim, nkv * head_dim, nkv * head_dim), dim=-1
)
tol = 1e-2 if dtype == torch.bfloat16 else 2e-3
for actual, src, weight, heads in ((q, q_in, qw, nq), (k, k_in, kw, nkv)):
expected = norm_rope_reference(
src.reshape(num_tokens, heads, head_dim),
weight,
cos_sin,
positions,
rotary_dim,
eps,
bias,
).reshape_as(actual)
torch.testing.assert_close(actual, expected, rtol=tol, atol=tol)
torch.testing.assert_close(v, v_in, rtol=0, atol=0)
torch.testing.assert_close(storage, original, rtol=0, atol=0)
expected_k, expected_v = torch.zeros_like(key_cache), torch.zeros_like(value_cache)
if num_slots:
ops.reshape_and_cache_flash(
k.reshape(num_tokens, nkv, head_dim),
v.reshape(num_tokens, nkv, head_dim),
expected_k,
expected_v,
slots,
kv_cache_dtype,
k_scale,
v_scale,
)
for actual, expected in ((key_cache, expected_k), (value_cache, expected_v)):
if kv_cache_dtype == "fp8":
# Reciprocal multiplication versus division may differ by one code.
assert (actual.int() - expected.int()).abs().max() <= 1
else:
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize(
"case",
[
pytest.param(dict(head_dim=64, rotary_dim=32, num_tokens=1), id="decode"),
pytest.param(
dict(block_size=7, fp32_weights=True, bias=1.0, strided_cache=True),
id="strided-cache-fp32-weights",
),
pytest.param(
dict(head_dim=192, rotary_dim=128, fp32_weights=True),
id="head192-partial-rope",
),
pytest.param(
dict(
head_dim=192,
rotary_dim=192,
block_size=7,
bias=1.0,
strided_cache=True,
kv_cache_dtype="fp8",
),
id="head192-full-rope-fp8",
),
pytest.param(
dict(
head_dim=256, rotary_dim=256, strided_cache=True, kv_cache_dtype="fp8"
),
id="head256-full-rope-fp8",
),
pytest.param(
dict(head_dim=192, rotary_dim=0, fp32_weights=True, bias=1.0),
id="no-rope",
),
pytest.param(
dict(head_dim=64, rotary_dim=8, fp32_weights=True),
id="minimum-rotary-dim",
),
],
)
def test_norm_rope_and_cache(dtype, case):
check_norm_rope_and_cache(dtype=dtype, **case)
@pytest.mark.parametrize(
"case",
[
pytest.param(dict(num_tokens=0), id="empty-input"),
pytest.param(dict(use_cuda_graph=True), id="graph-auto-cache"),
pytest.param(
dict(use_cuda_graph=True, kv_cache_dtype="fp8"), id="graph-fp8-cache"
),
],
)
def test_empty_input_and_graph_replay(case):
check_norm_rope_and_cache(**case)