Merge branch 'main' into patches
This commit is contained in:
@@ -18,13 +18,16 @@ from vllm.distributed import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
## Parallel Processes Utils
|
||||
|
||||
P = ParamSpec("P")
|
||||
|
||||
LAUNCH_TIMEOUT_S = 900
|
||||
# On ROCm the first AITER CK MoE launch JIT-compiles two stage modules that the
|
||||
# AITER wheel does not prebuild, ~10 min each (ROCm/aiter#5766).
|
||||
LAUNCH_TIMEOUT_S = 1800 if current_platform.is_rocm() else 900
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
|
||||
@@ -147,26 +147,15 @@ def test_fp4_env_defaults():
|
||||
"""ROCm FP4 env defaults should stay stable for the AITER gates."""
|
||||
import vllm.envs as envs
|
||||
|
||||
assert envs.VLLM_ROCM_USE_AITER_FP4_ASM_GEMM is False
|
||||
assert envs.VLLM_ROCM_USE_AITER_FP4BMM is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"use_aiter",
|
||||
"use_fp4_asm_gemm",
|
||||
"use_fp4bmm",
|
||||
),
|
||||
[
|
||||
(True, True, True),
|
||||
(True, True, False),
|
||||
(True, False, True),
|
||||
(False, True, True),
|
||||
],
|
||||
("use_aiter", "use_fp4bmm"),
|
||||
[(True, True), (True, False), (False, True)],
|
||||
)
|
||||
def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
|
||||
use_aiter,
|
||||
use_fp4_asm_gemm,
|
||||
use_fp4bmm,
|
||||
monkeypatch,
|
||||
):
|
||||
@@ -182,22 +171,16 @@ def test_rocm_aiter_fp4_enablement_follows_env_and_arch(
|
||||
_assert_aiter_supported()
|
||||
|
||||
on_gfx950_value = on_gfx950()
|
||||
expected_asm_gemm = use_aiter and use_fp4_asm_gemm and on_gfx950_value
|
||||
expected_asm_gemm = use_aiter and on_gfx950_value
|
||||
expected_fp4bmm = use_aiter and use_fp4bmm and on_gfx950_value
|
||||
|
||||
with monkeypatch.context() as mp:
|
||||
mp.setenv("VLLM_ROCM_USE_AITER", "1" if use_aiter else "0")
|
||||
mp.setenv(
|
||||
"VLLM_ROCM_USE_AITER_FP4_ASM_GEMM",
|
||||
"1" if use_fp4_asm_gemm else "0",
|
||||
)
|
||||
mp.setenv("VLLM_ROCM_USE_AITER_FP4BMM", "1" if use_fp4bmm else "0")
|
||||
_reload_envs()
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
assert (
|
||||
rocm_aiter_ops.is_asm_fp4_gemm_dynamic_quant_enabled() is expected_asm_gemm
|
||||
)
|
||||
assert rocm_aiter_ops.is_asm_fp4_gemm_supported() is expected_asm_gemm
|
||||
assert rocm_aiter_ops.is_fp4bmm_enabled() is expected_fp4bmm
|
||||
|
||||
|
||||
@@ -695,24 +678,52 @@ def test_asm_fp4_scale_swizzle_rejects_non_2d():
|
||||
|
||||
|
||||
@pytest.mark.skipif(not on_gfx950(), reason="gfx950 ROCm only")
|
||||
def test_aiter_mxfp4_process_weights_falls_back_to_triton_for_misaligned_scale():
|
||||
"""A misaligned weight_scale must disable ASM and take the Triton path."""
|
||||
@pytest.mark.parametrize(
|
||||
("asm_tuned", "preshuffle_tuned", "weight_shape", "scale_shape", "expected"),
|
||||
[
|
||||
(True, True, (1280, 4096), (1280, 256), True),
|
||||
(True, False, (1280, 4096), (1280, 256), False),
|
||||
(False, True, (1280, 4096), (1280, 256), False),
|
||||
# Tuned, but the scale can't be swizzled for ASM (columns % 8 != 0).
|
||||
(True, True, (2560, 160), (2560, 10), False),
|
||||
],
|
||||
ids=["tuned", "preshuffle_untuned", "asm_untuned", "misaligned_scale"],
|
||||
)
|
||||
def test_aiter_mxfp4_asm_layout_follows_aiter_tuned_configs(
|
||||
asm_tuned, preshuffle_tuned, weight_shape, scale_shape, expected, monkeypatch
|
||||
):
|
||||
"""A layer takes the ASM layout only when aiter has tuned configs for both
|
||||
the ASM GEMM and the preshuffled Triton GEMM, and the scale can be swizzled.
|
||||
Otherwise it stays on the Triton layout."""
|
||||
from torch.nn.parameter import Parameter
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.model_executor.kernels.linear.mxfp4.aiter import AiterMxfp4LinearKernel
|
||||
|
||||
monkeypatch.setattr(rocm_aiter_ops, "is_asm_fp4_gemm_tuned", lambda n, k: asm_tuned)
|
||||
monkeypatch.setattr(
|
||||
rocm_aiter_ops,
|
||||
"is_triton_gemm_afp4wfp4_presh_ws_tuned",
|
||||
lambda n, k: preshuffle_tuned,
|
||||
)
|
||||
kernel = object.__new__(AiterMxfp4LinearKernel)
|
||||
kernel.use_asm_gemm = True
|
||||
kernel.asm_gemm_supported = True
|
||||
kernel.use_asm_gemm = False
|
||||
kernel.out_dtype = torch.bfloat16
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight = Parameter(
|
||||
torch.zeros(weight_shape, dtype=torch.uint8, device="cuda"),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.weight_scale = Parameter(
|
||||
torch.zeros(2560, 10, dtype=torch.uint8, device="cpu"), requires_grad=False
|
||||
torch.zeros(scale_shape, dtype=torch.uint8, device="cuda"),
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
assert kernel.use_asm_gemm is False
|
||||
# Triton path stores the transposed, contiguous scale.
|
||||
assert tuple(layer.weight_scale.shape) == (10, 2560)
|
||||
assert layer.weight_scale.is_contiguous()
|
||||
assert kernel.use_asm_gemm is expected
|
||||
if not expected:
|
||||
# Triton path stores the transposed, contiguous scale.
|
||||
assert tuple(layer.weight_scale.shape) == scale_shape[::-1]
|
||||
assert layer.weight_scale.is_contiguous()
|
||||
|
||||
@@ -0,0 +1,340 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
HEAD_DIM = 64
|
||||
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike() or not torch.cuda.is_available(),
|
||||
reason="Muse Spark kernel requires a CUDA or ROCm GPU",
|
||||
)
|
||||
|
||||
|
||||
def make_cos_sin_cache(max_pos, rotary_dim, base, device, dtype=torch.float32):
|
||||
"""The table ``RotaryEmbedding`` builds: cos||sin, fp32 for Muse-Spark."""
|
||||
inv_freq = 1.0 / (
|
||||
base
|
||||
** (
|
||||
torch.arange(0, rotary_dim, 2, dtype=torch.float32, device=device)
|
||||
/ rotary_dim
|
||||
)
|
||||
)
|
||||
t = torch.arange(max_pos, dtype=torch.float32, device=device)
|
||||
freqs = torch.einsum("i,j->ij", t, inv_freq)
|
||||
return torch.cat((freqs.cos(), freqs.sin()), dim=-1).to(dtype)
|
||||
|
||||
|
||||
def qk_norm_rope_ref(x, positions, cos_sin_cache, rotary_dim, eps):
|
||||
"""fp32 reference for one q or k branch, rounded once; no table means NoPE."""
|
||||
out = x.float()
|
||||
out = out * torch.rsqrt(out.pow(2).mean(dim=-1, keepdim=True) + eps)
|
||||
if cos_sin_cache is not None:
|
||||
half = rotary_dim // 2
|
||||
cs = cos_sin_cache[positions].float()
|
||||
cos = cs[..., :half].unsqueeze(1)
|
||||
sin = cs[..., half:].unsqueeze(1)
|
||||
# GPT-J interleaved: the rotated pair is (2i, 2i+1).
|
||||
x0 = out[..., :rotary_dim:2]
|
||||
x1 = out[..., 1:rotary_dim:2]
|
||||
out = out.clone()
|
||||
out[..., :rotary_dim:2] = x0 * cos - x1 * sin
|
||||
out[..., 1:rotary_dim:2] = x0 * sin + x1 * cos
|
||||
return out.to(x.dtype)
|
||||
|
||||
|
||||
def assert_fp8_cache_close(got, expected, exact_fraction=0.999):
|
||||
"""Compare raw cache codes in the platform's FP8 encoding."""
|
||||
diff = (got.int() - expected.int()).abs()
|
||||
within = diff <= 1
|
||||
assert bool(within.all()), (
|
||||
f"fp8 cache differs by more than one code in {int((~within).sum())} "
|
||||
f"of {diff.numel()} elements"
|
||||
)
|
||||
exact = float((diff == 0).float().mean())
|
||||
assert exact >= exact_fraction, (
|
||||
f"only {exact:.5f} of the fp8 cache is bit-identical to "
|
||||
f"reshape_and_cache_flash, expected at least {exact_fraction}"
|
||||
)
|
||||
|
||||
|
||||
def make_case(num_tokens, nq, nkv, dtype, device, with_gate=True, seed=0):
|
||||
torch.manual_seed(seed)
|
||||
qsz, kvsz = nq * HEAD_DIM, nkv * HEAD_DIM
|
||||
gate = qsz if with_gate else 0
|
||||
qkv = torch.randn(num_tokens, qsz + 2 * kvsz + gate, dtype=dtype, device=device)
|
||||
return qkv, qsz, kvsz, gate
|
||||
|
||||
|
||||
def split_qkv(qkv, qsz, kvsz, gate):
|
||||
return qkv.split([qsz, kvsz, kvsz, gate], dim=-1)
|
||||
|
||||
|
||||
def check_qk_norm_rope(
|
||||
num_tokens,
|
||||
nq,
|
||||
nkv,
|
||||
dtype,
|
||||
use_rope,
|
||||
rotary_dim=HEAD_DIM,
|
||||
with_gate=True,
|
||||
rope_dtype=torch.float32,
|
||||
):
|
||||
device, eps, max_pos = "cuda", 1e-5, 4096
|
||||
|
||||
qkv, qsz, kvsz, gate = make_case(num_tokens, nq, nkv, dtype, device, with_gate)
|
||||
qkv_orig = qkv.clone()
|
||||
cos_sin = (
|
||||
make_cos_sin_cache(max_pos, rotary_dim, 500_000.0, device, rope_dtype)
|
||||
if use_rope
|
||||
else None
|
||||
)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
ops.fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
positions,
|
||||
nq,
|
||||
nkv,
|
||||
HEAD_DIM,
|
||||
eps,
|
||||
cos_sin_cache=cos_sin,
|
||||
rotary_dim=rotary_dim if use_rope else 0,
|
||||
)
|
||||
|
||||
q_out, k_out, v_out, og_out = split_qkv(qkv, qsz, kvsz, gate)
|
||||
q_in, k_in, v_in, og_in = split_qkv(qkv_orig, qsz, kvsz, gate)
|
||||
|
||||
q_ref = qk_norm_rope_ref(
|
||||
q_in.view(num_tokens, nq, HEAD_DIM), positions, cos_sin, rotary_dim, eps
|
||||
).view(num_tokens, qsz)
|
||||
k_ref = qk_norm_rope_ref(
|
||||
k_in.view(num_tokens, nkv, HEAD_DIM), positions, cos_sin, rotary_dim, eps
|
||||
).view(num_tokens, kvsz)
|
||||
|
||||
# rsqrtf and lane summation order cost a couple of fp32 ulp, well inside
|
||||
# one ulp of the stored dtype.
|
||||
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
|
||||
torch.testing.assert_close(og_out, og_in, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("use_rope", [True, False])
|
||||
def test_qk_norm_rope(dtype, use_rope):
|
||||
check_qk_norm_rope(17, 32, 4, dtype, use_rope)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens,variant",
|
||||
[
|
||||
pytest.param(64, {"rotary_dim": 32}, id="partial_rotary_dim"),
|
||||
# A [q | k | v] row (no o-gate) is addressed by its own stride.
|
||||
pytest.param(33, {"with_gate": False}, id="no_gate_row"),
|
||||
pytest.param(128, {"rope_dtype": torch.bfloat16}, id="bf16_cos_sin_cache"),
|
||||
],
|
||||
)
|
||||
def test_qk_norm_rope_layout_variants(num_tokens, variant):
|
||||
check_qk_norm_rope(num_tokens, 32, 4, torch.bfloat16, True, **variant)
|
||||
|
||||
|
||||
# Cover narrow/wide head groups for TP4/TP8 and the one-chunk-per-token case.
|
||||
@pytest.mark.parametrize(
|
||||
"num_tokens,nq,nkv,kv_cache_dtype,use_rope",
|
||||
[
|
||||
pytest.param(1, 32, 4, "auto", True, id="tp4-decode"),
|
||||
pytest.param(1, 16, 2, "fp8", False, id="tp8-decode-fp8-nope"),
|
||||
pytest.param(1025, 32, 4, "auto", True, id="tp4-group2"),
|
||||
pytest.param(1025, 32, 4, "fp8", True, id="tp4-group2-fp8"),
|
||||
pytest.param(1537, 32, 4, "auto", True, id="tp4-group4"),
|
||||
pytest.param(1537, 32, 4, "auto", False, id="tp4-nope"),
|
||||
pytest.param(3073, 16, 2, "auto", True, id="tp8-group4"),
|
||||
pytest.param(3073, 16, 2, "fp8", False, id="tp8-fp8-nope"),
|
||||
pytest.param(40000, 2, 1, "auto", True, id="single-chunk-per-token"),
|
||||
],
|
||||
)
|
||||
def test_kv_insert(num_tokens, nq, nkv, kv_cache_dtype, use_rope):
|
||||
device, dtype, eps = "cuda", torch.bfloat16, 1e-5
|
||||
rotary_dim, max_pos, block_size = HEAD_DIM, 4096, 16
|
||||
|
||||
qkv, qsz, kvsz, gate = make_case(num_tokens, nq, nkv, dtype, device, seed=1)
|
||||
qkv_orig = qkv.clone()
|
||||
cos_sin = (
|
||||
make_cos_sin_cache(max_pos, rotary_dim, 500_000.0, device) if use_rope else None
|
||||
)
|
||||
positions = torch.randint(
|
||||
0, max_pos, (num_tokens,), dtype=torch.int64, device=device
|
||||
)
|
||||
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
storage = torch.uint8 if kv_cache_dtype == "fp8" else dtype
|
||||
kv_cache = torch.zeros(
|
||||
num_blocks, nkv, block_size, 2 * HEAD_DIM, dtype=storage, device=device
|
||||
)
|
||||
slot_mapping = torch.randperm(
|
||||
num_blocks * block_size, dtype=torch.int64, device=device
|
||||
)[:num_tokens]
|
||||
if num_tokens > 1:
|
||||
# -1 marks a token the scheduler did not place; the kernel must skip it
|
||||
# and leave the cache alone.
|
||||
slot_mapping[::3] = -1
|
||||
|
||||
# Non-unit scales, so a kernel that ignored them would fail loudly.
|
||||
k_scale = torch.tensor(0.37, dtype=torch.float32, device=device)
|
||||
v_scale = torch.tensor(1.83, dtype=torch.float32, device=device)
|
||||
|
||||
ops.fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
positions,
|
||||
nq,
|
||||
nkv,
|
||||
HEAD_DIM,
|
||||
eps,
|
||||
cos_sin_cache=cos_sin,
|
||||
rotary_dim=rotary_dim if use_rope else 0,
|
||||
slot_mapping=slot_mapping,
|
||||
kv_cache=kv_cache,
|
||||
block_size=block_size,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
k_scale=k_scale if kv_cache_dtype != "auto" else None,
|
||||
v_scale=v_scale if kv_cache_dtype != "auto" else None,
|
||||
)
|
||||
|
||||
q_out, k_out, v_out, og_out = split_qkv(qkv, qsz, kvsz, gate)
|
||||
q_in, k_in, v_in, og_in = split_qkv(qkv_orig, qsz, kvsz, gate)
|
||||
|
||||
q_ref = qk_norm_rope_ref(
|
||||
q_in.view(num_tokens, nq, HEAD_DIM), positions, cos_sin, rotary_dim, eps
|
||||
).view(num_tokens, qsz)
|
||||
k_ref = qk_norm_rope_ref(
|
||||
k_in.view(num_tokens, nkv, HEAD_DIM), positions, cos_sin, rotary_dim, eps
|
||||
).view(num_tokens, kvsz)
|
||||
torch.testing.assert_close(q_out, q_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(k_out, k_ref, rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(v_out, v_in, rtol=0, atol=0)
|
||||
torch.testing.assert_close(og_out, og_in, rtol=0, atol=0)
|
||||
|
||||
expected = torch.zeros_like(kv_cache)
|
||||
exp_k, exp_v = expected.transpose(1, 2).split(HEAD_DIM, dim=-1)
|
||||
ops.reshape_and_cache_flash(
|
||||
k_out.view(num_tokens, nkv, HEAD_DIM),
|
||||
v_out.view(num_tokens, nkv, HEAD_DIM),
|
||||
exp_k,
|
||||
exp_v,
|
||||
slot_mapping,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
)
|
||||
if kv_cache_dtype == "auto":
|
||||
torch.testing.assert_close(kv_cache, expected, rtol=0, atol=0)
|
||||
else:
|
||||
assert_fp8_cache_close(kv_cache, expected)
|
||||
|
||||
|
||||
# The fused Q output requires E4M3FN; scaled_fp8_quant uses the platform dtype.
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm() and current_platform.fp8_dtype() != torch.float8_e4m3fn,
|
||||
reason="Fused E4M3FN Q output cannot be compared with ROCm FNUZ quantization",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"head_dim,num_tokens,rotary_fraction,use_cuda_graph",
|
||||
[
|
||||
pytest.param(64, 17, 0, False, id="head64-nope"),
|
||||
pytest.param(128, 17, 0.5, False, id="head128-partial-rope"),
|
||||
pytest.param(256, 17, 0, False, id="head256-nope"),
|
||||
pytest.param(256, 3073, 0.5, False, id="head256-partial-rope-large"),
|
||||
pytest.param(64, 3073, 0.5, True, id="graph-rope"),
|
||||
pytest.param(64, 3073, 0, True, id="graph-nope"),
|
||||
],
|
||||
)
|
||||
def test_fused_query_quant_matches_separate_quant(
|
||||
head_dim, num_tokens, rotary_fraction, use_cuda_graph
|
||||
):
|
||||
"""Preserve the rounding boundary, scales, padded Q, and KV/gate writes."""
|
||||
torch.manual_seed(42)
|
||||
nq, nkv, block_size = 16, 2, 16
|
||||
qsz, kvsz = nq * head_dim, nkv * head_dim
|
||||
qkv = torch.randn(num_tokens, 2 * (qsz + kvsz), device="cuda", dtype=torch.bfloat16)
|
||||
original = qkv.clone()
|
||||
reference = qkv.clone()
|
||||
q_fp8 = torch.empty(num_tokens, qsz, device="cuda", dtype=torch.float8_e4m3fn)
|
||||
positions = torch.arange(num_tokens, device="cuda", dtype=torch.int64)
|
||||
rotary_dim = int(head_dim * rotary_fraction)
|
||||
cos_sin = (
|
||||
make_cos_sin_cache(num_tokens, rotary_dim, 500_000.0, "cuda")
|
||||
if rotary_dim
|
||||
else None
|
||||
)
|
||||
cache = torch.zeros(
|
||||
(num_tokens + block_size - 1) // block_size,
|
||||
nkv,
|
||||
block_size,
|
||||
2 * head_dim,
|
||||
device="cuda",
|
||||
dtype=torch.uint8,
|
||||
)
|
||||
expected_cache = torch.zeros_like(cache)
|
||||
slots = positions.clone()
|
||||
slots[::3] = -1
|
||||
# Small enough that part of Q saturates the e4m3 range.
|
||||
scale = torch.tensor(0.001, device="cuda", dtype=torch.float32)
|
||||
kwargs = dict(
|
||||
cos_sin_cache=cos_sin,
|
||||
rotary_dim=rotary_dim,
|
||||
slot_mapping=slots,
|
||||
block_size=block_size,
|
||||
kv_cache_dtype="fp8",
|
||||
k_scale=torch.tensor(0.37, device="cuda"),
|
||||
v_scale=torch.tensor(1.83, device="cuda"),
|
||||
)
|
||||
ops.fused_muse_spark_qknorm_rope_kv_insert(
|
||||
reference,
|
||||
positions,
|
||||
nq,
|
||||
nkv,
|
||||
head_dim,
|
||||
1e-5,
|
||||
kv_cache=expected_cache,
|
||||
**kwargs,
|
||||
)
|
||||
expected_q, _ = ops.scaled_fp8_quant(reference[:, :qsz], scale=scale)
|
||||
|
||||
def run():
|
||||
qkv.copy_(original)
|
||||
ops.fused_muse_spark_qknorm_rope_kv_insert(
|
||||
qkv,
|
||||
positions,
|
||||
nq,
|
||||
nkv,
|
||||
head_dim,
|
||||
1e-5,
|
||||
kv_cache=cache,
|
||||
q_fp8_out=q_fp8,
|
||||
q_scale=scale,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
run()
|
||||
if use_cuda_graph:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
run()
|
||||
qkv.zero_()
|
||||
cache.zero_()
|
||||
q_fp8.zero_()
|
||||
graph.replay()
|
||||
torch.testing.assert_close(
|
||||
q_fp8.view(torch.uint8), expected_q.view(torch.uint8), rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(cache, expected_cache, rtol=0, atol=0)
|
||||
torch.testing.assert_close(qkv[:, :qsz], original[:, :qsz], rtol=0, atol=0)
|
||||
torch.testing.assert_close(qkv[:, qsz:], reference[:, qsz:], rtol=0, atol=0)
|
||||
@@ -0,0 +1,195 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Weighted RMSNorm + partial NeoX RoPE with separate paged K/V caches."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm._custom_ops as ops
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda() or not torch.cuda.is_available(),
|
||||
reason="Step-5 kernel is only built for CUDA",
|
||||
)
|
||||
|
||||
|
||||
def norm_rope_reference(x, weight, cos_sin, positions, rotary_dim, eps, bias):
|
||||
xf = x.float()
|
||||
normed = xf * torch.rsqrt(xf.square().mean(-1, keepdim=True) + eps)
|
||||
out = (normed * (weight.float() + bias)).to(x.dtype)
|
||||
half = rotary_dim // 2
|
||||
cos, sin = cos_sin[positions].float().unsqueeze(1).chunk(2, dim=-1)
|
||||
x0, x1 = out[..., :half].float(), out[..., half:rotary_dim].float()
|
||||
out[..., :half] = x0 * cos - x1 * sin
|
||||
out[..., half:rotary_dim] = x0 * sin + x1 * cos
|
||||
return out
|
||||
|
||||
|
||||
def check_norm_rope_and_cache(
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
head_dim=128,
|
||||
rotary_dim=64,
|
||||
num_tokens=19,
|
||||
block_size=16,
|
||||
fp32_weights=False,
|
||||
bias=0.0,
|
||||
strided_cache=False,
|
||||
use_cuda_graph=False,
|
||||
):
|
||||
"""Cover rounding, 192-wide RoPE, padded rows and short/negative slots."""
|
||||
torch.manual_seed(0)
|
||||
nq, nkv, max_pos, eps = 5, 2, 64, 1e-5
|
||||
width = (nq + 2 * nkv) * head_dim
|
||||
storage = torch.randn(num_tokens, width + 8, device="cuda", dtype=dtype)
|
||||
qkv = storage[:, :width]
|
||||
original = storage.clone()
|
||||
weight_dtype = torch.float32 if fp32_weights else dtype
|
||||
qw = torch.randn(head_dim, device="cuda", dtype=weight_dtype)
|
||||
kw = torch.randn_like(qw)
|
||||
angles = torch.randn(max_pos, rotary_dim // 2, device="cuda")
|
||||
rope_dtype = torch.float32 if fp32_weights else dtype
|
||||
cos_sin = torch.cat((angles.cos(), angles.sin()), dim=-1).to(rope_dtype)
|
||||
positions = torch.randint(max_pos, (num_tokens,), device="cuda")
|
||||
num_blocks = (num_tokens + block_size - 1) // block_size + 1
|
||||
cache_dtype = torch.uint8 if kv_cache_dtype == "fp8" else dtype
|
||||
shape = (num_blocks, block_size, nkv, head_dim)
|
||||
if strided_cache:
|
||||
shape = (num_blocks, nkv, block_size, head_dim)
|
||||
key_cache = torch.zeros(shape, device="cuda", dtype=cache_dtype)
|
||||
value_cache = torch.zeros_like(key_cache)
|
||||
if strided_cache:
|
||||
key_cache = key_cache.transpose(1, 2)
|
||||
value_cache = value_cache.transpose(1, 2)
|
||||
# A short mapping models graph padding; negative slots must stay unwritten.
|
||||
num_slots = max(0, num_tokens - 3) if num_tokens > 1 else num_tokens
|
||||
slots = torch.randperm(num_blocks * block_size, device="cuda")[:num_slots]
|
||||
slots[1::3] = -1
|
||||
k_scale = torch.tensor(0.37, device="cuda")
|
||||
v_scale = torch.tensor(1.83, device="cuda")
|
||||
|
||||
def run():
|
||||
return ops.fused_qknorm_rope_cache(
|
||||
qkv,
|
||||
qw,
|
||||
kw,
|
||||
cos_sin,
|
||||
positions,
|
||||
slots,
|
||||
key_cache,
|
||||
value_cache,
|
||||
nq,
|
||||
nkv,
|
||||
head_dim,
|
||||
rotary_dim,
|
||||
eps,
|
||||
norm_weight_bias=bias,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
)
|
||||
|
||||
q, k, v = run()
|
||||
if use_cuda_graph:
|
||||
graph = torch.cuda.CUDAGraph()
|
||||
with torch.cuda.graph(graph):
|
||||
q, k, v = run()
|
||||
key_cache.zero_()
|
||||
value_cache.zero_()
|
||||
graph.replay()
|
||||
|
||||
q_in, k_in, v_in = qkv.split(
|
||||
(nq * head_dim, nkv * head_dim, nkv * head_dim), dim=-1
|
||||
)
|
||||
tol = 1e-2 if dtype == torch.bfloat16 else 2e-3
|
||||
for actual, src, weight, heads in ((q, q_in, qw, nq), (k, k_in, kw, nkv)):
|
||||
expected = norm_rope_reference(
|
||||
src.reshape(num_tokens, heads, head_dim),
|
||||
weight,
|
||||
cos_sin,
|
||||
positions,
|
||||
rotary_dim,
|
||||
eps,
|
||||
bias,
|
||||
).reshape_as(actual)
|
||||
torch.testing.assert_close(actual, expected, rtol=tol, atol=tol)
|
||||
torch.testing.assert_close(v, v_in, rtol=0, atol=0)
|
||||
torch.testing.assert_close(storage, original, rtol=0, atol=0)
|
||||
|
||||
expected_k, expected_v = torch.zeros_like(key_cache), torch.zeros_like(value_cache)
|
||||
if num_slots:
|
||||
ops.reshape_and_cache_flash(
|
||||
k.reshape(num_tokens, nkv, head_dim),
|
||||
v.reshape(num_tokens, nkv, head_dim),
|
||||
expected_k,
|
||||
expected_v,
|
||||
slots,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
)
|
||||
for actual, expected in ((key_cache, expected_k), (value_cache, expected_v)):
|
||||
if kv_cache_dtype == "fp8":
|
||||
# Reciprocal multiplication versus division may differ by one code.
|
||||
assert (actual.int() - expected.int()).abs().max() <= 1
|
||||
else:
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
pytest.param(dict(head_dim=64, rotary_dim=32, num_tokens=1), id="decode"),
|
||||
pytest.param(
|
||||
dict(block_size=7, fp32_weights=True, bias=1.0, strided_cache=True),
|
||||
id="strided-cache-fp32-weights",
|
||||
),
|
||||
pytest.param(
|
||||
dict(head_dim=192, rotary_dim=128, fp32_weights=True),
|
||||
id="head192-partial-rope",
|
||||
),
|
||||
pytest.param(
|
||||
dict(
|
||||
head_dim=192,
|
||||
rotary_dim=192,
|
||||
block_size=7,
|
||||
bias=1.0,
|
||||
strided_cache=True,
|
||||
kv_cache_dtype="fp8",
|
||||
),
|
||||
id="head192-full-rope-fp8",
|
||||
),
|
||||
pytest.param(
|
||||
dict(
|
||||
head_dim=256, rotary_dim=256, strided_cache=True, kv_cache_dtype="fp8"
|
||||
),
|
||||
id="head256-full-rope-fp8",
|
||||
),
|
||||
pytest.param(
|
||||
dict(head_dim=192, rotary_dim=0, fp32_weights=True, bias=1.0),
|
||||
id="no-rope",
|
||||
),
|
||||
pytest.param(
|
||||
dict(head_dim=64, rotary_dim=8, fp32_weights=True),
|
||||
id="minimum-rotary-dim",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_norm_rope_and_cache(dtype, case):
|
||||
check_norm_rope_and_cache(dtype=dtype, **case)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
pytest.param(dict(num_tokens=0), id="empty-input"),
|
||||
pytest.param(dict(use_cuda_graph=True), id="graph-auto-cache"),
|
||||
pytest.param(
|
||||
dict(use_cuda_graph=True, kv_cache_dtype="fp8"), id="graph-fp8-cache"
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_empty_input_and_graph_replay(case):
|
||||
check_norm_rope_and_cache(**case)
|
||||
Reference in New Issue
Block a user