[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,898 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from importlib.util import find_spec
from types import SimpleNamespace
import pytest
import torch
import vllm.envs as envs
from tests.compile.backend import TestBackend
from tests.utils import TestFP8Layer, has_module_attribute, multi_gpu_test
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
from vllm.compilation.passes.fusion.allreduce_rms_fusion import (
AllReduceFusionPass,
RocmAiterAllReduceFusionPass,
_fused_ar_workspace_hidden_dim,
_select_flashinfer_allreduce_use_oneshot,
)
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.compilation.passes.utility.fix_functionalization import (
FixFunctionalizationPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CompilationConfig,
CompilationMode,
DeviceConfig,
ModelConfig,
PassConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.distributed.device_communicators.aiter_custom_all_reduce import (
AiterCustomAllreduce,
)
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
from vllm.utils.torch_utils import set_random_seed
DEVICE_TYPE = current_platform.device_type
@pytest.mark.parametrize(
("workspace_backend", "device_capability", "world_size", "tensor_size", "expected"),
[
("mnnvl", 103, 8, 2 * 1024 * 1024, None),
("trtllm", 103, 8, 2 * 1024 * 1024, True),
("trtllm", 103, 8, 2 * 1024 * 1024 + 1, False),
("trtllm", 100, 4, 4 * 1024 * 1024, True),
("trtllm", 100, 4, 4 * 1024 * 1024 + 1, False),
("trtllm", None, 8, 128 * 1024 * 1024, True),
],
)
def test_select_flashinfer_allreduce_use_oneshot(
workspace_backend: str,
device_capability: int | None,
world_size: int,
tensor_size: int,
expected: bool | None,
):
assert (
_select_flashinfer_allreduce_use_oneshot(
workspace_backend,
device_capability,
world_size,
tensor_size,
)
is expected
)
def _make_workspace_hidden_config(
target_hidden: int, draft_hidden: int | None
) -> SimpleNamespace:
"""Minimal stand-in exposing only what _fused_ar_workspace_hidden_dim reads."""
speculative_config = None
if draft_hidden is not None:
speculative_config = SimpleNamespace(
draft_model_config=SimpleNamespace(
get_hidden_size=lambda: draft_hidden,
)
)
return SimpleNamespace(
model_config=SimpleNamespace(get_hidden_size=lambda: target_hidden),
speculative_config=speculative_config,
)
@pytest.mark.parametrize(
("target_hidden", "draft_hidden", "expected"),
[
(2048, None, 2048), # no speculative decoding -> target only
(2048, 2560, 2560), # draft wider than target -> draft (issue #52023)
(2048, 1024, 2048), # draft narrower than target -> target
(2048, 2048, 2048), # equal -> target
],
)
def test_fused_ar_workspace_hidden_dim(
target_hidden: int, draft_hidden: int | None, expected: int
):
config = _make_workspace_hidden_config(target_hidden, draft_hidden)
assert _fused_ar_workspace_hidden_dim(config) == expected
def test_fused_ar_workspace_hidden_dim_spec_without_draft():
# speculative_config present but no draft model (e.g. ngram) -> target only.
config = SimpleNamespace(
model_config=SimpleNamespace(get_hidden_size=lambda: 2048),
speculative_config=SimpleNamespace(draft_model_config=None),
)
assert _fused_ar_workspace_hidden_dim(config) == 2048
class TestAllReduceRMSNormModel(torch.nn.Module):
def __init__(
self,
hidden_size=16,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.float16,
use_aiter: bool = False,
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
self.w = [torch.rand(hidden_size, hidden_size) for _ in range(3)]
self.use_aiter = use_aiter
def forward(self, x):
# avoid having graph input be an arg to a pattern directly
z = torch.relu(x)
x = resid = tensor_model_parallel_all_reduce(z)
y = self.norm[0](x)
z2 = torch.mm(y, self.w[0])
x2 = tensor_model_parallel_all_reduce(z2)
y2, resid = self.norm[1](x2, resid)
z3 = torch.mm(y2, self.w[1])
x3 = tensor_model_parallel_all_reduce(z3)
y3, resid = self.norm[2](x3, resid)
z4 = torch.mm(y3, self.w[2])
x4 = tensor_model_parallel_all_reduce(z4)
y4, resid = self.norm[3](x4, resid)
return y4
def ops_in_model_before(self):
return [torch.ops.vllm.all_reduce.default]
def ops_in_model_after(self):
if self.use_aiter:
return [rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()]
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
class TestAllReduceGemmaRMSNormModel(torch.nn.Module):
def __init__(
self,
hidden_size=16,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.float16,
use_aiter: bool = False,
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.use_aiter = use_aiter
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
# Non-trivial weight (~Gemma range) so (1 + w) exercises the scale path.
for n in self.norm:
n.weight.data.normal_(mean=0.0, std=0.1)
self.w = [torch.rand(hidden_size, hidden_size) for _ in range(3)]
def forward(self, x):
# avoid having graph input be an arg to a pattern directly
z = torch.relu(x)
x = resid = tensor_model_parallel_all_reduce(z)
y = self.norm[0](x)
z2 = torch.mm(y, self.w[0])
x2 = tensor_model_parallel_all_reduce(z2)
y2, resid = self.norm[1](x2, resid)
z3 = torch.mm(y2, self.w[1])
x3 = tensor_model_parallel_all_reduce(z3)
y3, resid = self.norm[2](x3, resid)
z4 = torch.mm(y3, self.w[2])
x4 = tensor_model_parallel_all_reduce(z4)
y4, resid = self.norm[3](x4, resid)
return y4
def ops_in_model_before(self):
return [torch.ops.vllm.all_reduce.default]
def ops_in_model_after(self):
if self.use_aiter:
return [rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()]
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
quant_key = kFp8StaticTensorSym
def __init__(
self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
self.fp8_linear_layers = [
TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
input_dtype=dtype,
)
for i in range(3)
]
def forward(self, hidden_states):
# avoid having graph input be an arg to a pattern directly
z = torch.relu(hidden_states)
x = resid = tensor_model_parallel_all_reduce(z)
y = self.norm[0](x)
z2 = self.fp8_linear_layers[0](y)
x2 = tensor_model_parallel_all_reduce(z2)
y2, resid = self.norm[1](x2, resid)
z3 = self.fp8_linear_layers[1](y2)
x3 = tensor_model_parallel_all_reduce(z3)
y3, resid = self.norm[2](x3, resid) # use resid here
z4 = self.fp8_linear_layers[2](y3)
x4 = tensor_model_parallel_all_reduce(z4)
y4, resid = self.norm[3](x4, resid) # use resid here
return y4
def ops_in_model_after(self):
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
def ops_in_model_before(self):
return [
torch.ops.vllm.all_reduce.default,
torch.ops._C.static_scaled_fp8_quant.default
if self.fp8_linear_layers[0].is_quant_fp8_enabled()
else torch.ops.aten.reciprocal.default,
]
class TestAllReduceGemmaRMSNormStaticQuantFP8Model(
TestAllReduceRMSNormStaticQuantFP8Model
):
def __init__(
self,
hidden_size=16,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.float16,
):
super().__init__(hidden_size, token_num, eps, dtype)
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
for norm in self.norm:
norm.weight.requires_grad_(False)
def ops_in_model_before(self):
return [torch.ops.vllm.all_reduce.default]
class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
"""Exercises the new ROCm AITER AR+RMS+per-group-FP8-quant patterns.
Four ``rms_norm`` sites that together hit every pattern registered by
``RocmAiterAllReduceFusionPass`` for the per-group FP8 quant path:
* ``norm[0]``: ``all_reduce -> rms_norm -> group_fp8_quant`` (no residual)
-> ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern``
* ``norm[1]``: ``all_reduce -> fused_add_rms_norm -> group_fp8_quant``
(single ``rms`` consumer)
-> ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern``
* ``norm[2..3]``: ``all_reduce -> fused_add_rms_norm
-> (group_fp8_quant + rocm_unquantized_gemm)`` (two ``rms`` consumers,
modeling the DSv3.2 indexer fan-out)
-> ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern``
Each site starts from the same bf16 activation and is returned separately.
This avoids compounding four quantize/dequantize errors while still testing
all four pattern firings without depending on a real FP8 block-scaled GEMM.
"""
quant_group_size = 128
indexer_out_dim = 8
def __init__(
self,
hidden_size=128,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.bfloat16,
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.dtype = dtype
assert hidden_size % self.quant_group_size == 0, (
f"hidden_size ({hidden_size}) must be a multiple of "
f"quant_group_size ({self.quant_group_size}) for per-group FP8 quant"
)
self.norm = [RMSNorm(hidden_size, eps) for _ in range(4)]
self.w = [torch.rand(hidden_size, hidden_size, dtype=dtype) for _ in range(3)]
self.indexer_w = [
torch.rand(self.indexer_out_dim, hidden_size, dtype=dtype) for _ in range(2)
]
def _group_quant(self, rms: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops.vllm.rocm_aiter_group_fp8_quant.default(
rms, self.quant_group_size
)
def _dequantize_to_bf16(self, q: torch.Tensor, s: torch.Tensor) -> torch.Tensor:
# Broadcast the per-group scale across each group of `quant_group_size`
# to validate the quantized result without depending on a real FP8
# block-scaled GEMM kernel in the test.
s_full = s.repeat_interleave(self.quant_group_size, dim=-1).to(self.dtype)
return q.to(self.dtype) * s_full
def forward(self, hidden_states):
z = torch.relu(hidden_states)
x0 = tensor_model_parallel_all_reduce(z)
rms0 = self.norm[0](x0)
q0, s0 = self._group_quant(rms0)
y0 = self._dequantize_to_bf16(q0, s0)
x1 = tensor_model_parallel_all_reduce(torch.mm(z, self.w[0]))
rms1, resid1 = self.norm[1](x1, z.clone())
q1, s1 = self._group_quant(rms1)
y1 = self._dequantize_to_bf16(q1, s1)
x2 = tensor_model_parallel_all_reduce(torch.mm(z, self.w[1]))
rms2, resid2 = self.norm[2](x2, z.clone())
q2, s2 = self._group_quant(rms2)
# Second consumer of ``rms2``: forces the with-indexer pattern.
idx2 = torch.ops.vllm.rocm_unquantized_gemm(rms2, self.indexer_w[0], None)
y2 = self._dequantize_to_bf16(q2, s2)
x3 = tensor_model_parallel_all_reduce(torch.mm(z, self.w[2]))
rms3, resid3 = self.norm[3](x3, z.clone())
q3, s3 = self._group_quant(rms3)
# Second consumer of ``rms3``: forces the with-indexer pattern.
idx3 = torch.ops.vllm.rocm_unquantized_gemm(rms3, self.indexer_w[1], None)
y3 = self._dequantize_to_bf16(q3, s3)
return y0, y1, y2, y3, resid1, resid2, resid3, idx2, idx3
def ops_in_model_before(self):
return [
torch.ops.vllm.all_reduce.default,
torch.ops.vllm.rocm_aiter_group_fp8_quant.default,
]
def ops_in_model_after(self):
return [
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op(),
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op(), # noqa: E501
]
class TestAiterAllReduceGemmaRMSNormGroupQuantFP8Model(
TestAiterAllReduceRMSNormGroupQuantFP8Model
):
"""Gemma norms feeding the per-group FP8 quant sites.
The fused AR+RMS+quant op has no ``gemma_norm`` mode, so these sites must
keep matching the quant-fused patterns with the pre-added ``1 + weight``.
The Gemma AR+RMS-only patterns match a prefix of the same subgraph, so if
they were registered ahead of the quant-fused ones they would consume the
``all_reduce`` and strand ``rocm_aiter_group_fp8_quant`` as a separate
kernel -- which the ``fully_replaced=True`` check below would catch.
"""
def __init__(
self,
hidden_size=128,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.bfloat16,
):
super().__init__(hidden_size, token_num, eps, dtype)
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
for norm in self.norm:
norm.weight.requires_grad_(False)
class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module):
def __init__(
self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
self.w = [torch.rand(hidden_size, hidden_size) for _ in range(3)]
self.agscale = [torch.rand(1, dtype=torch.float32) for _ in range(3)]
wgscale = [torch.rand(1, dtype=torch.float32) for _ in range(3)]
self.alpha = [1 / (w * a) for w, a in zip(wgscale, self.agscale)]
wq_gen, wscale_gen = zip(
*(scaled_fp4_quant(w, wg) for w, wg in zip(self.w, wgscale))
)
self.wq, self.wscale = list(wq_gen), list(wscale_gen)
def forward(self, hidden_states):
# avoid having graph input be an arg to a pattern directly
z = torch.relu(hidden_states)
x = resid = tensor_model_parallel_all_reduce(z)
y = self.norm[0](x)
yq, y_scale = scaled_fp4_quant(y, self.agscale[0])
z2 = cutlass_scaled_fp4_mm(
yq, self.wq[0], y_scale, self.wscale[0], self.alpha[0], out_dtype=y.dtype
)
x2 = tensor_model_parallel_all_reduce(z2)
y2, resid = self.norm[1](x2, resid)
yq2, y_scale2 = scaled_fp4_quant(y2, self.agscale[1])
z3 = cutlass_scaled_fp4_mm(
yq2, self.wq[1], y_scale2, self.wscale[1], self.alpha[1], out_dtype=y2.dtype
)
x3 = tensor_model_parallel_all_reduce(z3)
y3, resid = self.norm[2](x3, resid) # use resid here
yq3, y_scale3 = scaled_fp4_quant(y3, self.agscale[2])
z4 = cutlass_scaled_fp4_mm(
yq3, self.wq[2], y_scale3, self.wscale[2], self.alpha[2], out_dtype=y3.dtype
)
x4 = tensor_model_parallel_all_reduce(z4)
y4, resid = self.norm[3](x4, resid) # use resid here
return y4
def ops_in_model_after(self):
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
def ops_in_model_before(self):
return [
torch.ops.vllm.all_reduce.default,
torch.ops._C.scaled_fp4_quant.out,
]
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"test_model, enable_quant_fp8_custom_op, use_aiter",
[
(TestAllReduceRMSNormModel, False, IS_AITER_FOUND),
pytest.param(
TestAllReduceGemmaRMSNormModel,
False,
IS_AITER_FOUND,
marks=pytest.mark.skipif(
current_platform.is_rocm() and not IS_AITER_FOUND,
reason="AITER is required on ROCm",
),
),
pytest.param(
TestAllReduceRMSNormStaticQuantFP8Model,
True,
False,
marks=pytest.mark.skipif(
current_platform.is_rocm(),
reason="Not supported on ROCm platform",
),
),
pytest.param(
TestAllReduceGemmaRMSNormStaticQuantFP8Model,
True,
False,
marks=pytest.mark.skipif(
current_platform.is_rocm(),
reason="Not supported on ROCm platform",
),
),
pytest.param(
TestAllReduceRMSNormStaticQuantFP8Model,
False,
False,
marks=pytest.mark.skipif(
current_platform.is_rocm(),
reason="Not supported on ROCm platform",
),
),
pytest.param(
TestAllReduceFusedAddRMSNormStaticQuantFP4Model,
False,
False,
marks=pytest.mark.skipif(
current_platform.is_rocm(),
reason="Not supported on ROCm platform",
),
),
],
)
@pytest.mark.parametrize("batch_size", [8])
@pytest.mark.parametrize("seq_len", [8])
@pytest.mark.parametrize("hidden_size", [64])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
@pytest.mark.parametrize("flashinfer_allreduce_backend", ["trtllm", "mnnvl"])
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
@pytest.mark.skipif(
current_platform.is_rocm() and not IS_AITER_FOUND,
reason="aiter is not found",
)
@pytest.mark.skipif(
current_platform.is_cuda()
and (
not find_spec("flashinfer")
or not has_module_attribute("flashinfer.comm", "allreduce_fusion")
or not has_module_attribute(
"flashinfer.comm", "create_allreduce_fusion_workspace"
)
),
reason="flashinfer is not found or flashinfer "
"is not compiled with allreduce_fusion",
)
def test_all_reduce_fusion_pass_replace(
test_model: torch.nn.Module,
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
enable_rms_norm_custom_op,
enable_quant_fp8_custom_op,
flashinfer_allreduce_backend,
use_aiter: bool,
monkeypatch: pytest.MonkeyPatch,
):
if use_aiter:
with monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", str(use_aiter))
rocm_aiter_ops.refresh_env_variables()
num_processes = 2
if (
test_model == TestAllReduceFusedAddRMSNormStaticQuantFP4Model
and not current_platform.has_device_capability(100)
):
pytest.skip(
"Skip as nvfp4 is only supported on "
"devices with compute capability 10.0 (Blackwell)"
)
def run_torch_spawn(fn, nprocs):
master_port = get_open_port()
torch.multiprocessing.spawn(
fn,
args=(
num_processes,
master_port,
test_model,
batch_size,
seq_len,
hidden_size,
dtype,
enable_rms_norm_custom_op,
enable_quant_fp8_custom_op,
flashinfer_allreduce_backend,
use_aiter,
monkeypatch,
),
nprocs=nprocs,
)
run_torch_spawn(all_reduce_fusion_pass_on_test_model, num_processes)
def all_reduce_fusion_pass_on_test_model(
local_rank: int,
world_size: int,
master_port: int,
test_model_cls: torch.nn.Module,
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
enable_rms_norm_custom_op,
enable_quant_fp8_custom_op,
flashinfer_allreduce_backend,
use_aiter: bool,
monkeypatch: pytest.MonkeyPatch,
):
set_random_seed(0)
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": str(master_port),
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": flashinfer_allreduce_backend,
"VLLM_ROCM_USE_AITER": str(int(use_aiter)),
"VLLM_ROCM_USE_AITER_CUSTOM_AR": str(int(use_aiter)),
}
)
if use_aiter:
rocm_aiter_ops.refresh_env_variables()
init_distributed_environment()
custom_ops = []
if enable_rms_norm_custom_op:
custom_ops.append("+rms_norm")
if enable_quant_fp8_custom_op:
custom_ops.append("+quant_fp8")
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE, custom_ops=custom_ops
)
)
vllm_config.compilation_config.pass_config = PassConfig(
fuse_allreduce_rms=True, eliminate_noops=True
)
vllm_config.device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
vllm_config.parallel_config.rank = local_rank # Setup rank for debug path
# this is a fake model name to construct the model config
# in the vllm_config, it's not really used.
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
vllm_config.model_config = ModelConfig(
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
)
with set_current_vllm_config(vllm_config):
initialize_model_parallel(tensor_model_parallel_size=world_size)
all_reduce_fusion_pass = (
RocmAiterAllReduceFusionPass(vllm_config)
if use_aiter
else AllReduceFusionPass(vllm_config)
)
noop_pass = NoOpEliminationPass(vllm_config)
func_pass = FixFunctionalizationPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(
noop_pass, all_reduce_fusion_pass, func_pass, cleanup_pass
)
token_num = batch_size * seq_len
if test_model_cls in (
TestAllReduceRMSNormModel,
TestAllReduceGemmaRMSNormModel,
):
model = test_model_cls(
hidden_size, token_num, dtype=dtype, use_aiter=use_aiter
)
else:
model = test_model_cls(hidden_size, token_num, dtype=dtype)
hidden_states = torch.randn((token_num, hidden_size), requires_grad=False)
compiled_model = torch.compile(model, backend=backend)
compiled_model(hidden_states)
results_unfused = model(hidden_states)
results_fused = compiled_model(hidden_states)
torch.testing.assert_close(results_unfused, results_fused, atol=1e-2, rtol=1e-2)
assert all_reduce_fusion_pass.matched_count == 4, (
f"{all_reduce_fusion_pass.matched_count=}"
)
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=False)
backend.check_after_ops(model.ops_in_model_after())
if (
test_model_cls
in (
TestAllReduceGemmaRMSNormModel,
TestAllReduceGemmaRMSNormStaticQuantFP8Model,
)
and not use_aiter
):
fused_op = torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default
fused_nodes = list(find_op_nodes(fused_op, backend.graph_post_pass))
assert fused_nodes
assert all(n.kwargs.get("weight_bias") == 1.0 for n in fused_nodes)
del all_reduce_fusion_pass
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("batch_size", [8])
@pytest.mark.parametrize("seq_len", [8])
@pytest.mark.parametrize("hidden_size", [128])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
@pytest.mark.parametrize(
"test_model_cls",
[
TestAiterAllReduceRMSNormGroupQuantFP8Model,
TestAiterAllReduceGemmaRMSNormGroupQuantFP8Model,
],
)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm AITER AR+RMS+per-group-FP8-quant fusion is ROCm-only",
)
@pytest.mark.skipif(not IS_AITER_FOUND, reason="aiter is not found")
def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
enable_rms_norm_custom_op: bool,
test_model_cls: type[torch.nn.Module],
monkeypatch: pytest.MonkeyPatch,
):
"""Sibling of ``test_all_reduce_fusion_pass_replace`` for the new
ROCm AITER AR+RMS+per-group-FP8-quant fusion patterns.
Validates the three new ``VllmPatternReplacement`` patterns added to
``RocmAiterAllReduceFusionPass``:
* ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern`` (no-residual)
* ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern`` (with-residual,
single ``rms`` consumer)
* ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern`` (with-
residual, DSv3.2 indexer fan-out; parametrized over
``rocm_aiter_group_fp8_quant``
producer).
Parametrized over plain and Gemma norms: the Gemma AR+RMS-only patterns
match a prefix of these subgraphs, so this also pins their registration
order behind the quant-fused ones.
"""
with monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
if not AiterCustomAllreduce.build_supports_per_group_quant():
pytest.skip(
"aiter build is missing 'fused_ar_rms_per_group_quant' (needs "
"ROCm/aiter PR #2823); the new patterns aren't registered."
)
num_processes = 2
def run_torch_spawn(fn, nprocs):
master_port = get_open_port()
torch.multiprocessing.spawn(
fn,
args=(
num_processes,
master_port,
test_model_cls,
batch_size,
seq_len,
hidden_size,
dtype,
enable_rms_norm_custom_op,
monkeypatch,
),
nprocs=nprocs,
)
run_torch_spawn(rocm_aiter_group_quant_fusion_pass_on_test_model, num_processes)
def rocm_aiter_group_quant_fusion_pass_on_test_model(
local_rank: int,
world_size: int,
master_port: int,
test_model_cls: torch.nn.Module,
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
enable_rms_norm_custom_op: bool,
monkeypatch: pytest.MonkeyPatch,
):
set_random_seed(0)
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": str(master_port),
"VLLM_ROCM_USE_AITER": "1",
"VLLM_ROCM_USE_AITER_CUSTOM_AR": "1",
}
)
rocm_aiter_ops.refresh_env_variables()
init_distributed_environment()
custom_ops = []
if enable_rms_norm_custom_op:
custom_ops.append("+rms_norm")
# We always enable +quant_fp8 so the matcher's example
# trace finds the same form the test model uses.
custom_ops.append("+quant_fp8")
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE, custom_ops=custom_ops
)
)
vllm_config.compilation_config.pass_config = PassConfig(
fuse_allreduce_rms=True, eliminate_noops=True
)
vllm_config.device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
vllm_config.parallel_config.rank = local_rank
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
vllm_config.model_config = ModelConfig(
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
)
with set_current_vllm_config(vllm_config):
initialize_model_parallel(tensor_model_parallel_size=world_size)
all_reduce_fusion_pass = RocmAiterAllReduceFusionPass(vllm_config)
noop_pass = NoOpEliminationPass(vllm_config)
func_pass = FixFunctionalizationPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(
noop_pass, all_reduce_fusion_pass, func_pass, cleanup_pass
)
token_num = batch_size * seq_len
model = test_model_cls(hidden_size, token_num, dtype=dtype)
hidden_states = torch.randn((token_num, hidden_size), requires_grad=False)
compiled_model = torch.compile(model, backend=backend)
compiled_model(hidden_states)
results_unfused = model(hidden_states)
results_fused = compiled_model(hidden_states)
# AITER's kernel-level oracle permits up to 5% isolated FP8 values to
# fall outside a 5% allclose envelope after collective reordering.
for site, (reference, actual) in enumerate(
zip(results_unfused[:4], results_fused[:4])
):
close = torch.isclose(reference, actual, atol=5e-2, rtol=5e-2)
mismatch_ratio = float((~close).float().mean().item())
assert mismatch_ratio <= 0.05, (
f"FP8 site {site} mismatch ratio {mismatch_ratio:.2%} exceeds 5%"
)
# Residual and bf16 indexer outputs do not incur FP8 quantization noise.
torch.testing.assert_close(
results_unfused[4:], results_fused[4:], atol=1e-2, rtol=1e-2
)
# Four pattern firings: norm[0] (no-add quant), norm[1] (add quant,
# single ``rms`` consumer), norm[2..3] (add quant + indexer fan-out).
assert all_reduce_fusion_pass.matched_count == 4, (
f"{all_reduce_fusion_pass.matched_count=}"
)
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=True)
fused_quant_op, fused_indexer_op = model.ops_in_model_after()
assert backend.op_count(fused_quant_op) == 2
assert backend.op_count(fused_indexer_op) == 2
del all_reduce_fusion_pass
@@ -1,462 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for IR inplace functionalization pass integration.
This test suite verifies that the inplace functionalization pass, lowering pass,
and clone cleanup pass work together correctly with donated buffer tracking.
"""
from collections.abc import Callable
import pytest
import torch
import torch._dynamo.exc
from torch import nn
import vllm.kernels # noqa: F401 to register kernels
from vllm.compilation.passes.inductor_pass import InductorPass, get_pass_context
from vllm.compilation.passes.ir.clone_elimination import (
UnsafeCloneEliminationPass,
)
from vllm.compilation.passes.ir.inplace_functionalization import (
VllmIRInplaceFunctionalizationPass,
)
from vllm.compilation.passes.ir.lowering_pass import VllmIRLoweringPass
from vllm.config import VllmConfig
from vllm.ir import ops
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON, tl, triton
from ...backend import TestBackend
class StoreDonationInfoPass(InductorPass):
def __init__(self):
self.donated_input_ids_sets: list[set[int]] = []
def __call__(self, *args, **kwargs):
ctx = get_pass_context()
self.donated_input_ids_sets += [ctx.donated_input_ids]
class MaybeInplaceModel(nn.Module):
"""Model using only maybe_inplace variants."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
):
# First maybe_inplace - x & residual1 are donated
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
x, residual1, self.weight1, 1e-5
)
# Second maybe_inplace - residual2 is donated
x_normed2, residual_out2 = ops.fused_add_rms_norm.maybe_inplace(
x_normed1, residual2, self.weight2, 1e-5
)
return x_normed2, residual_out1, residual_out2
class FunctionalModel(nn.Module):
"""Model using only functional (default) variants."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
):
# First functional - no donation
x_normed1, residual_out1 = ops.fused_add_rms_norm(
x, residual1, self.weight1, 1e-5
)
# Second functional - no donation
x_normed2, residual_out2 = ops.fused_add_rms_norm(
x_normed1, residual2, self.weight2, 1e-5
)
return x_normed2, residual_out1, residual_out2
class MixedModel(nn.Module):
"""Model mixing maybe_inplace and functional variants."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
):
# First maybe_inplace - x & residual1 are donated
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
x, residual1, self.weight1, 1e-5
)
# Second functional - no donation, x_normed1 must be preserved as it's returned
x_normed2, residual_out2 = ops.fused_add_rms_norm(
x_normed1, residual2, self.weight2, 1e-5
)
# Return both to prevent x_normed1 from being optimized away
return x_normed1, x_normed2, residual_out1, residual_out2
class ModelWithTritonAfterMaybeInplace(nn.Module):
"""Model using maybe_inplace followed by a Triton kernel.
Test clone elimination can handle Triton in the graph
"""
def __init__(self, hidden_size=16):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
@triton.jit
def _triton_add_kernel(
x_ptr,
y_ptr,
n_elements,
BLOCK_SIZE: tl.constexpr,
):
pid = tl.program_id(axis=0)
block_start = pid * BLOCK_SIZE
offsets = block_start + tl.arange(0, BLOCK_SIZE)
mask = offsets < n_elements
x = tl.load(x_ptr + offsets, mask=mask)
y = x + 0.1
tl.store(y_ptr + offsets, y, mask=mask)
def triton_add(x: torch.Tensor) -> torch.Tensor:
"""Simple Triton add kernel."""
y = torch.empty_like(x)
n_elements = x.numel()
grid = (triton.cdiv(n_elements, 256),)
_triton_add_kernel[grid](x, y, n_elements, BLOCK_SIZE=256)
return y
self.triton_add = triton_add
def forward(self, x: torch.Tensor, residual: torch.Tensor, residual2: torch.Tensor):
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
x, residual, self.weight, 1e-5
)
x_processed = self.triton_add(x_normed)
# x_processed does not need to be cloned, residual2 does
x_normed2, residual_out2 = ops.fused_add_rms_norm(
x_processed, residual2, self.weight, 1e-5
)
return x_normed2, residual_out2
skipif_no_triton = pytest.mark.skipif(not HAS_TRITON, reason="Requires Triton")
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
@pytest.mark.parametrize(
"model_class,expected_functionalized,expected_donated,expected_clones",
[
# 2 inplace calls, all activations donated, all clones eliminated
(MaybeInplaceModel, 2, 3, 0),
# No inplace calls, no donations, 3 clones (one eliminated)
(FunctionalModel, 0, 0, 3),
# One inplace call, two donated activations, 2 clones
(MixedModel, 1, 2, 2),
# One inplace call, two donated, 1 clone remaining
pytest.param(ModelWithTritonAfterMaybeInplace, 1, 2, 1, marks=skipif_no_triton),
],
)
def test_inplace_functionalization(
default_vllm_config: VllmConfig,
model_class,
expected_functionalized: int,
expected_clones: int,
expected_donated: int,
):
"""Test inplace functionalization, lowering, and clone cleanup."""
torch.set_default_device(current_platform.device_type)
# Use vllm_c so inplace path is triggered
default_vllm_config.kernel_config.ir_op_priority.fused_add_rms_norm = [
"vllm_c",
"native",
]
# Create passes in order they run during compilation
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
lowering_pass = VllmIRLoweringPass(default_vllm_config)
donated_info_pass = StoreDonationInfoPass()
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
# Set up backend with pre-grad pass
backend = TestBackend(lowering_pass, donated_info_pass, cleanup_pass)
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
model = model_class()
x = torch.randn(8, 16, dtype=torch.bfloat16)
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
with default_vllm_config.kernel_config.ir_op_priority.set_priority():
# Reference output without optimization
ref_output = model(x.clone(), residual1.clone(), residual2.clone())
# Compile with inplace optimization
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
output = compiled_model(x.clone(), residual1.clone(), residual2.clone())
# Verify correctness (relaxed tolerance for bfloat16)
for i in range(len(ref_output)):
torch.testing.assert_close(output[i], ref_output[i], rtol=1e-2, atol=1e-2)
# Verify expected number of ops were functionalized
func_ops = functionalization_pass.functionalized_ops
assert len(func_ops) == int(bool(expected_functionalized))
if expected_functionalized > 0:
assert "fused_add_rms_norm" in func_ops
assert func_ops["fused_add_rms_norm"] == expected_functionalized
# Verify lowering happened (2 ops in all cases)
assert "fused_add_rms_norm" in lowering_pass.selected_impls
assert len(lowering_pass.selected_impls["fused_add_rms_norm"]) == 2
assert all(
provider == "vllm_c"
for node, provider in lowering_pass.selected_impls["fused_add_rms_norm"].items()
), lowering_pass.selected_impls
# Verify correct number of donated IDs
assert len(donated_info_pass.donated_input_ids_sets) == 1
assert len(donated_info_pass.donated_input_ids_sets[0]) == expected_donated
# Verify expected number of clones after cleanup
actual_clones = backend.op_count(torch.ops.aten.clone.default, before=False)
assert actual_clones == expected_clones, (
f"Expected {expected_clones} clones, got {actual_clones}:"
f"{backend.print_graphs()}"
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_donated_buffer_context_propagation(default_vllm_config):
"""Test that donated_input_ids propagates correctly through pass_context."""
torch.set_default_device(current_platform.device_type)
# Create a custom backend that inspects pass_context in cleanup pass
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
lowering_pass = VllmIRLoweringPass(default_vllm_config)
donation_info_pass = StoreDonationInfoPass()
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
backend = TestBackend(lowering_pass, donation_info_pass, cleanup_pass)
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
model = MaybeInplaceModel()
x = torch.randn(8, 16, dtype=torch.bfloat16)
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
compiled_model(x.clone(), residual1.clone(), residual2.clone())
donated_ids_seen = donation_info_pass.donated_input_ids_sets
# Verify donated_input_ids was set and propagated
assert len(donated_ids_seen) == 1
# Should have donated inputs (exact indices depend on AOTAutograd)
assert len(donated_ids_seen[0]) == 3
# All donated ids should be valid non-negative integers
for idx in donated_ids_seen[0]:
assert isinstance(idx, int) and idx >= 0, f"Invalid donated index: {idx}"
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_maybe_inplace_reuse_error(default_vllm_config):
"""Test that reusing a donated activation input raises ValueError."""
torch.set_default_device(current_platform.device_type)
class ReuseModel(nn.Module):
"""Model that incorrectly reuses a donated activation input."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(self, x: torch.Tensor, residual: torch.Tensor):
# x is donated to maybe_inplace
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
x, residual, self.weight, 1e-5
)
# ERROR: x is used again after being donated
return x_normed + x # This should raise ValueError
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
lowering_pass = VllmIRLoweringPass(default_vllm_config)
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
backend = TestBackend(lowering_pass, cleanup_pass)
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
model = ReuseModel()
x = torch.randn(8, 16, dtype=torch.bfloat16)
residual = torch.randn(8, 16, dtype=torch.bfloat16)
# Compilation should raise BackendCompilerFailed wrapping ValueError
with pytest.raises(
torch._dynamo.exc.BackendCompilerFailed,
match="is used again after the node",
):
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
compiled_model(x.clone(), residual.clone())
# Piecewise compilation tests with graph splitting
@torch.library.custom_op("vllm::test_split_marker", mutates_args=())
def test_split_marker(x: torch.Tensor) -> torch.Tensor:
"""Identity op that marks a split point for piecewise compilation."""
return x.clone()
@test_split_marker.register_fake
def _fake_split_marker(x: torch.Tensor) -> torch.Tensor:
return torch.empty_like(x)
class TransformerBlockWithSplits(nn.Module):
"""Transformer block with explicit split points for piecewise compilation."""
def __init__(self, hidden_size=32, intermediate_size=128):
super().__init__()
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
# Attention-like projection
self.attn_proj = nn.Linear(
hidden_size, hidden_size, bias=False, dtype=torch.bfloat16
)
# Post-attention norm
self.post_attn_norm = nn.Parameter(
torch.ones(hidden_size, dtype=torch.bfloat16)
)
# MLP
self.gate_proj = nn.Linear(
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
)
self.up_proj = nn.Linear(
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
)
self.down_proj = nn.Linear(
intermediate_size, hidden_size, bias=False, dtype=torch.bfloat16
)
# Post-MLP norm
self.post_mlp_norm = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(self, x: torch.Tensor):
# Attention block with residual
residual1 = x
attn_out = self.attn_proj(x)
# Fused add + norm (maybe_inplace: residual1 is donated)
normed1, residual1 = ops.fused_add_rms_norm.maybe_inplace(
attn_out, residual1, self.post_attn_norm, 1e-5
)
# Force a graph split here
normed1 = torch.ops.vllm.test_split_marker(normed1)
# MLP block
gate = self.gate_proj(normed1)
up = self.up_proj(normed1)
mlp_out = self.down_proj(gate * torch.nn.functional.silu(up))
# Fused add + norm (maybe_inplace: residual1 is donated)
normed2, residual2 = ops.fused_add_rms_norm.maybe_inplace(
mlp_out, residual1, self.post_mlp_norm, 1e-5
)
return normed2, residual2
def with_dyn_arg(fn: Callable, arg_index: int, dim_index: int):
def inner(*args):
torch._dynamo.mark_dynamic(args[arg_index], dim_index)
return fn(*args)
return inner
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_piecewise_compilation_with_donated_buffers(monkeypatch, fresh_vllm_cache):
"""Test piecewise compilation with donated buffers across graph splits.
Utilizes a custom splitting op. Uses fresh cache to avoid compilation caching.
"""
torch.set_default_device(current_platform.device_type)
# Disable compilation cache to avoid serialization issues
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
from vllm.compilation.backends import VllmBackend
from vllm.config import CompilationConfig, VllmConfig
# Create config with custom splitting op
store_donation_info = StoreDonationInfoPass()
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
custom_ops=["all"],
splitting_ops=["vllm::test_split_marker"],
inductor_compile_config={"post_grad_custom_post_pass": store_donation_info},
)
)
backend = VllmBackend(vllm_config)
model = TransformerBlockWithSplits()
x = torch.randn(8, 32, dtype=torch.bfloat16)
# Reference output
ref_output = with_dyn_arg(model, 0, 0)(x.clone())
# Compile with piecewise compilation (graph will split at split_marker)
compiled_model = torch.compile(model, backend=backend, fullgraph=False)
output = with_dyn_arg(compiled_model, 0, 0)(x.clone())
# Verify correctness (relaxed tolerance for bfloat16)
torch.testing.assert_close(output[0], ref_output[0], rtol=1e-2, atol=1e-2)
torch.testing.assert_close(output[1], ref_output[1], rtol=1e-2, atol=1e-2)
# Verify the model was split into multiple submodules
assert hasattr(backend, "split_gm"), "Backend should have split graph module"
# Should have at least 2 submodules (split by test_split_marker op)
submodules = list(backend.split_gm.named_children())
num_submodules = len(submodules)
assert num_submodules >= 2, (
f"Expected at least 2 submodules (split), got {num_submodules}"
)
# Check that donation info was propagated correctly
donated_inputs_sets = store_donation_info.donated_input_ids_sets
assert len(donated_inputs_sets) == 2
assert len(donated_inputs_sets[0]) == 1
assert len(donated_inputs_sets[1]) == 1
-103
View File
@@ -1,103 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from torch import nn
import vllm.kernels # noqa: F401 to register kernels
from vllm import ir
from vllm.compilation.passes.ir.lowering_pass import (
VllmIRLoweringPass,
)
from vllm.config import get_current_vllm_config
from vllm.ir import ops
from vllm.model_executor.layers.activation import GeluAndMulSparse
from vllm.platforms import current_platform
from ...backend import TestBackend
class Model(nn.Module):
def __init__(self, hidden_size=16, *args, **kwargs):
super().__init__(*args, **kwargs)
self.hidden_size = hidden_size
self.weight = torch.ones(hidden_size, dtype=torch.bfloat16)
def forward(self, x):
x1 = x + 4.0
x2 = ops.rms_norm(x1, self.weight, 1e-5)
x3 = x2 * 5.0
# no weight
x4 = ops.rms_norm(x3, None, 1e-5)
x5 = x4 / 2.0
# dispatch to native due to variance_size parameter
x6 = ops.rms_norm(x5, self.weight, 1e-5, self.hidden_size // 2)
return x6 + 3.0
class SparseActivationModel(nn.Module):
def __init__(self):
super().__init__()
self.activation = GeluAndMulSparse(0.95, "tanh")
def forward(self, x):
return self.activation(x)
@pytest.mark.parametrize("rms_provider", ops.rms_norm.supported_providers())
def test_lowering_rms_norm(rms_provider, default_vllm_config):
torch.set_default_device(current_platform.device_type)
lowering_pass = VllmIRLoweringPass(get_current_vllm_config())
backend = TestBackend(lowering_pass)
backend_unlowered = TestBackend()
model = Model()
x = torch.randn(8, 16, dtype=torch.bfloat16)
with (
ops.rms_norm.set_priority([rms_provider, "native"]),
ir.enable_torch_wrap(True),
):
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
compiled_unlowered_model = torch.compile(
model, backend=backend_unlowered, fullgraph=True
)
output = compiled_model(x)
output_unlowered = compiled_unlowered_model(x)
selected = lowering_pass.selected_impls["rms_norm"]
assert len(selected) == 3
assert selected["rms_norm"] == rms_provider
assert selected["rms_norm_1"] == rms_provider
assert selected["rms_norm_2"] == "native"
# Compiled function guards on global value, avoid recompilation
with ir.enable_torch_wrap(True):
output2 = compiled_model(x)
torch.testing.assert_close(output_unlowered, output)
torch.testing.assert_close(output_unlowered, output2)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_lowering_gelu_and_mul_sparse(default_vllm_config):
torch.set_default_device("cuda")
lowering_pass = VllmIRLoweringPass(get_current_vllm_config())
backend = TestBackend(lowering_pass)
model = SparseActivationModel()
x = torch.randn(8, 32768, dtype=torch.bfloat16)
with (
ops.gelu_and_mul_sparse.set_priority(["triton", "native"]),
ir.enable_torch_wrap(True),
):
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
output = compiled_model(x)
selected = lowering_pass.selected_impls["gelu_and_mul_sparse"]
assert selected == {"gelu_and_mul_sparse": "triton"}
expected = ops.gelu_and_mul_sparse.impls["native"].impl_fn(
x, 1.6448536269514722, "tanh"
)
torch.testing.assert_close(output, expected, rtol=1.6e-2, atol=1e-3)
@@ -1,340 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import pytest
import torch
from tests.compile.backend import TestBackend
from tests.utils import TestFP8Layer
from vllm.compilation.passes.fusion.act_quant_fusion import (
ActivationQuantFusionPass,
)
from vllm.compilation.passes.fusion.rms_quant_fusion import RMSNormQuantFusionPass
from vllm.compilation.passes.fx_utils import find_auto_fn, find_auto_fn_maybe, is_func
from vllm.compilation.passes.utility.fix_functionalization import (
FixFunctionalizationPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CompilationConfig,
ModelConfig,
PassConfig,
VllmConfig,
get_current_vllm_config,
set_current_vllm_config,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
TEST_FP8 = current_platform.supports_fp8()
FP8_DTYPE = current_platform.fp8_dtype()
class TestSiluMul(torch.nn.Module):
quant_key = kFp8StaticTensorSym
def __init__(self, hidden_size: int = 128):
super().__init__()
self.silu_and_mul = SiluAndMul()
if TEST_FP8:
self.fp8_linear = TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
input_dtype=get_current_vllm_config().model_config.dtype,
)
def forward(self, x):
y = self.silu_and_mul(x)
if TEST_FP8:
return self.fp8_linear(y)
else:
return y
def example_inputs(self, num_tokens=32, hidden_size=128):
return (torch.rand(num_tokens, hidden_size * 2),)
def ops_in_model(self, do_fusion):
if TEST_FP8 and do_fusion:
return [torch.ops._C.silu_and_mul_quant.default]
else:
return [torch.ops._C.silu_and_mul.default]
def ops_not_in_model(self):
return []
class TestFusedAddRMSNorm(torch.nn.Module):
quant_key = kFp8StaticTensorSym
def __init__(self, hidden_size=16, intermediate_size=32):
super().__init__()
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.gate_proj = torch.nn.Parameter(
torch.empty((intermediate_size, hidden_size))
)
self.norm = RMSNorm(intermediate_size, 1e-05)
self.norm.weight = torch.nn.Parameter(torch.ones(intermediate_size))
torch.nn.init.normal_(self.gate_proj, std=0.02)
if TEST_FP8:
self.fp8_linear = TestFP8Layer(
weight_shape=(hidden_size, intermediate_size),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
input_dtype=get_current_vllm_config().model_config.dtype,
)
def forward(self, hidden_states, residual):
# Reshape input
view = hidden_states.reshape(-1, self.hidden_size)
# matrix multiplication
permute = self.gate_proj.permute(1, 0)
mm = torch.mm(view, permute)
# layer normalization
norm_output, residual_output = self.norm(mm, residual)
if TEST_FP8:
# scaled_mm with static input quantization
fp8_linear_result = self.fp8_linear(norm_output)
return fp8_linear_result, residual_output
else:
return norm_output, residual_output
def example_inputs(self, batch_size=8, seq_len=16):
hidden_states = torch.randn((batch_size * seq_len, self.hidden_size))
residual = torch.randn((batch_size * seq_len, self.intermediate_size))
return (hidden_states, residual)
def ops_in_model(self, do_fusion):
if TEST_FP8 and do_fusion:
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
else:
return []
def ops_not_in_model(self):
return []
class TestRotaryEmbedding(torch.nn.Module):
def __init__(self, head_dim=64, max_position=2048, base=10000):
super().__init__()
self.head_dim = head_dim
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position,
rope_parameters={"rope_type": "default", "rope_theta": base},
)
def forward(self, positions, q, k):
q_rotated, k_rotated = self.rotary_emb(positions, q, k)
return q_rotated, k_rotated
def example_inputs(self, num_tokens=32, head_dim=64):
positions = torch.arange(num_tokens, dtype=torch.long)
q = torch.randn(num_tokens, head_dim)
k = torch.randn(num_tokens, head_dim)
return (positions, q, k)
def ops_in_model(self, do_fusion):
return [torch.ops._C.rotary_embedding.default]
def ops_not_in_model(self):
return []
class TestRotaryEmbeddingSliceScatter(torch.nn.Module):
def __init__(self, head_dim=64, num_heads=4, max_position=2048, base=10000):
super().__init__()
self.head_dim = head_dim
self.num_heads = num_heads
self.hidden_size = head_dim * num_heads
self.qkv_proj = torch.nn.Linear(
self.hidden_size, self.hidden_size * 3, bias=False
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position,
rope_parameters={"rope_type": "default", "rope_theta": base},
)
def forward(self, positions, hidden_states):
# Simulate the pattern: mm -> split_with_sizes -> rotary_embedding
# -> slice_scatter -> split_with_sizes
qkv = self.qkv_proj(hidden_states)
split_sizes = [self.hidden_size, self.hidden_size, self.hidden_size]
q, k, v = torch.split(qkv, split_sizes, dim=-1)
q_rotated, k_rotated = self.rotary_emb(positions, q, k)
qkv_updated = torch.cat([q_rotated, k_rotated, v], dim=-1)
return qkv_updated
def example_inputs(self, num_tokens=32, head_dim=64, num_heads=4):
hidden_size = head_dim * num_heads
positions = torch.arange(num_tokens, dtype=torch.long)
hidden_states = torch.randn(num_tokens, hidden_size)
return (positions, hidden_states)
def ops_in_model(self, do_fusion):
return [torch.ops._C.rotary_embedding.default]
def ops_not_in_model(self):
return [torch.ops.aten.slice_scatter.default]
class TestFunctionWithMutatedArgsAndReturn(torch.nn.Module):
OP_REGISTERED = False
def __init__(self):
super().__init__()
self.register_test_custom_op()
@classmethod
def register_test_custom_op(cls):
if not cls.OP_REGISTERED:
def function_with_mutated_args_and_return_impl(
x: torch.Tensor,
) -> torch.Tensor:
ret = x + 1
x.add_(2)
return ret
def function_with_mutated_args_and_return_fake(
x: torch.Tensor,
) -> torch.Tensor:
return torch.empty_like(x)
direct_register_custom_op(
op_name="function_with_mutated_args_and_return",
op_func=function_with_mutated_args_and_return_impl,
mutates_args=["x"],
fake_impl=function_with_mutated_args_and_return_fake,
)
cls.OP_REGISTERED = True
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
# Clone x to avoid mutating the original tensor
ret = torch.ops.vllm.function_with_mutated_args_and_return(x)
return x, ret
def example_inputs(self, num_tokens=32):
hidden_states = torch.randn(num_tokens)
return (hidden_states,)
def ops_in_model(self, do_fusion):
return [torch.ops.vllm.function_with_mutated_args_and_return.default]
def ops_not_in_model(self):
return []
MODELS_AND_DO_FUSION = {
TestSiluMul: [True, False],
TestFusedAddRMSNorm: [True, False],
TestRotaryEmbedding: [False],
TestRotaryEmbeddingSliceScatter: [False],
TestFunctionWithMutatedArgsAndReturn: [False],
}
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize(
"model_class, do_fusion",
[
(model_class, do_fusion)
for model_class, fusions in MODELS_AND_DO_FUSION.items()
for do_fusion in fusions
],
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_fix_functionalization(
model_class: torch.nn.Module, do_fusion: bool, dtype: torch.dtype
):
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(0)
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
custom_ops=["all"],
pass_config=PassConfig(
fuse_norm_quant=do_fusion,
fuse_act_quant=do_fusion,
eliminate_noops=True,
),
),
)
with set_current_vllm_config(vllm_config):
assert RMSNorm.enabled()
noop_pass = NoOpEliminationPass(vllm_config)
fusion_pass = RMSNormQuantFusionPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
act_quant_fusion_pass = ActivationQuantFusionPass(vllm_config)
passes = (
[noop_pass, fusion_pass, act_quant_fusion_pass, cleanup_pass]
if do_fusion
else [noop_pass, cleanup_pass]
)
func_pass = FixFunctionalizationPass(vllm_config)
backend_func = TestBackend(*passes, func_pass)
backend_no_func = TestBackend(*passes)
model = model_class()
inputs_func = model.example_inputs()
inputs_no_func = copy.deepcopy(inputs_func)
model_func = copy.deepcopy(model)
model_no_func = copy.deepcopy(model)
model_func = torch.compile(model_func, backend=backend_func)
model_no_func = torch.compile(model_no_func, backend=backend_no_func)
# deepcopy inputs to prevent potential in place mutation
outputs_func = model_func(*copy.deepcopy(inputs_func))
outputs_no_func = model_no_func(*copy.deepcopy(inputs_no_func))
torch.testing.assert_close(outputs_func, outputs_no_func)
# check if the functionalization pass is applied
for op in model.ops_in_model(do_fusion):
find_auto_fn(backend_no_func.graph_post_pass.nodes, op)
assert find_auto_fn_maybe(backend_func.graph_post_pass.nodes, op) is None
# make sure the ops were all de-functionalized
found = dict()
for node in backend_func.graph_post_pass.nodes:
for op in model.ops_in_model(do_fusion):
if is_func(node, op):
found[op] = True
for op in model.ops_not_in_model():
if is_func(node, op):
found[op] = True
assert all(found[op] for op in model.ops_in_model(do_fusion))
assert all(not found.get(op) for op in model.ops_not_in_model())
@@ -1,418 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit test for the MLADualRMSNormFusionPass.
The pass fuses paired q/kv RMS norms in MLA attention into a single
fused_mla_dual_rms_norm op backed by AITER's fused_qk_rmsnorm kernel.
"""
import pytest
import torch
import vllm.config
from tests.compile.backend import TestBackend
from vllm._aiter_ops import (
is_aiter_found_and_supported,
rocm_aiter_ops,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.platforms import current_platform
# MLA attention geometry for DeepSeek-V3 / Kimi-K2
Q_DIM = 1536
KV_C_DIM = 512
K_PE_DIM = 64
EPS = 1e-6
FP8_DTYPE = current_platform.fp8_dtype()
class MLADualRMSNormTestModel(torch.nn.Module):
"""Minimal model reproducing the MLA dual RMS norm pattern:
linear -> split([q_dim, kv_dim])
+-- q_c (getitem 0) -> rms_norm(q_w, eps) -> linear
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
+-- kv_c (getitem 0) -> rms_norm(kv_w, eps)
+-- k_pe
"""
def __init__(
self,
hidden_size: int,
q_dim: int = Q_DIM,
kv_c_dim: int = KV_C_DIM,
k_pe_dim: int = K_PE_DIM,
eps: float = EPS,
):
super().__init__()
self.q_dim = q_dim
self.kv_dim = kv_c_dim + k_pe_dim
self.kv_c_dim = kv_c_dim
self.k_pe_dim = k_pe_dim
self.proj = torch.nn.Linear(hidden_size, q_dim + self.kv_dim, bias=False)
self.q_norm = RMSNorm(q_dim, eps=eps)
self.kv_norm = RMSNorm(kv_c_dim, eps=eps)
self.q_b_proj = torch.nn.Linear(q_dim, hidden_size, bias=False)
def forward(
self, x: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
# Avoid graph input being a direct arg to a matched pattern node
x = torch.relu(x)
projected = self.proj(x)
q_c, kv_lora = projected.split([self.q_dim, self.kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([self.kv_c_dim, self.k_pe_dim], dim=-1)
q_normed = self.q_norm(q_c)
kv_normed = self.kv_norm(kv_c)
q_out = self.q_b_proj(q_normed)
return q_out, kv_normed, k_pe
def ops_in_model_before(self):
return [torch.ops.vllm_ir.rms_norm.default]
def ops_in_model_after(self):
return [torch.ops.vllm.fused_mla_dual_rms_norm.default]
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [7168])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def test_fuse_mla_dual_rms_norm(
dtype: torch.dtype,
hidden_size: int,
monkeypatch: pytest.MonkeyPatch,
):
torch._dynamo.reset()
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm"],
pass_config=PassConfig(
fuse_mla_dual_rms_norm=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
MLADualRMSNormFusionPass,
)
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(42)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
fusion_pass = MLADualRMSNormFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
model = MLADualRMSNormTestModel(hidden_size)
x = torch.randn(1, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
outputs_unfused = model(x)
model_fused = torch.compile(model, backend=backend)
outputs_fused = model_fused(x)
torch.testing.assert_close(outputs_unfused, outputs_fused, atol=1e-2, rtol=1e-2)
assert fusion_pass.matched_count == 1, (
f"Expected 1 fused pair, got {fusion_pass.matched_count}"
)
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
class MLADualRMSNormFp8PerTokenTestModel(torch.nn.Module):
"""Minimal model reproducing the FP8 MLA attention path with *per-token* quant:
linear -> split([q_dim, kv_dim])
+-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fused_dynamic_quant -> dequant
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
+-- kv_c (getitem 0) -> rms_norm (bf16)
+-- k_pe
"""
def __init__(
self,
hidden_size: int,
q_dim: int = Q_DIM,
kv_c_dim: int = KV_C_DIM,
k_pe_dim: int = K_PE_DIM,
eps: float = EPS,
):
super().__init__()
self.q_dim = q_dim
self.kv_dim = kv_c_dim + k_pe_dim
self.kv_c_dim = kv_c_dim
self.k_pe_dim = k_pe_dim
self.eps = eps
self.proj = torch.nn.Linear(hidden_size, q_dim + self.kv_dim, bias=False)
self.q_weight = torch.nn.Parameter(torch.ones(q_dim))
self.kv_norm = RMSNorm(kv_c_dim, eps=eps)
def _dequant(self, x_fp8: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
# Per-token: a single (M, 1) scale broadcast across the row.
return (x_fp8.to(torch.float32) * scale).to(torch.bfloat16)
def forward(self, x: torch.Tensor):
# Avoid graph input being a direct arg to a matched pattern node
x = torch.relu(x)
projected = self.proj(x)
q_c, kv_lora = projected.split([self.q_dim, self.kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([self.kv_c_dim, self.k_pe_dim], dim=-1)
q_fp8, q_scale = torch.ops.vllm.rocm_aiter_rmsnorm_fused_dynamic_quant(
q_c, self.q_weight, self.eps, FP8_DTYPE
)
kv_normed = self.kv_norm(kv_c)
return self._dequant(q_fp8, q_scale), kv_normed, k_pe
def ops_in_model_before(self):
return [
torch.ops.vllm.rocm_aiter_rmsnorm_fused_dynamic_quant.default,
torch.ops.vllm_ir.rms_norm.default,
]
def ops_in_model_after(self):
return [torch.ops.vllm.fused_mla_dual_rms_norm_per_token_quant.default]
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [7168])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def test_fuse_mla_dual_rms_norm_fp8_per_token(
dtype: torch.dtype,
hidden_size: int,
monkeypatch: pytest.MonkeyPatch,
):
torch._dynamo.reset()
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm"],
pass_config=PassConfig(
fuse_mla_dual_rms_norm=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
MLADualRMSNormFusionPass,
)
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(42)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
fusion_pass = MLADualRMSNormFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
model = MLADualRMSNormFp8PerTokenTestModel(hidden_size)
x = torch.randn(4, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
with torch.inference_mode():
outputs_unfused = model(x)
model_fused = torch.compile(model, backend=backend)
outputs_fused = model_fused(x)
q_deq_u, kv_normed_u, k_pe_u = outputs_unfused
q_deq_f, kv_normed_f, k_pe_f = outputs_fused
torch.testing.assert_close(k_pe_u, k_pe_f, atol=0, rtol=0)
torch.testing.assert_close(kv_normed_u, kv_normed_f, atol=1e-2, rtol=1e-2)
E4M3_STEP = 0.125
exact_frac = (q_deq_u == q_deq_f).float().mean().item()
assert exact_frac > 0.99, (
f"q: only {exact_frac:.4f} of elements bit-exact; scales likely differ"
)
torch.testing.assert_close(q_deq_u, q_deq_f, atol=1e-2, rtol=E4M3_STEP)
assert fusion_pass.matched_count == 1, (
f"Expected 1 fused pair, got {fusion_pass.matched_count}"
)
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
class MLADualRMSNormFp8GroupTestModel(torch.nn.Module):
"""Minimal model reproducing the FP8 MLA attention path with *group* quant
(block-scaled fp8 weights, 128-wide activation groups):
linear -> split([q_dim, kv_dim])
+-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fp8_group_quant -> dequant
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
+-- kv_c (getitem 0) -> rms_norm (bf16)
+-- k_pe
"""
GROUP_SIZE = 128
def __init__(
self,
hidden_size: int,
q_dim: int = Q_DIM,
kv_c_dim: int = KV_C_DIM,
k_pe_dim: int = K_PE_DIM,
eps: float = EPS,
):
super().__init__()
self.q_dim = q_dim
self.kv_dim = kv_c_dim + k_pe_dim
self.kv_c_dim = kv_c_dim
self.k_pe_dim = k_pe_dim
self.eps = eps
self.proj = torch.nn.Linear(hidden_size, q_dim + self.kv_dim, bias=False)
self.q_weight = torch.nn.Parameter(torch.ones(q_dim))
self.kv_norm = RMSNorm(kv_c_dim, eps=eps)
def _dequant(self, x_fp8: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
# Group: one (M, N / 128) scale per 128-column group of the row.
m = x_fp8.shape[0]
x = x_fp8.to(torch.float32).view(m, -1, self.GROUP_SIZE)
return (x * scale.view(m, -1, 1)).view(m, -1).to(torch.bfloat16)
def forward(self, x: torch.Tensor):
# Avoid graph input being a direct arg to a matched pattern node
x = torch.relu(x)
projected = self.proj(x)
q_c, kv_lora = projected.split([self.q_dim, self.kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([self.kv_c_dim, self.k_pe_dim], dim=-1)
q_fp8, q_scale = torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant(
q_c, self.q_weight, self.eps, self.GROUP_SIZE
)
kv_normed = self.kv_norm(kv_c)
# Also return the group scales so the test can check them directly:
# equal scales pin any fp8 payload difference to rounding alone.
return self._dequant(q_fp8, q_scale), q_scale, kv_normed, k_pe
def ops_in_model_before(self):
return [
torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant.default,
torch.ops.vllm_ir.rms_norm.default,
]
def ops_in_model_after(self):
return [torch.ops.vllm.fused_mla_dual_rms_norm_group_quant.default]
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [7168])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def test_fuse_mla_dual_rms_norm_fp8_group(
dtype: torch.dtype,
hidden_size: int,
monkeypatch: pytest.MonkeyPatch,
):
torch._dynamo.reset()
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm"],
pass_config=PassConfig(
fuse_mla_dual_rms_norm=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
MLADualRMSNormFusionPass,
)
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(42)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
fusion_pass = MLADualRMSNormFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
model = MLADualRMSNormFp8GroupTestModel(hidden_size)
x = torch.randn(64, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
with torch.inference_mode():
outputs_unfused = model(x)
model_fused = torch.compile(model, backend=backend)
outputs_fused = model_fused(x)
q_deq_u, q_scale_u, kv_normed_u, k_pe_u = outputs_unfused
q_deq_f, q_scale_f, kv_normed_f, k_pe_f = outputs_fused
torch.testing.assert_close(k_pe_u, k_pe_f, atol=0, rtol=0)
torch.testing.assert_close(kv_normed_u, kv_normed_f, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(q_scale_u, q_scale_f, atol=0, rtol=1e-5)
E4M3_STEP = 0.125
exact_frac = (q_deq_u == q_deq_f).float().mean().item()
assert exact_frac > 0.99, f"q: only {exact_frac:.4f} of elements bit-exact"
torch.testing.assert_close(q_deq_u, q_deq_f, atol=1e-2, rtol=E4M3_STEP)
assert fusion_pass.matched_count == 1, (
f"Expected 1 fused pair, got {fusion_pass.matched_count}"
)
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
-813
View File
@@ -1,813 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.config
import vllm.ir.ops
import vllm.plugins
from tests.compile.backend import TestBackend
from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor
from tests.utils import TestFP8Layer
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
from vllm.compilation.passes.fusion.matcher_utils import QUANT_OPS
from vllm.compilation.passes.fusion.rms_quant_fusion import (
FUSED_OPS,
FusedRMSQuantKey,
RMSNormQuantFusionPass,
)
from vllm.compilation.passes.fx_utils import find_auto_fn_maybe
from vllm.compilation.passes.utility.fix_functionalization import (
FixFunctionalizationPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
from vllm.model_executor.kernels.linear import (
AiterFp8BlockScaledMMKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
CutlassFp8BlockScaledMMKernel,
CutlassFP8ScaledMMLinearKernel,
DeepGemmFp8BlockScaledMMKernel,
FlashInferFp8DeepGEMMDynamicBlockScaledKernel,
FlashInferFP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ROCmFP8ScaledMMLinearKernel,
RowWiseTorchFP8ScaledMMLinearKernel,
TritonFp8BlockScaledMMKernel,
_KernelT,
)
from vllm.model_executor.layers.layernorm import RMSNorm, RMSNormGated
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
create_fp8_quant_key,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
cutlass_block_fp8_supported,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
is_deep_gemm_e8m0_used,
is_deep_gemm_supported,
)
# Kernel and group_shape combinations: (kernel, group_shape)
# CUDA kernels
CUDA_KERNEL_GROUPSHAPE_COMBINATIONS = [
# FlashInferFP8ScaledMMLinearKernel supports both per-tensor only
(FlashInferFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR),
# CutlassFP8ScaledMMLinearKernel supports both per-tensor and per-token
(CutlassFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN),
(CutlassFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR),
# PerTensorTorchFP8ScaledMMLinearKernel only supports per-tensor
(PerTensorTorchFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR),
# ChannelWiseTorchFP8ScaledMMLinearKernel only supports per-token
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN),
# Blockwise group shapes
(FlashInferFp8DeepGEMMDynamicBlockScaledKernel, GroupShape(1, 128)),
(CutlassFp8BlockScaledMMKernel, GroupShape(1, 128)),
(DeepGemmFp8BlockScaledMMKernel, GroupShape(1, 128)),
(TritonFp8BlockScaledMMKernel, GroupShape(1, 128)),
(TritonFp8BlockScaledMMKernel, GroupShape(1, 64)),
]
# ROCm kernels
ROCM_KERNEL_GROUPSHAPE_COMBINATIONS = [
# ROCmFP8ScaledMMLinearKernel supports per-tensor only
(ROCmFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR),
# RowWiseTorchFP8ScaledMMLinearKernel only supports per-token
(RowWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN),
# ChannelWiseTorchFP8ScaledMMLinearKernel only supports per-token
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN),
# Blockwise group shapes (no kernel abstraction)
(TritonFp8BlockScaledMMKernel, GroupShape(1, 128)),
(TritonFp8BlockScaledMMKernel, GroupShape(1, 64)),
]
KERNEL_GROUPSHAPE_COMBINATIONS = (
CUDA_KERNEL_GROUPSHAPE_COMBINATIONS
if current_platform.is_cuda()
else ROCM_KERNEL_GROUPSHAPE_COMBINATIONS
)
# For Aiter tests we toggle use_aiter_quant_op
AITER_KERNEL_GROUPSHAPE_COMBINATIONS = [
# Per-token with RowWiseTorchFP8ScaledMMLinearKernel
(RowWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, True),
(RowWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, False),
# Per-token with ChannelWiseTorchFP8ScaledMMLinearKernel
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, True),
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, False),
# Static per-tensor: only the aiter quant op is matched by the pass
(ROCmFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR, True),
# Blockwise
(AiterFp8BlockScaledMMKernel, GroupShape(1, 128), True),
]
class TestModel(torch.nn.Module):
def __init__(
self,
hidden_size: int,
eps: float,
force_kernel: type[_KernelT] | None,
group_shape: GroupShape,
dtype: torch.dtype,
use_aiter_fusion: bool = False,
use_aiter_quant: bool = False,
*args,
**kwargs,
):
super().__init__(*args, **kwargs)
self.fp8_linear_layers: list[torch.nn.Module]
self.group_shape = group_shape
self.use_aiter_quant_op = use_aiter_quant
self.use_aiter_fusion = use_aiter_fusion
self.norm = [RMSNorm(hidden_size, eps) for _ in range(4)]
self.enable_rms_norm_custom_op = self.norm[0].enabled()
# Determine if blockwise based on group_shape
is_blockwise = group_shape.is_per_group()
if is_blockwise:
block_size = group_shape.col
self.activation_quant_key = create_fp8_quant_key(
static=False, group_shape=group_shape
)
self.weight_quant_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(block_size, block_size)
)
else:
is_static = group_shape == GroupShape.PER_TENSOR
self.activation_quant_key = create_fp8_quant_key(
is_static, group_shape=group_shape
)
self.weight_quant_key = create_fp8_quant_key(
static=True, group_shape=group_shape
)
self.fp8_linear_layers = [
TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
force_kernel=force_kernel,
transpose_weights=use_aiter_fusion,
input_dtype=dtype,
)
for _ in range(3)
]
# Enable aiter quantization if requested
for layer in self.fp8_linear_layers:
layer.kernel.quant_fp8.use_aiter = use_aiter_quant
self.enable_quant_fp8_custom_op = self.fp8_linear_layers[
0
].is_quant_fp8_enabled()
def forward(self, x):
# avoid having graph input be an arg to a pattern directly
x = resid = torch.relu(x)
y = self.norm[0](x)
x2 = self.fp8_linear_layers[0](y)
# make sure resid is used for replacement to work
y2, resid = self.norm[1](x2, resid)
x3 = self.fp8_linear_layers[1](y2)
y3, resid = self.norm[2](x3, resid) # use resid here
x4 = self.fp8_linear_layers[2](y3)
y4, resid = self.norm[3](x4, resid) # use resid here
return y4
def ops_in_model_before(self):
if self.group_shape.is_per_group():
# Blockwise path
if self.use_aiter_fusion and self.use_aiter_quant_op:
return [rocm_aiter_ops.get_group_quant_op()]
elif self.group_shape.is_per_tensor():
if self.use_aiter_quant_op:
return [rocm_aiter_ops.get_per_tensor_quant_op()]
else:
if self.use_aiter_quant_op:
return [rocm_aiter_ops.get_per_token_quant_op()]
# Common path
return (
[QUANT_OPS[self.activation_quant_key]]
if self.enable_quant_fp8_custom_op
else [torch.ops.aten.reciprocal]
)
def ops_in_model_after(self):
if self.use_aiter_fusion:
if self.group_shape.is_per_group():
# Blockwise aiter fusion
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
AiterFusedAddRMSFp8GroupQuantPattern,
AiterRMSFp8GroupQuantPattern,
)
return [
AiterFusedAddRMSFp8GroupQuantPattern.FUSED_OP,
AiterRMSFp8GroupQuantPattern.FUSED_OP,
]
elif self.group_shape.is_per_tensor():
# Static per-tensor aiter fusion
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
AiterFusedAddRMSNormStaticQuantPattern,
AiterRMSNormStaticQuantPattern,
)
return [
AiterFusedAddRMSNormStaticQuantPattern.FUSED_OP,
AiterRMSNormStaticQuantPattern.FUSED_OP,
]
else:
# Per-token aiter fusion
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
AiterFusedAddRMSNormDynamicQuantPattern,
AiterRMSNormDynamicQuantPattern,
)
return [
AiterFusedAddRMSNormDynamicQuantPattern.FUSED_OP,
AiterRMSNormDynamicQuantPattern.FUSED_OP,
]
# Regular fusion
return [
FUSED_OPS[FusedRMSQuantKey(self.activation_quant_key, True)],
FUSED_OPS[FusedRMSQuantKey(self.activation_quant_key, False)],
]
def ops_in_model_before_partial(self):
return [
torch.ops.vllm_ir.rms_norm,
torch.ops.vllm_ir.fused_add_rms_norm.default,
]
class AddRMSNormNvfp4Model(torch.nn.Module):
def __init__(self, hidden_size: int, eps: float):
super().__init__()
self.norm = RMSNorm(hidden_size, eps)
self.activation_scale = torch.rand((1, 1), dtype=torch.float32)
weight = torch.rand((hidden_size, hidden_size))
self.weight, self.weight_scale, weight_global_scale = quant_nvfp4_tensor(weight)
self.alpha = (1 / (weight_global_scale * self.activation_scale)).reshape(1)
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
x = residual = torch.relu(x)
x, residual = self.norm(x, residual)
x, block_scale = scaled_fp4_quant(x, self.activation_scale)
x = cutlass_scaled_fp4_mm(
x,
self.weight,
block_scale,
self.weight_scale,
self.alpha,
out_dtype=torch.get_default_dtype(),
)
return x, residual
def _run_fusion_test(
model,
fusion_pass,
vllm_config,
dtype,
hidden_size,
num_tokens,
):
"""Helper function for common fusion test logic.
Must be called within vllm_config context.
"""
noop_pass = NoOpEliminationPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(noop_pass, fusion_pass, cleanup_pass)
backend2 = TestBackend(noop_pass, cleanup_pass)
x = torch.rand(num_tokens, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
model_fused = torch.compile(model, backend=backend)
result_fused = model_fused(x)
model_unfused = torch.compile(model, backend=backend2)
result_unfused = model_unfused(x)
if dtype == torch.float16:
ATOL, RTOL = (2e-3, 2e-3)
else:
ATOL, RTOL = (1e-2, 1e-2)
torch.testing.assert_close(result_fused, result_unfused, atol=ATOL, rtol=RTOL)
assert fusion_pass.matched_count == 3
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
return backend, backend2
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [256])
@pytest.mark.parametrize("num_tokens", [257])
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
@pytest.mark.parametrize("kernel_groupshape", KERNEL_GROUPSHAPE_COMBINATIONS)
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
@pytest.mark.parametrize("enable_quant_fp8_custom_op", [True, False])
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Only test on CUDA and ROCm"
)
def test_fusion_rmsnorm_quant(
dtype,
hidden_size,
num_tokens,
eps,
kernel_groupshape,
enable_rms_norm_custom_op,
enable_quant_fp8_custom_op,
):
force_kernel, group_shape = kernel_groupshape
if not enable_quant_fp8_custom_op and group_shape.is_per_group():
pytest.skip("Unsupported unwrapped quant fp8 op for blockwise quantization")
if group_shape == GroupShape(1, 64) and (
cutlass_block_fp8_supported() or is_deep_gemm_supported()
):
pytest.skip("Unsupported group shape 64 for CUTLASS/DeepGemm")
# TODO(quant-rms-fusion): DeepGEMM UE8M0 activation quant on B200 lowers
# to a packed int32-scale op (per_token_group_quant_fp8_packed_for_deepgemm),
# but the rms+quant fusion pattern only matches the fp32-scale variant, so
# the fused output gets a mismatched scale layout and produces NaN. Only
# reproduces on bf16 (DeepGEMM UE8M0 on B200 is bf16-only).
# To re-enable: make rms_norm_per_block_quant emit packed UE8M0 scales
# and extend the fusion pattern to rewrite the packed activation quant.
deepgemm_kernels = (
DeepGemmFp8BlockScaledMMKernel,
FlashInferFp8DeepGEMMDynamicBlockScaledKernel,
)
if (
dtype == torch.bfloat16
and force_kernel in deepgemm_kernels
and is_deep_gemm_e8m0_used()
):
pytest.skip(
"rms+quant fusion does not yet match the packed UE8M0 DeepGEMM path"
)
custom_ops = []
if enable_rms_norm_custom_op:
custom_ops.append("+rms_norm")
if enable_quant_fp8_custom_op:
custom_ops.append("+quant_fp8")
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops,
pass_config=PassConfig(
fuse_norm_quant=True, fuse_act_quant=True, eliminate_noops=True
),
),
)
with (
vllm.config.set_current_vllm_config(vllm_config),
vllm_config.kernel_config.ir_op_priority.set_priority(),
):
# Setup device before model creation
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(1)
fusion_pass = RMSNormQuantFusionPass(vllm_config)
model = TestModel(
hidden_size=hidden_size,
eps=eps,
force_kernel=force_kernel,
group_shape=group_shape,
dtype=dtype,
use_aiter_fusion=False,
use_aiter_quant=False,
)
if any(
type(layer.kernel) is not force_kernel for layer in model.fp8_linear_layers
):
pytest.skip(f"{force_kernel.__name__} is not supported on this platform")
backend, _ = _run_fusion_test(
model, fusion_pass, vllm_config, dtype, hidden_size, num_tokens
)
backend.check_before_ops(
model.ops_in_model_before_partial(), fully_replaced=False
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
@pytest.mark.parametrize(
("num_tokens", "hidden_size"),
[(1, 256), (32, 1024), (257, 4096)],
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
def test_fusion_add_rmsnorm_nvfp4_quant(
dtype: torch.dtype, eps: float, num_tokens: int, hidden_size: int
):
fused_op = getattr(
torch.ops.vllm,
"flashinfer_fused_add_rms_norm_nvfp4_quant",
None,
)
if not current_platform.has_device_capability(100) or fused_op is None:
pytest.skip("FlashInfer add-RMSNorm NVFP4 fusion is not available")
assert fused_op is not None
fused_op_default = fused_op.default
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(
fuse_norm_quant=True,
eliminate_noops=True,
),
),
)
with (
vllm.config.set_current_vllm_config(vllm_config),
vllm_config.kernel_config.ir_op_priority.set_priority(),
):
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(1)
fusion_pass = RMSNormQuantFusionPass(vllm_config)
fused_backend = TestBackend(
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
FixFunctionalizationPass(vllm_config),
)
unfused_backend = TestBackend(
NoOpEliminationPass(vllm_config),
PostCleanupPass(vllm_config),
FixFunctionalizationPass(vllm_config),
)
model = AddRMSNormNvfp4Model(hidden_size=hidden_size, eps=eps)
x = torch.rand(num_tokens, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
result_fused = torch.compile(model, backend=fused_backend)(x)
result_unfused = torch.compile(model, backend=unfused_backend)(x)
torch.testing.assert_close(result_fused, result_unfused, atol=2e-1, rtol=2e-1)
assert fusion_pass.matched_count == 1
fused_backend.check_before_ops([torch.ops._C.scaled_fp4_quant.out])
fused_backend.check_after_ops([fused_op_default])
assert (
find_auto_fn_maybe(
fused_backend.graph_post_pass.nodes,
fused_op_default,
)
is None
)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [256])
@pytest.mark.parametrize("num_tokens", [257])
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
@pytest.mark.parametrize(
"kernel_groupshape_quant", AITER_KERNEL_GROUPSHAPE_COMBINATIONS
)
@pytest.mark.skipif(
(not current_platform.is_rocm() or not IS_AITER_FOUND),
reason="Only test on ROCm with aiter package installed",
)
def test_aiter_fusion_rmsnorm_quant(
dtype: torch.dtype,
hidden_size: int,
num_tokens: int,
eps: float,
kernel_groupshape_quant: tuple,
monkeypatch: pytest.MonkeyPatch,
):
force_kernel, group_shape, use_aiter_quant_op = kernel_groupshape_quant
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm", "+quant_fp8"],
pass_config=PassConfig(fuse_norm_quant=True, eliminate_noops=True),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
RocmAiterRMSNormQuantFusionPass,
)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(1)
fusion_pass = RocmAiterRMSNormQuantFusionPass(vllm_config)
model = TestModel(
hidden_size=hidden_size,
eps=eps,
force_kernel=force_kernel,
group_shape=group_shape,
dtype=dtype,
use_aiter_fusion=True, # Always use aiter fusion ops in aiter test
use_aiter_quant=use_aiter_quant_op, # Toggle aiter quantization
)
_run_fusion_test(
model, fusion_pass, vllm_config, dtype, hidden_size, num_tokens
)
class TestGatedModel(torch.nn.Module):
"""Model that uses RMSNormGated + reshape + group FP8 quant + linear.
Mimics GatedDeltaNetAttention's output projection path where:
- RMSNormGated operates on per-head tensors, either (N, H, D) as
QwenGatedDeltaNetAttention passes them or flattened to (N*H, D) as
OlmoHybridGatedDeltaNetAttention does
- Output is reshaped to (N, H*D) before group quantization + linear
"""
def __init__(
self,
num_heads: int,
head_dim: int,
eps: float,
force_kernel: type[_KernelT],
group_shape: GroupShape,
dtype: torch.dtype,
use_aiter_quant: bool = True,
flatten_heads: bool = True,
):
super().__init__()
self.num_heads = num_heads
self.head_dim = head_dim
self.flatten_heads = flatten_heads
hidden_dim = num_heads * head_dim
self.norm = RMSNormGated(
head_dim,
eps=eps,
group_size=None,
norm_before_gate=True,
)
self.activation_quant_key = create_fp8_quant_key(
static=False, group_shape=group_shape
)
self.weight_quant_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(group_shape.col, group_shape.col)
)
self.fp8_linear = TestFP8Layer(
weight_shape=(hidden_dim, hidden_dim),
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
force_kernel=force_kernel,
transpose_weights=True,
input_dtype=dtype,
)
self.fp8_linear.kernel.quant_fp8.use_aiter = use_aiter_quant
def forward(self, x, z):
num_heads = self.num_heads
head_dim = self.head_dim
hidden_dim = num_heads * head_dim
x = torch.relu(x)
z = torch.relu(z)
x_heads = x.reshape(-1, num_heads, head_dim)
z_heads = z.reshape(-1, num_heads, head_dim)
if self.flatten_heads:
x_heads = x_heads.reshape(-1, head_dim)
z_heads = z_heads.reshape(-1, head_dim)
normed = self.norm(x_heads, z_heads)
merged = normed.reshape(-1, hidden_dim)
out = self.fp8_linear(merged)
return out
def ops_in_model_after(self):
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
AiterRMSNormGatedFp8GroupQuantPattern,
)
return [AiterRMSNormGatedFp8GroupQuantPattern.FUSED_OP]
class _MockGDNLayer:
"""Minimal mock to populate static_forward_context for pass discovery.
Uses __class__ assignment to pass isinstance checks against
GatedDeltaNetAttention without requiring a full config-based init.
"""
def __init__(self, num_v_heads: int, head_v_dim: int, tp_size: int = 1):
self.num_v_heads = num_v_heads
self.head_v_dim = head_v_dim
self.tp_size = tp_size
from vllm.model_executor.layers.mamba.gdn.base import (
GatedDeltaNetAttention,
)
self.__class__ = GatedDeltaNetAttention
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("num_heads", [2])
@pytest.mark.parametrize("head_dim", [128])
@pytest.mark.parametrize("num_tokens", [8])
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
@pytest.mark.parametrize("flatten_heads", [True, False])
@pytest.mark.parametrize("quant_fp8_op", ["-quant_fp8", "+quant_fp8"])
@pytest.mark.skipif(
(not current_platform.is_rocm() or not IS_AITER_FOUND),
reason="Only test on ROCm with aiter package installed",
)
def test_aiter_fusion_rmsnorm_gated_quant(
dtype: torch.dtype,
num_heads: int,
head_dim: int,
num_tokens: int,
eps: float,
flatten_heads: bool,
quant_fp8_op: str,
monkeypatch: pytest.MonkeyPatch,
):
group_shape = GroupShape(1, 128)
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["-rms_norm", "-silu_and_mul", quant_fp8_op],
pass_config=PassConfig(fuse_norm_quant=True, eliminate_noops=True),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
RocmAiterRMSNormQuantFusionPass,
)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
# Register a mock GDN layer so the pass discovers num_heads/head_dim
mock_gdn = _MockGDNLayer(num_v_heads=num_heads, head_v_dim=head_dim, tp_size=1)
vllm_config.compilation_config.static_forward_context["mock_gdn_layer"] = (
mock_gdn
)
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(1)
fusion_pass = RocmAiterRMSNormQuantFusionPass(vllm_config)
model = TestGatedModel(
num_heads=num_heads,
head_dim=head_dim,
eps=eps,
force_kernel=AiterFp8BlockScaledMMKernel,
group_shape=group_shape,
dtype=dtype,
use_aiter_quant=True,
flatten_heads=flatten_heads,
)
noop_pass = NoOpEliminationPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(noop_pass, fusion_pass, cleanup_pass)
backend2 = TestBackend(noop_pass, cleanup_pass)
hidden_dim = num_heads * head_dim
x = torch.rand(num_tokens, hidden_dim)
z = torch.rand(num_tokens, hidden_dim)
torch._dynamo.mark_dynamic(x, 0)
torch._dynamo.mark_dynamic(z, 0)
model_fused = torch.compile(model, backend=backend)
result_fused = model_fused(x, z)
model_unfused = torch.compile(model, backend=backend2)
result_unfused = model_unfused(x, z)
# The opaque AITER quant op preserves a BF16 intermediate in the
# unfused path, while the fused kernel normalizes and quantizes
# directly. Allow for that extra rounding before the FP8 GEMM.
tol = 2e-2 if quant_fp8_op == "+quant_fp8" else 1e-2
torch.testing.assert_close(result_fused, result_unfused, atol=tol, rtol=tol)
assert fusion_pass.matched_count == 1
backend.check_after_ops(model.ops_in_model_after())
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("num_heads", [2])
@pytest.mark.parametrize("head_dim", [128])
@pytest.mark.parametrize("num_tokens", [8])
@pytest.mark.parametrize("eps", [1e-6])
@pytest.mark.skipif(
(not current_platform.is_rocm() or not IS_AITER_FOUND),
reason="Only test on ROCm with aiter package installed",
)
def test_aiter_fusion_rmsnorm_gated_quant_no_gdn_layers(
dtype: torch.dtype,
num_heads: int,
head_dim: int,
num_tokens: int,
eps: float,
monkeypatch: pytest.MonkeyPatch,
):
"""Verify that without GDN layers in static_forward_context,
the gated pattern is not registered and no matches occur."""
group_shape = GroupShape(1, 128)
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["-rms_norm", "-silu_and_mul", "-quant_fp8"],
pass_config=PassConfig(fuse_norm_quant=True, eliminate_noops=True),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
RocmAiterRMSNormQuantFusionPass,
)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(1)
# No mock GDN layer registered -- pass should not register gated pattern
fusion_pass = RocmAiterRMSNormQuantFusionPass(vllm_config)
model = TestGatedModel(
num_heads=num_heads,
head_dim=head_dim,
eps=eps,
force_kernel=AiterFp8BlockScaledMMKernel,
group_shape=group_shape,
dtype=dtype,
use_aiter_quant=True,
)
noop_pass = NoOpEliminationPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(noop_pass, fusion_pass, cleanup_pass)
hidden_dim = num_heads * head_dim
x = torch.rand(num_tokens, hidden_dim)
z = torch.rand(num_tokens, hidden_dim)
torch._dynamo.mark_dynamic(x, 0)
torch._dynamo.mark_dynamic(z, 0)
model_fused = torch.compile(model, backend=backend)
model_fused(x, z)
assert fusion_pass.matched_count == 0
-582
View File
@@ -1,582 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import pytest
import torch._dynamo
from tests.compile.backend import LazyInitPass, TestBackend
from tests.utils import TestFP8Layer, flat_product
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
dense_kv_cache_views,
)
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
from vllm.compilation.passes.fusion.attn_quant_fusion import (
ATTN_OP,
AttnQuantFusionPass,
)
from vllm.compilation.passes.fusion.matcher_utils import QUANT_OPS
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
AttentionConfig,
CacheConfig,
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
SchedulerConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.backends.utils import (
get_supported_kv_cache_layouts,
resolve_kv_cache_layout,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
get_kv_quant_mode,
)
DEVICE_TYPE = current_platform.device_type
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
class AttentionQuantPatternModel(torch.nn.Module):
"""Base model for AttentionQuantPattern fusion."""
def __init__(
self,
num_qo_heads: int,
num_kv_heads: int,
head_size: int,
device: torch.device,
vllm_config: VllmConfig,
block_size: int,
**kwargs,
):
super().__init__()
self.num_qo_heads = num_qo_heads
self.num_kv_heads = num_kv_heads
self.head_size = head_size
self.device = device
self.vllm_config = vllm_config
self.dtype = vllm_config.model_config.dtype
self.attn = Attention(
num_heads=self.num_qo_heads,
head_size=self.head_size,
scale=1.0 / (self.head_size**0.5),
num_kv_heads=self.num_kv_heads,
cache_config=vllm_config.cache_config,
prefix="model.layers.0.self_attn.attn",
)
self.attn._k_scale = self.attn._k_scale.to(device)
self.attn._v_scale = self.attn._v_scale.to(device)
self.block_size = block_size
# Initialize attn MetadataBuilder (match Attention.get_kv_cache_spec)
self.builder = self.attn.attn_backend.get_builder_cls()(
kv_cache_spec=AttentionSpec(
block_size=self.block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
dtype=self.attn.kv_cache_torch_dtype,
kv_quant_mode=get_kv_quant_mode(self.attn.kv_cache_dtype),
),
layer_names=[self.attn.layer_name],
vllm_config=self.vllm_config,
device=self.device,
)
def build_attn_metadata(self, batch_size: int) -> AttentionMetadata:
"""Initialize attention metadata."""
# TODO (Rohan138) reuse utils from vllm/v1/worker/gpu/attn_utils.py
# Create common attn metadata
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
common_attn_metadata = create_common_attn_metadata(
batch_spec, self.block_size, self.device, arange_block_indices=True
)
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
num_blocks = batch_size * max_blocks
spec = self.attn.attn_backend.customize_spec(
AttentionSpec(
block_size=self.block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
dtype=self.attn.kv_cache_torch_dtype,
kv_quant_mode=get_kv_quant_mode(self.attn.kv_cache_dtype),
)
)
supported = get_supported_kv_cache_layouts([self.attn.attn_backend])
layout = resolve_kv_cache_layout(
self.vllm_config, [[m.name for m in supported]]
)
raw_tensor = torch.zeros(
num_blocks * spec.page_size_bytes,
dtype=torch.int8,
device=self.device,
)
kv_cache = dense_kv_cache_views(
raw_tensor,
spec,
num_blocks,
num_layers=1,
layout=layout,
)[0]
self.attn.kv_cache = kv_cache
# Build attn metadata
self.attn_metadata = self.builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
return self.attn_metadata
class TestAttentionFp8StaticQuantPatternModel(AttentionQuantPatternModel):
"""Test model for AttentionFp8StaticQuantPattern fusion."""
quant_key = kFp8StaticTensorSym
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
hidden_size = self.num_qo_heads * self.head_size
self.fp8_linear = TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
device=self.device,
input_dtype=self.dtype,
)
w = kwargs.get("w")
if w is not None:
self.fp8_linear.weight = w["weight"]
self.fp8_linear.weight_scale = w["wscale"]
self.fp8_linear.input_scale = w["scale"]
self.w = {
"weight": self.fp8_linear.weight,
"wscale": self.fp8_linear.weight_scale,
"scale": self.fp8_linear.input_scale,
}
def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor):
"""Forward pass that creates the pattern to be fused."""
attn_output = self.attn(q, k, v)
return self.fp8_linear(attn_output)
class TestAttentionAiterFp8StaticQuantPatternModel(
TestAttentionFp8StaticQuantPatternModel
):
aiter_quant = True
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fp8_linear.kernel.quant_fp8.use_aiter = True
assert self.fp8_linear.input_scale is not None
self.fp8_linear.input_scale.fill_(0.125)
class TestAttentionNvfp4QuantPatternModel(AttentionQuantPatternModel):
"""Test model for AttentionNvfp4QuantPattern fusion."""
quant_key = kNvfp4Dynamic
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
hidden_size = self.num_qo_heads * self.head_size
self.w = kwargs.get(
"w",
{
"weight": torch.randint(
256,
(hidden_size, hidden_size // 2),
dtype=FP4_DTYPE,
device=self.device,
),
"wscale_swizzled": torch.randn(hidden_size, hidden_size // 16).to(
dtype=FP8_DTYPE, device=self.device
),
"wscale": torch.tensor([500], dtype=torch.float32, device=self.device),
"scale": torch.tensor([0.002], dtype=torch.float32, device=self.device),
},
)
def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor):
"""Forward pass that creates the pattern to be fused."""
attn_output = self.attn(q, k, v)
quant_output, output_block_scale = scaled_fp4_quant(
attn_output, 1 / self.w["scale"]
)
return cutlass_scaled_fp4_mm(
a=quant_output,
b=self.w["weight"],
block_scale_a=output_block_scale,
block_scale_b=self.w["wscale_swizzled"],
alpha=self.w["scale"] * self.w["wscale"],
out_dtype=attn_output.dtype,
)
PATTERN_TEST_MODELS_FP8: list[tuple[str, type]] = []
PATTERN_TEST_MODELS_FP4: list[tuple[str, type]] = []
HEADS: list[tuple[int, int]] = []
BACKENDS_FP8: list[AttentionBackendEnum] = []
BACKENDS_FP4: list[AttentionBackendEnum] = []
if current_platform.is_cuda():
HEADS = [(64, 8), (40, 8)]
PATTERN_TEST_MODELS_FP8 = [
(
"RedHatAI/Meta-Llama-3.1-8B-FP8",
TestAttentionFp8StaticQuantPatternModel,
)
]
PATTERN_TEST_MODELS_FP4 = [
(
"nvidia/Llama-3.1-8B-Instruct-NVFP4",
TestAttentionNvfp4QuantPatternModel,
)
]
BACKENDS_FP8 = [AttentionBackendEnum.TRITON_ATTN, AttentionBackendEnum.FLASHINFER]
BACKENDS_FP4 = [AttentionBackendEnum.FLASHINFER]
elif current_platform.is_rocm():
HEADS = [(32, 8), (40, 8)]
PATTERN_TEST_MODELS_FP8 = [
("amd/Llama-3.1-8B-Instruct-FP8-KV", TestAttentionFp8StaticQuantPatternModel)
]
BACKENDS_FP8 = [
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
AttentionBackendEnum.ROCM_ATTN,
AttentionBackendEnum.TRITON_ATTN,
]
@pytest.mark.parametrize("num_qo_heads, num_kv_heads", HEADS)
@pytest.mark.parametrize("head_size", [128])
@pytest.mark.parametrize(
"batch_size", [7, 256, 533] if current_platform.is_cuda() else [8]
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "nvfp4"])
@pytest.mark.parametrize(
"backend, model_name, model_class, custom_ops",
# Test attention+quant_fp8 fusion with custom and torch impls of QuantFP8
list(
flat_product(
BACKENDS_FP8, PATTERN_TEST_MODELS_FP8, ["+quant_fp8", "-quant_fp8"]
)
)
# quant_fp4 only has the custom impl
+ list(flat_product(BACKENDS_FP4, PATTERN_TEST_MODELS_FP4, [""])),
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Only test ROCm or CUDA"
)
@pytest.mark.skipif(not current_platform.supports_fp8(), reason="Need FP8")
def test_attention_quant_pattern(
num_qo_heads: int,
num_kv_heads: int,
head_size: int,
batch_size: int,
dtype: torch.dtype,
kv_cache_dtype: CacheDType,
custom_ops: str,
model_name: str,
model_class: type[AttentionQuantPatternModel],
backend: AttentionBackendEnum,
dist_init,
disable_vllm_compile_cache,
):
"""Test AttentionStaticQuantPattern fusion pass."""
if backend == AttentionBackendEnum.FLASHINFER and (
not current_platform.is_device_capability((10, 0)) or not has_flashinfer()
):
# This also captures the FP4 case
pytest.skip("FlashInfer attn fusion requires Blackwell and flashinfer")
if kv_cache_dtype == "nvfp4" and backend != AttentionBackendEnum.FLASHINFER:
pytest.skip("NVFP4 KV cache is only supported by FlashInfer")
# trtllm-gen has no NVFP4-KV kernel with NVFP4 output, so with an NVFP4 KV
# cache the FP4 output fusion must be rejected: the standalone
# scaled_fp4_quant stays in the graph and the result is unchanged. FP8 output
# fusion with NVFP4 KV is supported.
expect_fusion = not (
kv_cache_dtype == "nvfp4" and model_class.quant_key is kNvfp4Dynamic
)
custom_ops_list = custom_ops.split(",") if custom_ops else []
device = torch.device(f"{DEVICE_TYPE}:0")
torch.set_default_dtype(dtype)
torch.manual_seed(42)
backend_cls = backend.get_class()
block_size = backend_cls.get_preferred_block_size(16)
model_config = ModelConfig(
model=model_name,
max_model_len=2048,
dtype=dtype,
)
vllm_config = VllmConfig(
model_config=model_config,
scheduler_config=SchedulerConfig(
max_num_seqs=1024,
max_model_len=model_config.max_model_len,
is_encoder_decoder=model_config.is_encoder_decoder,
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops_list,
),
cache_config=CacheConfig(cache_dtype=kv_cache_dtype),
attention_config=AttentionConfig(backend=backend),
)
# Create test inputs
q = torch.randn(batch_size, num_qo_heads * head_size, dtype=dtype, device=device)
k = torch.randn(batch_size, num_kv_heads * head_size, dtype=dtype, device=device)
v = torch.randn(batch_size, num_kv_heads * head_size, dtype=dtype, device=device)
# Mark first dimension as dynamic for realistic testing
torch._dynamo.mark_dynamic(q, 0)
torch._dynamo.mark_dynamic(k, 0)
torch._dynamo.mark_dynamic(v, 0)
# Run model directly without compilation and fusion
vllm_config_unfused = copy.deepcopy(vllm_config)
with (
set_current_vllm_config(vllm_config_unfused),
set_forward_context(attn_metadata=None, vllm_config=vllm_config_unfused),
):
model_unfused = model_class(
num_qo_heads=num_qo_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
device=device,
vllm_config=vllm_config_unfused,
block_size=block_size,
)
model_unfused = model_unfused.to(device)
result_unfused_0 = model_unfused(q, k, v) # noqa: F841 HACK: See #131044
forward_ctx = get_forward_context()
forward_ctx.attn_metadata = model_unfused.build_attn_metadata(batch_size)
# Run model directly without fusion
# Still compile so query QuantFP8 has closer numerics
compiled_unfused = torch.compile(model_unfused, fullgraph=True)
result_unfused = compiled_unfused(q, k, v)
# Run model with attn fusion enabled
vllm_config.compilation_config.pass_config = PassConfig(
fuse_attn_quant=True, eliminate_noops=True
)
with (
set_current_vllm_config(vllm_config),
set_forward_context(attn_metadata=None, vllm_config=vllm_config),
):
model_fused = model_class(
num_qo_heads=num_qo_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
device=device,
vllm_config=vllm_config,
w=model_unfused.w,
block_size=block_size,
)
model_fused = model_fused.to(device)
forward_ctx = get_forward_context()
forward_ctx.attn_metadata = model_fused.build_attn_metadata(batch_size)
# Create test backend with fusion passes enabled
noop_pass = NoOpEliminationPass(vllm_config)
attn_pass = LazyInitPass(AttnQuantFusionPass, vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
test_backend = TestBackend(noop_pass, attn_pass, cleanup_pass)
# HACK: See https://github.com/vllm-project/vllm/issues/31044
result_fused_0 = model_fused(q, k, v) # noqa: F841
# Compile model with fusion enabled
compiled_fused = torch.compile(
model_fused, backend=test_backend, fullgraph=True
)
assert compiled_fused.attn._o_scale_float is None
result_fused = compiled_fused(q, k, v)
if backend == AttentionBackendEnum.FLASHINFER and expect_fusion:
# With the Flashinfer backend after the 1st round of the forward
# pass, output quant scale should be loaded into the attn layer's
# _o_scale_float, the 2nd round should reuse the loaded
# _o_scale_float
assert compiled_fused.attn._o_scale_float is not None
result_fused_2 = compiled_fused(q, k, v)
assert compiled_fused.attn._o_scale_float is not None
torch.testing.assert_close(
result_unfused, result_fused_2, atol=1e-2, rtol=1e-2
)
elif not expect_fusion:
assert compiled_fused.attn._o_scale_float is None
# Check attn fusion support
quant_key: QuantKey = model_class.quant_key
attn_fusion_supported = [
layer.impl.fused_output_quant_supported(quant_key)
for key, layer in vllm_config.compilation_config.static_forward_context.items()
]
if expect_fusion:
assert sum(attn_fusion_supported) == len(attn_fusion_supported), (
"All layers should support attention fusion"
)
else:
assert sum(attn_fusion_supported) == 0, (
"No layer should support attention fusion"
)
# Check quantization ops in the graph before and after fusion
if "-quant_fp8" in custom_ops_list:
quant_op = torch.ops.aten.reciprocal
elif getattr(model_class, "aiter_quant", False):
quant_op = torch.ops.vllm.rocm_aiter_per_tensor_quant.default
else:
quant_op = QUANT_OPS[quant_key]
if expect_fusion:
# Note: for fp8, fully_replaced=False because query quant ops remain in
# graph. Only output quant ops are fused into attention.
test_backend.check_before_ops(
[quant_op], fully_replaced=quant_key is kNvfp4Dynamic
)
else:
# Fusion rejected: the standalone output quant must survive untouched.
num_pre = test_backend.op_count(quant_op, before=True)
assert num_pre > 0, f"Op {quant_op} not found in pre-pass graph"
assert test_backend.op_count(quant_op) == num_pre, (
"Output quant op should remain when fusion is rejected"
)
# access the underlying `AttnQuantFusionPass` on the `LazyInitPass`
assert attn_pass.pass_.matched_count == sum(attn_fusion_supported)
# Check attention ops in the graph before and after fusion
attn_nodes_pre = list(find_op_nodes(ATTN_OP, test_backend.graph_pre_pass))
attn_nodes_post = list(find_op_nodes(ATTN_OP, test_backend.graph_post_pass))
assert len(attn_nodes_pre) > 0, "Should have attention nodes before fusion"
assert len(attn_nodes_pre) == len(attn_nodes_post), (
"Should have same number of attention nodes before and after fusion"
)
assert attn_nodes_pre[0].kwargs.get("output_scale") is None, (
"Attention should not have output_scale before fusion"
)
if expect_fusion:
assert attn_nodes_post[0].kwargs.get("output_scale") is not None, (
"Attention should have output_scale after fusion"
)
else:
assert attn_nodes_post[0].kwargs.get("output_scale") is None, (
"Attention should not have output_scale when fusion is rejected"
)
assert attn_nodes_pre[0].kwargs.get("output_block_scale") is None, (
"Attention should not have output_block_scale before fusion"
)
kv_cache_dummy_dep_pre_is_none = (
attn_nodes_pre[0].kwargs.get("kv_cache_dummy_dep") is None
)
kv_cache_dummy_dep_post_is_none = (
attn_nodes_post[0].kwargs.get("kv_cache_dummy_dep") is None
)
assert not (kv_cache_dummy_dep_pre_is_none ^ kv_cache_dummy_dep_post_is_none), (
"The kv_cache_dummy_dep should be consistent before and after fusion"
)
if quant_key.dtype == FP8_DTYPE or not expect_fusion:
assert attn_nodes_post[0].kwargs.get("output_block_scale") is None, (
"Attention should not have output_block_scale unless FP4 fusion happened"
)
elif quant_key.dtype == FP4_DTYPE:
assert attn_nodes_post[0].kwargs.get("output_block_scale") is not None, (
"Attention should have output_block_scale after FP4 fusion"
)
# Check that results are close
torch.testing.assert_close(result_unfused, result_fused, atol=1e-2, rtol=1e-2)
if getattr(model_class, "aiter_quant", False):
# Query quantization remains; exactly the output quant op is removed.
assert test_backend.op_count(quant_op, before=True) == (
test_backend.op_count(quant_op) + 1
)
for model in (model_unfused, model_fused):
torch.testing.assert_close(
model.fp8_linear.input_scale,
torch.full_like(model.fp8_linear.input_scale, 0.125),
atol=0,
rtol=0,
)
@pytest.mark.skipif(
not current_platform.is_rocm() or not is_aiter_found_and_supported(),
reason="Requires ROCm AITER attention and quantization",
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_rocm_aiter_static_quant_attention(
dtype, dist_init, disable_vllm_compile_cache, monkeypatch
):
"""Fuse the actual AITER op without changing its static scale or outputs."""
try:
with monkeypatch.context() as patch:
patch.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
test_attention_quant_pattern(
num_qo_heads=32,
num_kv_heads=8,
head_size=128,
batch_size=8,
dtype=dtype,
kv_cache_dtype="fp8",
custom_ops="+quant_fp8",
model_name="amd/Llama-3.1-8B-Instruct-FP8-KV",
model_class=TestAttentionAiterFp8StaticQuantPatternModel,
backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
dist_init=dist_init,
disable_vllm_compile_cache=disable_vllm_compile_cache,
)
finally:
rocm_aiter_ops.refresh_env_variables()
@@ -1,590 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import pytest
import torch._dynamo
from tests.compile.backend import LazyInitPass, TestBackend
from tests.utils import TestFP8Layer, flat_product
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
from vllm.compilation.passes.fusion.matcher_utils import QUANT_OPS
from vllm.compilation.passes.fusion.mla_attn_quant_fusion import (
MLA_ATTN_OP,
MLAAttnQuantFusionPass,
)
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
AttentionConfig,
CacheConfig,
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
SchedulerConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
CutlassFp8BlockScaledMMKernel,
)
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.modelopt import ModelOptNvFp4Config
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
create_fp8_quant_key,
kFp8Dynamic128Sym,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import MLAAttentionSpec
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
DEVICE_TYPE = current_platform.device_type
class MLAAttentionQuantPatternModel(torch.nn.Module):
"""Base model for MLA AttentionQuantPattern fusion."""
def __init__(
self,
num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
kv_lora_rank: int,
kv_cache_dtype: torch.dtype,
device: torch.device,
vllm_config: VllmConfig,
**kwargs,
):
super().__init__()
self.num_heads = num_heads
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_rope_head_dim = qk_rope_head_dim
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
self.v_head_dim = v_head_dim
self.kv_lora_rank = kv_lora_rank
self.output_dim = num_heads * v_head_dim
self.head_size = kv_lora_rank + qk_rope_head_dim
self.kv_cache_dtype = kv_cache_dtype
self.device = device
self.vllm_config = vllm_config
self.dtype = vllm_config.model_config.dtype
kv_b_proj = ColumnParallelLinear(
input_size=kv_lora_rank,
output_size=num_heads * (qk_nope_head_dim + v_head_dim),
bias=False,
prefix="model.layers.0.self_attn.kv_b_proj",
).to(device)
kv_b_proj_weight = kwargs.get("kv_b_proj_weight")
if kv_b_proj_weight is not None:
kv_b_proj.weight.data.copy_(kv_b_proj_weight)
else:
kv_b_proj.weight.data.normal_()
# Create MLAAttention
self.mla_attn = MLAAttention(
num_heads=num_heads,
scale=1.0 / (self.qk_head_dim**0.5),
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
kv_b_proj=kv_b_proj,
cache_config=vllm_config.cache_config,
quant_config=self.quant_config,
prefix="model.layers.0.self_attn.attn",
)
self.mla_attn._k_scale = self.mla_attn._k_scale.to(device)
self.mla_attn._v_scale = self.mla_attn._v_scale.to(device)
# Initialize W_UK_T and W_UV from kv_b_proj weights
self.mla_attn.process_weights_after_loading(torch.get_default_dtype())
self.kv_b_proj_weight = kv_b_proj.weight.data.clone()
self.block_size = 16
# Initialize MLA MetadataBuilder
self.builder = self.mla_attn.attn_backend.get_builder_cls()(
kv_cache_spec=MLAAttentionSpec(
block_size=self.block_size,
num_kv_heads=1,
head_size=self.head_size,
dtype=self.kv_cache_dtype,
),
layer_names=[self.mla_attn.layer_name],
vllm_config=self.vllm_config,
device=self.device,
)
def build_attn_metadata(self, batch_size: int) -> AttentionMetadata:
"""Initialize MLA attention metadata.
NOTE: Uses decode-only batch (query_len=1 per request). The prefill
(forward_mha) path is not separately tested here because it requires
FlashAttention availability and different input tensor shapes. The
quant logic in forward_impl is identical for both paths — it quantizes
the full output[:num_actual_toks] buffer after both forward_mha and
forward_mqa have written their results.
"""
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
common_attn_metadata = create_common_attn_metadata(
batch_spec, self.block_size, self.device, arange_block_indices=True
)
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
num_blocks = batch_size * max_blocks
# MLA KV cache is 4D: (num_blocks, num_heads=1, block_size, head_size)
kv_cache = torch.zeros(
(num_blocks, 1, self.block_size, self.head_size),
dtype=self.kv_cache_dtype,
device=self.device,
)
self.mla_attn.bind_kv_cache(kv_cache)
self.attn_metadata = self.builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
return self.attn_metadata
class TestMLAAttentionFp8StaticQuantPatternModel(MLAAttentionQuantPatternModel):
"""Test model for MLA Attention + FP8 static quant fusion."""
quant_key = kFp8StaticTensorSym
quant_config = Fp8Config()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fp8_linear = TestFP8Layer(
weight_shape=(self.output_dim, self.output_dim),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
device=self.device,
input_dtype=self.dtype,
)
w = kwargs.get("w")
if w is not None:
self.fp8_linear.weight = w["weight"]
self.fp8_linear.weight_scale = w["wscale"]
self.fp8_linear.input_scale = w["scale"]
self.w = {
"weight": self.fp8_linear.weight,
"wscale": self.fp8_linear.weight_scale,
"scale": self.fp8_linear.input_scale,
}
def forward(
self,
q: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
):
"""Forward pass that creates the MLA attention + FP8 quant pattern."""
attn_output = self.mla_attn(
q,
kv_c_normed,
k_pe,
output_shape=(q.shape[0], self.output_dim),
)
return self.fp8_linear(attn_output)
class TestMLAAttentionNvfp4QuantPatternModel(MLAAttentionQuantPatternModel):
"""Test model for MLA Attention + NVFP4 quant fusion."""
quant_key = kNvfp4Dynamic
quant_config = ModelOptNvFp4Config(
is_checkpoint_nvfp4_serialized=False,
kv_cache_quant_algo=None,
exclude_modules=[],
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.w = kwargs.get(
"w",
{
"weight": torch.randint(
256,
(self.output_dim, self.output_dim // 2),
dtype=FP4_DTYPE,
device=self.device,
),
"wscale_swizzled": torch.randn(
self.output_dim, self.output_dim // 16
).to(dtype=FP8_DTYPE, device=self.device),
"wscale": torch.tensor([500], dtype=torch.float32, device=self.device),
"scale": torch.tensor([0.002], dtype=torch.float32, device=self.device),
},
)
def forward(
self,
q: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
):
"""Forward pass that creates the MLA attention + NVFP4 quant pattern."""
attn_output = self.mla_attn(
q,
kv_c_normed,
k_pe,
output_shape=(q.shape[0], self.output_dim),
)
quant_output, output_block_scale = scaled_fp4_quant(
attn_output, 1 / self.w["scale"]
)
return cutlass_scaled_fp4_mm(
a=quant_output,
b=self.w["weight"],
block_scale_a=output_block_scale,
block_scale_b=self.w["wscale_swizzled"],
alpha=self.w["scale"] * self.w["wscale"],
out_dtype=attn_output.dtype,
)
class TestMLAAttentionFp8GroupQuantPatternModel(MLAAttentionQuantPatternModel):
"""Test model for MLA Attention + per-group FP8 (block quant) fusion."""
quant_key = kFp8Dynamic128Sym
quant_config = Fp8Config(
weight_block_size=[128, 128],
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
weight_quant_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(128, 128)
)
device = kwargs.get("device", torch.device("cuda:0"))
# Subclass to set weight_block_size before process_weights_after_loading
class _BlockFP8Layer(TestFP8Layer):
def __init__(self, *a, **kw):
self.weight_block_size = [128, 128]
super().__init__(*a, **kw)
# Force CutlassFp8BlockScaledMMKernel to ensure the graph uses
# per_token_group_fp8_quant (not the deepgemm packed variant).
self.block_fp8_linear = _BlockFP8Layer(
weight_shape=(self.output_dim, self.output_dim),
activation_quant_key=self.quant_key,
weight_quant_key=weight_quant_key,
input_dtype=self.dtype,
device=device,
force_kernel=CutlassFp8BlockScaledMMKernel,
)
w = kwargs.get("w")
if w is not None:
self.block_fp8_linear.weight = w["weight"]
# Block-wise uses weight_scale_inv, not weight_scale
self.block_fp8_linear.weight_scale_inv = w["wscale"]
self.w = {
"weight": self.block_fp8_linear.weight,
"wscale": self.block_fp8_linear.weight_scale_inv,
}
def forward(
self,
q: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
):
"""Forward pass: MLA attention -> block FP8 linear (group quant)."""
attn_output = self.mla_attn(
q,
kv_c_normed,
k_pe,
output_shape=(q.shape[0], self.output_dim),
)
return self.block_fp8_linear(attn_output)
def is_nvfp4_supported():
return current_platform.has_device_capability(100)
# MLA test configuration
MLA_DIMS: list[tuple[int, int, int, int, int]] = []
PATTERN_TEST_MODELS_MLA_FP8: list[tuple[str, type]] = []
PATTERN_TEST_MODELS_MLA_GROUP_FP8: list[tuple[str, type]] = []
PATTERN_TEST_MODELS_MLA_FP4: list[tuple[str, type]] = []
BACKENDS_MLA_FP8: list[AttentionBackendEnum] = []
BACKENDS_MLA_FP4: list[AttentionBackendEnum] = []
if current_platform.is_cuda():
# (num_heads, qk_nope_head_dim, qk_rope_head_dim, v_head_dim, kv_lora_rank)
MLA_DIMS = [(16, 128, 64, 128, 512)]
PATTERN_TEST_MODELS_MLA_FP8 = [
(
"deepseek-ai/DeepSeek-V2-Lite",
TestMLAAttentionFp8StaticQuantPatternModel,
)
]
PATTERN_TEST_MODELS_MLA_GROUP_FP8 = [
(
"deepseek-ai/DeepSeek-V3",
TestMLAAttentionFp8GroupQuantPatternModel,
)
]
PATTERN_TEST_MODELS_MLA_FP4 = [
(
"deepseek-ai/DeepSeek-V2-Lite",
TestMLAAttentionNvfp4QuantPatternModel,
)
]
BACKENDS_MLA_FP8 = [AttentionBackendEnum.TRITON_MLA]
BACKENDS_MLA_FP4 = [AttentionBackendEnum.TRITON_MLA]
elif current_platform.is_rocm():
# ROCm supports the static-FP8 output-quant fusion through AITER MLA.
# Per-group FP8 still relies on the CUDA-only CUTLASS block-scaled
# kernel, and NVFP4 is NVIDIA-specific.
MLA_DIMS = [(16, 128, 64, 128, 512)]
PATTERN_TEST_MODELS_MLA_FP8 = [
(
"deepseek-ai/DeepSeek-V2-Lite",
TestMLAAttentionFp8StaticQuantPatternModel,
)
]
BACKENDS_MLA_FP8 = [AttentionBackendEnum.ROCM_AITER_MLA]
@pytest.mark.parametrize(
"num_heads, qk_nope_head_dim, qk_rope_head_dim, v_head_dim, kv_lora_rank",
MLA_DIMS,
)
@pytest.mark.parametrize(
"batch_size", [7, 256] if current_platform.is_cuda_alike() else [8]
)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize(
"backend, model_name, model_class, custom_ops",
list(
flat_product(
BACKENDS_MLA_FP8,
PATTERN_TEST_MODELS_MLA_FP8,
["+quant_fp8", "-quant_fp8"],
)
)
+ list(
flat_product(
BACKENDS_MLA_FP8,
PATTERN_TEST_MODELS_MLA_GROUP_FP8,
["+quant_fp8"],
)
)
+ list(flat_product(BACKENDS_MLA_FP4, PATTERN_TEST_MODELS_MLA_FP4, [""])),
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Only test ROCm or CUDA"
)
@pytest.mark.skipif(not current_platform.supports_fp8(), reason="Need FP8")
def test_mla_attention_quant_pattern(
num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
kv_lora_rank: int,
batch_size: int,
dtype: torch.dtype,
custom_ops: str,
model_name: str,
model_class: type[MLAAttentionQuantPatternModel],
backend: AttentionBackendEnum,
dist_init,
disable_vllm_compile_cache,
workspace_init,
):
"""Test MLA AttentionQuantPattern fusion pass."""
if (
model_class is TestMLAAttentionNvfp4QuantPatternModel
and not is_nvfp4_supported()
):
pytest.skip("NVFP4 is not supported on this GPU (requires SM 100+).")
custom_ops_list = custom_ops.split(",") if custom_ops else []
device = torch.device(f"{DEVICE_TYPE}:0")
torch.set_default_dtype(dtype)
torch.manual_seed(42)
model_config = ModelConfig(
model=model_name,
max_model_len=2048,
dtype=dtype,
)
vllm_config = VllmConfig(
model_config=model_config,
scheduler_config=SchedulerConfig(
max_num_seqs=1024,
max_model_len=model_config.max_model_len,
is_encoder_decoder=model_config.is_encoder_decoder,
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops_list,
),
cache_config=CacheConfig(cache_dtype="auto"),
attention_config=AttentionConfig(backend=backend),
)
# MLA inputs: q(B, N, qk_head_dim), kv_c_normed(B, L), k_pe(B, 1, R)
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
q = torch.randn(batch_size, num_heads, qk_head_dim, dtype=dtype, device=device)
kv_c_normed = torch.randn(batch_size, kv_lora_rank, dtype=dtype, device=device)
k_pe = torch.randn(batch_size, 1, qk_rope_head_dim, dtype=dtype, device=device)
# Mark first dimension as dynamic
torch._dynamo.mark_dynamic(q, 0)
torch._dynamo.mark_dynamic(kv_c_normed, 0)
torch._dynamo.mark_dynamic(k_pe, 0)
# Run model without fusion
vllm_config_unfused = copy.deepcopy(vllm_config)
with (
set_current_vllm_config(vllm_config_unfused),
set_forward_context(attn_metadata=None, vllm_config=vllm_config_unfused),
):
model_unfused = model_class(
num_heads=num_heads,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
kv_lora_rank=kv_lora_rank,
kv_cache_dtype=dtype,
device=device,
vllm_config=vllm_config_unfused,
)
if (
model_class is TestMLAAttentionFp8GroupQuantPatternModel
and type(model_unfused.block_fp8_linear.kernel)
is not CutlassFp8BlockScaledMMKernel
):
pytest.skip("CUTLASS FP8 block kernel is not supported on this platform")
model_unfused = model_unfused.to(device)
# HACK: See #131044
result_unfused_0 = model_unfused(q, kv_c_normed, k_pe) # noqa: F841
forward_ctx = get_forward_context()
forward_ctx.attn_metadata = model_unfused.build_attn_metadata(batch_size)
compiled_unfused = torch.compile(model_unfused, fullgraph=True)
result_unfused = compiled_unfused(q, kv_c_normed, k_pe)
# Run model with attn fusion enabled
vllm_config.compilation_config.pass_config = PassConfig(
fuse_attn_quant=True, eliminate_noops=True
)
with (
set_current_vllm_config(vllm_config),
set_forward_context(attn_metadata=None, vllm_config=vllm_config),
):
model_fused = model_class(
num_heads=num_heads,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
kv_lora_rank=kv_lora_rank,
kv_cache_dtype=dtype,
device=device,
vllm_config=vllm_config,
w=model_unfused.w,
kv_b_proj_weight=model_unfused.kv_b_proj_weight,
)
model_fused = model_fused.to(device)
forward_ctx = get_forward_context()
forward_ctx.attn_metadata = model_fused.build_attn_metadata(batch_size)
# Create test backend with fusion passes
noop_pass = NoOpEliminationPass(vllm_config)
attn_pass = LazyInitPass(MLAAttnQuantFusionPass, vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
test_backend = TestBackend(noop_pass, attn_pass, cleanup_pass)
# HACK: See https://github.com/vllm-project/vllm/issues/31044
result_fused_0 = model_fused(q, kv_c_normed, k_pe) # noqa: F841
compiled_fused = torch.compile(
model_fused, backend=test_backend, fullgraph=True
)
result_fused = compiled_fused(q, kv_c_normed, k_pe)
# Check attn fusion support
quant_key: QuantKey = model_class.quant_key
attn_fusion_supported = [
layer.impl.fused_output_quant_supported(quant_key)
for key, layer in vllm_config.compilation_config.static_forward_context.items()
if isinstance(layer, MLAAttention)
]
assert sum(attn_fusion_supported) == len(attn_fusion_supported), (
"All MLA layers should support attention fusion"
)
# Check quantization ops in the graph
is_per_group = quant_key.scale.group_shape.is_per_group()
quant_op = (
torch.ops.aten.reciprocal
if "-quant_fp8" in custom_ops_list
else QUANT_OPS[quant_key]
)
test_backend.check_before_ops([quant_op], fully_replaced=is_per_group)
assert attn_pass.pass_.matched_count == sum(attn_fusion_supported)
# Check MLA attention ops in the graph
attn_nodes_pre = list(find_op_nodes(MLA_ATTN_OP, test_backend.graph_pre_pass))
attn_nodes_post = list(find_op_nodes(MLA_ATTN_OP, test_backend.graph_post_pass))
assert len(attn_nodes_pre) > 0, "Should have MLA attention nodes before fusion"
assert len(attn_nodes_pre) == len(attn_nodes_post), (
"Should have same number of MLA attention nodes before and after fusion"
)
# Before fusion: neither scale should be set
assert attn_nodes_pre[0].kwargs.get("output_scale") is None
assert attn_nodes_pre[0].kwargs.get("output_block_scale") is None
# After fusion: derive expected scale presence from quant_key properties.
# - output_scale: present for static quant or non-FP8 (NVFP4 carries input_scale)
# - output_block_scale: present when quant uses per-group/block scaling
has_output_scale = attn_nodes_post[0].kwargs.get("output_scale") is not None
has_block_scale = attn_nodes_post[0].kwargs.get("output_block_scale") is not None
expects_output_scale = quant_key.scale.static or quant_key.dtype != FP8_DTYPE
assert has_output_scale == expects_output_scale, (
f"output_scale: expected present={expects_output_scale}, got {has_output_scale}"
)
assert has_block_scale == is_per_group, (
f"output_block_scale: expected present={is_per_group}, got {has_block_scale}"
)
# Check numerical correctness
torch.testing.assert_close(result_unfused, result_fused, atol=1e-2, rtol=1e-2)
@@ -1,399 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.config
from tests.compile.backend import TestBackend
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.compilation.passes.fusion.mla_rope_kvcache_cat_fusion import (
MLARoPEKVCacheCatFusionPass,
)
from vllm.compilation.passes.utility.fix_functionalization import (
FixFunctionalizationPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CacheConfig,
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.rotary_embedding import (
DeepseekScalingRotaryEmbedding,
RotaryEmbedding,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import _encode_layer_name
from vllm.v1.attention.backend import (
AttentionBackend,
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
from vllm.v1.attention.backends.registry import AttentionBackendEnum
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
VLLM_UNIFIED_MLA_KV_CACHE_UPDATE_OP = torch.ops.vllm.unified_mla_kv_cache_update
FP8_DTYPE = current_platform.fp8_dtype()
class MLARoPEKVCacheCatTestModel(torch.nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
attn_backend: AttentionBackendEnum,
use_deepseek_scaling_rope: bool,
num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
q_lora_rank: int,
kv_lora_rank: int,
is_neox: bool,
dtype: torch.dtype,
device: torch.device,
prefix: str = "model.layers.0.self_attn.attn",
):
super().__init__()
self.num_heads = num_heads
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_rope_head_dim = qk_rope_head_dim
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
self.v_head_dim = v_head_dim
self.q_lora_rank = q_lora_rank
self.kv_lora_rank = kv_lora_rank
self.dtype = dtype
self.device = device
self.layer_name = prefix
self.num_kv_heads = 1
self.head_size = kv_lora_rank + qk_rope_head_dim
self.block_size = vllm_config.cache_config.block_size
self.scale = self.qk_head_dim**-0.5
if use_deepseek_scaling_rope:
self.rotary_emb = DeepseekScalingRotaryEmbedding(
head_size=qk_rope_head_dim,
rotary_dim=qk_rope_head_dim,
max_position_embeddings=4096,
base=10000,
is_neox_style=is_neox,
scaling_factor=1.0,
dtype=dtype,
)
else:
self.rotary_emb = RotaryEmbedding(
head_size=qk_rope_head_dim,
rotary_dim=qk_rope_head_dim,
max_position_embeddings=4096,
base=10000,
is_neox_style=is_neox,
dtype=dtype,
)
# Initialize intermediate mm layers for unit test
self.q_b_proj = ColumnParallelLinear(
self.q_lora_rank,
self.num_heads * self.qk_head_dim,
bias=False,
prefix=f"{prefix}.q_b_proj",
).to(device)
self.kv_b_proj = ColumnParallelLinear(
self.kv_lora_rank,
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
bias=False,
prefix=f"{prefix}.kv_b_proj",
).to(device)
# ColumnParallelLinear default init in bf16 with seed 0 produces
# near-zero weights (7/4.7M nonzero), making the GEMM output almost
# entirely zero and masking correctness bugs. Reinitialize to get
# dense outputs.
with torch.no_grad():
torch.nn.init.normal_(self.q_b_proj.weight, std=0.02)
torch.nn.init.normal_(self.kv_b_proj.weight, std=0.02)
# Register layer metadata for the fusion pass via MLAAttention
self.mla_attn = MLAAttention(
num_heads=self.num_heads,
scale=self.scale,
qk_nope_head_dim=self.qk_nope_head_dim,
qk_rope_head_dim=self.qk_rope_head_dim,
v_head_dim=self.v_head_dim,
q_lora_rank=self.q_lora_rank,
kv_lora_rank=self.kv_lora_rank,
kv_b_proj=self.kv_b_proj,
cache_config=vllm_config.cache_config,
quant_config=vllm_config.quant_config,
prefix=prefix,
attn_backend=attn_backend.get_class(),
)
self.attn_backend: type[AttentionBackend] = self.mla_attn.get_attn_backend()
self.mla_attn._k_scale = self.mla_attn._k_scale.to(device)
self.mla_attn._v_scale = self.mla_attn._v_scale.to(device)
# Keep both the string dtype (for ops) and torch dtype (for tensors)
self.kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
self.kv_cache_dtype = (
FP8_DTYPE if self.kv_cache_dtype_str.startswith("fp8") else self.dtype
)
# Initialize attn MetadataBuilder
self.builder = self.attn_backend.get_builder_cls()(
kv_cache_spec=self.mla_attn.get_kv_cache_spec(vllm_config),
layer_names=[self.mla_attn.layer_name],
vllm_config=vllm_config,
device=device,
)
def build_attn_metadata(self, batch_size: int) -> CommonAttentionMetadata:
"""Initialize attention metadata."""
# Create common attn metadata
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
common_attn_metadata = create_common_attn_metadata(
batch_spec, self.block_size, self.device, arange_block_indices=True
)
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
num_blocks = batch_size * max_blocks
# MLA KV cache is 4D: (num_blocks, num_heads=1, block_size, head_size)
kv_cache = torch.zeros(
(num_blocks, 1, self.block_size, self.head_size),
dtype=self.kv_cache_dtype,
device=self.device,
)
self.mla_attn.bind_kv_cache(kv_cache)
# Build attn metadata
attn_metadata = self.builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
return attn_metadata
def forward(
self, qkv_lora: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
qkv_lora = qkv_lora.clone()
q_c, kv_lora = qkv_lora.split(
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
dim=-1,
)
q = self.q_b_proj(q_c)[0]
kv_c, k_pe = kv_lora.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
q = q.view(-1, self.num_heads, self.qk_head_dim)
k_pe = k_pe.unsqueeze(1)
q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb(
positions, q[..., self.qk_nope_head_dim :], k_pe
)
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
kv_c,
k_pe,
_encode_layer_name(self.layer_name),
self.kv_cache_dtype_str,
self.mla_attn._k_scale,
)
return q, kv_c, k_pe, dummy
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
ops = [
INDEX_SELECT_OP,
torch.ops.vllm.unified_mla_kv_cache_update.default,
]
return ops
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.vllm.fused_rope_unified_mla_kv_cache_update.default]
MLA_BACKENDS = [AttentionBackendEnum.TRITON_MLA]
if flash_attn_supports_mla():
MLA_BACKENDS += [AttentionBackendEnum.FLASH_ATTN_MLA]
if is_aiter_found_and_supported():
MLA_BACKENDS += [AttentionBackendEnum.ROCM_AITER_MLA]
@pytest.mark.parametrize("attn_backend", MLA_BACKENDS)
@pytest.mark.parametrize("use_deepseek_scaling_rope", [True])
@pytest.mark.parametrize("num_heads", [16])
@pytest.mark.parametrize("qk_nope_head_dim", [128])
@pytest.mark.parametrize("qk_rope_head_dim", [64])
@pytest.mark.parametrize("v_head_dim", [128])
@pytest.mark.parametrize("q_lora_rank", [1536])
@pytest.mark.parametrize("kv_lora_rank", [512])
@pytest.mark.parametrize("block_size", [16])
@pytest.mark.parametrize("is_neox", [True, False])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="MLA RoPE+KVCache+Cat fusion is only supported on CUDA and ROCm.",
)
def test_mla_rope_kvcache_cat_fusion(
attn_backend: AttentionBackendEnum,
use_deepseek_scaling_rope: bool,
num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
q_lora_rank: int,
kv_lora_rank: int,
block_size: int,
is_neox: bool,
dtype: torch.dtype,
kv_cache_dtype: CacheDType,
monkeypatch: pytest.MonkeyPatch,
):
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(0)
vllm_config = VllmConfig(
model_config=ModelConfig(
model="deepseek-ai/DeepSeek-V2-Lite",
dtype=dtype,
),
cache_config=CacheConfig(
block_size=block_size,
cache_dtype=kv_cache_dtype,
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(
fuse_rope_kvcache_cat_mla=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
if not torch.distributed.is_initialized():
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.utils.system_utils import update_environment_variables
update_environment_variables(
{
"RANK": "0",
"LOCAL_RANK": "0",
"WORLD_SIZE": "1",
"MASTER_ADDR": "localhost",
"MASTER_PORT": "54321",
}
)
init_distributed_environment()
initialize_model_parallel()
if attn_backend == AttentionBackendEnum.ROCM_AITER_MLA:
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
model = MLARoPEKVCacheCatTestModel(
vllm_config=vllm_config,
attn_backend=attn_backend,
use_deepseek_scaling_rope=use_deepseek_scaling_rope,
num_heads=num_heads,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
q_lora_rank=q_lora_rank,
kv_lora_rank=kv_lora_rank,
is_neox=is_neox,
dtype=dtype,
device=torch.get_default_device(),
)
fusion_pass = MLARoPEKVCacheCatFusionPass(vllm_config)
# note: FixFunctionalizationPass is required to correctly lower
# the fused op to its inplace version with auto-functionalization v1.
# Without it, decompose_auto_functionalized calls clone_preserve_strides
# on the non-contiguous q_pe slice directly, and inductor's lowering
# of the resulting as_strided chain incorrectly drops the storage offset.
# auto-functionalization v2 avoids this: it clones the contiguous base
# tensor (_all_bases) and reconstructs the slice as a view, so the
# offset is never passed through as_strided lowering.
passes = [
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
FixFunctionalizationPass(vllm_config),
]
backend = TestBackend(*passes)
T = 5
qkv_lora = torch.randn(
T,
q_lora_rank + kv_lora_rank + qk_rope_head_dim,
dtype=dtype,
)
pos = torch.arange(T, dtype=torch.long)
qkv_unfused = qkv_lora.clone()
pos_unfused = pos.clone()
# Run unfused version
with set_forward_context(None, vllm_config):
forward_context = get_forward_context()
attn_metadata = model.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_unfused, kv_c_unfused, k_pe_unfused, dummy = model(
qkv_unfused, pos_unfused
)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_unfused = attn_layer.kv_cache.clone()
del dummy
# Run fused version (compiled)
torch._dynamo.mark_dynamic(qkv_lora, 0)
torch._dynamo.mark_dynamic(pos, 0)
with set_forward_context(None, vllm_config):
model_fused = torch.compile(model, backend=backend)
forward_context = get_forward_context()
attn_metadata = model.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_fused, kv_c_fused, k_pe_fused, dummy = model_fused(qkv_lora, pos)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_fused = attn_layer.kv_cache
del dummy
assert fusion_pass.matched_count == 1
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
if dtype == torch.float16:
ATOL, RTOL = (2e-3, 2e-3)
else:
ATOL, RTOL = (1e-2, 1e-2)
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(kv_c_unfused, kv_c_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(k_pe_unfused, k_pe_fused, atol=ATOL, rtol=RTOL)
# Cannot compare fp8_* directly here, cast to model dtype instead
torch.testing.assert_close(
kv_cache_unfused.view(dtype),
kv_cache_fused.view(dtype),
atol=ATOL,
rtol=RTOL,
)
@@ -1,120 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm
from tests.compile.backend import TestBackend
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.config import CompilationConfig, CompilationMode, PassConfig, VllmConfig
from vllm.platforms import current_platform
DEVICE_TYPE = current_platform.device_type
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
# Important edge case is when `num_tokens == buffer_size`
@pytest.mark.parametrize(
("num_tokens", "buffer_size"), [(256, 256), (256, 512), (1024, 1024), (1024, 1025)]
)
@pytest.mark.parametrize("hidden_size", [64, 4096])
def test_noop_elimination(dtype, num_tokens, hidden_size, buffer_size):
torch.set_default_device(DEVICE_TYPE)
torch.set_default_dtype(dtype)
torch.manual_seed(1)
class Model(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
# Avoid using empty, since on rocm torch.empty
# does not initialize the memory.
self.pos_embed = torch.randn(buffer_size, hidden_size, dtype=dtype)
def forward(self, x):
# Avoid += to prevent inplace addition.
x = x + self.pos_embed[: x.shape[0]]
# Chain of reshapes
y = x.reshape(-1, 128, 32)
z = y.reshape(-1, 4096)
# No-op reshape
a = z.reshape(-1, 4096)
# Final reshape that should remain
b = a.reshape(-1, 128, 32)
# No-op slice
c = b[0 : b.shape[0]]
# The pass should replace the result of this op with `c`.
d = torch.slice_scatter(
torch.ones_like(c), # Dummy tensor to be scattered into
c, # Source tensor
0, # dim
0, # start
c.shape[0], # end
)
return d
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(eliminate_noops=True),
)
)
with vllm.config.set_current_vllm_config(vllm_config):
noop_pass = NoOpEliminationPass(vllm_config)
backend = TestBackend(noop_pass)
model = Model()
# First dimension dynamic
x = torch.rand(num_tokens, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
result = model(x)
model2 = torch.compile(model, backend=backend)
result2 = model2(x)
ATOL, RTOL = (2e-3, 2e-3)
torch.testing.assert_close(result, result2, atol=ATOL, rtol=RTOL)
# The no-op reshape and slice should be eliminated.
# The initial slice on the positional embedding should remain.
# The chain of reshapes should be fused into a single reshape.
assert backend.op_count(torch.ops.aten.reshape.default) == 1
assert backend.op_count(torch.ops.aten.slice.Tensor) == 1
assert backend.op_count(torch.ops.aten.slice_scatter.default) == 0
def test_non_noop_slice_preserved():
"""Ensure that a slice with end=-1 (dropping last row) is NOT eliminated.
Regression test for a bug where end=-1 was treated like an inferred
dimension (reshape semantics) leading to incorrect elimination.
"""
torch.set_default_device(DEVICE_TYPE)
x = torch.randn(16, 16)
class SliceModel(torch.nn.Module):
def forward(self, x):
base = x.clone()
src = torch.ones(15, 16)
y = torch.slice_scatter(base, src, dim=0, start=0, end=-1)
return x[0:-1, :], y
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(eliminate_noops=True),
)
)
with vllm.config.set_current_vllm_config(vllm_config):
noop_pass = NoOpEliminationPass(vllm_config)
backend = TestBackend(noop_pass)
model = SliceModel()
ref = model(x)
compiled = torch.compile(model, backend=backend)
out = compiled(x)
torch.testing.assert_close(ref, out)
# The slice should remain (not a no-op).
assert backend.op_count(torch.ops.aten.slice.Tensor) == 1
assert backend.op_count(torch.ops.aten.slice_scatter.default) == 1
@@ -1,284 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from torch._ops import OpOverload, OpOverloadPacket
from tests.compile.backend import TestBackend
from vllm.compilation.passes.fusion.matcher_utils import (
FLASHINFER_ROTARY_OP,
ROTARY_OP,
)
from vllm.compilation.passes.fusion.qk_norm_rope_fusion import (
FUSED_QK_ROPE_OP,
QKNormRoPEFusionPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
from vllm.config import (
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.platforms import current_platform
from vllm.v1.attention.backend import AttentionType
RSQRT_OP = torch.ops.aten.rsqrt.default
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
class QKNormRoPETestModel(torch.nn.Module):
def __init__(
self,
*,
num_heads: int,
num_kv_heads: int,
head_dim: int,
eps: float,
is_neox: bool,
vllm_config: VllmConfig,
dtype: torch.dtype,
test_scattered_split: bool = False,
prefix: str = "model.layers.0.self_attn.attn",
) -> None:
super().__init__()
self.num_heads = num_heads
self.num_kv_heads = num_kv_heads
self.head_dim = head_dim
self.q_size = num_heads * head_dim
self.kv_size = num_kv_heads * head_dim
self.rotary_dim = head_dim
self.eps = eps
self.dtype = dtype
# Register layer metadata for the fusion pass via Attention.
self.attn = Attention(
num_heads=self.num_heads,
head_size=self.head_dim,
scale=1.0 / self.head_dim**0.5,
num_kv_heads=self.num_kv_heads,
cache_config=vllm_config.cache_config,
prefix=prefix,
attn_type=AttentionType.DECODER,
)
self.q_norm = RMSNorm(self.head_dim, eps=self.eps)
self.k_norm = RMSNorm(self.head_dim, eps=self.eps)
self.rotary_emb = RotaryEmbedding(
self.head_dim,
rotary_dim=self.rotary_dim,
max_position_embeddings=4096,
base=10000,
is_neox_style=is_neox,
dtype=self.dtype,
)
self.test_scattered_split = test_scattered_split
self.enable_rms_norm_custom_op = self.q_norm.enabled()
self.enable_rope_custom_op = self.rotary_emb.enabled()
def forward(self, qkv: torch.Tensor, positions: torch.Tensor):
if self.test_scattered_split:
q, _, _ = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
_, k, _ = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
_, _, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
else:
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q_by_head = q.view(*q.shape[:-1], q.shape[-1] // self.head_dim, self.head_dim)
q_by_head = self.q_norm(q_by_head)
q = q_by_head.view(q.shape)
k_by_head = k.view(*k.shape[:-1], k.shape[-1] // self.head_dim, self.head_dim)
k_by_head = self.k_norm(k_by_head)
k = k_by_head.view(k.shape)
q, k = self.rotary_emb(positions, q, k)
return q, k, v
def ops_in_model_before(self) -> list[OpOverload | OpOverloadPacket]:
ops: list[OpOverload | OpOverloadPacket] = [torch.ops.vllm_ir.rms_norm]
if self.enable_rope_custom_op:
if self.rotary_emb.use_flashinfer:
ops.append(FLASHINFER_ROTARY_OP)
else:
ops.append(ROTARY_OP)
else:
ops.append(INDEX_SELECT_OP)
return ops
def ops_in_model_after(self) -> list[OpOverload | OpOverloadPacket]:
return [FUSED_QK_ROPE_OP]
@pytest.mark.parametrize("scattered_split", [True, False])
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
@pytest.mark.parametrize("is_neox", [True, False])
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
@pytest.mark.parametrize("enable_rope_custom_op", [True])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Only test on cuda and rocm platform",
)
def test_qk_norm_rope_fusion(
eps,
is_neox,
enable_rms_norm_custom_op,
enable_rope_custom_op,
dtype,
scattered_split,
):
if not hasattr(torch.ops._C, "fused_qk_norm_rope"):
pytest.skip("fused_qk_norm_rope custom op not available")
torch.set_default_device(current_platform.device_type)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
custom_ops: list[str] = []
if enable_rms_norm_custom_op:
custom_ops.append("+rms_norm")
if enable_rope_custom_op:
custom_ops.append("+rotary_embedding")
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops,
pass_config=PassConfig(
enable_qk_norm_rope_fusion=True,
eliminate_noops=True,
),
),
)
num_heads, num_kv_heads, head_dim = 16, 4, 128
T = 5
with (
set_current_vllm_config(vllm_config),
vllm_config.kernel_config.ir_op_priority.set_priority(),
):
model = QKNormRoPETestModel(
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
eps=eps,
is_neox=is_neox,
vllm_config=vllm_config,
dtype=dtype,
test_scattered_split=scattered_split,
)
noop_pass = NoOpEliminationPass(vllm_config)
coalesce_pass = SplitCoalescingPass(vllm_config)
fusion_pass = QKNormRoPEFusionPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(noop_pass, coalesce_pass, fusion_pass, cleanup_pass)
backend_baseline = TestBackend(noop_pass, cleanup_pass)
qkv = torch.randn(T, model.q_size + 2 * model.kv_size)
pos = torch.arange(T, dtype=torch.long, device=qkv.device)
qkv_unfused = qkv.clone()
pos_unfused = pos.clone()
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(pos, 0)
model_fused = torch.compile(model, backend=backend)
q_fused, k_fused, v_fused = model_fused(qkv, pos)
torch._dynamo.mark_dynamic(qkv_unfused, 0)
torch._dynamo.mark_dynamic(pos_unfused, 0)
model_unfused = torch.compile(model, backend=backend_baseline)
q_unfused, k_unfused, v_unfused = model_unfused(qkv_unfused, pos_unfused)
if dtype == torch.float16:
ATOL, RTOL = (2e-3, 2e-3)
else:
ATOL, RTOL = (1e-2, 1e-2)
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(v_unfused, v_fused, atol=ATOL, rtol=RTOL)
assert fusion_pass.matched_count == 1
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on CUDA and ROCm platforms",
)
def test_qk_norm_rope_fusion_with_heterogeneous_draft_model():
"""The fusion pass should support target and draft attention geometries."""
if not hasattr(torch.ops._C, "fused_qk_norm_rope"):
pytest.skip("fused_qk_norm_rope custom op not available")
dtype = torch.bfloat16
torch.set_default_device(current_platform.device_type)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm", "+rotary_embedding"],
pass_config=PassConfig(
enable_qk_norm_rope_fusion=True,
eliminate_noops=True,
),
),
)
with (
set_current_vllm_config(vllm_config),
vllm_config.kernel_config.ir_op_priority.set_priority(),
):
target_model = QKNormRoPETestModel(
num_heads=32,
num_kv_heads=8,
head_dim=128,
eps=1e-6,
is_neox=True,
vllm_config=vllm_config,
dtype=dtype,
prefix="model.layers.0.self_attn.attn",
)
draft_model = QKNormRoPETestModel(
num_heads=16,
num_kv_heads=8,
head_dim=128,
eps=1e-6,
is_neox=True,
vllm_config=vllm_config,
dtype=dtype,
prefix="draft_model.layers.0.self_attn.attn",
)
fusion_pass = QKNormRoPEFusionPass(vllm_config)
backend = TestBackend(
NoOpEliminationPass(vllm_config),
SplitCoalescingPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
)
for model in (target_model, draft_model):
num_tokens = 5
qkv = torch.randn(num_tokens, model.q_size + 2 * model.kv_size)
positions = torch.arange(num_tokens, dtype=torch.long, device=qkv.device)
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(positions, 0)
torch.compile(model, backend=backend)(qkv, positions)
assert fusion_pass.matched_count == 1
@@ -1,105 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.ir.ops
from tests.compile.backend import TestBackend
from vllm.compilation.passes.fusion.add_rms_fusion import (
AddRMSNormFusionPass,
RMSNormReshapeFusionPass,
)
from vllm.compilation.passes.fx_utils import find_op_nodes, is_func
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import CompilationConfig, CompilationMode, VllmConfig
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Requires CUDA or ROCm"
)
class RMSNormModel(torch.nn.Module):
def __init__(self, hidden_size: int) -> None:
super().__init__()
self.weight = torch.nn.Parameter(torch.randn(hidden_size))
def forward(self, x: torch.Tensor) -> torch.Tensor:
rms = vllm.ir.ops.rms_norm(x, self.weight, 1e-6)
return rms.reshape(-1, rms.shape[-1])
class AddRMSNormModel(torch.nn.Module):
def __init__(self, hidden_size: int, residual_first: bool) -> None:
super().__init__()
self.weight = torch.nn.Parameter(torch.randn(hidden_size))
self.residual_first = residual_first
def forward(
self, x: torch.Tensor, residual: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
residual = residual + x if self.residual_first else x + residual
rms = vllm.ir.ops.rms_norm(residual, self.weight, 1e-6)
return rms.reshape(-1, rms.shape[-1]), residual
def ops_in_model_before(self):
return [torch.ops.aten.add, torch.ops.vllm_ir.rms_norm]
def ops_in_model_after(self):
return [torch.ops.vllm_ir.fused_add_rms_norm]
def _run_fusion_test(model, config, passes, *inputs):
backend = TestBackend(NoOpEliminationPass(config), *passes, PostCleanupPass(config))
outputs_unfused = model(*inputs)
outputs_fused = torch.compile(model, backend=backend)(*inputs)
torch.testing.assert_close(outputs_fused, outputs_unfused)
return backend
def _is_reshape(node):
return isinstance(node, torch.fx.Node) and is_func(
node, torch.ops.aten.reshape.default
)
@pytest.fixture
def vllm_config():
config = VllmConfig(
compilation_config=CompilationConfig(mode=CompilationMode.VLLM_COMPILE)
)
with vllm.config.set_current_vllm_config(config):
torch.set_default_device("cuda")
torch.set_default_dtype(torch.bfloat16)
torch.manual_seed(0)
yield config
def test_rmsnorm_reshape_fusion(vllm_config):
fusion_pass = RMSNormReshapeFusionPass(vllm_config)
model = RMSNormModel(hidden_size=32)
x = torch.randn(2, 7, 32)
backend = _run_fusion_test(model, vllm_config, [fusion_pass], x)
assert fusion_pass.matched_count == 1
(rms_node,) = find_op_nodes(torch.ops.vllm_ir.rms_norm, backend.graph_post_pass)
assert _is_reshape(rms_node.args[0])
@pytest.mark.parametrize("residual_first", [True, False])
def test_add_rmsnorm_reshape_fusion(vllm_config, residual_first):
add_fusion = AddRMSNormFusionPass(vllm_config)
reshape_fusion = RMSNormReshapeFusionPass(vllm_config)
model = AddRMSNormModel(hidden_size=32, residual_first=residual_first)
x = torch.randn(2, 7, 32)
residual = torch.randn_like(x)
backend = _run_fusion_test(
model, vllm_config, [add_fusion, reshape_fusion], x, residual
)
assert add_fusion.matched_count == 1
assert reshape_fusion.matched_count == 1
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
@@ -1,901 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from importlib.metadata import version
import pytest
import torch
from packaging.version import Version
import vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion as fusion_module
import vllm.config
from tests.compile.backend import TestBackend
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
dense_kv_cache_views,
)
from vllm._aiter_ops import (
IS_AITER_FOUND,
is_aiter_found_and_supported,
rocm_aiter_ops,
)
from vllm.compilation.passes.fusion.matcher_utils import MROPE_OP, ROTARY_OP
from vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion import (
QkNormRopeKvCacheFusionPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.compilation.passes.utility.scatter_split_replace import (
ScatterSplitReplacementPass,
)
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
from vllm.config import (
CacheConfig,
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
from vllm.platforms import current_platform
from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name
from vllm.v1.attention.backend import (
AttentionBackend,
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
pytestmark = pytest.mark.skip_global_cleanup
if not is_aiter_found_and_supported():
pytest.skip(
"ROCm with supported AITER is required",
allow_module_level=True,
)
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
FP8_DTYPE = current_platform.fp8_dtype()
@pytest.fixture(scope="module", autouse=True)
def module_global_cleanup():
# Cleanup once at the end of the module
from vllm.distributed import cleanup_dist_env_and_memory
yield
cleanup_dist_env_and_memory()
class QKNormRoPEKVCacheTestModel(torch.nn.Module):
"""Minimal model that reproduces the QK-norm + RoPE + KV cache update
pattern matched by QkNormRopeKvCacheFusionPass:
q, k, v = split(qkv)
q = rms_norm(q.view(heads, dim), q_weight).view(flat)
k = rms_norm(k.view(heads, dim), k_weight).view(flat)
q, k = rotary_emb(positions, q, k)
q = q.view(num_heads, head_dim)
k = k.view(num_kv_heads, head_dim)
v = v.view(num_kv_heads, head_dim)
dummy = unified_kv_cache_update(k, v, layer_name)
"""
def __init__(
self,
vllm_config: VllmConfig,
attn_backend: AttentionBackendEnum,
num_heads: int,
num_kv_heads: int,
head_size: int,
is_neox: bool,
rms_norm_eps: float,
dtype: torch.dtype,
device: torch.device,
rotary_dim: int | None = None,
prefix: str = "model.layers.0.self_attn.attn",
expected_query_quant_group_shape: GroupShape | None = None,
):
super().__init__()
self.num_heads = num_heads
self.num_kv_heads = num_kv_heads
self.head_size = head_size
self.rotary_dim = rotary_dim if rotary_dim is not None else head_size
self.block_size = vllm_config.cache_config.block_size
self.q_size = num_heads * head_size
self.kv_size = num_kv_heads * head_size
self.is_neox = is_neox
self.dtype = dtype
self.device = device
self.layer_name = prefix
self.q_norm = RMSNorm(head_size, eps=rms_norm_eps)
self.k_norm = RMSNorm(head_size, eps=rms_norm_eps)
self.rotary_emb = RotaryEmbedding(
head_size,
rotary_dim=self.rotary_dim,
max_position_embeddings=4096,
base=10000,
is_neox_style=is_neox,
dtype=self.dtype,
)
self.enable_rope_custom_op = self.rotary_emb.enabled()
self.attn = Attention(
num_heads=num_heads,
head_size=head_size,
scale=1.0 / head_size**0.5,
num_kv_heads=num_kv_heads,
cache_config=vllm_config.cache_config,
quant_config=vllm_config.quant_config,
prefix=prefix,
attn_backend=attn_backend.get_class(),
)
self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend()
if expected_query_quant_group_shape is not None:
assert self.attn.query_quant is not None
assert self.attn.query_quant.group_shape == expected_query_quant_group_shape
assert not self.attn_backend.forward_includes_kv_cache_update, (
f"Attention backend {self.attn_backend} does not support "
"fuse_qk_norm_rope_kvcache."
)
kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
self.kv_cache_dtype = (
FP8_DTYPE if kv_cache_dtype_str.startswith("fp8") else self.dtype
)
if self.kv_cache_dtype != self.dtype:
# Non-unit scales so the fused and unfused writers must agree on them.
self.attn._k_scale = torch.tensor(0.5, dtype=torch.float32, device=device)
self.attn._v_scale = torch.tensor(0.25, dtype=torch.float32, device=device)
self.attn._k_scale_cpu = torch.tensor(
0.5, dtype=torch.float32, device="cpu"
)
self.attn._v_scale_cpu = torch.tensor(
0.25, dtype=torch.float32, device="cpu"
)
self.attn._k_scale_float = 0.5
self.attn._v_scale_float = 0.25
else:
self.attn._k_scale = self.attn._k_scale.to(device)
self.attn._v_scale = self.attn._v_scale.to(device)
# Mirror the worker spec-collection loop: the backend publishes the
# packing it needs.
self.kv_cache_spec = self.attn.attn_backend.customize_spec(
AttentionSpec(
block_size=self.block_size,
num_kv_heads=self.num_kv_heads,
head_size=head_size,
dtype=self.kv_cache_dtype,
)
)
self.builder = self.attn.attn_backend.get_builder_cls()(
kv_cache_spec=self.kv_cache_spec,
layer_names=[self.attn.layer_name],
vllm_config=vllm_config,
device=device,
)
def build_attn_metadata(
self, batch_size: int, layout: KVCacheLayout
) -> CommonAttentionMetadata:
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
common_attn_metadata = create_common_attn_metadata(
batch_spec, self.block_size, self.device, arange_block_indices=True
)
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
num_blocks = batch_size * max_blocks
raw_tensor = torch.zeros(
num_blocks * self.kv_cache_spec.page_size_bytes,
dtype=torch.int8,
device=self.device,
)
kv_cache = dense_kv_cache_views(
raw_tensor,
self.kv_cache_spec,
num_blocks,
num_layers=1,
layout=layout,
)[0]
# Store as a bare tensor (not wrapped in a list) to match production
# `bind_kv_cache` behavior. `get_attention_context` returns this
# attribute directly to the fused/unfused `do_kv_cache_update` impls,
# which call `kv_cache.unbind(0)` and therefore require a tensor.
self.attn.kv_cache = kv_cache
attn_metadata = self.builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
return attn_metadata
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
qkv = qkv.clone()
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
# QK-norm: RMSNorm on per-head Q and K
q = q.view(-1, self.num_heads, self.head_size)
q = self.q_norm(q)
q = q.view(-1, self.q_size)
k = k.view(-1, self.num_kv_heads, self.head_size)
k = self.k_norm(k)
k = k.view(-1, self.kv_size)
# RoPE
q, k = self.rotary_emb(positions, q, k)
# Mirror Attention.forward: quant-query impls consume an fp8 q.
if (
self.kv_cache_dtype != self.dtype
and self.attn.impl.supports_quant_query_input
):
assert self.attn.query_quant is not None
q, _ = self.attn.query_quant(q, self.attn._q_scale)
# Final views + KV cache update
q = q.view(-1, self.num_heads, self.head_size)
k = k.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size)
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
k, v, _encode_layer_name(self.layer_name)
)
return q, k, v, kv_cache_dummy_dep
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
ops: list[torch._ops.OpOverload] = []
# RoPE is not yet IR-migrated, so its custom op still surfaces
# directly in the graph based on `enable_rope_custom_op`.
if self.enable_rope_custom_op:
if rocm_aiter_ops.is_triton_rotary_embed_enabled():
ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default)
else:
ops.append(ROTARY_OP)
else:
ops.append(INDEX_SELECT_OP)
ops.append(torch.ops.vllm.unified_kv_cache_update.default)
return ops
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default]
class QKNormMRoPEKVCacheTestModel(QKNormRoPEKVCacheTestModel):
def __init__(
self,
*,
mrope_section: tuple[int, int, int],
mrope_interleaved: bool,
**kwargs,
):
super().__init__(**kwargs)
self.rotary_emb = MRotaryEmbedding(
self.head_size,
rotary_dim=self.rotary_dim,
max_position_embeddings=4096,
base=10000,
is_neox_style=self.is_neox,
dtype=self.dtype,
mrope_section=list(mrope_section),
mrope_interleaved=mrope_interleaved,
)
self.enable_rope_custom_op = self.rotary_emb.enabled()
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q, k, v, kv_cache_dummy_dep = super().forward(qkv, positions)
output = torch.empty(
q.shape[0],
self.num_heads * self.head_size,
device=q.device,
dtype=self.dtype,
)
torch.ops.vllm.unified_attention_with_output(
q,
k,
v,
output,
_encode_layer_name(self.layer_name),
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
# Production decoder attention consumes K from the cache rather than
# its direct K argument. Returning `output` keeps that production user
# in the graph without exposing K to an unsupported consumer.
return q, output, v, kv_cache_dummy_dep
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
return [MROPE_OP, torch.ops.vllm.unified_kv_cache_update.default]
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.vllm.fused_qk_norm_mrope_and_unified_kv_cache_update.default]
class QKNormMRoPEObservableKTestModel(QKNormMRoPEKVCacheTestModel):
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
return QKNormRoPEKVCacheTestModel.forward(self, qkv, positions)
def _run_qk_norm_rope_kvcache_fusion_test(
*,
attn_backend: AttentionBackendEnum,
enable_aiter_triton_rope: bool,
num_tokens: int,
num_heads: int,
num_kv_heads: int,
head_size: int,
rotary_dim: int,
block_size: int,
is_neox: bool,
use_shuffle_kv_layout: str,
kv_layout: KVCacheLayout,
dtype: torch.dtype,
kv_cache_dtype: CacheDType,
rms_norm_eps: float,
custom_op: str,
monkeypatch: pytest.MonkeyPatch,
mrope_section: tuple[int, int, int] | None = None,
mrope_interleaved: bool = False,
observe_mrope_k: bool = False,
expect_fusion: bool = True,
use_quark_scalar_query_scale: bool = False,
) -> None:
device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda")
torch.set_default_device(device)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
# Set AITER toggles before VllmConfig initialization enables env caching.
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
rocm_aiter_ops.refresh_env_variables()
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
cache_config=CacheConfig(
block_size=block_size,
cache_dtype=kv_cache_dtype,
),
quant_config=(
QuarkConfig(quant_config={"exclude": []})
if use_quark_scalar_query_scale
else None
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=[custom_op],
pass_config=PassConfig(
fuse_qk_norm_rope_kvcache=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
m.setenv(
"VLLM_ROCM_USE_AITER_TRITON_ROPE",
"1" if enable_aiter_triton_rope else "0",
)
m.setenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", use_shuffle_kv_layout)
rocm_aiter_ops.refresh_env_variables()
expected_query_quant_group_shape = None
if use_quark_scalar_query_scale:
expected_query_quant_group_shape = (
GroupShape(-1, num_heads * head_size)
if num_kv_heads == 1
else GroupShape.PER_TENSOR
)
model_kwargs = {
"vllm_config": vllm_config,
"attn_backend": attn_backend,
"num_heads": num_heads,
"num_kv_heads": num_kv_heads,
"head_size": head_size,
"rotary_dim": rotary_dim,
"is_neox": is_neox,
"rms_norm_eps": rms_norm_eps,
"dtype": dtype,
"device": torch.get_default_device(),
"expected_query_quant_group_shape": expected_query_quant_group_shape,
}
if mrope_section is None:
model = QKNormRoPEKVCacheTestModel(**model_kwargs)
else:
alternate_section = (8, 8, 8)
m.setattr(
fusion_module,
"_discover_mrope_configs",
lambda _: (
(mrope_section, mrope_interleaved),
(alternate_section, not mrope_interleaved),
),
)
model_cls = (
QKNormMRoPEObservableKTestModel
if observe_mrope_k
else QKNormMRoPEKVCacheTestModel
)
model = model_cls(
**model_kwargs,
mrope_section=mrope_section,
mrope_interleaved=mrope_interleaved,
)
fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
SplitCoalescingPass(vllm_config),
ScatterSplitReplacementPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
qkv = torch.randn(
num_tokens,
num_heads * head_size + 2 * num_kv_heads * head_size,
dtype=dtype,
)
if mrope_section is None:
pos = torch.arange(num_tokens, dtype=torch.long)
else:
# Real Qwen-VL position IDs can be a strided [3, N] view.
pos = torch.randint(0, 4096, (3, 2 * num_tokens), dtype=torch.long)[:, ::2]
assert not pos.is_contiguous()
qkv_unfused = qkv.clone()
pos_unfused = pos.clone()
# Run unfused (eager) forward
with set_forward_context(None, vllm_config):
forward_context = get_forward_context()
attn_metadata = model.build_attn_metadata(num_tokens, kv_layout)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_unfused = attn_layer.kv_cache
del dummy
# Run fused (compiled) forward
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(pos, 0 if mrope_section is None else 1)
with set_forward_context(None, vllm_config):
model_fused = torch.compile(model, backend=backend)
forward_context = get_forward_context()
attn_metadata = model_fused.build_attn_metadata(num_tokens, kv_layout)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_fused = attn_layer.kv_cache
del dummy
assert fusion_pass.matched_count == int(expect_fusion)
if expect_fusion:
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
else:
for op in model.ops_in_model_before():
assert backend.op_count(op, before=True) > 0
assert backend.op_count(op) > 0
if use_quark_scalar_query_scale:
expected_quant_op = (
torch.ops.vllm.rocm_aiter_per_tensor_quant.default
if num_kv_heads != 1
else torch.ops._C.static_scaled_fp8_quant.default
)
assert backend.op_count(expected_quant_op, before=True) > 0
# Query quantization remains separate after the prologue fusion.
assert backend.op_count(expected_quant_op) > 0
# Sweep-backed (18.2k pts, PR #42749): native-rope ref worst 7.7e-3 -> 1e-2;
# AITER-triton-rope ref is itself approximate (plateau 1.28e-2) -> 2e-2.
ATOL, RTOL = (
(2e-2, 2e-2)
if enable_aiter_triton_rope or mrope_section is not None
else (1e-2, 1e-2)
)
is_fp8_cache = model.kv_cache_dtype != dtype
if q_fused.dtype == FP8_DTYPE:
# Quant-query path: both q are fp8; compare dequant within 1 fp8 ULP.
torch.testing.assert_close(
q_unfused.float(), q_fused.float(), atol=1.25e-1, rtol=1.25e-1
)
else:
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
# Fused and unfused arithmetic can straddle an FP8 rounding boundary.
# Allow one E4M3 quantization step while still comparing every block.
if is_fp8_cache:
cache_atol = cache_rtol = 1.25e-1
else:
cache_atol, cache_rtol = ATOL, RTOL
# The fused k_out is the dequantized cache write, so it carries the same
# rounding; FA and ROCM_ATTN prefill consume it directly. Unified reads K
# from the cache and asks the kernel not to write k_out at all.
if attn_backend != AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN:
torch.testing.assert_close(
k_unfused, k_fused, atol=cache_atol, rtol=cache_rtol
)
# Should be bit exact since no processing had been done on v for both paths
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
# Whole cache, so the interleaved V write (ROCM_ATTN kv_cache[1]) is checked.
torch.testing.assert_close(
kv_cache_unfused.float(),
kv_cache_fused.float(),
atol=cache_atol,
rtol=cache_rtol,
)
_FUSION_CONFIGS = [
# Full rotary, both neox styles (the original coverage).
pytest.param(64, 8, 64, 64, True, id="full-neox"),
pytest.param(64, 8, 64, 64, False, id="full-non_neox"),
# GLM-4.5/4.6/4.7 (glm4_moe.py:275 partial_rotary_factor=0.5, neox-style)
pytest.param(32, 8, 128, 64, True, id="glm4_moe"),
# GLM-4 dense (glm4.py:97,124 partial_rotary_factor=0.5, non-neox)
pytest.param(32, 2, 128, 64, False, id="glm4_dense"),
# Moondream3-style small head (head_size=64, rotary_dim=32)
pytest.param(16, 2, 64, 32, True, id="partial_small_head"),
]
@pytest.mark.parametrize(
"num_heads, num_kv_heads, head_size, rotary_dim, is_neox",
_FUSION_CONFIGS,
)
@pytest.mark.parametrize(
"attn_backend, use_shuffle_kv_layout, kv_layout",
[
# ROCM_ATTN always writes the shuffled, interleaved-V layout and needs each
# K/V side contiguous per block, so head-major only.
pytest.param(
AttentionBackendEnum.ROCM_ATTN,
"0",
KVCacheLayout.LBHNC,
id="rocm_attn",
),
# Unified never reads the shuffle env; its strided views take any layout.
pytest.param(
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
"0",
KVCacheLayout.LBHNC,
id="unified-head_major",
),
pytest.param(
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
"0",
KVCacheLayout.LBNHC,
id="unified-token_major",
),
# FA takes any layout unshuffled; its shuffle writer needs each K/V side
# contiguous per block, so head-major only.
pytest.param(
AttentionBackendEnum.ROCM_AITER_FA,
"0",
KVCacheLayout.LBHNC,
id="fa-head_major",
),
pytest.param(
AttentionBackendEnum.ROCM_AITER_FA,
"0",
KVCacheLayout.LBNHC,
id="fa-token_major",
),
pytest.param(
AttentionBackendEnum.ROCM_AITER_FA,
"1",
KVCacheLayout.LBHNC,
id="fa-shuffle",
),
],
)
@pytest.mark.parametrize("num_tokens", [5, 2048])
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
@pytest.mark.parametrize("block_size", [16, 32, 64])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("rms_norm_eps", [1e-6])
@pytest.mark.parametrize("custom_op", ["+rotary_embedding", "+rms_norm"])
# The fused_qk_norm_rope_cache kernel used by this test aborts on AITER < 0.1.20
# (fused_qk_norm_rope_cache_quant.cu: "k_cache/v_cache must be contiguous within
# a block")
@pytest.mark.skipif(
not Version(version("amd_aiter")) >= Version("0.1.20"),
reason=(
"Requires AITER >= 0.1.20; older kernels abort on "
"fused_qk_norm_rope_cache (k_cache/v_cache contiguity)"
),
)
def test_qk_norm_rope_kvcache_fusion(
num_tokens: int,
num_heads: int,
num_kv_heads: int,
head_size: int,
rotary_dim: int,
is_neox: bool,
attn_backend: AttentionBackendEnum,
enable_aiter_triton_rope: bool,
kv_layout: KVCacheLayout,
block_size: int,
dtype: torch.dtype,
kv_cache_dtype: CacheDType,
rms_norm_eps: float,
custom_op: str,
use_shuffle_kv_layout: str,
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=attn_backend,
enable_aiter_triton_rope=enable_aiter_triton_rope,
num_tokens=num_tokens,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
rotary_dim=rotary_dim,
block_size=block_size,
is_neox=is_neox,
use_shuffle_kv_layout=use_shuffle_kv_layout,
kv_layout=kv_layout,
dtype=dtype,
kv_cache_dtype=kv_cache_dtype,
rms_norm_eps=rms_norm_eps,
custom_op=custom_op,
monkeypatch=monkeypatch,
)
@pytest.mark.parametrize(
"mrope_section",
[
pytest.param((16, 24, 24), id="full_rotary"),
pytest.param((4, 6, 6), id="partial_rotary"),
],
)
@pytest.mark.parametrize("mrope_interleaved", [False, True])
@pytest.mark.parametrize("is_neox", [False, True])
@pytest.mark.parametrize(
"kv_layout",
[
pytest.param(KVCacheLayout.LBHNC, id="head_major"),
pytest.param(KVCacheLayout.LBNHC, id="token_major"),
pytest.param(KVCacheLayout.LHBNC, id="block_major"),
],
)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.skipif(
not IS_AITER_FOUND,
reason="Requires AITER",
)
def test_qk_norm_mrope_kvcache_fusion(
mrope_section: tuple[int, int, int],
mrope_interleaved: bool,
is_neox: bool,
kv_layout: KVCacheLayout,
kv_cache_dtype: CacheDType,
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
enable_aiter_triton_rope=False,
num_tokens=5,
num_heads=8,
num_kv_heads=2,
head_size=128,
rotary_dim=2 * sum(mrope_section),
block_size=16,
is_neox=is_neox,
use_shuffle_kv_layout="0",
kv_layout=kv_layout,
dtype=torch.bfloat16,
kv_cache_dtype=kv_cache_dtype,
rms_norm_eps=1e-6,
custom_op="+rotary_embedding",
monkeypatch=monkeypatch,
mrope_section=mrope_section,
mrope_interleaved=mrope_interleaved,
)
@pytest.mark.parametrize(
("num_heads", "num_kv_heads"),
[(32, 2), (16, 1), (8, 1)],
ids=["tp2", "tp4", "tp8"],
)
def test_qk_norm_mrope_kvcache_fusion_quark_scalar_query_scale(
num_heads: int,
num_kv_heads: int,
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
enable_aiter_triton_rope=False,
num_tokens=5,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=128,
rotary_dim=128,
block_size=16,
is_neox=True,
use_shuffle_kv_layout="0",
kv_layout=KVCacheLayout.LBHNC,
dtype=torch.bfloat16,
kv_cache_dtype="fp8",
rms_norm_eps=1e-6,
custom_op="+rotary_embedding",
monkeypatch=monkeypatch,
mrope_section=(24, 20, 20),
mrope_interleaved=True,
use_quark_scalar_query_scale=True,
)
def test_mrope_fusion_rejects_observable_k(monkeypatch: pytest.MonkeyPatch):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
enable_aiter_triton_rope=False,
num_tokens=5,
num_heads=8,
num_kv_heads=2,
head_size=128,
rotary_dim=128,
block_size=16,
is_neox=True,
use_shuffle_kv_layout="0",
kv_layout=KVCacheLayout.LBHNC,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
rms_norm_eps=1e-6,
custom_op="+rotary_embedding",
monkeypatch=monkeypatch,
mrope_section=(16, 24, 24),
observe_mrope_k=True,
expect_fusion=False,
)
def test_mrope_fusion_rejects_unaligned_rotary_dim(
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
enable_aiter_triton_rope=False,
num_tokens=5,
num_heads=8,
num_kv_heads=2,
head_size=128,
rotary_dim=6,
block_size=16,
is_neox=True,
use_shuffle_kv_layout="0",
kv_layout=KVCacheLayout.LBHNC,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
rms_norm_eps=1e-6,
custom_op="+rotary_embedding",
monkeypatch=monkeypatch,
mrope_section=(1, 1, 1),
expect_fusion=False,
)
@pytest.mark.skipif(not _USE_LAYERNAME, reason="requires opaque LayerName")
def test_opaque_layer_name_patterns_register_once(monkeypatch: pytest.MonkeyPatch):
device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda")
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=torch.bfloat16),
cache_config=CacheConfig(block_size=16, cache_dtype="fp8_e4m3"),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rotary_embedding"],
pass_config=PassConfig(fuse_qk_norm_rope_kvcache=True),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
model_kwargs = {
"vllm_config": vllm_config,
"attn_backend": AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
"num_heads": 8,
"num_kv_heads": 2,
"head_size": 128,
"is_neox": True,
"rms_norm_eps": 1e-6,
"dtype": torch.bfloat16,
"device": torch.device(device),
}
models = [
QKNormRoPEKVCacheTestModel(
**model_kwargs,
prefix=f"model.layers.{index}.self_attn.attn",
)
for index in range(2)
]
# Opaque layer names are wildcards in the pattern graph. Registering
# once per physical layer raises "Duplicate pattern" on torch >= 2.11.
fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config)
assert models
assert fusion_pass.patterns
def _call_fused_mrope_impl(q_out: torch.Tensor) -> None:
device = q_out.device
fusion_module.fused_qk_norm_mrope_and_unified_kv_cache_update_impl(
q_out=q_out,
qkv=torch.zeros(2, 16, device=device),
positions=torch.arange(2, device=device),
q_weight=torch.ones(4, device=device),
k_weight=torch.ones(4, device=device),
rms_norm_eps=1e-6,
cos_sin_cache=torch.zeros(16, 4, device=device),
is_neox=True,
mrope_section_t=1,
mrope_section_h=1,
mrope_section_w=0,
is_interleaved=False,
rotary_dim=4,
layer_name="model.layers.0.self_attn.attn",
)
def test_mrope_profiling_pass_initializes_query(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(
fusion_module,
"get_attention_context",
lambda _: (None, None, None, None),
)
q_out = torch.full((2, 8), torch.nan, device="cpu")
_call_fused_mrope_impl(q_out)
torch.testing.assert_close(q_out, torch.zeros_like(q_out))
def test_mrope_attention_context_failure_propagates(
monkeypatch: pytest.MonkeyPatch,
):
def fail_context(_):
raise RuntimeError("missing attention context")
monkeypatch.setattr(fusion_module, "get_attention_context", fail_context)
with pytest.raises(RuntimeError, match="missing attention context"):
_call_fused_mrope_impl(torch.empty(2, 8, device="cpu"))
@@ -1,635 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from torch._higher_order_ops import auto_functionalized
import vllm.config
from tests.compile.backend import TestBackend
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
dense_kv_cache_views,
)
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.compilation.passes.fusion import rope_kvcache_fusion
from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP
from vllm.compilation.passes.fusion.rope_kvcache_fusion import RopeKVCacheFusionPass
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.compilation.passes.utility.scatter_split_replace import (
ScatterSplitReplacementPass,
)
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
from vllm.config import (
CacheConfig,
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
from vllm.config.cache import CacheDType
from vllm.config.utils import Range
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention import attention as attention_module
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.platforms import current_platform
from vllm.utils.torch_utils import _encode_layer_name
from vllm.v1.attention.backend import (
AttentionBackend,
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheLayout,
)
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
VLLM_UNIFIED_KV_CACHE_UPDATE_OP = torch.ops.vllm.unified_kv_cache_update
FP8_DTYPE = current_platform.fp8_dtype()
def test_kv_cache_update_ops_fake_tensor_metadata(monkeypatch: pytest.MonkeyPatch):
query = torch.empty(1, dtype=torch.bfloat16)
kv_cache = torch.empty(1, dtype=torch.uint8)
context = (None, None, kv_cache, None)
monkeypatch.setattr(attention_module, "get_attention_context", lambda _: context)
monkeypatch.setattr(rope_kvcache_fusion, "get_attention_context", lambda _: context)
runtime_output = attention_module.unified_kv_cache_update(query, query, "layer")
fake_output = attention_module.unified_kv_cache_update_fake(query, query, "layer")
assert runtime_output.shape == fake_output.shape
assert runtime_output.dtype == fake_output.dtype
assert runtime_output.device == fake_output.device
args = (
query,
query,
query,
torch.empty(1, dtype=torch.int64),
torch.empty(1),
True,
"layer",
)
runtime_output = rope_kvcache_fusion.fused_rope_and_unified_kv_cache_update_impl(
*args
)
fake_output = rope_kvcache_fusion.fused_rope_and_unified_kv_cache_update_fake(*args)
assert runtime_output.shape == fake_output.shape
assert runtime_output.dtype == fake_output.dtype
assert runtime_output.device == fake_output.device
def test_rope_kvcache_fusion_default_keeps_large_ranges_unfused():
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(fuse_rope_kvcache=True),
),
)
fusion_pass = RopeKVCacheFusionPass(vllm_config)
assert fusion_pass.is_applicable_for_range(Range(1, 256))
assert not fusion_pass.is_applicable_for_range(Range(257, 11650))
assert not fusion_pass.is_applicable_for_range(Range(11651, 16384))
class QKRoPEKVCacheTestModel(torch.nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
attn_backend: AttentionBackendEnum,
num_heads: int,
num_kv_heads: int,
head_size: int,
is_neox: bool,
dtype: torch.dtype,
device: torch.device,
prefix: str = "model.layers.0.self_attn.attn",
):
super().__init__()
self.num_heads = num_heads
self.num_kv_heads = num_kv_heads
self.head_size = head_size
self.block_size = vllm_config.cache_config.block_size
self.q_size = num_heads * head_size
self.kv_size = num_kv_heads * head_size
self.is_neox = is_neox
self.dtype = dtype
self.device = device
self.layer_name = prefix
self.rotary_emb = RotaryEmbedding(
head_size,
rotary_dim=head_size,
max_position_embeddings=4096,
base=10000,
is_neox_style=is_neox,
dtype=self.dtype,
)
# Whether to check for the RoPE custom op or component index_select
self.enable_rope_custom_op = self.rotary_emb.enabled()
# Register layer metadata for the fusion pass via Attention.
self.attn = Attention(
num_heads=num_heads,
head_size=head_size,
scale=1.0 / head_size**0.5,
num_kv_heads=num_kv_heads,
cache_config=vllm_config.cache_config,
quant_config=vllm_config.quant_config,
prefix=prefix,
attn_backend=attn_backend.get_class(),
)
self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend()
assert not self.attn_backend.forward_includes_kv_cache_update, (
f"Attention backend {self.attn_backend} does not support fuse_rope_kvcache."
)
self.attn._k_scale = self.attn._k_scale.to(device)
self.attn._v_scale = self.attn._v_scale.to(device)
kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
self.kv_cache_dtype = (
FP8_DTYPE if kv_cache_dtype_str.startswith("fp8") else self.dtype
)
# Mirror the worker spec-collection loop: the backend publishes its
# packing (e.g. ROCM_ATTN's separate K/V head groups).
self.kv_cache_spec = self.attn_backend.customize_spec(
FullAttentionSpec(
block_size=self.block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
dtype=self.kv_cache_dtype,
)
)
# Initialize attn MetadataBuilder
self.builder = self.attn_backend.get_builder_cls()(
kv_cache_spec=self.kv_cache_spec,
layer_names=[self.attn.layer_name],
vllm_config=vllm_config,
device=device,
)
def build_attn_metadata(self, batch_size: int) -> CommonAttentionMetadata:
"""Initialize attention metadata."""
# Create common attn metadata
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
common_attn_metadata = create_common_attn_metadata(
batch_spec, self.block_size, self.device, arange_block_indices=True
)
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
num_blocks = batch_size * max_blocks
# A backend's preferred supported layout wins over the default, mirroring
# selector-time resolution (e.g. ROCM_ATTN's head groups force LHBNC).
layout = KVCacheLayout.LBNHC
supported = self.attn_backend.supported_kv_cache_layouts()
if supported:
layout = supported[0]
raw_tensor = torch.zeros(
num_blocks * self.kv_cache_spec.page_size_bytes,
dtype=torch.int8,
device=self.device,
)
self.attn.kv_cache = dense_kv_cache_views(
raw_tensor,
self.kv_cache_spec,
num_blocks,
num_layers=1,
layout=layout,
)[0]
# Build attn metadata
attn_metadata = self.builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
return attn_metadata
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# Create copy so inplace ops do not modify the original tensors
qkv = qkv.clone()
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
# Instead of a full forward pass, match only the KV cache update op here
q = q.view(-1, self.num_heads, self.head_size)
k = k.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size)
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
k, v, _encode_layer_name(self.layer_name)
)
return q, k, v, kv_cache_dummy_dep
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
ops = []
if self.enable_rope_custom_op:
if rocm_aiter_ops.is_triton_rotary_embed_enabled():
ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default)
else:
ops.append(ROTARY_OP)
else:
ops.append(INDEX_SELECT_OP)
ops.append(torch.ops.vllm.unified_kv_cache_update.default)
return ops
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default]
class QKRoPEStaticQKVCacheTestModel(QKRoPEKVCacheTestModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.q_scale = torch.ones((), dtype=torch.float32, device=self.device)
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# Create copy so inplace ops do not modify the original tensors
qkv = qkv.clone()
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
q_fp8 = torch.empty(q.shape, device=q.device, dtype=FP8_DTYPE)
_, q_fp8 = auto_functionalized(
torch.ops._C.static_scaled_fp8_quant.default,
result=q_fp8,
input=q,
scale=self.q_scale,
group_shape=(-1, -1),
)
q = q_fp8.view(-1, self.num_heads, self.head_size)
k = k.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size)
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
k, v, _encode_layer_name(self.layer_name)
)
return q, k, v, kv_cache_dummy_dep
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
ops = []
if self.enable_rope_custom_op:
if rocm_aiter_ops.is_triton_rotary_embed_enabled():
ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default)
else:
ops.append(ROTARY_OP)
else:
ops.append(INDEX_SELECT_OP)
ops.append(torch.ops.vllm.unified_kv_cache_update.default)
return ops
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default]
@pytest.mark.parametrize(
"attn_backend",
[
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
AttentionBackendEnum.TRITON_ATTN,
AttentionBackendEnum.ROCM_ATTN,
AttentionBackendEnum.ROCM_AITER_FA,
],
)
@pytest.mark.parametrize("enable_rope_custom_op", [True]) # [True, False])
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
@pytest.mark.parametrize("num_heads", [64])
@pytest.mark.parametrize("num_kv_heads", [8])
@pytest.mark.parametrize("head_size", [64])
@pytest.mark.parametrize("block_size", [16])
@pytest.mark.parametrize("is_neox", [True, False])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def test_rope_kvcache_fusion(
attn_backend: AttentionBackendEnum,
enable_rope_custom_op: bool,
enable_aiter_triton_rope: bool,
num_heads: int,
num_kv_heads: int,
head_size: int,
block_size: int,
is_neox: bool,
dtype: torch.dtype,
kv_cache_dtype: CacheDType,
monkeypatch: pytest.MonkeyPatch,
):
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(0)
custom_ops: list[str] = []
if enable_rope_custom_op:
custom_ops.append("+rotary_embedding")
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
cache_config=CacheConfig(
block_size=block_size,
cache_dtype=kv_cache_dtype,
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops,
pass_config=PassConfig(
fuse_rope_kvcache=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv(
"VLLM_ROCM_USE_AITER_TRITON_ROPE", "1" if enable_aiter_triton_rope else "0"
)
rocm_aiter_ops.refresh_env_variables()
model = QKRoPEKVCacheTestModel(
vllm_config=vllm_config,
attn_backend=attn_backend,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
is_neox=is_neox,
dtype=dtype,
device=torch.get_default_device(),
)
fusion_pass = RopeKVCacheFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
SplitCoalescingPass(vllm_config),
ScatterSplitReplacementPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
T = 5
qkv = torch.randn(
T, num_heads * head_size + 2 * num_kv_heads * head_size, dtype=dtype
)
pos = torch.arange(T, dtype=torch.long)
qkv_unfused = qkv.clone()
pos_unfused = pos.clone()
with set_forward_context(None, vllm_config):
forward_context = get_forward_context()
attn_metadata = model.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_unfused = attn_layer.kv_cache
del dummy
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(pos, 0)
with set_forward_context(None, vllm_config):
model_fused = torch.compile(model, backend=backend)
forward_context = get_forward_context()
attn_metadata = model_fused.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_fused = attn_layer.kv_cache
del dummy
assert fusion_pass.matched_count == 1
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
if dtype == torch.float16:
ATOL, RTOL = (2e-3, 2e-3)
else:
ATOL, RTOL = (1e-2, 1e-2)
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(v_unfused, v_fused, atol=ATOL, rtol=RTOL)
# Cannot compare fp8_* directly here, cast to model dtype instead
# TODO(charlifu): switch back to ATOL, RTOL after aiter fix is merged.
torch.testing.assert_close(
kv_cache_unfused.to(dtype),
kv_cache_fused.to(dtype),
atol=1e-1,
rtol=1e-1,
)
@pytest.mark.parametrize(
"attn_backend",
[AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN],
)
@pytest.mark.parametrize("enable_rope_custom_op", [True])
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
@pytest.mark.parametrize("num_heads", [64])
@pytest.mark.parametrize("num_kv_heads", [8])
@pytest.mark.parametrize("head_size", [64])
@pytest.mark.parametrize("block_size", [16])
@pytest.mark.parametrize("is_neox", [True, False])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
@pytest.mark.skipif(
not hasattr(torch.ops._C, "static_scaled_fp8_quant"),
reason="static fp8 quant op not available on this build",
)
def test_rope_static_qquant_kvcache_fusion(
attn_backend: AttentionBackendEnum,
enable_rope_custom_op: bool,
enable_aiter_triton_rope: bool,
num_heads: int,
num_kv_heads: int,
head_size: int,
block_size: int,
is_neox: bool,
dtype: torch.dtype,
kv_cache_dtype: CacheDType,
monkeypatch: pytest.MonkeyPatch,
):
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(0)
custom_ops: list[str] = []
if enable_rope_custom_op:
custom_ops.append("+rotary_embedding")
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
cache_config=CacheConfig(
block_size=block_size,
cache_dtype=kv_cache_dtype,
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops,
pass_config=PassConfig(
fuse_rope_kvcache=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv(
"VLLM_ROCM_USE_AITER_TRITON_ROPE", "1" if enable_aiter_triton_rope else "0"
)
rocm_aiter_ops.refresh_env_variables()
model = QKRoPEStaticQKVCacheTestModel(
vllm_config=vllm_config,
attn_backend=attn_backend,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
is_neox=is_neox,
dtype=dtype,
device=torch.get_default_device(),
)
fusion_pass = RopeKVCacheFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
SplitCoalescingPass(vllm_config),
ScatterSplitReplacementPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
T = 5
qkv = torch.randn(
T, num_heads * head_size + 2 * num_kv_heads * head_size, dtype=dtype
)
pos = torch.arange(T, dtype=torch.long)
qkv_unfused = qkv.clone()
pos_unfused = pos.clone()
with set_forward_context(None, vllm_config):
forward_context = get_forward_context()
attn_metadata = model.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_unfused = attn_layer.kv_cache
del dummy
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(pos, 0)
with set_forward_context(None, vllm_config):
model_fused = torch.compile(model, backend=backend)
forward_context = get_forward_context()
attn_metadata = model_fused.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_fused = attn_layer.kv_cache
del dummy
assert fusion_pass.matched_count == 1
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
static_quant_pre = backend.op_count(
torch.ops._C.static_scaled_fp8_quant.default, before=True
)
static_quant_post = backend.op_count(
torch.ops._C.static_scaled_fp8_quant.default
)
assert static_quant_pre > 0
# The replacement still emits static quant, so count is expected to
# remain non-zero after fusion.
assert static_quant_post > 0
# Negative control: without the static-Q pattern, the generic RoPE+KV
# pattern cannot match this rope -> static-quant -> kv graph, so the
# fusion above is attributable solely to RopeStaticQQuantKVCachePattern.
# This is a structural property independent of the rope/dtype/neox axes,
# so run the (extra compile) check only once on a representative combo.
if is_neox and enable_aiter_triton_rope and kv_cache_dtype == "auto":
m.setattr(
rope_kvcache_fusion,
"_supports_static_q_fp8_quant_fusion",
lambda: False,
)
generic_pass = RopeKVCacheFusionPass(vllm_config)
generic_backend = TestBackend(
NoOpEliminationPass(vllm_config),
SplitCoalescingPass(vllm_config),
ScatterSplitReplacementPass(vllm_config),
generic_pass,
PostCleanupPass(vllm_config),
)
# Reset dynamo so the model is recompiled through generic_backend
# instead of reusing the cached compilation from above.
torch._dynamo.reset()
with set_forward_context(None, vllm_config):
model_generic = torch.compile(model, backend=generic_backend)
forward_context = get_forward_context()
attn_metadata = model_generic.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
model_generic(qkv, pos)
# op_count reads the post-pass graph, so it also confirms the pass ran
# (a no-op pass would raise instead of silently passing on count 0).
assert generic_pass.matched_count == 0
assert (
generic_backend.op_count(
torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default
)
== 0
)
if dtype == torch.float16:
ATOL, RTOL = (2e-3, 2e-3)
else:
ATOL, RTOL = (1e-2, 1e-2)
# TODO(charlifu): switch back to ATOL, RTOL after aiter fix is merged.
torch.testing.assert_close(
q_unfused.to(torch.float32),
q_fused.to(torch.float32),
atol=1e-1,
rtol=1e-1,
)
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(v_unfused, v_fused, atol=ATOL, rtol=RTOL)
# TODO(charlifu): switch back to ATOL, RTOL after aiter fix is merged.
torch.testing.assert_close(
kv_cache_unfused.to(dtype),
kv_cache_fused.to(dtype),
atol=1e-1,
rtol=1e-1,
)
@@ -1,110 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn as nn
import vllm
from tests.compile.backend import TestBackend
from vllm.compilation.passes.utility.scatter_split_replace import (
ScatterSplitReplacementPass,
)
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
from vllm.config import CompilationConfig, CompilationMode, VllmConfig
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.platforms import current_platform
DEVICE_TYPE = current_platform.device_type
class ScatterSplitReplacementModel(nn.Module):
"""Model with a rope+getitem+slice_scatter+split_with_sizes sequence."""
def __init__(
self,
num_heads: int,
num_kv_heads: int,
head_size: int,
dtype: torch.dtype,
):
super().__init__()
self.q_size = num_heads * head_size
self.kv_size = num_kv_heads * head_size
self.rotary_emb = RotaryEmbedding(
head_size,
rotary_dim=head_size,
max_position_embeddings=4096,
base=10000,
is_neox_style=True,
dtype=dtype,
)
def forward(self, qkv: torch.Tensor, positions: torch.Tensor):
# Create copy so inplace ops do not modify the original tensors
qkv = qkv.clone()
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
q = q + 1
k = k + 2
v = v + 3
return q, k, v
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
return [
torch.ops.aten.slice_scatter.default,
torch.ops.aten.split_with_sizes.default,
torch.ops.aten.getitem.default,
]
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.aten.getitem.default]
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_scatter_split_replace(dtype):
torch.set_default_device(DEVICE_TYPE)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
num_heads = 8
num_kv_heads = 4
head_size = 64
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rotary_embedding"],
),
)
with vllm.config.set_current_vllm_config(vllm_config):
# ScatterSplitReplacementPass requires SplitCoalescingPass to be run before it
coalesce_pass = SplitCoalescingPass(vllm_config)
replace_pass = ScatterSplitReplacementPass(vllm_config)
passes = [coalesce_pass, replace_pass]
backend = TestBackend(*passes)
model = ScatterSplitReplacementModel(num_heads, num_kv_heads, head_size, dtype)
T = 5
qkv = torch.randn(
T, num_heads * head_size + 2 * num_kv_heads * head_size, dtype=dtype
)
pos = torch.arange(T, dtype=torch.long)
qkv_eager = qkv.clone()
pos_eager = pos.clone()
result_eager = model(qkv_eager, pos_eager)
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(pos, 0)
model_compiled = torch.compile(model, backend=backend)
result_compiled = model_compiled(qkv, pos)
for eager, compiled in zip(result_eager, result_compiled):
torch.testing.assert_close(eager, compiled)
assert backend.op_count(torch.ops.aten.slice_scatter.default) == 0
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
@@ -1,373 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from functools import partial
import pytest
import torch
import vllm.envs as envs
from tests.compile.backend import TestBackend
from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor
from tests.utils import TestFP8Layer
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
from vllm.compilation.passes.fusion.act_quant_fusion import (
FUSED_OPS,
SILU_MUL_OP,
ActivationQuantFusionPass,
)
from vllm.compilation.passes.fusion.rms_quant_fusion import QUANT_OPS
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CompilationConfig,
CompilationMode,
PassConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.model_executor.kernels.linear import (
CutlassFP8ScaledMMLinearKernel,
FlashInferFP8ScaledMMLinearKernel,
FP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ROCmFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
create_fp8_quant_key,
kFp8Dynamic128Sym,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
def is_nvfp4_supported():
return current_platform.has_device_capability(100)
class TestSiluMulFp8QuantModel(torch.nn.Module):
quant_key = kFp8StaticTensorSym
def __init__(
self,
hidden_size: int,
force_kernel: type[FP8ScaledMMLinearKernel],
dtype: torch.dtype,
**kwargs,
):
super().__init__()
self.silu_and_mul = SiluAndMul()
self.fp8_linear = TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
force_kernel=force_kernel,
input_dtype=dtype,
)
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
self.enable_quant_fp8_custom_op = self.fp8_linear.is_quant_fp8_enabled()
def forward(self, x):
y = self.silu_and_mul(x)
x2 = self.fp8_linear(y)
return x2
def ops_in_model_before(self):
return [
SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul,
(
QUANT_OPS[kFp8StaticTensorSym]
if self.enable_quant_fp8_custom_op
else torch.ops.aten.reciprocal
),
]
def ops_in_model_after(self):
return [FUSED_OPS[kFp8StaticTensorSym]]
class TestSiluMulNvfp4QuantModel(torch.nn.Module):
def __init__(self, hidden_size: int, x: torch.Tensor, **kwargs):
super().__init__()
from vllm.compilation.passes.fusion.act_quant_fusion import (
silu_and_mul_nvfp4_quant_supported,
)
assert silu_and_mul_nvfp4_quant_supported
self.silu_and_mul = SiluAndMul()
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
# create nvfp4 weight
w = torch.rand((hidden_size, hidden_size))
self.w, self.w_block_scale, self.w_global_scale = quant_nvfp4_tensor(w)
# get global scale offline
_, _, self.y_global_scale = quant_nvfp4_tensor(self.silu_and_mul(x))
self.alpha = 1.0 / (self.w_global_scale * self.y_global_scale)
def forward(self, x):
y = self.silu_and_mul(x)
y_quant, y_block_scale = scaled_fp4_quant(y, self.y_global_scale)
out = cutlass_scaled_fp4_mm(
a=y_quant,
b=self.w,
block_scale_a=y_block_scale,
block_scale_b=self.w_block_scale,
alpha=self.alpha,
out_dtype=y.dtype,
)
return out
def ops_in_model_before(self):
return [
SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul,
QUANT_OPS[kNvfp4Dynamic],
]
def ops_in_model_after(self):
return [FUSED_OPS[kNvfp4Dynamic]]
class TestSiluMulGroupFp8QuantModel(torch.nn.Module):
act_quant_key = kFp8Dynamic128Sym
def __init__(self, hidden_size: int, dtype: torch.dtype, **kwargs):
super().__init__()
self.silu_and_mul = SiluAndMul()
self.weight_quant_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(hidden_size, hidden_size)
)
self.w8a8_block_fp8_linear = TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
weight_quant_key=self.weight_quant_key,
activation_quant_key=self.act_quant_key,
input_dtype=dtype,
)
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
def forward(self, x):
y = self.silu_and_mul(x)
x2 = self.w8a8_block_fp8_linear(y)
return x2
def ops_in_model_before(self):
return [
SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul,
rocm_aiter_ops.get_group_quant_op(),
]
def ops_in_model_after(self):
return [torch.ops.vllm.rocm_aiter_act_mul_and_fp8_group_quant]
class TestSiluMulBlockQuantModel(torch.nn.Module):
quant_key = kFp8Dynamic128Sym
def __init__(self, hidden_size: int, is_scale_transposed: bool = False, **kwargs):
super().__init__()
self.silu_and_mul = SiluAndMul()
self.is_scale_transposed = is_scale_transposed
self.quant_fp8 = QuantFP8(
static=False,
group_shape=GroupShape(1, 128),
column_major_scales=is_scale_transposed,
compile_native=False,
)
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
self.enable_quant_fp8_custom_op = self.quant_fp8.enabled()
def forward(self, x):
y = self.silu_and_mul(x)
out, scale = self.quant_fp8(y)
group_size = self.quant_key.scale.group_shape[1]
scale_expanded = scale.repeat_interleave(group_size, dim=1)
dequant = out.to(dtype=torch.float32) * scale_expanded
return (dequant,)
def ops_in_model_before(self):
ops = []
if self.enable_silu_mul_custom_op:
ops.append(SILU_MUL_OP)
# When silu custom op is disabled, aten.mul.Tensor also appears
# in dequant code, so we skip checking it to avoid false positives.
ops.append(
QUANT_OPS[self.quant_key]
if self.enable_quant_fp8_custom_op
else torch.ops.aten.reciprocal.default
)
return ops
def ops_in_model_after(self):
return [FUSED_OPS[self.quant_key]]
ROCM_KERNELS = [ROCmFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel]
CUDA_KERNELS = [
FlashInferFP8ScaledMMLinearKernel,
CutlassFP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
]
TEST_KERNELS = ROCM_KERNELS if current_platform.is_rocm() else CUDA_KERNELS
@pytest.mark.parametrize("num_tokens", [32, 64])
@pytest.mark.parametrize("hidden_size", [128, 256])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("enable_silu_mul_custom_op", [True, False])
@pytest.mark.parametrize(
"model_class, enable_quant_fp8_custom_op, force_kernel",
list(itertools.product([TestSiluMulFp8QuantModel], [True, False], TEST_KERNELS))
+ [
pytest.param(
TestSiluMulNvfp4QuantModel,
False,
None,
marks=pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUDA only"
),
),
# GroupFP8Quant fusion only works with AITER on ROCm.
# and the enable_quant_fp8_custom_op must be True.
pytest.param(
TestSiluMulGroupFp8QuantModel,
True,
None,
marks=pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm only"
),
),
# Block quant fusion for per-group FP8 (CUDA only).
*[
pytest.param(
partial(TestSiluMulBlockQuantModel, is_scale_transposed=transposed),
True,
None,
marks=pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUDA only"
),
id=f"TestSiluMulBlockQuant-transposed={transposed}",
)
for transposed in [False, True]
],
],
)
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_fusion_silu_and_mul_quant(
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
model_class: type[
TestSiluMulFp8QuantModel
| TestSiluMulNvfp4QuantModel
| TestSiluMulGroupFp8QuantModel
| TestSiluMulBlockQuantModel
],
enable_silu_mul_custom_op: bool,
enable_quant_fp8_custom_op: bool,
force_kernel: type[FP8ScaledMMLinearKernel] | None,
monkeypatch: pytest.MonkeyPatch,
):
if model_class is TestSiluMulNvfp4QuantModel and not is_nvfp4_supported():
pytest.skip("NVFP4 is not supported on this GPU.")
if model_class is TestSiluMulGroupFp8QuantModel and not IS_AITER_FOUND:
pytest.skip("AITER is not supported on this GPU.")
if (
isinstance(model_class, partial)
and model_class.func is TestSiluMulBlockQuantModel
and is_deep_gemm_supported()
):
pytest.skip("SiluMul+BlockQuant fusion not applicable with DeepGemm")
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
x = torch.rand(num_tokens, hidden_size * 2)
# Reshape pass is needed for the fusion pass to work
custom_ops = ["none"]
if enable_silu_mul_custom_op:
custom_ops.append("+silu_and_mul")
if enable_quant_fp8_custom_op:
custom_ops.append("+quant_fp8")
config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops,
backend="eager", # avoid compilation for SiluAndMul and QuantFP8
pass_config=PassConfig(fuse_act_quant=True, eliminate_noops=True),
),
)
with set_current_vllm_config(config), monkeypatch.context() as m:
fusion_passes = [ActivationQuantFusionPass(config)]
if IS_AITER_FOUND and model_class is TestSiluMulGroupFp8QuantModel:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
RocmAiterSiluMulFp8GroupQuantFusionPass,
)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
fusion_passes += [RocmAiterSiluMulFp8GroupQuantFusionPass(config)]
passes = [NoOpEliminationPass(config), *fusion_passes, PostCleanupPass(config)]
backend = TestBackend(*passes)
model = model_class(
hidden_size=hidden_size,
force_kernel=force_kernel,
x=x,
dtype=dtype,
)
# First dimension dynamic
torch._dynamo.mark_dynamic(x, 0)
result = model(x)
model2 = torch.compile(model, backend=backend)
result2 = model2(x)
# Check that it gives the same answer
if isinstance(model, TestSiluMulFp8QuantModel):
atol, rtol = 1e-3, 1e-3
elif isinstance(model, TestSiluMulNvfp4QuantModel):
atol, rtol = 1e-1, 1e-1
elif isinstance(model, TestSiluMulGroupFp8QuantModel):
atol, rtol = 5e-2, 5e-2
elif isinstance(model, TestSiluMulBlockQuantModel):
if current_platform.is_rocm():
atol, rtol = 1e-3, 1e-3
else:
# CUDA fused kernel computes silu*mul in fp32 while the reference
# goes through bf16/fp16 storage, so group maxima (and thus scales)
# can shift by one FP8-e4m3 code (~1/8 relative step).
atol, rtol = 5e-2, 5e-2
torch.testing.assert_close(
result[0].to(dtype=dtype), result2[0].to(dtype=dtype), atol=atol, rtol=rtol
)
assert sum([p.matched_count for p in fusion_passes]) == 1
# In pre-nodes, quant op should be present and fused kernels should not
backend.check_before_ops(model.ops_in_model_before())
# In post-nodes, fused kernels should be present and quant op should not
backend.check_after_ops(model.ops_in_model_after())
@@ -1,379 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for manual fusion via maybe_fused_act_quant.
Tests all fusion paths in _FUSED_ACT_QUANT:
- kFp8StaticTensorSym: all platforms
- kFp8Dynamic128Sym: CUDA only
- kNvfp4Dynamic: CUDA SM100+ only
"""
import pytest
import torch
import vllm.envs as envs
from tests.utils import TestFP8Layer
from vllm.config import (
CompilationConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.model_executor.kernels.linear import (
CutlassFP8ScaledMMLinearKernel,
FlashInferFP8ScaledMMLinearKernel,
FP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ROCmFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fusion.fused_act_quant import (
_FUSED_ACT_QUANT,
maybe_fused_act_quant,
)
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
expose_input_quant_key,
get_input_quant_key,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
# Mock linear layer for testing fusion paths that don't have real kernel support
class MockLinearForFusion(torch.nn.Module):
"""Mock linear layer that exposes an input key for fusion testing."""
def __init__(
self,
quant_key,
input_scale=None,
input_global_scale=None,
input_global_scale_inv=None,
):
super().__init__()
self._input_quant_key = quant_key
if input_scale is not None:
self.input_scale = input_scale
if input_global_scale is not None:
self.input_global_scale = input_global_scale
if input_global_scale_inv is not None:
self.input_global_scale_inv = input_global_scale_inv
ROCM_KERNELS = [ROCmFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel]
CUDA_KERNELS = [
FlashInferFP8ScaledMMLinearKernel,
CutlassFP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
]
TEST_KERNELS = ROCM_KERNELS if current_platform.is_rocm() else CUDA_KERNELS
@pytest.mark.parametrize("num_tokens", [32, 64])
@pytest.mark.parametrize("hidden_size", [128, 256])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("force_kernel", TEST_KERNELS)
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_fp8_static_with_linear(
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
force_kernel: type[FP8ScaledMMLinearKernel],
):
"""Test manual fusion with real FP8 linear layer (kFp8StaticTensorSym).
This is an end-to-end test that verifies the full flow:
unfused (silu_and_mul -> in-kernel quant) vs fused (silu_and_mul_quant).
"""
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
x = torch.rand(num_tokens, hidden_size * 2)
config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["none"]),
)
with set_current_vllm_config(config):
silu_and_mul = SiluAndMul()
fp8_linear = TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=kFp8StaticTensorSym,
weight_quant_key=kFp8StaticTensorSym,
force_kernel=force_kernel,
input_dtype=dtype,
)
# Run without fusion: silu_and_mul returns plain tensor
y_unfused = maybe_fused_act_quant(silu_and_mul, x, fp8_linear)
assert isinstance(y_unfused, torch.Tensor)
result_unfused = fp8_linear(y_unfused)
# Enable fusion
expose_input_quant_key(fp8_linear, fp8_linear.kernel)
if get_input_quant_key(fp8_linear) is None:
pytest.skip(
f"Kernel {force_kernel.__name__} doesn't support input_quant_key"
)
# Run with fusion: silu_and_mul returns QuantizedActivation
y_fused = maybe_fused_act_quant(silu_and_mul, x, fp8_linear)
assert isinstance(y_fused, QuantizedActivation)
assert y_fused.quant_key == kFp8StaticTensorSym
assert y_fused.data.dtype == current_platform.fp8_dtype()
assert y_fused.data.shape == (num_tokens, hidden_size)
assert y_fused.orig_dtype == dtype
assert y_fused.orig_shape == (num_tokens, hidden_size)
result_fused = fp8_linear(y_fused)
torch.testing.assert_close(
result_fused.to(dtype=dtype),
result_unfused.to(dtype=dtype),
atol=5e-2,
rtol=5e-2,
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Dynamic block quant CUDA only"
)
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_fp8_dynamic_128(dtype: torch.dtype):
"""Test kFp8Dynamic128Sym fusion path (group_size=128).
Compares fused (silu_and_mul_per_block_quant) vs unfused (silu_and_mul)
by dequantizing the fused result and comparing with unfused.
"""
if (SiluAndMul, kFp8Dynamic128Sym) not in _FUSED_ACT_QUANT:
pytest.skip("kFp8Dynamic128Sym fusion not available")
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
# hidden_size must be divisible by group_size (128)
num_tokens, hidden_size = 32, 256
group_size = 128
x = torch.rand(num_tokens, hidden_size * 2)
config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["none"]),
)
with set_current_vllm_config(config):
silu_and_mul = SiluAndMul()
# Unfused path: just apply silu_and_mul
mock_linear_no_key = torch.nn.Linear(hidden_size, hidden_size)
result_unfused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_no_key)
assert isinstance(result_unfused, torch.Tensor)
# Fused path: apply silu_and_mul + per-block quantization
mock_linear_with_key = MockLinearForFusion(kFp8Dynamic128Sym)
result_fused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_with_key)
# Verify fused result structure
assert isinstance(result_fused, QuantizedActivation)
assert result_fused.quant_key == kFp8Dynamic128Sym
assert result_fused.data.dtype == current_platform.fp8_dtype()
assert result_fused.data.shape == (num_tokens, hidden_size)
assert result_fused.orig_dtype == dtype
# Check scale shape
expected_num_groups = hidden_size // group_size
assert result_fused.scale.shape == (num_tokens, expected_num_groups)
# Dequantize fused result and compare with unfused
# Per-block dequant: data * scale (broadcast scale across group)
dequant_data = result_fused.data.to(dtype).view(
num_tokens, expected_num_groups, group_size
)
scales_expanded = result_fused.scale.unsqueeze(
-1
) # (num_tokens, num_groups, 1)
dequant_result = (
(dequant_data * scales_expanded).view(num_tokens, hidden_size).to(dtype)
)
torch.testing.assert_close(
dequant_result,
result_unfused,
atol=5e-2,
rtol=5e-2,
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
# Cover M < 128 (padded swizzled SF tiles) and hidden_size that makes the
# kernel launch grid.y > 1 (num_packed_cols = hidden_size / 16 > 512).
@pytest.mark.parametrize("num_tokens", [1, 127, 128])
@pytest.mark.parametrize("hidden_size", [256, 14336])
@pytest.mark.skipif(not current_platform.is_cuda(), reason="NVFP4 CUDA only")
@pytest.mark.skipif(
not current_platform.has_device_capability(100), reason="NVFP4 requires SM100+"
)
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_nvfp4_dynamic(
dtype: torch.dtype, num_tokens: int, hidden_size: int
):
"""Test kNvfp4Dynamic fusion path.
Compares fused (silu_and_mul_nvfp4_quant) vs unfused (silu_and_mul)
by dequantizing the fused result and comparing with unfused.
"""
if (SiluAndMul, kNvfp4Dynamic) not in _FUSED_ACT_QUANT:
pytest.skip("kNvfp4Dynamic fusion not available")
from tests.kernels.quantization.nvfp4_utils import dequantize_nvfp4_to_dtype
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
# NVFP4 requires hidden_size divisible by 16 (block size) and by 2 (packing).
x = torch.rand(num_tokens, hidden_size * 2)
# Non-1.0 global scale so the test is sensitive to the scale direction:
# the fused producer must quantize with input_global_scale_inv (the GEMM's
# alpha divides by input_global_scale), matching the unfused path.
input_global_scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
input_global_scale_inv = 1.0 / input_global_scale
config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["none"]),
)
with set_current_vllm_config(config):
silu_and_mul = SiluAndMul()
# Unfused path: just apply silu_and_mul
mock_linear_no_key = torch.nn.Linear(hidden_size, hidden_size)
result_unfused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_no_key)
assert isinstance(result_unfused, torch.Tensor)
# Fused path: apply silu_and_mul + NVFP4 quantization
mock_linear_with_key = MockLinearForFusion(
kNvfp4Dynamic,
input_global_scale=input_global_scale,
input_global_scale_inv=input_global_scale_inv,
)
result_fused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_with_key)
# Verify fused result structure
assert isinstance(result_fused, QuantizedActivation)
assert result_fused.quant_key == kNvfp4Dynamic
# NVFP4 packs 2 values into 1 byte
assert result_fused.data.dtype == torch.uint8
assert result_fused.data.shape == (num_tokens, hidden_size // 2)
assert result_fused.orig_dtype == dtype
assert result_fused.orig_shape == (num_tokens, hidden_size)
# The scale tensor must use the padded 128x4-tile swizzled layout,
# same as scaled_fp4_quant's allocator; the kernel writes a full
# 128-row tile even when M is smaller.
from vllm.utils.math_utils import round_up
assert result_fused.scale.dtype == current_platform.fp8_dtype()
assert result_fused.scale.shape == (
round_up(num_tokens, 128),
round_up(hidden_size // 16, 4),
)
# Dequantize fused result and compare with unfused. The block scales
# must have been folded with input_global_scale_inv, so dequantize
# with the same inverse scale to recover the activation.
dequant_result = dequantize_nvfp4_to_dtype(
tensor_fp4=result_fused.data,
tensor_sf=result_fused.scale,
global_scale=input_global_scale_inv,
dtype=dtype,
device="cuda",
block_size=16,
is_sf_128x4_layout=True,
)
torch.testing.assert_close(
dequant_result,
result_unfused,
atol=3e-1,
rtol=3e-1,
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_fallback_no_key(dtype: torch.dtype):
"""Test that maybe_fused_act_quant falls back without an input key."""
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
x = torch.rand(32, 256)
config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["none"]),
)
with set_current_vllm_config(config):
silu_and_mul = SiluAndMul()
# Linear without an input quantization key
mock_linear = torch.nn.Linear(128, 128)
result = maybe_fused_act_quant(silu_and_mul, x, mock_linear)
# Should fall back to plain silu_and_mul
assert isinstance(result, torch.Tensor)
assert result.shape == (32, 128)
assert result.dtype == dtype
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_fallback_unsupported_key(dtype: torch.dtype):
"""Test that maybe_fused_act_quant falls back for unsupported quant keys."""
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
ScaleDesc,
)
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
x = torch.rand(32, 256)
config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["none"]),
)
with set_current_vllm_config(config):
# Create an unsupported quant key
unsupported_key = QuantKey(
dtype=torch.int8,
scale=ScaleDesc(
dtype=torch.float32, static=False, group_shape=GroupShape(1, 1)
),
)
silu_and_mul = SiluAndMul()
mock_linear = MockLinearForFusion(unsupported_key)
result = maybe_fused_act_quant(silu_and_mul, x, mock_linear)
# Should fall back to plain silu_and_mul since key not in _FUSED_ACT_QUANT
assert isinstance(result, torch.Tensor)
assert result.shape == (32, 128)
assert result.dtype == dtype
@@ -1,273 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import operator
import pytest
import torch
import vllm
from tests.compile.backend import TestBackend
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
from vllm.config import CompilationConfig, CompilationMode, PassConfig, VllmConfig
from vllm.platforms import current_platform
DEVICE_TYPE = current_platform.device_type
class SplitCoalescingModel(torch.nn.Module):
"""Model with 3 separate split_with_sizes calls on the same input,
simulating the B200+FP8 graph where CSE fails to merge them."""
def __init__(self, q_size: int, kv_size: int) -> None:
super().__init__()
self.q_size = q_size
self.kv_size = kv_size
def forward(self, qkv: torch.Tensor):
q, _, _ = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
_, k, _ = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
_, _, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
return q + 1, k + 2, v + 3
class SliceCoalescingModel(torch.nn.Module):
"""Model with a complete QKV partition represented as contiguous slices."""
def __init__(self, q_size: int, kv_size: int) -> None:
super().__init__()
self.q_size = q_size
self.kv_size = kv_size
def forward(self, qkv: torch.Tensor):
q_end = self.q_size
k_end = q_end + self.kv_size
v_end = k_end + self.kv_size
q = torch.ops.aten.slice.Tensor(qkv, -1, 0, q_end)
k = torch.ops.aten.slice.Tensor(qkv, -1, q_end, k_end)
v = torch.ops.aten.slice.Tensor(qkv, -1, k_end, v_end)
return q + 1, k + 2, v + 3
def make_vllm_config() -> VllmConfig:
return VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(),
)
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_split_coalescing(dtype):
torch.set_default_device(DEVICE_TYPE)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
q_size, kv_size = 2048, 512
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
coalesce_pass = SplitCoalescingPass(vllm_config)
backend = TestBackend(coalesce_pass)
model = SplitCoalescingModel(q_size, kv_size)
T = 5
qkv = torch.randn(T, q_size + 2 * kv_size)
torch._dynamo.mark_dynamic(qkv, 0)
result_eager = model(qkv)
model_compiled = torch.compile(model, backend=backend)
result_compiled = model_compiled(qkv)
ATOL, RTOL = (2e-3, 2e-3)
for eager, compiled in zip(result_eager, result_compiled):
torch.testing.assert_close(eager, compiled, atol=ATOL, rtol=RTOL)
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_slice_partition_canonicalization(dtype):
torch.set_default_device(DEVICE_TYPE)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
q_size, kv_size = 2048, 512
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
coalesce_pass = SplitCoalescingPass(vllm_config)
backend = TestBackend(coalesce_pass)
model = SliceCoalescingModel(q_size, kv_size)
qkv = torch.randn(5, q_size + 2 * kv_size)
torch._dynamo.mark_dynamic(qkv, 0)
result_eager = model(qkv)
result_compiled = torch.compile(model, backend=backend, fullgraph=True)(qkv)
for eager, compiled in zip(result_eager, result_compiled):
torch.testing.assert_close(eager, compiled, atol=2e-3, rtol=2e-3)
assert backend.op_count(torch.ops.aten.slice.Tensor, before=True) == 3
assert backend.op_count(torch.ops.aten.slice.Tensor) == 0
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
split = next(
node
for node in backend.final_graph.nodes
if node.target == torch.ops.aten.split_with_sizes.default
)
assert list(split.args[1]) == [q_size, kv_size, kv_size]
assert split.args[2] == -1
getitems = [
node
for node in split.users
if node.op == "call_function" and node.target == operator.getitem
]
assert sorted(node.args[1] for node in getitems) == [0, 1, 2]
assert isinstance(split.meta["val"], list)
for getitem in getitems:
index = getitem.args[1]
assert getitem.meta["val"] is split.meta["val"][index]
def make_slice_graph(
ranges: list[tuple[int, int, int]],
) -> torch.fx.Graph:
graph = torch.fx.Graph()
source = graph.placeholder("qkv")
source.meta["val"] = torch.empty((2, 8), device="meta")
slices = []
for start, end, step in ranges:
node = graph.call_function(
torch.ops.aten.slice.Tensor,
args=(source, -1, start, end, step),
)
node.meta["val"] = torch.empty(
(2, len(range(start, min(end, 8), step))), device="meta"
)
slices.append(node)
graph.output(tuple(slices))
return graph
def test_slice_partition_accepts_safe_group():
graph = make_slice_graph([(0, 4, 1), (4, 8, 1)])
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
graph.lint()
assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0
assert (
sum(
node.target == torch.ops.aten.split_with_sizes.default
for node in graph.nodes
)
== 1
)
@pytest.mark.parametrize(
"ranges",
[
[(0, 3, 1), (4, 8, 1)], # gap
[(0, 5, 1), (4, 8, 1)], # overlap
[(1, 4, 1), (4, 8, 1)], # missing prefix
[(0, 4, 1), (4, 7, 1)], # missing suffix
[(0, 8, 1)], # single full-range slice
[(0, 8, 2)], # non-unit step
],
)
def test_slice_partition_rejects_unsafe_groups(ranges):
graph = make_slice_graph(ranges)
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
slice_count = sum(
node.target == torch.ops.aten.slice.Tensor for node in graph.nodes
)
split_count = sum(
node.target == torch.ops.aten.split_with_sizes.default for node in graph.nodes
)
assert slice_count == len(ranges)
assert split_count == 0
def test_nested_slice_partitions_use_live_source():
"""Use the rewired source when a slice partition consumes another slice."""
graph = torch.fx.Graph()
source = graph.placeholder("input")
source.meta["val"] = torch.empty((2, 8), device="meta")
outer_left = graph.call_function(
torch.ops.aten.slice.Tensor, args=(source, -1, 0, 4)
)
outer_left.meta["val"] = torch.empty((2, 4), device="meta")
outer_right = graph.call_function(
torch.ops.aten.slice.Tensor, args=(source, -1, 4, 8)
)
outer_right.meta["val"] = torch.empty((2, 4), device="meta")
inner_left = graph.call_function(
torch.ops.aten.slice.Tensor, args=(outer_left, -1, 0, 2)
)
inner_left.meta["val"] = torch.empty((2, 2), device="meta")
inner_right = graph.call_function(
torch.ops.aten.slice.Tensor, args=(outer_left, -1, 2, 4)
)
inner_right.meta["val"] = torch.empty((2, 2), device="meta")
graph.output((inner_left, inner_right, outer_right))
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
graph.lint()
assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0
splits = [
node
for node in graph.nodes
if node.target == torch.ops.aten.split_with_sizes.default
]
assert len(splits) == 2
outer_split = next(node for node in splits if list(node.args[1]) == [4, 4])
inner_split = next(node for node in splits if list(node.args[1]) == [2, 2])
assert outer_split.args[0] is source
assert inner_split.args[0].target == operator.getitem
assert inner_split.args[0].args == (outer_split, 0)
def test_split_coalescing_preserves_different_dimensions():
"""Keep equal-sized splits on different dimensions as separate nodes."""
graph = torch.fx.Graph()
source = graph.placeholder("input")
source.meta["val"] = torch.empty((8, 8), device="meta")
outputs = []
for dim in (0, 1):
split = graph.call_function(
torch.ops.aten.split_with_sizes.default,
args=(source, [4, 4], dim),
)
for index in range(2):
outputs.append(graph.call_function(operator.getitem, args=(split, index)))
graph.output(tuple(outputs))
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
splits = [
node
for node in graph.nodes
if node.target == torch.ops.aten.split_with_sizes.default
]
assert len(splits) == 2
assert all(split.args[0] is source for split in splits)
assert {split.args[2] for split in splits} == {0, 1}
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.config
from tests.compile.backend import TestBackend
from vllm.compilation.passes.vllm_inductor_pass import (
VllmFusionPatternMatcherPass,
VllmPatternMatcherPass,
VllmPatternReplacement,
)
from vllm.config import CompilationConfig, CompilationMode, VllmConfig
from vllm.platforms import current_platform
class ReluToAbsPattern(VllmPatternReplacement):
"""Replaces relu(x) with abs(x) — a minimal test fixture."""
@property
def pattern(self):
def _pattern(x: torch.Tensor) -> torch.Tensor:
return torch.ops.aten.relu.default(x)
return _pattern
@property
def replacement(self):
def _replacement(x: torch.Tensor) -> torch.Tensor:
return torch.ops.aten.abs.default(x)
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
return [self.empty_fp32(4)]
class ExpToSqrtPattern(VllmPatternReplacement):
"""A second distinct pattern type — used to test uuid differentiation."""
@property
def pattern(self):
def _pattern(x: torch.Tensor) -> torch.Tensor:
return torch.ops.aten.exp.default(x)
return _pattern
@property
def replacement(self):
def _replacement(x: torch.Tensor) -> torch.Tensor:
return torch.ops.aten.sqrt.default(x)
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
return [self.empty_fp32(4)]
class ReluFusionPass(VllmFusionPatternMatcherPass):
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "test_relu_fusion")
self.register(ReluToAbsPattern())
class TwoPatternFusionPass(VllmFusionPatternMatcherPass):
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "test_two_pattern_fusion")
self.register(ReluToAbsPattern())
self.register(ExpToSqrtPattern())
@pytest.fixture
def vllm_config():
return VllmConfig(
compilation_config=CompilationConfig(mode=CompilationMode.VLLM_COMPILE),
)
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Requires CUDA")
def test_register_tracks_patterns(vllm_config):
"""register() appends each VllmPatternReplacement to _pattern_replacements."""
with vllm.config.set_current_vllm_config(vllm_config):
single = ReluFusionPass(vllm_config)
two = TwoPatternFusionPass(vllm_config)
assert len(single._pattern_replacements) == 1
assert len(two._pattern_replacements) == 2
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Requires CUDA")
def test_uuid_stable(vllm_config):
"""Two instances of the same pass class produce identical uuids."""
with vllm.config.set_current_vllm_config(vllm_config):
p1 = ReluFusionPass(vllm_config)
p2 = ReluFusionPass(vllm_config)
p3 = TwoPatternFusionPass(vllm_config)
assert p1.uuid() == p2.uuid()
assert p1.uuid() != p3.uuid()
assert p2.uuid() != p3.uuid()
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Requires CUDA")
@pytest.mark.parametrize("N", [1, 2, 4])
def test_matched_count_and_match_table(vllm_config, N):
"""matched_count and match_table reflect the number of matched patterns."""
class Model(torch.nn.Module):
def forward(self, *inputs):
# N independent relus
return sum(torch.relu(x) for x in inputs)
with vllm.config.set_current_vllm_config(vllm_config):
torch.set_default_device("cuda")
torch.set_default_dtype(torch.float32)
fusion_pass = ReluFusionPass(vllm_config)
backend = TestBackend(fusion_pass)
model = torch.compile(Model(), backend=backend)
inputs = [torch.rand(8) for _ in range(N)]
model(*inputs)
assert fusion_pass.matched_count == N
assert VllmPatternMatcherPass.match_table["test_relu_fusion"] >= N