[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests

- Drop dead model modules, keeping files live imports require
- Remove anthropic, cohere, codegen entrypoints and batch STT modes
- Delete unused attention backends, MoE experts, and NPU/XPU util leaves
- Strip dead third_party trees and ROCm leaves with no CUDA path
- Remove mooncake EC connectors: dependency missing since branch base
- Re-audit verified by full import sweep and Qwen3.5 smoke test

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 00:20:48 +07:00
co-authored by Hinata
parent 6a95f62e2f
commit 81074e94a3
940 changed files with 11 additions and 380533 deletions
@@ -1,91 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import logging
import pytest
import regex as re
from tests.utils import compare_two_settings
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.config import CompilationMode
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
@pytest.mark.skipif(
not current_platform.is_rocm() or not is_aiter_found_and_supported(),
reason="Requires ROCm AITER attention and quantization",
)
def test_rocm_aiter_static_attention_output_correctness(caplog_mp_spawn):
"""Trained FP8 outputs agree when actual AITER output quantization is fused."""
model = hf_api().snapshot_download(
"RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
revision="870a1177ed082c0ba5dbefd62573ab8f91803f6a",
allow_patterns=["*.json", "*.safetensors", "*.model", "*.txt", "*.jinja"],
)
common_args = [
"--dtype",
"bfloat16",
"--load-format",
"auto",
"--max-model-len",
"512",
"--max-num-batched-tokens",
"512",
"--max-num-seqs",
"8",
"--kv-cache-memory-bytes",
str(256 * 1024**2),
"--gpu-memory-utilization",
"0.01",
"--attention-backend",
"ROCM_AITER_UNIFIED_ATTN",
]
def settings(enabled):
return [
*common_args,
"--compilation-config",
json.dumps(
{
"mode": CompilationMode.VLLM_COMPILE,
"cudagraph_mode": "NONE",
"splitting_ops": [],
"inductor_compile_config": {"force_disable_caches": True},
"custom_ops": ["+quant_fp8", "-rms_norm"],
"pass_config": {
"fuse_attn_quant": enabled,
"fuse_norm_quant": False,
"fuse_act_quant": False,
"fuse_allreduce_rms": False,
},
}
),
]
env = {
"VLLM_ROCM_USE_AITER": "1",
"VLLM_ROCM_USE_AITER_LINEAR": "1",
"VLLM_DISABLE_COMPILE_CACHE": "1",
}
with caplog_mp_spawn(logging.DEBUG) as logs:
compare_two_settings(
model,
settings(False),
settings(True),
env1=env,
env2=env,
method="generate",
# Fusing output quant drops a bf16 round-trip, so the two settings
# agree on the argmax but not on the exact distribution. Greedy
# decoding survives that, sampling at temperature 1.0 does not.
include_seeded_sampling=False,
)
matches = [
int(count) for count in re.findall(r"'attn_quant_fusion': (\d+)", logs.text)
]
print(f"Attention output quantization rewrite counts: {matches}")
assert matches and all(count == 32 for count in matches), (
f"Expected output quantization fused in all 32 Llama layers, found {matches}"
)
@@ -1,306 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tempfile
from contextlib import contextmanager
from dataclasses import dataclass
import pytest
import torch
from tests.models.utils import check_logprobs_close
from vllm import SamplingParams
from vllm.compilation.decorators import support_torch_compile
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
from vllm.config.compilation import (
CompilationMode,
DynamicShapesConfig,
DynamicShapesType,
)
from vllm.forward_context import set_forward_context
from vllm.utils.torch_utils import is_torch_equal_or_newer
def get_test_models():
"""Get list of models to test based on PyTorch version."""
models = [
"Qwen/Qwen3-0.6B",
"openai-community/gpt2",
]
return models
@dataclass(frozen=True)
class DynamicShapesTestCase:
model_name: str
shapes_type: DynamicShapesType
use_aot_compile: bool
use_bytecode_hook: bool
def __str__(self) -> str:
model_id = self.model_name.rsplit("/", 1)[-1]
return (
f"{model_id}-{self.shapes_type.value}-"
f"aot{int(self.use_aot_compile)}-hook{int(self.use_bytecode_hook)}"
)
def get_dynamic_shapes_test_cases():
"""Pairwise-cover compilation options and smoke-test model classes."""
reference_model = "Qwen/Qwen3-0.6B"
cases = [
DynamicShapesTestCase(reference_model, DynamicShapesType.BACKED, False, True),
DynamicShapesTestCase(reference_model, DynamicShapesType.BACKED, True, False),
DynamicShapesTestCase(
reference_model, DynamicShapesType.UNBACKED, False, False
),
DynamicShapesTestCase(reference_model, DynamicShapesType.UNBACKED, True, True),
DynamicShapesTestCase(
reference_model,
DynamicShapesType.BACKED_SIZE_OBLIVIOUS,
False,
True,
),
DynamicShapesTestCase(
reference_model,
DynamicShapesType.BACKED_SIZE_OBLIVIOUS,
True,
False,
),
]
cases.extend(
DynamicShapesTestCase(model_name, DynamicShapesType.BACKED, False, True)
for model_name in get_test_models()
if model_name != reference_model
)
return cases
TEST_PROMPTS = ["Hello, my name is", "The capital of France is"]
def generate_outputs(vllm_model):
sampling_params = SamplingParams(max_tokens=5, temperature=0, logprobs=10)
outputs = []
for prompt in TEST_PROMPTS:
output = vllm_model.llm.generate(prompt, sampling_params)[0].outputs[0]
assert len(output.text.strip()) > 0, "Model produced empty output"
outputs.append((output.token_ids, output.text, output.logprobs))
return outputs
@pytest.fixture(scope="module")
def get_eager_outputs(vllm_runner):
cache = {}
def get(model_name):
if model_name not in cache:
with vllm_runner(
model_name,
enforce_eager=True,
max_model_len=1024,
enable_chunked_prefill=None,
) as vllm_model:
cache[model_name] = generate_outputs(vllm_model)
return cache[model_name]
return get
@pytest.mark.parametrize("test_case", get_dynamic_shapes_test_cases(), ids=str)
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
def test_dynamic_shapes_compilation(
monkeypatch,
vllm_runner,
get_eager_outputs,
test_case,
):
"""Test representative dynamic-shapes configurations end to end."""
if (
test_case.shapes_type == DynamicShapesType.UNBACKED
and not is_torch_equal_or_newer("2.11.0")
):
# NOTE[ROCm]: shape_id (used by Qwen2/Llama to relate input dims) only
# landed in torch 2.11, but the ROCm CI still runs torch 2.10.x. On
# older torch there's no way to express it, so unbacked shapes go
# data-dependent and compilation blows up -- nothing to test.
pytest.skip("unbacked dynamic shapes with shape_id require torch>=2.11")
monkeypatch.setenv(
"VLLM_USE_AOT_COMPILE", "1" if test_case.use_aot_compile else "0"
)
monkeypatch.setenv(
"VLLM_USE_BYTECODE_HOOK", "1" if test_case.use_bytecode_hook else "0"
)
print(f"Testing {test_case.shapes_type.name} dynamic shapes...")
# The compiled engine shuts down before an uncached eager baseline starts.
with vllm_runner(
test_case.model_name,
compilation_config={
"mode": CompilationMode.VLLM_COMPILE,
"dynamic_shapes_config": {
"type": test_case.shapes_type.value,
},
},
max_model_len=1024,
enable_chunked_prefill=None,
) as vllm_model:
compiled_outputs = generate_outputs(vllm_model)
check_logprobs_close(
outputs_0_lst=get_eager_outputs(test_case.model_name),
outputs_1_lst=compiled_outputs,
name_0="eager",
name_1="compiled",
)
@pytest.mark.parametrize("use_aot_compile", [False, True])
@pytest.mark.parametrize(
"dynamic_shapes_type",
[
DynamicShapesType.BACKED,
DynamicShapesType.BACKED_SIZE_OBLIVIOUS,
],
)
@pytest.mark.parametrize("evaluate_guards", [False, True])
def test_model_specialization_with_evaluate_guards(
monkeypatch, use_aot_compile, dynamic_shapes_type, evaluate_guards
):
"""Test that evaluate_guards correctly detects shape specialization
violations.
"""
if (
use_aot_compile
and dynamic_shapes_type == DynamicShapesType.BACKED
and evaluate_guards
):
pytest.skip("evaluate_guards for backed does not work with aot_compile=1")
@support_torch_compile
class ModelWithSizeCheck(torch.nn.Module):
def __init__(self, **kwargs):
super().__init__()
def forward(self, x: torch.Tensor):
# This will cause specialization - torch.compile will guard on
# sx.shape[0]
if x.shape[0] >= 10:
return x * 10
else:
return x * 10
@support_torch_compile
class ModelWithOneSizeCheck(torch.nn.Module):
def __init__(self, **kwargs):
super().__init__()
def forward(self, x: torch.Tensor):
# This will cause 0/1 specializations.
if x.shape[0] == 0:
return x * 10
if x.shape[0] == 1:
return x * 10
else:
return x * 10
@contextmanager
def use_vllm_config(vllm_config: VllmConfig):
with set_forward_context({}, vllm_config), set_current_vllm_config(vllm_config):
yield
monkeypatch.setenv("TOKENIZERS_PARALLELISM", "true")
monkeypatch.setenv("VLLM_USE_AOT_COMPILE", "1" if use_aot_compile else "0")
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "0")
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
dynamic_shapes_config=DynamicShapesConfig(
type=dynamic_shapes_type,
evaluate_guards=evaluate_guards,
),
)
)
def test(model_class, input1, input2, is_01_specialization=False):
with (
torch.no_grad(),
use_vllm_config(vllm_config),
tempfile.TemporaryDirectory() as tmpdirname,
):
monkeypatch.setenv("VLLM_CACHE_ROOT", tmpdirname)
model = model_class(vllm_config=vllm_config).cuda()
model(input1)
if evaluate_guards and (
not (
is_01_specialization
and dynamic_shapes_type == DynamicShapesType.BACKED
)
):
# This should fail because guards were added.
with pytest.raises(RuntimeError) as excinfo:
model(input2)
# Expected failure - guard was violated
error_msg = str(excinfo.value)
assert (
"GuardManager check failed" in error_msg
or "Detected recompile when torch.compile stance" in error_msg
), error_msg
else:
model(input2)
test(ModelWithSizeCheck, torch.randn(20, 10).cuda(), torch.randn(5, 10).cuda())
test(ModelWithSizeCheck, torch.randn(5, 10).cuda(), torch.randn(20, 10).cuda())
test(
ModelWithOneSizeCheck,
torch.randn(20, 10).cuda(),
torch.randn(1, 10).cuda(),
is_01_specialization=True,
)
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
def test_piecewise_backend_empty_sym_shape_indices(vllm_runner):
"""Test that PiecewiseBackend handles empty sym_shape_indices correctly.
When all inputs have static shapes (no torch.SymInt), sym_shape_indices
will be empty. The fix in PiecewiseBackend.__call__ handles this case
by using the first compiled range_entry.
"""
# Use small max_model_len and max_num_batched_tokens to encourage
# static shape compilation with empty sym_shape_indices
with vllm_runner(
"Qwen/Qwen3-0.6B",
max_model_len=512,
max_num_batched_tokens=1,
enable_chunked_prefill=None,
compilation_config={
"mode": CompilationMode.VLLM_COMPILE,
"dynamic_shapes_config": {
"type": DynamicShapesType.BACKED.value,
},
},
) as vllm_model:
sampling_params = SamplingParams(temperature=0, top_p=0.95, max_tokens=10)
# Generate with static shape inputs
output = vllm_model.llm.generate(
"Hello, my name is", sampling_params=sampling_params
)
result = output[0].outputs[0].text
assert len(result) > 0, "Should generate non-empty output"
# Generate again to verify compilation works with empty sym_shape_indices
output = vllm_model.llm.generate(
"The capital of France is", sampling_params=sampling_params
)
result = output[0].outputs[0].text
assert len(result) > 0, "Should generate non-empty output on second run"
@@ -1,117 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import dataclasses
import pytest
from vllm.config import CompilationMode
from vllm.platforms import current_platform
from ...utils import compare_all_settings
ATTN_BACKEND = "FLASH_ATTN" if not current_platform.is_rocm() else "ROCM_ATTN"
@dataclasses.dataclass
class TestSetting:
model: str
model_args: list[str]
pp_size: int
tp_size: int
attn_backend: str
method: str
# we cannot afford testing the full Cartesian product
# of all models and all modes
@pytest.mark.parametrize(
"test_setting",
[
# MoE model
TestSetting(
model="ibm-granite/granite-3.0-1b-a400m-instruct",
model_args=["--max-model-len", "2048"],
pp_size=1,
tp_size=1,
attn_backend=ATTN_BACKEND,
method="generate",
),
# embedding model
TestSetting(
model="BAAI/bge-multilingual-gemma2",
model_args=[
"--runner",
"pooling",
"--dtype",
"bfloat16",
"--max-model-len",
"2048",
"--gpu-memory-utilization",
"0.98",
],
pp_size=1,
tp_size=1,
attn_backend=ATTN_BACKEND,
method="encode",
),
],
)
@pytest.mark.parametrize("use_v2_model_runner", [False, True])
def test_compile_correctness(
test_setting: TestSetting,
use_v2_model_runner: bool,
):
model = test_setting.model
model_args = test_setting.model_args
pp_size = test_setting.pp_size
tp_size = test_setting.tp_size
attn_backend = test_setting.attn_backend
method = test_setting.method
final_args = [
*model_args,
"-pp",
str(pp_size),
"-tp",
str(tp_size),
"-cc.cudagraph_mode=none",
f"--attention-backend={attn_backend}",
]
# Pin every compared setting to the same model runner so that only the
# compilation mode differs within a comparison.
runner_env = {"VLLM_USE_V2_MODEL_RUNNER": str(int(use_v2_model_runner))}
all_args: list[list[str]] = []
all_envs: list[dict[str, str] | None] = []
# Test all compilation modes with inductor backend
for mode in [
CompilationMode.NONE,
CompilationMode.STOCK_TORCH_COMPILE,
CompilationMode.DYNAMO_TRACE_ONCE,
CompilationMode.VLLM_COMPILE,
]:
all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=inductor"])
all_envs.append(dict(runner_env))
# inductor will change the output, so we only compare if the output
# is close, not exactly the same.
compare_all_settings(
model,
all_args,
all_envs,
method=method if method != "generate" else "generate_close",
)
all_envs.clear()
all_args.clear()
# Test all compilation modes with eager backend
for mode in [
CompilationMode.NONE,
CompilationMode.STOCK_TORCH_COMPILE,
CompilationMode.DYNAMO_TRACE_ONCE,
CompilationMode.VLLM_COMPILE,
]:
all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=eager"])
all_envs.append(dict(runner_env))
compare_all_settings(model, all_args, all_envs, method=method)
@@ -1,191 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import os
import weakref
import pytest
from tests.utils import wait_for_gpu_memory_to_clear
from tests.v1.attention.utils import full_cg_backend_configs as backend_configs
from vllm import LLM, SamplingParams
from vllm.config import CompilationConfig, CUDAGraphMode
from vllm.outputs import RequestOutput
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_torch_equal_or_newer
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@contextlib.contextmanager
def temporary_environ(env_vars):
"""Temporarily set environment variables and restore them afterward.
We have to do this vs monkeypatch because monkeypatch doesn't work
with "module" scoped fixtures.
"""
original_env = {k: os.environ.get(k) for k in env_vars}
try:
os.environ.update(env_vars)
yield
finally:
for k, v in original_env.items():
if v is None:
os.environ.pop(k, None)
else:
os.environ[k] = v
model_backends_full_cudagraph = []
# deepseek-ai/DeepSeek-V2-Lite with MLA
MLA_backends = ["FlashMLA", "FlashAttentionMLA", "CutlassMLA", "FlashInferMLA"]
for mla_backend in MLA_backends:
model_backends_full_cudagraph.append(
("deepseek-ai/DeepSeek-V2-Lite", backend_configs[mla_backend])
)
# Qwen/Qwen2-1.5B-Instruct with other backends
other_backend_configs = [
backend_configs[c] for c in backend_configs if c not in MLA_backends
]
for backend_config in other_backend_configs:
model_backends_full_cudagraph.append(("Qwen/Qwen2-1.5B-Instruct", backend_config))
@pytest.fixture(scope="class")
def llm_pair(request):
model, backend_config, use_inductor_graph_partition = request.param
if use_inductor_graph_partition and not is_torch_equal_or_newer("2.9.0.dev"):
pytest.skip("Inductor graph partition only supported in torch>=2.9")
backend = AttentionBackendEnum[backend_config.attention_config["backend"]]
if current_platform.is_rocm() and backend not in (
AttentionBackendEnum.TRITON_ATTN,
AttentionBackendEnum.ROCM_ATTN,
):
pytest.skip(f"{backend_config.name} is a CUDA-only attention backend")
if backend == AttentionBackendEnum.ROCM_ATTN and not current_platform.is_rocm():
pytest.skip("ROCM_ATTN requires ROCm")
# Dynamically skip test if GPU capability is not met
if backend_config.specific_gpu_arch and (
not current_platform.is_cuda()
or backend_config.specific_gpu_arch != current_platform.get_device_capability()
):
if backend_config.specific_gpu_arch == (9, 0):
pytest.skip("Only Hopper GPUs support FA3 and FlashMLA")
elif backend_config.specific_gpu_arch == (10, 0):
pytest.skip("Only Blackwell GPUs support Cutlass MLA")
env_vars = {
# Force native sampler to avoid potential nondeterminism in FlashInfer
# when per-request generators are not used in V1.
"VLLM_USE_FLASHINFER_SAMPLER": "0",
}
with temporary_environ(env_vars):
full = LLM(
model=model,
gpu_memory_utilization=0.43,
trust_remote_code=True,
max_model_len=1024,
max_num_seqs=128,
attention_config=backend_config.attention_config,
compilation_config=CompilationConfig(
**backend_config.comp_config,
use_inductor_graph_partition=use_inductor_graph_partition,
),
generation_config="vllm",
seed=42,
)
piecewise = LLM(
model=model,
gpu_memory_utilization=0.43,
trust_remote_code=True,
max_model_len=1024,
max_num_seqs=128,
attention_config=backend_config.attention_config,
compilation_config=CompilationConfig(
cudagraph_mode=CUDAGraphMode.PIECEWISE,
use_inductor_graph_partition=use_inductor_graph_partition,
),
generation_config="vllm",
seed=42,
)
# PyTest caches the fixture values so we use weakref.proxy to enable GC
yield weakref.proxy(full), weakref.proxy(piecewise)
del full
del piecewise
wait_for_gpu_memory_to_clear(
devices=[0],
threshold_ratio=0.1,
)
@pytest.mark.parametrize(
"llm_pair",
[
pytest.param(
(model, backend_config, use_inductor_graph_partition),
id=f"{backend_config.name}-partition={use_inductor_graph_partition}",
)
for model, backend_config in model_backends_full_cudagraph
for use_inductor_graph_partition in [True, False]
],
indirect=True,
)
class TestFullCUDAGraph:
"""Use a class such that an llm pair is constructed once for all
batch_size/max_tokens combinations and released immediately after.
Module-scope fixtures would stick around the whole time,
meaning there would be multiple LLM instances hogging memory simultaneously.
"""
@pytest.mark.parametrize(
("batch_size", "max_tokens"),
[
(1, 10),
(7, 10),
(16, 10),
(25, 10),
(32, 10),
(45, 10),
(64, 10),
(123, 10),
(8, 5),
(8, 30),
],
)
def test_full_cudagraph(self, batch_size, max_tokens, llm_pair: tuple[LLM, LLM]):
"""Test various batch sizes and max_tokens to ensure that the
full cudagraph compilation works for padded cases too.
"""
full_cudagraph_llm, piecewise_llm = llm_pair
prompts = ["the quick brown fox"] * batch_size
# Use purely greedy decoding to avoid top-p truncation sensitivity
# that can amplify tiny numeric differences across runtimes.
sampling_params = SamplingParams(
temperature=0.0, max_tokens=max_tokens, top_p=1.0
)
responses = []
for llm in (piecewise_llm, full_cudagraph_llm):
# Queue the whole batch before scheduling so both engines exercise
# the requested batch size instead of timing-dependent sub-batches.
llm.sleep(level=0, mode="keep")
try:
llm.enqueue(prompts, sampling_params)
finally:
llm.wake_up(tags=["scheduling"])
responses.append(llm.wait_for_completion(output_type=RequestOutput))
piecewise_responses, full_responses = responses
# Check that all responses are the same
assert len(piecewise_responses) == len(full_responses) == batch_size
for piecewise_res, full_res in zip(piecewise_responses, full_responses):
assert (
piecewise_res.outputs[0].text.lower()
== full_res.outputs[0].text.lower()
)
-201
View File
@@ -1,201 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import tempfile
from pathlib import Path
from typing import Any
import pytest
import torch
from tests.quantization.utils import is_quant_method_supported
from vllm import LLM, SamplingParams
from vllm.config import (
CacheConfig,
CompilationConfig,
CompilationMode,
CUDAGraphMode,
PassConfig,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_torch_equal_or_newer
from ...utils import create_new_process_for_each_test, wait_for_memory_to_settle
def models_list(*, all: bool = True, keywords: list[str] | None = None):
TEST_MODELS: list[tuple[str, dict[str, Any]]] = [
("facebook/opt-125m", {}),
(
"RedHatAI/Llama-3.2-1B-Instruct-FP8-dynamic",
{"dtype": torch.float16},
),
]
if all:
if is_quant_method_supported("gptq_marlin"):
TEST_MODELS.append(
(
"LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
{"quantization": "gptq_marlin"},
)
)
if not current_platform.is_rocm() and is_quant_method_supported("awq"):
TEST_MODELS.append(
("TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", {"quantization": "AWQ"})
)
if keywords is None:
return TEST_MODELS
# filter by keywords
pred = lambda model: any(keyword in model[0] for keyword in keywords)
return list(filter(pred, TEST_MODELS))
@pytest.mark.parametrize(
"compilation_mode",
[CompilationMode.DYNAMO_TRACE_ONCE, CompilationMode.VLLM_COMPILE],
)
@pytest.mark.parametrize("model, model_kwargs", models_list(all=True))
@create_new_process_for_each_test()
def test_full_graph(
monkeypatch: pytest.MonkeyPatch,
model: str,
model_kwargs: dict[str, Any],
compilation_mode: CompilationMode,
):
with monkeypatch.context():
print(f"MODEL={model}")
run_model(compilation_mode, model, **model_kwargs)
# TODO(luka) add other supported compilation config scenarios here
@pytest.mark.parametrize(
"compilation_config, model, model_kwargs",
[
# additional compile sizes, only some of the models
(
CompilationConfig(mode=CompilationMode.VLLM_COMPILE, compile_sizes=[1, 2]),
*model_info,
)
for model_info in models_list(all=False)
]
+ [
# RMSNorm + quant fusion, only 8-bit quant models
(
CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm"],
pass_config=PassConfig(
fuse_norm_quant=True, fuse_act_quant=True, eliminate_noops=True
),
),
*model_info,
)
for model_info in models_list(keywords=["FP8-dynamic"])
]
+ [
# Test depyf integration works
(
CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
debug_dump_path=Path(tempfile.gettempdir()),
),
"facebook/opt-125m",
{},
),
]
+ [
# graph inductor partition
(
CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
# inductor graph partition uses
# torch._C.Tag.cudagraph_unsafe to specify splitting ops
use_inductor_graph_partition=True,
cudagraph_mode=CUDAGraphMode.PIECEWISE,
compile_sizes=[1, 2],
),
*model_info,
)
for model_info in models_list(all=False)
if is_torch_equal_or_newer("2.9.0.dev")
]
+ [
# Test get_raw_stream patch with compile_sizes
# This tests that TorchInductor autotune works correctly with get_raw_stream
# patch in torch 2.9 and without patch in torch 2.10+
(
CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
compile_sizes=[1, 2], # Triggers autotune which uses get_raw_stream
cudagraph_mode=CUDAGraphMode.NONE,
),
"facebook/opt-125m",
{},
),
],
)
# only test some of the models
@create_new_process_for_each_test()
def test_custom_compile_config(
compilation_config: CompilationConfig,
model: str,
model_kwargs: dict[str, Any],
):
if compilation_config.use_inductor_graph_partition and not is_torch_equal_or_newer(
"2.9.0.dev"
):
pytest.skip("inductor graph partition is only available in PyTorch 2.9+")
print(f"MODEL={model}")
run_model(compilation_config, model, **model_kwargs)
def run_model(
compile_config: CompilationMode | CompilationConfig, model: str, **model_kwargs
):
compilation_config = (
compile_config
if isinstance(compile_config, CompilationConfig)
else CompilationConfig(mode=compile_config)
)
prompts = [
"Hello, my name is",
"The president of the United States is",
"The capital of France is",
"The future of AI is",
]
sampling_params = SamplingParams(temperature=0)
# Allow override from model_kwargs
model_kwargs = {"tensor_parallel_size": 1, **model_kwargs}
model_kwargs = {"disable_custom_all_reduce": True, **model_kwargs}
# No cudagraphs by default
if compilation_config.cudagraph_mode is None:
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
# V1 startup requires free_memory >= total * gpu_memory_utilization, so a
# device that is not free yet fails engine init rather than waiting for it.
# VllmRunner guards this; building LLM directly does not. No-op off ROCm.
wait_for_memory_to_settle(
threshold_ratio=1.0
- model_kwargs.get("gpu_memory_utilization", CacheConfig.gpu_memory_utilization)
)
llm = LLM(
model=model,
compilation_config=compilation_config,
**model_kwargs,
)
outputs = llm.generate(prompts, sampling_params)
# Print the outputs.
for output in outputs:
prompt = output.prompt
generated_text = output.outputs[0].text
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
@@ -1,184 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
import pytest
from tests.models.utils import dummy_hf_overrides
from vllm.compilation.counter import compilation_counter
from vllm.config import VllmConfig
from vllm.config.compilation import CompilationMode, CUDAGraphMode
from vllm.platforms import current_platform
def test_compile():
vllm_config = VllmConfig()
# Default configuration does not compile mm encoder
assert not vllm_config.compilation_config.compile_mm_encoder
@pytest.mark.forked
@pytest.mark.core_model
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
@pytest.mark.parametrize(
("model", "tokenizer", "config_format", "tokenizer_mode", "model_arch"),
[
pytest.param(
"mistralai/Pixtral-12B-2409",
None,
"mistral",
"mistral",
"PixtralForConditionalGeneration",
id="pixtral",
),
pytest.param(
"mistralai/Mistral-Small-3.1-24B-Instruct-2503",
"mistral-community/pixtral-12b",
"hf",
"auto",
"Mistral3ForConditionalGeneration",
id="mistral3-hf",
),
],
)
@pytest.mark.parametrize(
("compile_mm_encoder", "expected_models"),
[
# The dummy model has two compiled text components. Enabling encoder
# compilation adds its single vision transformer block.
pytest.param(False, 2, id="disabled"),
pytest.param(True, 3, id="enabled"),
],
)
def test_pixtral_compilation(
vllm_runner,
monkeypatch,
model: str,
tokenizer: str | None,
config_format: str,
tokenizer_mode: str,
model_arch: str,
compile_mm_encoder: bool,
expected_models: int,
):
"""Test compilation for both Pixtral vision encoder implementations."""
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
with (
compilation_counter.expect(num_models_seen=expected_models),
vllm_runner(
model,
tokenizer_name=tokenizer,
tokenizer_mode=tokenizer_mode,
config_format=config_format,
load_format="dummy",
hf_overrides=partial(dummy_hf_overrides, model_arch=model_arch),
max_model_len=4096,
limit_mm_per_prompt={"image": 1},
gpu_memory_utilization=0.8,
attention_backend="FLASH_ATTN",
compilation_config={
"mode": CompilationMode.VLLM_COMPILE,
"compile_mm_encoder": compile_mm_encoder,
"cudagraph_mode": CUDAGraphMode.NONE,
},
) as _,
):
pass
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
@pytest.mark.forked
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
def test_qwen2_5_vl_compilation(vllm_runner, monkeypatch):
"""Test that Qwen2.5-VL vision submodules are compiled.
This test verifies that the 3 vision submodules (Qwen2_5_VisionPatchEmbed,
Qwen2_5_VisionBlock, and Qwen2_5_VisionPatchMerger) are properly tagged
for compilation by checking that num_models_seen increases by at least 3.
"""
# Disable multiprocessing so that the counter is in the same process
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
with (
# NOTE: Qwen2.5-VL has 35 models in total - the LLM backend
# Vision Patch Embed, Vision Patch Merger, and then 32 Vision Blocks
# (one for each layer) - in the future, we should fix vLLM compilation
# logic to handle this case and only compile the Vision submodules once
# and reuse the compiled code for all layers
# See https://github.com/vllm-project/vllm/issues/27590
compilation_counter.expect(num_models_seen=35),
vllm_runner(
"Qwen/Qwen2.5-VL-3B-Instruct",
max_model_len=2048,
gpu_memory_utilization=0.8,
compilation_config={
"mode": CompilationMode.VLLM_COMPILE,
"compile_mm_encoder": True,
},
) as _,
):
pass
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
@pytest.mark.forked
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
def test_qwen2_5_vl_no_vit_compilation(vllm_runner, monkeypatch):
"""Test that Qwen2.5-VL vision submodules are not compiled when the
config is passed off
"""
# Disable multiprocessing so that the counter is in the same process
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
with (
compilation_counter.expect(num_models_seen=1),
vllm_runner(
"Qwen/Qwen2.5-VL-3B-Instruct",
max_model_len=2048,
gpu_memory_utilization=0.8,
compilation_config={
"mode": CompilationMode.VLLM_COMPILE,
"compile_mm_encoder": False,
},
) as _,
):
pass
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
# Requires Cuda and 8 gpus as well
@pytest.mark.forked
@pytest.mark.skip(reason="Skipping due to CI resource constraints")
def test_mllama4_vit_compilation(vllm_runner, monkeypatch):
"""Test that Mllama4 vision submodules are compiled.
This test verifies that the 2 vision submodules (Llama4VisionEncoder,
Llama4VisionPixelShuffleMLP) are properly tagged
for compilation by checking that num_models_seen increases to 3.
However since we are using TP=8, we compilation_counter will not
work properly so we will just check the run succeeds rn
"""
# Disable multiprocessing so that the counter is in the same process
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
with (
monkeypatch.context(),
# TODO: Since we require TP=8, this messes with the compilation
# counter. We should fix this in the future, but leave for now
# to make sure that compilation runs (no crash) with llama vision encoder
compilation_counter.expect(num_models_seen=0),
vllm_runner(
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
max_model_len=512,
gpu_memory_utilization=0.8,
tensor_parallel_size=8,
compilation_config={
"mode": CompilationMode.VLLM_COMPILE,
"compile_mm_encoder": True,
},
),
):
pass
-144
View File
@@ -1,144 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from collections.abc import Callable, Iterable
from typing import Any, NamedTuple
import pytest
import regex as re
from vllm.platforms import current_platform
from vllm.v1.attention.backends.registry import AttentionBackendEnum
class Matches(NamedTuple):
# simple pointwise
aiter_rms_quant_fusion: int = 0
rms_quant_fusion: int = 0
act_quant_fusion: int = 0
norm_rope_fusion: int = 0
attn_quant_fusion: int = 0
# distributed
ar_rms_fusion: int = 0
aiter_ar_rms_fusion: int = 0
class ModelFusionInfo(NamedTuple):
model_name: str
matches: Callable[[int], Matches]
"""Given number of hidden layers, produces the matches object"""
model_kwargs: dict[str, Any] = {}
hf_overrides: Callable[[int], dict] = lambda n: {"num_hidden_layers": n}
class AttentionBackendCase(NamedTuple):
backend: AttentionBackendEnum
model_kwargs: dict[str, Any] = {}
"""Additional args required for attn+quant fusion"""
is_blackwell = lambda: current_platform.is_device_capability_family(100)
"""Are we running on Blackwell, a lot of tests depend on it"""
def nvfp4_kernel_exposes_input_quant_key() -> bool:
"""Check if the NVFP4 kernel selected on this platform exposes input_quant_key.
FlashInferCuteDslNvFp4LinearKernel does not expose input_quant_key() due to
layout incompatibility between the manual fusion kernel (silu_and_mul_nvfp4_quant)
output format and the cutedsl backend's expected input format.
FlashInferCutlassNvFp4LinearKernel does expose input_quant_key() and supports
manual fusion.
"""
if not current_platform.is_cuda():
return False
try:
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
FlashInferCuteDslNvFp4LinearKernel,
)
is_supported, _ = FlashInferCuteDslNvFp4LinearKernel.is_supported()
if is_supported:
return False
except ImportError:
pass
try:
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
FlashInferCutlassNvFp4LinearKernel,
)
is_supported, _ = FlashInferCutlassNvFp4LinearKernel.is_supported()
if is_supported:
return True
except ImportError:
pass
# Fallback: assume no manual fusion support
return False
def custom_ops_combos(*custom_ops: str) -> Iterable[str]:
"""Generate all combinations of custom ops for parametrization."""
custom_ops_lists = [[f"-{op}", f"+{op}"] for op in custom_ops]
for op_list in itertools.product(*custom_ops_lists):
yield ",".join(op_list)
# Quick inline validation
assert list(custom_ops_combos("silu_and_mul")) == ["-silu_and_mul", "+silu_and_mul"]
assert list(custom_ops_combos("quant_fp8", "rms_norm")) == [
"-quant_fp8,-rms_norm",
"-quant_fp8,+rms_norm",
"+quant_fp8,-rms_norm",
"+quant_fp8,+rms_norm",
]
def has_cuda_graph_wrapper_metadata() -> bool:
from importlib import import_module
try:
module = import_module("torch._inductor.utils")
module.CUDAGraphWrapperMetadata # noqa B018
except AttributeError:
return False
return True
INDUCTOR_GRAPH_PARTITION = [
pytest.param(
True,
marks=pytest.mark.skipif(
not has_cuda_graph_wrapper_metadata(),
reason="torch version does not support Inductor partition",
),
id="inductor_partition",
),
pytest.param(False, id="dynamo_partition"),
]
FUSION_LOG_PATTERNS: dict[str, re.Pattern] = {
"aiter_rms_quant_fusion": re.compile(
r"RocmAiterRMSNormQuantFusionPass Replaced (\d+) patterns"
),
"rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"),
"act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"),
# The combined pass can consume these sites before the standalone pass.
"norm_rope_fusion": re.compile(
r"(?:qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on|"
r"qk_norm_rope_kvcache_fusion.py:\d+] "
r"QK-Norm\+RoPE(?:/MRoPE)?\+KVCache fusion: replaced) (\d+)"
),
"attn_quant_fusion": re.compile(
r"attn_quant_fusion.py:\d+] Fused quant onto (\d+) attention nodes"
),
"ar_rms_fusion": re.compile(
r"allreduce_rms_fusion.py:\d+] Replaced (\d+) patterns"
),
"aiter_ar_rms_fusion": re.compile(
r"RocmAiterAllReduceFusionPass Replaced (\d+) patterns"
),
}
-205
View File
@@ -1,205 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from .common import AttentionBackendCase, Matches, ModelFusionInfo, is_blackwell
# Attn backends
FLASHINFER_ATTN = pytest.param(
AttentionBackendCase(
backend=AttentionBackendEnum.FLASHINFER,
model_kwargs=dict(kv_cache_dtype="fp8"),
),
id="FLASHINFER",
marks=pytest.mark.skipif(
not is_blackwell() or not has_flashinfer(),
reason="FI backend requires Blackwell and FlashInfer",
),
)
TRITON_ATTN = pytest.param(
AttentionBackendCase(backend=AttentionBackendEnum.TRITON_ATTN), id="TRITON_ATTN"
)
ROCM_ATTN = pytest.param(
AttentionBackendCase(backend=AttentionBackendEnum.ROCM_ATTN),
id="ROCM_ATTN",
marks=pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm attention only for AMD",
),
)
ROCM_AITER_UNIFIED_ATTN = pytest.param(
AttentionBackendCase(backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN),
id="ROCM_AITER_UNIFIED_ATTN",
marks=pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="ROCM_AITER_UNIFIED_ATTN only for AMD when AITER is installed",
),
)
ROCM_AITER_MLA_ATTN = pytest.param(
AttentionBackendCase(backend=AttentionBackendEnum.ROCM_AITER_MLA),
id="ROCM_AITER_MLA",
marks=pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="ROCM_AITER_MLA only for AMD when AITER is installed",
),
)
FLASHINFER_MLA_ATTN = pytest.param(
AttentionBackendCase(backend=AttentionBackendEnum.FLASHINFER_MLA),
id="FLASHINFER_MLA",
marks=pytest.mark.skipif(
not is_blackwell() or not has_flashinfer(),
reason="FI backend requires Blackwell and FlashInfer",
),
)
TRITON_MLA_ATTN = pytest.param(
AttentionBackendCase(backend=AttentionBackendEnum.TRITON_MLA),
id="TRITON_MLA",
)
# Models
llama3_8b = ModelFusionInfo(
model_name="meta-llama/Llama-3.1-8B-Instruct",
matches=lambda n_layers: Matches(
ar_rms_fusion=n_layers * 2 + 1,
aiter_ar_rms_fusion=n_layers * 2 + 1,
),
)
llama3_8b_fp8 = ModelFusionInfo(
model_name="RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
matches=lambda n_layers: Matches(
rms_quant_fusion=n_layers * 2,
act_quant_fusion=n_layers,
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
),
)
llama3_8b_fp4 = ModelFusionInfo(
model_name="nvidia/Llama-3.1-8B-Instruct-FP4",
matches=lambda n_layers: Matches(
act_quant_fusion=n_layers,
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
),
)
# ModelOpt MIXED_PRECISION: FP8 attention projections, NVFP4 MLPs. It resolves to
# modelopt_mixed, not modelopt_fp4, and its MLP has no manual act+quant fusion, so
# every MLP boundary is left to ActivationQuantFusionPass.
qwen3_8_27b_mixed_fp4 = ModelFusionInfo(
model_name="nvidia/Qwen3.8-27B-NVFP4",
hf_overrides=lambda n_layers: {
"text_config": {
"num_hidden_layers": n_layers,
"layer_types": ["linear_attention"] * (n_layers - 1) + ["full_attention"],
}
},
matches=lambda n_layers: Matches(act_quant_fusion=n_layers),
)
# MoEs cannot do act+quant fusion because those ops are hidden from torch.compile.
# MoEs also only expose 1 rms+quant fusion because the quant for up_proj is hidden.
# TODO(luka): https://github.com/vllm-project/vllm/issues/31985
# Also, for MoEs, gemm+collective fusion only happens for dense GEMMs (o_proj/qkv proj)
llama4_scout_fp8 = ModelFusionInfo(
model_name="nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",
hf_overrides=lambda n_layers: {"text_config": {"num_hidden_layers": n_layers}},
matches=lambda n_layers: Matches(
rms_quant_fusion=n_layers,
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2,
),
)
llama4_scout_fp4 = ModelFusionInfo(
model_name="nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4",
hf_overrides=lambda n_layers: {"text_config": {"num_hidden_layers": n_layers}},
matches=lambda n_layers: Matches(
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2,
),
)
qwen3_a3b = ModelFusionInfo(
model_name="Qwen/Qwen3-30B-A3B",
matches=lambda n_layers: Matches(
norm_rope_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
aiter_ar_rms_fusion=n_layers * 2 + 1,
),
)
qwen3_a3b_fp8 = ModelFusionInfo(
model_name="Qwen/Qwen3-30B-A3B-FP8",
matches=lambda n_layers: Matches(
rms_quant_fusion=n_layers,
norm_rope_fusion=n_layers,
attn_quant_fusion=0, # attn + group quant not supported
ar_rms_fusion=n_layers * 2 + 1,
),
)
deepseek_coder_v2_lite_fp8 = ModelFusionInfo(
model_name="RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8",
matches=lambda n_layers: Matches(
# first_k_dense_replace=1; MoE hides most rms+quant sites
rms_quant_fusion=1,
act_quant_fusion=min(1, n_layers), # dense layers only
# MLA attn + static FP8 quant
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
),
)
deepseek_v3_fp8 = ModelFusionInfo(
model_name="deepseek-ai/DeepSeek-V3",
matches=lambda n_layers: Matches(
# 3 per dense layer (first 3):
# - input_rms + qkv_proj
# - q_a_layernorm + q_b_proj (inside MLA wrapper)
# - post_attn_layernorm + MLP
# 2 per MoE layer (remaining) due to MoE wrapping
rms_quant_fusion=n_layers * 2 + min(3, n_layers), # add for 3 dense layers
# silu+block quant
act_quant_fusion=min(3, n_layers), # dense layers only
# MLA attn + per-group FP8 quant
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
),
)
deepseek_r1_fp4 = ModelFusionInfo(
model_name="nvidia/DeepSeek-R1-0528-NVFP4-v2",
matches=lambda n_layers: Matches(
rms_quant_fusion=0,
act_quant_fusion=min(3, n_layers),
attn_quant_fusion=n_layers,
ar_rms_fusion=n_layers * 2 + 1,
),
)
gpt_oss_20b = ModelFusionInfo(
model_name="openai/gpt-oss-20b",
matches=lambda n_layers: Matches(
ar_rms_fusion=n_layers * 2 + 1,
aiter_ar_rms_fusion=n_layers + 1,
),
model_kwargs=(
{"quantization_config": {"moe": {"activation": "mxfp8"}}}
if is_blackwell()
else {}
),
)
-248
View File
@@ -1,248 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import pytest
from vllm.config import PassConfig
from vllm.platforms import current_platform
from vllm.utils.flashinfer import is_flashinfer_fp8_blockscale_gemm_supported
from .common import (
INDUCTOR_GRAPH_PARTITION,
AttentionBackendCase,
Matches,
custom_ops_combos,
is_blackwell,
nvfp4_kernel_exposes_input_quant_key,
)
from .models import (
FLASHINFER_ATTN,
FLASHINFER_MLA_ATTN,
ROCM_AITER_UNIFIED_ATTN,
ROCM_ATTN,
TRITON_ATTN,
TRITON_MLA_ATTN,
deepseek_coder_v2_lite_fp8,
deepseek_r1_fp4,
deepseek_v3_fp8,
llama3_8b_fp4,
llama3_8b_fp8,
llama4_scout_fp4,
llama4_scout_fp8,
qwen3_8_27b_mixed_fp4,
qwen3_a3b_fp8,
)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides, use_deepgemm",
[
(*llama3_8b_fp8, False),
(*qwen3_a3b_fp8, False),
(*qwen3_a3b_fp8, True),
(*deepseek_coder_v2_lite_fp8, False),
(*deepseek_v3_fp8, False),
(*deepseek_v3_fp8, True),
pytest.param(
*llama4_scout_fp8,
False,
marks=pytest.mark.skipif(
not current_platform.is_cuda(),
reason="Llama4 Scout FP8 only supported on CUDA",
),
),
],
)
@pytest.mark.parametrize(
"attn_backend",
[
TRITON_ATTN,
FLASHINFER_ATTN,
ROCM_ATTN,
ROCM_AITER_UNIFIED_ATTN,
FLASHINFER_MLA_ATTN,
TRITON_MLA_ATTN,
],
)
@pytest.mark.parametrize("n_layers", [6])
@pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm"))
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
def test_tp1_fp8_fusions(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
attn_backend: AttentionBackendCase,
n_layers: int,
custom_ops: str,
inductor_graph_partition: bool,
use_deepgemm: bool,
run_e2e_fusion_test,
):
if use_deepgemm and not current_platform.is_cuda():
pytest.skip("DeepGemm only supported on CUDA")
if use_deepgemm and is_flashinfer_fp8_blockscale_gemm_supported():
# Flashinfer block FP8 GEMM has internal quantization, so it can't
# be fused with other ops.
pytest.skip("FlashInfer block FP8 GEMM not supported")
if use_deepgemm and is_blackwell():
# TODO(luka) DeepGEMM uses different quants, matching not supported
# - on Blackwell, uses a special quant fp8, currently not supported
pytest.skip("DeepGEMM & quant matching not currently supported")
matches = matches_fn(n_layers)
block_fp8 = "qwen" in model_name.lower() or "deepseek" in model_name.lower()
if block_fp8 and "-quant_fp8" in custom_ops:
# This is why config forces +quant_fp8 by default
pytest.skip("native QuantFP8 matching not supported for group quant")
# Reduce size of model and skip weight loading time
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=custom_ops.split(","),
pass_config=PassConfig(
fuse_norm_quant=True,
fuse_act_quant=False,
fuse_attn_quant=True,
enable_qk_norm_rope_fusion=True,
),
)
use_aiter = current_platform.is_rocm() and ("qwen" in model_name.lower())
matches_check = [
"rms_quant_fusion",
"norm_rope_fusion",
"attn_quant_fusion",
]
if use_aiter:
matches_check[0] = "aiter_rms_quant_fusion"
matches = matches._replace(aiter_rms_quant_fusion=matches.rms_quant_fusion)
# TODO: enable the `norm_rope_fusion` test,
# On ROCm norm_rope_fusion is only supported without
# enabling AITER.
matches_check.remove("norm_rope_fusion")
run_e2e_fusion_test(
model_name,
matches,
model_kwargs,
attn_backend,
compilation_config,
matches_check,
use_deepgemm=use_deepgemm,
use_aiter=use_aiter,
)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4],
)
@pytest.mark.parametrize(
"attn_backend",
[FLASHINFER_ATTN, FLASHINFER_MLA_ATTN],
)
@pytest.mark.parametrize("n_layers", [6])
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
def test_tp1_fp4_fusions(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
attn_backend: AttentionBackendCase,
n_layers: int,
custom_ops: str,
inductor_graph_partition: bool,
run_e2e_fusion_test,
):
if nvfp4_kernel_exposes_input_quant_key():
pytest.skip(
"NVFP4 kernel exposes input_quant_key; manual fusion fires "
"instead of compiler pass-based fusion"
)
matches = matches_fn(n_layers)
# Reduce size of model and skip weight loading time
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=custom_ops.split(","),
pass_config=PassConfig(
fuse_norm_quant=True,
fuse_act_quant=False,
fuse_attn_quant=True,
enable_qk_norm_rope_fusion=True,
),
)
matches_check = ["attn_quant_fusion", "norm_rope_fusion"]
run_e2e_fusion_test(
model_name,
matches,
model_kwargs,
attn_backend,
compilation_config,
matches_check,
)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides", [qwen3_8_27b_mixed_fp4]
)
@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN])
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
def test_tp1_fp4_mixed_precision_act_fusion(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
attn_backend: AttentionBackendCase,
n_layers: int,
inductor_graph_partition: bool,
run_e2e_fusion_test,
):
"""Act+quant fusion matches on a MIXED_PRECISION NVFP4 checkpoint."""
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
pass_config=PassConfig(
fuse_norm_quant=False,
fuse_act_quant=True,
fuse_attn_quant=False,
enable_qk_norm_rope_fusion=False,
),
)
run_e2e_fusion_test(
model_name,
matches_fn(n_layers),
model_kwargs,
attn_backend,
compilation_config,
["act_quant_fusion"],
)
@@ -1,313 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
import pytest
from vllm.config import PassConfig
from vllm.platforms import current_platform
from ...utils import multi_gpu_test
from .common import (
INDUCTOR_GRAPH_PARTITION,
AttentionBackendCase,
Matches,
custom_ops_combos,
is_blackwell,
nvfp4_kernel_exposes_input_quant_key,
)
from .models import (
FLASHINFER_ATTN,
FLASHINFER_MLA_ATTN,
ROCM_AITER_MLA_ATTN,
ROCM_AITER_UNIFIED_ATTN,
ROCM_ATTN,
TRITON_ATTN,
deepseek_coder_v2_lite_fp8,
deepseek_r1_fp4,
deepseek_v3_fp8,
gpt_oss_20b,
llama3_8b,
llama3_8b_fp4,
llama3_8b_fp8,
llama4_scout_fp4,
llama4_scout_fp8,
qwen3_a3b,
qwen3_a3b_fp8,
)
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Only test CUDA/ROCm"
)
# qwen3 & dsv3 should still fuse AR+rms even though group quant is not yet
# supported. ROCm carries only the payloads its two backends support.
TP2_FP8_MODELS = (
[
llama3_8b_fp8,
llama4_scout_fp8,
qwen3_a3b_fp8,
deepseek_coder_v2_lite_fp8,
deepseek_v3_fp8,
]
if current_platform.is_cuda()
else [llama3_8b_fp8, qwen3_a3b_fp8, deepseek_coder_v2_lite_fp8]
)
TP2_FP8_ATTN_BACKENDS = (
[TRITON_ATTN, FLASHINFER_ATTN, FLASHINFER_MLA_ATTN]
if current_platform.is_cuda()
else [ROCM_AITER_UNIFIED_ATTN, ROCM_AITER_MLA_ATTN]
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides", TP2_FP8_MODELS
)
@pytest.mark.parametrize("attn_backend", TP2_FP8_ATTN_BACKENDS)
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm"))
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
def test_tp2_ar_rms_fp8_fusions(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
attn_backend: AttentionBackendCase,
n_layers: int,
custom_ops: str,
inductor_graph_partition: bool,
run_e2e_fusion_test,
monkeypatch,
):
matches = matches_fn(n_layers)
model_name_lower = model_name.lower()
# DeepSeek-Coder on AITER MLA is the one static-FP8 payload here, validated
# against the native QuantFP8 pattern; every other DeepSeek stays group
# quantized as on CUDA.
rocm_static_fp8_mla = (
current_platform.is_rocm()
and "deepseek-coder" in model_name_lower
and attn_backend.backend.name == "ROCM_AITER_MLA"
)
block_fp8 = "qwen" in model_name_lower or "deepseek" in model_name_lower
if rocm_static_fp8_mla:
block_fp8 = False
if custom_ops != "-quant_fp8,-rms_norm":
pytest.skip("ROCm static-FP8 MLA parity uses native quant and RMSNorm")
# AITER's MoE GEMM rejects this reduced dummy model shape; keep AITER
# for MLA and the all-reduce, and let the MoE run on Triton.
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "0")
if block_fp8 and "-quant_fp8" in custom_ops:
# This is why config forces +quant_fp8 by default
pytest.skip("native QuantFP8 matching not supported for group quant")
# Reduce size of model and skip weight loading time
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
model_kwargs["disable_custom_all_reduce"] = False
if current_platform.is_rocm() and (
"llama-3" in model_name_lower or "qwen" in model_name_lower
):
model_kwargs.update(
gpu_memory_utilization=0.01,
kv_cache_memory_bytes=64 * 1024**2,
max_num_batched_tokens=1024,
max_num_seqs=32,
)
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=custom_ops.split(","),
pass_config=PassConfig(
fuse_norm_quant=True,
fuse_act_quant=False,
fuse_attn_quant=True,
enable_qk_norm_rope_fusion=True,
fuse_allreduce_rms=True,
),
)
matches_check = [
"rms_quant_fusion",
"norm_rope_fusion",
"attn_quant_fusion",
"ar_rms_fusion",
]
use_aiter = current_platform.is_rocm()
if use_aiter:
matches = matches._replace(aiter_ar_rms_fusion=matches.ar_rms_fusion)
# The AITER all-reduce pass registers the AR+RMS(+group-quant) patterns
# before its AR+RMS-only ones, so the standalone AITER RMS-quant pass
# does not own the post-AR sites here. ROCm's contract is native
# QK-norm+RoPE and attention-output quant fusion plus AITER AR+RMS.
matches_check = [
"norm_rope_fusion",
"attn_quant_fusion",
"aiter_ar_rms_fusion",
]
run_e2e_fusion_test(
model_name,
matches,
model_kwargs,
attn_backend,
compilation_config,
matches_check,
tp_size=2,
use_aiter=use_aiter,
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides",
[llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4],
)
@pytest.mark.parametrize(
"attn_backend",
[FLASHINFER_ATTN, FLASHINFER_MLA_ATTN],
)
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
def test_tp2_ar_rms_fp4_fusions(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
attn_backend: AttentionBackendCase,
n_layers: int,
custom_ops: str,
inductor_graph_partition: bool,
run_e2e_fusion_test,
):
if nvfp4_kernel_exposes_input_quant_key():
pytest.skip(
"NVFP4 kernel exposes input_quant_key; manual fusion fires "
"instead of compiler pass-based fusion"
)
matches = matches_fn(n_layers)
# Reduce size of model and skip weight loading time
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
model_kwargs["disable_custom_all_reduce"] = False
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=custom_ops.split(","),
pass_config=PassConfig(
fuse_act_quant=False,
fuse_attn_quant=True,
fuse_allreduce_rms=True,
),
)
matches_check = [
"attn_quant_fusion",
"ar_rms_fusion",
]
run_e2e_fusion_test(
model_name,
matches,
model_kwargs,
attn_backend,
compilation_config,
matches_check,
tp_size=2,
)
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"model_name, matches_fn, model_kwargs, hf_overrides, model_impl",
[
(*llama3_8b, "auto"),
(*llama3_8b, "transformers"),
(*qwen3_a3b, "auto"),
(*gpt_oss_20b, "auto"),
],
)
@pytest.mark.parametrize(
"attn_backend",
[
TRITON_ATTN,
FLASHINFER_ATTN,
ROCM_ATTN,
ROCM_AITER_UNIFIED_ATTN,
],
)
@pytest.mark.parametrize("n_layers", [4])
@pytest.mark.parametrize("custom_ops", tuple(custom_ops_combos("rms_norm")))
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Only test CUDA/ROCm")
def test_tp2_ar_rms_fusions(
model_name: str,
matches_fn: Callable[[int], Matches],
model_kwargs: dict,
hf_overrides: Callable[[int], dict],
model_impl: str,
attn_backend: AttentionBackendCase,
n_layers: int,
custom_ops: str,
inductor_graph_partition: bool,
run_e2e_fusion_test,
):
if model_impl == "transformers" and not current_platform.is_rocm():
pytest.skip("Transformers 3D AR+RMS regression is ROCm-only")
matches = matches_fn(n_layers)
if model_impl == "transformers":
# Transformers add+RMSNorm canonicalization exposes every generic
# AR+RMS fusion site, including the final norm.
matches = matches._replace(aiter_ar_rms_fusion=matches.ar_rms_fusion)
# Reduce size of model and skip weight loading time
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
model_kwargs["load_format"] = "dummy"
model_kwargs["model_impl"] = model_impl
model_kwargs["max_model_len"] = 1024
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
model_kwargs["disable_custom_all_reduce"] = False
compilation_config = dict(
use_inductor_graph_partition=inductor_graph_partition,
custom_ops=custom_ops.split(","),
pass_config=PassConfig(
enable_qk_norm_rope_fusion=True,
fuse_allreduce_rms=True,
),
)
matches_check = [
"norm_rope_fusion",
]
if current_platform.is_rocm():
matches_check.append("aiter_ar_rms_fusion")
else:
matches_check.append("ar_rms_fusion")
run_e2e_fusion_test(
model_name,
matches,
model_kwargs,
attn_backend,
compilation_config,
matches_check,
tp_size=2,
use_aiter=current_platform.is_rocm(),
)
@@ -1,898 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from importlib.util import find_spec
from types import SimpleNamespace
import pytest
import torch
import vllm.envs as envs
from tests.compile.backend import TestBackend
from tests.utils import TestFP8Layer, has_module_attribute, multi_gpu_test
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
from vllm.compilation.passes.fusion.allreduce_rms_fusion import (
AllReduceFusionPass,
RocmAiterAllReduceFusionPass,
_fused_ar_workspace_hidden_dim,
_select_flashinfer_allreduce_use_oneshot,
)
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.compilation.passes.utility.fix_functionalization import (
FixFunctionalizationPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CompilationConfig,
CompilationMode,
DeviceConfig,
ModelConfig,
PassConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.distributed import tensor_model_parallel_all_reduce
from vllm.distributed.device_communicators.aiter_custom_all_reduce import (
AiterCustomAllreduce,
)
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
from vllm.utils.system_utils import update_environment_variables
from vllm.utils.torch_utils import set_random_seed
DEVICE_TYPE = current_platform.device_type
@pytest.mark.parametrize(
("workspace_backend", "device_capability", "world_size", "tensor_size", "expected"),
[
("mnnvl", 103, 8, 2 * 1024 * 1024, None),
("trtllm", 103, 8, 2 * 1024 * 1024, True),
("trtllm", 103, 8, 2 * 1024 * 1024 + 1, False),
("trtllm", 100, 4, 4 * 1024 * 1024, True),
("trtllm", 100, 4, 4 * 1024 * 1024 + 1, False),
("trtllm", None, 8, 128 * 1024 * 1024, True),
],
)
def test_select_flashinfer_allreduce_use_oneshot(
workspace_backend: str,
device_capability: int | None,
world_size: int,
tensor_size: int,
expected: bool | None,
):
assert (
_select_flashinfer_allreduce_use_oneshot(
workspace_backend,
device_capability,
world_size,
tensor_size,
)
is expected
)
def _make_workspace_hidden_config(
target_hidden: int, draft_hidden: int | None
) -> SimpleNamespace:
"""Minimal stand-in exposing only what _fused_ar_workspace_hidden_dim reads."""
speculative_config = None
if draft_hidden is not None:
speculative_config = SimpleNamespace(
draft_model_config=SimpleNamespace(
get_hidden_size=lambda: draft_hidden,
)
)
return SimpleNamespace(
model_config=SimpleNamespace(get_hidden_size=lambda: target_hidden),
speculative_config=speculative_config,
)
@pytest.mark.parametrize(
("target_hidden", "draft_hidden", "expected"),
[
(2048, None, 2048), # no speculative decoding -> target only
(2048, 2560, 2560), # draft wider than target -> draft (issue #52023)
(2048, 1024, 2048), # draft narrower than target -> target
(2048, 2048, 2048), # equal -> target
],
)
def test_fused_ar_workspace_hidden_dim(
target_hidden: int, draft_hidden: int | None, expected: int
):
config = _make_workspace_hidden_config(target_hidden, draft_hidden)
assert _fused_ar_workspace_hidden_dim(config) == expected
def test_fused_ar_workspace_hidden_dim_spec_without_draft():
# speculative_config present but no draft model (e.g. ngram) -> target only.
config = SimpleNamespace(
model_config=SimpleNamespace(get_hidden_size=lambda: 2048),
speculative_config=SimpleNamespace(draft_model_config=None),
)
assert _fused_ar_workspace_hidden_dim(config) == 2048
class TestAllReduceRMSNormModel(torch.nn.Module):
def __init__(
self,
hidden_size=16,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.float16,
use_aiter: bool = False,
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
self.w = [torch.rand(hidden_size, hidden_size) for _ in range(3)]
self.use_aiter = use_aiter
def forward(self, x):
# avoid having graph input be an arg to a pattern directly
z = torch.relu(x)
x = resid = tensor_model_parallel_all_reduce(z)
y = self.norm[0](x)
z2 = torch.mm(y, self.w[0])
x2 = tensor_model_parallel_all_reduce(z2)
y2, resid = self.norm[1](x2, resid)
z3 = torch.mm(y2, self.w[1])
x3 = tensor_model_parallel_all_reduce(z3)
y3, resid = self.norm[2](x3, resid)
z4 = torch.mm(y3, self.w[2])
x4 = tensor_model_parallel_all_reduce(z4)
y4, resid = self.norm[3](x4, resid)
return y4
def ops_in_model_before(self):
return [torch.ops.vllm.all_reduce.default]
def ops_in_model_after(self):
if self.use_aiter:
return [rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()]
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
class TestAllReduceGemmaRMSNormModel(torch.nn.Module):
def __init__(
self,
hidden_size=16,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.float16,
use_aiter: bool = False,
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.use_aiter = use_aiter
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
# Non-trivial weight (~Gemma range) so (1 + w) exercises the scale path.
for n in self.norm:
n.weight.data.normal_(mean=0.0, std=0.1)
self.w = [torch.rand(hidden_size, hidden_size) for _ in range(3)]
def forward(self, x):
# avoid having graph input be an arg to a pattern directly
z = torch.relu(x)
x = resid = tensor_model_parallel_all_reduce(z)
y = self.norm[0](x)
z2 = torch.mm(y, self.w[0])
x2 = tensor_model_parallel_all_reduce(z2)
y2, resid = self.norm[1](x2, resid)
z3 = torch.mm(y2, self.w[1])
x3 = tensor_model_parallel_all_reduce(z3)
y3, resid = self.norm[2](x3, resid)
z4 = torch.mm(y3, self.w[2])
x4 = tensor_model_parallel_all_reduce(z4)
y4, resid = self.norm[3](x4, resid)
return y4
def ops_in_model_before(self):
return [torch.ops.vllm.all_reduce.default]
def ops_in_model_after(self):
if self.use_aiter:
return [rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()]
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
quant_key = kFp8StaticTensorSym
def __init__(
self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
self.fp8_linear_layers = [
TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
input_dtype=dtype,
)
for i in range(3)
]
def forward(self, hidden_states):
# avoid having graph input be an arg to a pattern directly
z = torch.relu(hidden_states)
x = resid = tensor_model_parallel_all_reduce(z)
y = self.norm[0](x)
z2 = self.fp8_linear_layers[0](y)
x2 = tensor_model_parallel_all_reduce(z2)
y2, resid = self.norm[1](x2, resid)
z3 = self.fp8_linear_layers[1](y2)
x3 = tensor_model_parallel_all_reduce(z3)
y3, resid = self.norm[2](x3, resid) # use resid here
z4 = self.fp8_linear_layers[2](y3)
x4 = tensor_model_parallel_all_reduce(z4)
y4, resid = self.norm[3](x4, resid) # use resid here
return y4
def ops_in_model_after(self):
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
def ops_in_model_before(self):
return [
torch.ops.vllm.all_reduce.default,
torch.ops._C.static_scaled_fp8_quant.default
if self.fp8_linear_layers[0].is_quant_fp8_enabled()
else torch.ops.aten.reciprocal.default,
]
class TestAllReduceGemmaRMSNormStaticQuantFP8Model(
TestAllReduceRMSNormStaticQuantFP8Model
):
def __init__(
self,
hidden_size=16,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.float16,
):
super().__init__(hidden_size, token_num, eps, dtype)
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
for norm in self.norm:
norm.weight.requires_grad_(False)
def ops_in_model_before(self):
return [torch.ops.vllm.all_reduce.default]
class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
"""Exercises the new ROCm AITER AR+RMS+per-group-FP8-quant patterns.
Four ``rms_norm`` sites that together hit every pattern registered by
``RocmAiterAllReduceFusionPass`` for the per-group FP8 quant path:
* ``norm[0]``: ``all_reduce -> rms_norm -> group_fp8_quant`` (no residual)
-> ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern``
* ``norm[1]``: ``all_reduce -> fused_add_rms_norm -> group_fp8_quant``
(single ``rms`` consumer)
-> ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern``
* ``norm[2..3]``: ``all_reduce -> fused_add_rms_norm
-> (group_fp8_quant + rocm_unquantized_gemm)`` (two ``rms`` consumers,
modeling the DSv3.2 indexer fan-out)
-> ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern``
Each site starts from the same bf16 activation and is returned separately.
This avoids compounding four quantize/dequantize errors while still testing
all four pattern firings without depending on a real FP8 block-scaled GEMM.
"""
quant_group_size = 128
indexer_out_dim = 8
def __init__(
self,
hidden_size=128,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.bfloat16,
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.dtype = dtype
assert hidden_size % self.quant_group_size == 0, (
f"hidden_size ({hidden_size}) must be a multiple of "
f"quant_group_size ({self.quant_group_size}) for per-group FP8 quant"
)
self.norm = [RMSNorm(hidden_size, eps) for _ in range(4)]
self.w = [torch.rand(hidden_size, hidden_size, dtype=dtype) for _ in range(3)]
self.indexer_w = [
torch.rand(self.indexer_out_dim, hidden_size, dtype=dtype) for _ in range(2)
]
def _group_quant(self, rms: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
return torch.ops.vllm.rocm_aiter_group_fp8_quant.default(
rms, self.quant_group_size
)
def _dequantize_to_bf16(self, q: torch.Tensor, s: torch.Tensor) -> torch.Tensor:
# Broadcast the per-group scale across each group of `quant_group_size`
# to validate the quantized result without depending on a real FP8
# block-scaled GEMM kernel in the test.
s_full = s.repeat_interleave(self.quant_group_size, dim=-1).to(self.dtype)
return q.to(self.dtype) * s_full
def forward(self, hidden_states):
z = torch.relu(hidden_states)
x0 = tensor_model_parallel_all_reduce(z)
rms0 = self.norm[0](x0)
q0, s0 = self._group_quant(rms0)
y0 = self._dequantize_to_bf16(q0, s0)
x1 = tensor_model_parallel_all_reduce(torch.mm(z, self.w[0]))
rms1, resid1 = self.norm[1](x1, z.clone())
q1, s1 = self._group_quant(rms1)
y1 = self._dequantize_to_bf16(q1, s1)
x2 = tensor_model_parallel_all_reduce(torch.mm(z, self.w[1]))
rms2, resid2 = self.norm[2](x2, z.clone())
q2, s2 = self._group_quant(rms2)
# Second consumer of ``rms2``: forces the with-indexer pattern.
idx2 = torch.ops.vllm.rocm_unquantized_gemm(rms2, self.indexer_w[0], None)
y2 = self._dequantize_to_bf16(q2, s2)
x3 = tensor_model_parallel_all_reduce(torch.mm(z, self.w[2]))
rms3, resid3 = self.norm[3](x3, z.clone())
q3, s3 = self._group_quant(rms3)
# Second consumer of ``rms3``: forces the with-indexer pattern.
idx3 = torch.ops.vllm.rocm_unquantized_gemm(rms3, self.indexer_w[1], None)
y3 = self._dequantize_to_bf16(q3, s3)
return y0, y1, y2, y3, resid1, resid2, resid3, idx2, idx3
def ops_in_model_before(self):
return [
torch.ops.vllm.all_reduce.default,
torch.ops.vllm.rocm_aiter_group_fp8_quant.default,
]
def ops_in_model_after(self):
return [
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op(),
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op(), # noqa: E501
]
class TestAiterAllReduceGemmaRMSNormGroupQuantFP8Model(
TestAiterAllReduceRMSNormGroupQuantFP8Model
):
"""Gemma norms feeding the per-group FP8 quant sites.
The fused AR+RMS+quant op has no ``gemma_norm`` mode, so these sites must
keep matching the quant-fused patterns with the pre-added ``1 + weight``.
The Gemma AR+RMS-only patterns match a prefix of the same subgraph, so if
they were registered ahead of the quant-fused ones they would consume the
``all_reduce`` and strand ``rocm_aiter_group_fp8_quant`` as a separate
kernel -- which the ``fully_replaced=True`` check below would catch.
"""
def __init__(
self,
hidden_size=128,
token_num=16,
eps=1e-6,
dtype: torch.dtype = torch.bfloat16,
):
super().__init__(hidden_size, token_num, eps, dtype)
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
for norm in self.norm:
norm.weight.requires_grad_(False)
class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module):
def __init__(
self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16
):
super().__init__()
self.hidden_size = hidden_size
self.eps = eps
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
self.w = [torch.rand(hidden_size, hidden_size) for _ in range(3)]
self.agscale = [torch.rand(1, dtype=torch.float32) for _ in range(3)]
wgscale = [torch.rand(1, dtype=torch.float32) for _ in range(3)]
self.alpha = [1 / (w * a) for w, a in zip(wgscale, self.agscale)]
wq_gen, wscale_gen = zip(
*(scaled_fp4_quant(w, wg) for w, wg in zip(self.w, wgscale))
)
self.wq, self.wscale = list(wq_gen), list(wscale_gen)
def forward(self, hidden_states):
# avoid having graph input be an arg to a pattern directly
z = torch.relu(hidden_states)
x = resid = tensor_model_parallel_all_reduce(z)
y = self.norm[0](x)
yq, y_scale = scaled_fp4_quant(y, self.agscale[0])
z2 = cutlass_scaled_fp4_mm(
yq, self.wq[0], y_scale, self.wscale[0], self.alpha[0], out_dtype=y.dtype
)
x2 = tensor_model_parallel_all_reduce(z2)
y2, resid = self.norm[1](x2, resid)
yq2, y_scale2 = scaled_fp4_quant(y2, self.agscale[1])
z3 = cutlass_scaled_fp4_mm(
yq2, self.wq[1], y_scale2, self.wscale[1], self.alpha[1], out_dtype=y2.dtype
)
x3 = tensor_model_parallel_all_reduce(z3)
y3, resid = self.norm[2](x3, resid) # use resid here
yq3, y_scale3 = scaled_fp4_quant(y3, self.agscale[2])
z4 = cutlass_scaled_fp4_mm(
yq3, self.wq[2], y_scale3, self.wscale[2], self.alpha[2], out_dtype=y3.dtype
)
x4 = tensor_model_parallel_all_reduce(z4)
y4, resid = self.norm[3](x4, resid) # use resid here
return y4
def ops_in_model_after(self):
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
def ops_in_model_before(self):
return [
torch.ops.vllm.all_reduce.default,
torch.ops._C.scaled_fp4_quant.out,
]
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize(
"test_model, enable_quant_fp8_custom_op, use_aiter",
[
(TestAllReduceRMSNormModel, False, IS_AITER_FOUND),
pytest.param(
TestAllReduceGemmaRMSNormModel,
False,
IS_AITER_FOUND,
marks=pytest.mark.skipif(
current_platform.is_rocm() and not IS_AITER_FOUND,
reason="AITER is required on ROCm",
),
),
pytest.param(
TestAllReduceRMSNormStaticQuantFP8Model,
True,
False,
marks=pytest.mark.skipif(
current_platform.is_rocm(),
reason="Not supported on ROCm platform",
),
),
pytest.param(
TestAllReduceGemmaRMSNormStaticQuantFP8Model,
True,
False,
marks=pytest.mark.skipif(
current_platform.is_rocm(),
reason="Not supported on ROCm platform",
),
),
pytest.param(
TestAllReduceRMSNormStaticQuantFP8Model,
False,
False,
marks=pytest.mark.skipif(
current_platform.is_rocm(),
reason="Not supported on ROCm platform",
),
),
pytest.param(
TestAllReduceFusedAddRMSNormStaticQuantFP4Model,
False,
False,
marks=pytest.mark.skipif(
current_platform.is_rocm(),
reason="Not supported on ROCm platform",
),
),
],
)
@pytest.mark.parametrize("batch_size", [8])
@pytest.mark.parametrize("seq_len", [8])
@pytest.mark.parametrize("hidden_size", [64])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
@pytest.mark.parametrize("flashinfer_allreduce_backend", ["trtllm", "mnnvl"])
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
@pytest.mark.skipif(
current_platform.is_rocm() and not IS_AITER_FOUND,
reason="aiter is not found",
)
@pytest.mark.skipif(
current_platform.is_cuda()
and (
not find_spec("flashinfer")
or not has_module_attribute("flashinfer.comm", "allreduce_fusion")
or not has_module_attribute(
"flashinfer.comm", "create_allreduce_fusion_workspace"
)
),
reason="flashinfer is not found or flashinfer "
"is not compiled with allreduce_fusion",
)
def test_all_reduce_fusion_pass_replace(
test_model: torch.nn.Module,
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
enable_rms_norm_custom_op,
enable_quant_fp8_custom_op,
flashinfer_allreduce_backend,
use_aiter: bool,
monkeypatch: pytest.MonkeyPatch,
):
if use_aiter:
with monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", str(use_aiter))
rocm_aiter_ops.refresh_env_variables()
num_processes = 2
if (
test_model == TestAllReduceFusedAddRMSNormStaticQuantFP4Model
and not current_platform.has_device_capability(100)
):
pytest.skip(
"Skip as nvfp4 is only supported on "
"devices with compute capability 10.0 (Blackwell)"
)
def run_torch_spawn(fn, nprocs):
master_port = get_open_port()
torch.multiprocessing.spawn(
fn,
args=(
num_processes,
master_port,
test_model,
batch_size,
seq_len,
hidden_size,
dtype,
enable_rms_norm_custom_op,
enable_quant_fp8_custom_op,
flashinfer_allreduce_backend,
use_aiter,
monkeypatch,
),
nprocs=nprocs,
)
run_torch_spawn(all_reduce_fusion_pass_on_test_model, num_processes)
def all_reduce_fusion_pass_on_test_model(
local_rank: int,
world_size: int,
master_port: int,
test_model_cls: torch.nn.Module,
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
enable_rms_norm_custom_op,
enable_quant_fp8_custom_op,
flashinfer_allreduce_backend,
use_aiter: bool,
monkeypatch: pytest.MonkeyPatch,
):
set_random_seed(0)
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": str(master_port),
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": flashinfer_allreduce_backend,
"VLLM_ROCM_USE_AITER": str(int(use_aiter)),
"VLLM_ROCM_USE_AITER_CUSTOM_AR": str(int(use_aiter)),
}
)
if use_aiter:
rocm_aiter_ops.refresh_env_variables()
init_distributed_environment()
custom_ops = []
if enable_rms_norm_custom_op:
custom_ops.append("+rms_norm")
if enable_quant_fp8_custom_op:
custom_ops.append("+quant_fp8")
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE, custom_ops=custom_ops
)
)
vllm_config.compilation_config.pass_config = PassConfig(
fuse_allreduce_rms=True, eliminate_noops=True
)
vllm_config.device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
vllm_config.parallel_config.rank = local_rank # Setup rank for debug path
# this is a fake model name to construct the model config
# in the vllm_config, it's not really used.
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
vllm_config.model_config = ModelConfig(
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
)
with set_current_vllm_config(vllm_config):
initialize_model_parallel(tensor_model_parallel_size=world_size)
all_reduce_fusion_pass = (
RocmAiterAllReduceFusionPass(vllm_config)
if use_aiter
else AllReduceFusionPass(vllm_config)
)
noop_pass = NoOpEliminationPass(vllm_config)
func_pass = FixFunctionalizationPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(
noop_pass, all_reduce_fusion_pass, func_pass, cleanup_pass
)
token_num = batch_size * seq_len
if test_model_cls in (
TestAllReduceRMSNormModel,
TestAllReduceGemmaRMSNormModel,
):
model = test_model_cls(
hidden_size, token_num, dtype=dtype, use_aiter=use_aiter
)
else:
model = test_model_cls(hidden_size, token_num, dtype=dtype)
hidden_states = torch.randn((token_num, hidden_size), requires_grad=False)
compiled_model = torch.compile(model, backend=backend)
compiled_model(hidden_states)
results_unfused = model(hidden_states)
results_fused = compiled_model(hidden_states)
torch.testing.assert_close(results_unfused, results_fused, atol=1e-2, rtol=1e-2)
assert all_reduce_fusion_pass.matched_count == 4, (
f"{all_reduce_fusion_pass.matched_count=}"
)
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=False)
backend.check_after_ops(model.ops_in_model_after())
if (
test_model_cls
in (
TestAllReduceGemmaRMSNormModel,
TestAllReduceGemmaRMSNormStaticQuantFP8Model,
)
and not use_aiter
):
fused_op = torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default
fused_nodes = list(find_op_nodes(fused_op, backend.graph_post_pass))
assert fused_nodes
assert all(n.kwargs.get("weight_bias") == 1.0 for n in fused_nodes)
del all_reduce_fusion_pass
@multi_gpu_test(num_gpus=2)
@pytest.mark.parametrize("batch_size", [8])
@pytest.mark.parametrize("seq_len", [8])
@pytest.mark.parametrize("hidden_size", [128])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
@pytest.mark.parametrize(
"test_model_cls",
[
TestAiterAllReduceRMSNormGroupQuantFP8Model,
TestAiterAllReduceGemmaRMSNormGroupQuantFP8Model,
],
)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm AITER AR+RMS+per-group-FP8-quant fusion is ROCm-only",
)
@pytest.mark.skipif(not IS_AITER_FOUND, reason="aiter is not found")
def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
enable_rms_norm_custom_op: bool,
test_model_cls: type[torch.nn.Module],
monkeypatch: pytest.MonkeyPatch,
):
"""Sibling of ``test_all_reduce_fusion_pass_replace`` for the new
ROCm AITER AR+RMS+per-group-FP8-quant fusion patterns.
Validates the three new ``VllmPatternReplacement`` patterns added to
``RocmAiterAllReduceFusionPass``:
* ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern`` (no-residual)
* ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern`` (with-residual,
single ``rms`` consumer)
* ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern`` (with-
residual, DSv3.2 indexer fan-out; parametrized over
``rocm_aiter_group_fp8_quant``
producer).
Parametrized over plain and Gemma norms: the Gemma AR+RMS-only patterns
match a prefix of these subgraphs, so this also pins their registration
order behind the quant-fused ones.
"""
with monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
if not AiterCustomAllreduce.build_supports_per_group_quant():
pytest.skip(
"aiter build is missing 'fused_ar_rms_per_group_quant' (needs "
"ROCm/aiter PR #2823); the new patterns aren't registered."
)
num_processes = 2
def run_torch_spawn(fn, nprocs):
master_port = get_open_port()
torch.multiprocessing.spawn(
fn,
args=(
num_processes,
master_port,
test_model_cls,
batch_size,
seq_len,
hidden_size,
dtype,
enable_rms_norm_custom_op,
monkeypatch,
),
nprocs=nprocs,
)
run_torch_spawn(rocm_aiter_group_quant_fusion_pass_on_test_model, num_processes)
def rocm_aiter_group_quant_fusion_pass_on_test_model(
local_rank: int,
world_size: int,
master_port: int,
test_model_cls: torch.nn.Module,
batch_size: int,
seq_len: int,
hidden_size: int,
dtype: torch.dtype,
enable_rms_norm_custom_op: bool,
monkeypatch: pytest.MonkeyPatch,
):
set_random_seed(0)
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
torch.set_default_dtype(dtype)
update_environment_variables(
{
"RANK": str(local_rank),
"LOCAL_RANK": str(local_rank),
"WORLD_SIZE": str(world_size),
"MASTER_ADDR": "localhost",
"MASTER_PORT": str(master_port),
"VLLM_ROCM_USE_AITER": "1",
"VLLM_ROCM_USE_AITER_CUSTOM_AR": "1",
}
)
rocm_aiter_ops.refresh_env_variables()
init_distributed_environment()
custom_ops = []
if enable_rms_norm_custom_op:
custom_ops.append("+rms_norm")
# We always enable +quant_fp8 so the matcher's example
# trace finds the same form the test model uses.
custom_ops.append("+quant_fp8")
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE, custom_ops=custom_ops
)
)
vllm_config.compilation_config.pass_config = PassConfig(
fuse_allreduce_rms=True, eliminate_noops=True
)
vllm_config.device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
vllm_config.parallel_config.rank = local_rank
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
vllm_config.model_config = ModelConfig(
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
)
with set_current_vllm_config(vllm_config):
initialize_model_parallel(tensor_model_parallel_size=world_size)
all_reduce_fusion_pass = RocmAiterAllReduceFusionPass(vllm_config)
noop_pass = NoOpEliminationPass(vllm_config)
func_pass = FixFunctionalizationPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(
noop_pass, all_reduce_fusion_pass, func_pass, cleanup_pass
)
token_num = batch_size * seq_len
model = test_model_cls(hidden_size, token_num, dtype=dtype)
hidden_states = torch.randn((token_num, hidden_size), requires_grad=False)
compiled_model = torch.compile(model, backend=backend)
compiled_model(hidden_states)
results_unfused = model(hidden_states)
results_fused = compiled_model(hidden_states)
# AITER's kernel-level oracle permits up to 5% isolated FP8 values to
# fall outside a 5% allclose envelope after collective reordering.
for site, (reference, actual) in enumerate(
zip(results_unfused[:4], results_fused[:4])
):
close = torch.isclose(reference, actual, atol=5e-2, rtol=5e-2)
mismatch_ratio = float((~close).float().mean().item())
assert mismatch_ratio <= 0.05, (
f"FP8 site {site} mismatch ratio {mismatch_ratio:.2%} exceeds 5%"
)
# Residual and bf16 indexer outputs do not incur FP8 quantization noise.
torch.testing.assert_close(
results_unfused[4:], results_fused[4:], atol=1e-2, rtol=1e-2
)
# Four pattern firings: norm[0] (no-add quant), norm[1] (add quant,
# single ``rms`` consumer), norm[2..3] (add quant + indexer fan-out).
assert all_reduce_fusion_pass.matched_count == 4, (
f"{all_reduce_fusion_pass.matched_count=}"
)
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=True)
fused_quant_op, fused_indexer_op = model.ops_in_model_after()
assert backend.op_count(fused_quant_op) == 2
assert backend.op_count(fused_indexer_op) == 2
del all_reduce_fusion_pass
@@ -1,462 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for IR inplace functionalization pass integration.
This test suite verifies that the inplace functionalization pass, lowering pass,
and clone cleanup pass work together correctly with donated buffer tracking.
"""
from collections.abc import Callable
import pytest
import torch
import torch._dynamo.exc
from torch import nn
import vllm.kernels # noqa: F401 to register kernels
from vllm.compilation.passes.inductor_pass import InductorPass, get_pass_context
from vllm.compilation.passes.ir.clone_elimination import (
UnsafeCloneEliminationPass,
)
from vllm.compilation.passes.ir.inplace_functionalization import (
VllmIRInplaceFunctionalizationPass,
)
from vllm.compilation.passes.ir.lowering_pass import VllmIRLoweringPass
from vllm.config import VllmConfig
from vllm.ir import ops
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON, tl, triton
from ...backend import TestBackend
class StoreDonationInfoPass(InductorPass):
def __init__(self):
self.donated_input_ids_sets: list[set[int]] = []
def __call__(self, *args, **kwargs):
ctx = get_pass_context()
self.donated_input_ids_sets += [ctx.donated_input_ids]
class MaybeInplaceModel(nn.Module):
"""Model using only maybe_inplace variants."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
):
# First maybe_inplace - x & residual1 are donated
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
x, residual1, self.weight1, 1e-5
)
# Second maybe_inplace - residual2 is donated
x_normed2, residual_out2 = ops.fused_add_rms_norm.maybe_inplace(
x_normed1, residual2, self.weight2, 1e-5
)
return x_normed2, residual_out1, residual_out2
class FunctionalModel(nn.Module):
"""Model using only functional (default) variants."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
):
# First functional - no donation
x_normed1, residual_out1 = ops.fused_add_rms_norm(
x, residual1, self.weight1, 1e-5
)
# Second functional - no donation
x_normed2, residual_out2 = ops.fused_add_rms_norm(
x_normed1, residual2, self.weight2, 1e-5
)
return x_normed2, residual_out1, residual_out2
class MixedModel(nn.Module):
"""Model mixing maybe_inplace and functional variants."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
):
# First maybe_inplace - x & residual1 are donated
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
x, residual1, self.weight1, 1e-5
)
# Second functional - no donation, x_normed1 must be preserved as it's returned
x_normed2, residual_out2 = ops.fused_add_rms_norm(
x_normed1, residual2, self.weight2, 1e-5
)
# Return both to prevent x_normed1 from being optimized away
return x_normed1, x_normed2, residual_out1, residual_out2
class ModelWithTritonAfterMaybeInplace(nn.Module):
"""Model using maybe_inplace followed by a Triton kernel.
Test clone elimination can handle Triton in the graph
"""
def __init__(self, hidden_size=16):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
@triton.jit
def _triton_add_kernel(
x_ptr,
y_ptr,
n_elements,
BLOCK_SIZE: tl.constexpr,
):
pid = tl.program_id(axis=0)
block_start = pid * BLOCK_SIZE
offsets = block_start + tl.arange(0, BLOCK_SIZE)
mask = offsets < n_elements
x = tl.load(x_ptr + offsets, mask=mask)
y = x + 0.1
tl.store(y_ptr + offsets, y, mask=mask)
def triton_add(x: torch.Tensor) -> torch.Tensor:
"""Simple Triton add kernel."""
y = torch.empty_like(x)
n_elements = x.numel()
grid = (triton.cdiv(n_elements, 256),)
_triton_add_kernel[grid](x, y, n_elements, BLOCK_SIZE=256)
return y
self.triton_add = triton_add
def forward(self, x: torch.Tensor, residual: torch.Tensor, residual2: torch.Tensor):
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
x, residual, self.weight, 1e-5
)
x_processed = self.triton_add(x_normed)
# x_processed does not need to be cloned, residual2 does
x_normed2, residual_out2 = ops.fused_add_rms_norm(
x_processed, residual2, self.weight, 1e-5
)
return x_normed2, residual_out2
skipif_no_triton = pytest.mark.skipif(not HAS_TRITON, reason="Requires Triton")
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
@pytest.mark.parametrize(
"model_class,expected_functionalized,expected_donated,expected_clones",
[
# 2 inplace calls, all activations donated, all clones eliminated
(MaybeInplaceModel, 2, 3, 0),
# No inplace calls, no donations, 3 clones (one eliminated)
(FunctionalModel, 0, 0, 3),
# One inplace call, two donated activations, 2 clones
(MixedModel, 1, 2, 2),
# One inplace call, two donated, 1 clone remaining
pytest.param(ModelWithTritonAfterMaybeInplace, 1, 2, 1, marks=skipif_no_triton),
],
)
def test_inplace_functionalization(
default_vllm_config: VllmConfig,
model_class,
expected_functionalized: int,
expected_clones: int,
expected_donated: int,
):
"""Test inplace functionalization, lowering, and clone cleanup."""
torch.set_default_device(current_platform.device_type)
# Use vllm_c so inplace path is triggered
default_vllm_config.kernel_config.ir_op_priority.fused_add_rms_norm = [
"vllm_c",
"native",
]
# Create passes in order they run during compilation
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
lowering_pass = VllmIRLoweringPass(default_vllm_config)
donated_info_pass = StoreDonationInfoPass()
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
# Set up backend with pre-grad pass
backend = TestBackend(lowering_pass, donated_info_pass, cleanup_pass)
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
model = model_class()
x = torch.randn(8, 16, dtype=torch.bfloat16)
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
with default_vllm_config.kernel_config.ir_op_priority.set_priority():
# Reference output without optimization
ref_output = model(x.clone(), residual1.clone(), residual2.clone())
# Compile with inplace optimization
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
output = compiled_model(x.clone(), residual1.clone(), residual2.clone())
# Verify correctness (relaxed tolerance for bfloat16)
for i in range(len(ref_output)):
torch.testing.assert_close(output[i], ref_output[i], rtol=1e-2, atol=1e-2)
# Verify expected number of ops were functionalized
func_ops = functionalization_pass.functionalized_ops
assert len(func_ops) == int(bool(expected_functionalized))
if expected_functionalized > 0:
assert "fused_add_rms_norm" in func_ops
assert func_ops["fused_add_rms_norm"] == expected_functionalized
# Verify lowering happened (2 ops in all cases)
assert "fused_add_rms_norm" in lowering_pass.selected_impls
assert len(lowering_pass.selected_impls["fused_add_rms_norm"]) == 2
assert all(
provider == "vllm_c"
for node, provider in lowering_pass.selected_impls["fused_add_rms_norm"].items()
), lowering_pass.selected_impls
# Verify correct number of donated IDs
assert len(donated_info_pass.donated_input_ids_sets) == 1
assert len(donated_info_pass.donated_input_ids_sets[0]) == expected_donated
# Verify expected number of clones after cleanup
actual_clones = backend.op_count(torch.ops.aten.clone.default, before=False)
assert actual_clones == expected_clones, (
f"Expected {expected_clones} clones, got {actual_clones}:"
f"{backend.print_graphs()}"
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_donated_buffer_context_propagation(default_vllm_config):
"""Test that donated_input_ids propagates correctly through pass_context."""
torch.set_default_device(current_platform.device_type)
# Create a custom backend that inspects pass_context in cleanup pass
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
lowering_pass = VllmIRLoweringPass(default_vllm_config)
donation_info_pass = StoreDonationInfoPass()
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
backend = TestBackend(lowering_pass, donation_info_pass, cleanup_pass)
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
model = MaybeInplaceModel()
x = torch.randn(8, 16, dtype=torch.bfloat16)
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
compiled_model(x.clone(), residual1.clone(), residual2.clone())
donated_ids_seen = donation_info_pass.donated_input_ids_sets
# Verify donated_input_ids was set and propagated
assert len(donated_ids_seen) == 1
# Should have donated inputs (exact indices depend on AOTAutograd)
assert len(donated_ids_seen[0]) == 3
# All donated ids should be valid non-negative integers
for idx in donated_ids_seen[0]:
assert isinstance(idx, int) and idx >= 0, f"Invalid donated index: {idx}"
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_maybe_inplace_reuse_error(default_vllm_config):
"""Test that reusing a donated activation input raises ValueError."""
torch.set_default_device(current_platform.device_type)
class ReuseModel(nn.Module):
"""Model that incorrectly reuses a donated activation input."""
def __init__(self, hidden_size=16):
super().__init__()
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(self, x: torch.Tensor, residual: torch.Tensor):
# x is donated to maybe_inplace
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
x, residual, self.weight, 1e-5
)
# ERROR: x is used again after being donated
return x_normed + x # This should raise ValueError
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
lowering_pass = VllmIRLoweringPass(default_vllm_config)
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
backend = TestBackend(lowering_pass, cleanup_pass)
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
model = ReuseModel()
x = torch.randn(8, 16, dtype=torch.bfloat16)
residual = torch.randn(8, 16, dtype=torch.bfloat16)
# Compilation should raise BackendCompilerFailed wrapping ValueError
with pytest.raises(
torch._dynamo.exc.BackendCompilerFailed,
match="is used again after the node",
):
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
compiled_model(x.clone(), residual.clone())
# Piecewise compilation tests with graph splitting
@torch.library.custom_op("vllm::test_split_marker", mutates_args=())
def test_split_marker(x: torch.Tensor) -> torch.Tensor:
"""Identity op that marks a split point for piecewise compilation."""
return x.clone()
@test_split_marker.register_fake
def _fake_split_marker(x: torch.Tensor) -> torch.Tensor:
return torch.empty_like(x)
class TransformerBlockWithSplits(nn.Module):
"""Transformer block with explicit split points for piecewise compilation."""
def __init__(self, hidden_size=32, intermediate_size=128):
super().__init__()
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
# Attention-like projection
self.attn_proj = nn.Linear(
hidden_size, hidden_size, bias=False, dtype=torch.bfloat16
)
# Post-attention norm
self.post_attn_norm = nn.Parameter(
torch.ones(hidden_size, dtype=torch.bfloat16)
)
# MLP
self.gate_proj = nn.Linear(
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
)
self.up_proj = nn.Linear(
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
)
self.down_proj = nn.Linear(
intermediate_size, hidden_size, bias=False, dtype=torch.bfloat16
)
# Post-MLP norm
self.post_mlp_norm = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
def forward(self, x: torch.Tensor):
# Attention block with residual
residual1 = x
attn_out = self.attn_proj(x)
# Fused add + norm (maybe_inplace: residual1 is donated)
normed1, residual1 = ops.fused_add_rms_norm.maybe_inplace(
attn_out, residual1, self.post_attn_norm, 1e-5
)
# Force a graph split here
normed1 = torch.ops.vllm.test_split_marker(normed1)
# MLP block
gate = self.gate_proj(normed1)
up = self.up_proj(normed1)
mlp_out = self.down_proj(gate * torch.nn.functional.silu(up))
# Fused add + norm (maybe_inplace: residual1 is donated)
normed2, residual2 = ops.fused_add_rms_norm.maybe_inplace(
mlp_out, residual1, self.post_mlp_norm, 1e-5
)
return normed2, residual2
def with_dyn_arg(fn: Callable, arg_index: int, dim_index: int):
def inner(*args):
torch._dynamo.mark_dynamic(args[arg_index], dim_index)
return fn(*args)
return inner
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_piecewise_compilation_with_donated_buffers(monkeypatch, fresh_vllm_cache):
"""Test piecewise compilation with donated buffers across graph splits.
Utilizes a custom splitting op. Uses fresh cache to avoid compilation caching.
"""
torch.set_default_device(current_platform.device_type)
# Disable compilation cache to avoid serialization issues
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
from vllm.compilation.backends import VllmBackend
from vllm.config import CompilationConfig, VllmConfig
# Create config with custom splitting op
store_donation_info = StoreDonationInfoPass()
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
custom_ops=["all"],
splitting_ops=["vllm::test_split_marker"],
inductor_compile_config={"post_grad_custom_post_pass": store_donation_info},
)
)
backend = VllmBackend(vllm_config)
model = TransformerBlockWithSplits()
x = torch.randn(8, 32, dtype=torch.bfloat16)
# Reference output
ref_output = with_dyn_arg(model, 0, 0)(x.clone())
# Compile with piecewise compilation (graph will split at split_marker)
compiled_model = torch.compile(model, backend=backend, fullgraph=False)
output = with_dyn_arg(compiled_model, 0, 0)(x.clone())
# Verify correctness (relaxed tolerance for bfloat16)
torch.testing.assert_close(output[0], ref_output[0], rtol=1e-2, atol=1e-2)
torch.testing.assert_close(output[1], ref_output[1], rtol=1e-2, atol=1e-2)
# Verify the model was split into multiple submodules
assert hasattr(backend, "split_gm"), "Backend should have split graph module"
# Should have at least 2 submodules (split by test_split_marker op)
submodules = list(backend.split_gm.named_children())
num_submodules = len(submodules)
assert num_submodules >= 2, (
f"Expected at least 2 submodules (split), got {num_submodules}"
)
# Check that donation info was propagated correctly
donated_inputs_sets = store_donation_info.donated_input_ids_sets
assert len(donated_inputs_sets) == 2
assert len(donated_inputs_sets[0]) == 1
assert len(donated_inputs_sets[1]) == 1
-103
View File
@@ -1,103 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from torch import nn
import vllm.kernels # noqa: F401 to register kernels
from vllm import ir
from vllm.compilation.passes.ir.lowering_pass import (
VllmIRLoweringPass,
)
from vllm.config import get_current_vllm_config
from vllm.ir import ops
from vllm.model_executor.layers.activation import GeluAndMulSparse
from vllm.platforms import current_platform
from ...backend import TestBackend
class Model(nn.Module):
def __init__(self, hidden_size=16, *args, **kwargs):
super().__init__(*args, **kwargs)
self.hidden_size = hidden_size
self.weight = torch.ones(hidden_size, dtype=torch.bfloat16)
def forward(self, x):
x1 = x + 4.0
x2 = ops.rms_norm(x1, self.weight, 1e-5)
x3 = x2 * 5.0
# no weight
x4 = ops.rms_norm(x3, None, 1e-5)
x5 = x4 / 2.0
# dispatch to native due to variance_size parameter
x6 = ops.rms_norm(x5, self.weight, 1e-5, self.hidden_size // 2)
return x6 + 3.0
class SparseActivationModel(nn.Module):
def __init__(self):
super().__init__()
self.activation = GeluAndMulSparse(0.95, "tanh")
def forward(self, x):
return self.activation(x)
@pytest.mark.parametrize("rms_provider", ops.rms_norm.supported_providers())
def test_lowering_rms_norm(rms_provider, default_vllm_config):
torch.set_default_device(current_platform.device_type)
lowering_pass = VllmIRLoweringPass(get_current_vllm_config())
backend = TestBackend(lowering_pass)
backend_unlowered = TestBackend()
model = Model()
x = torch.randn(8, 16, dtype=torch.bfloat16)
with (
ops.rms_norm.set_priority([rms_provider, "native"]),
ir.enable_torch_wrap(True),
):
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
compiled_unlowered_model = torch.compile(
model, backend=backend_unlowered, fullgraph=True
)
output = compiled_model(x)
output_unlowered = compiled_unlowered_model(x)
selected = lowering_pass.selected_impls["rms_norm"]
assert len(selected) == 3
assert selected["rms_norm"] == rms_provider
assert selected["rms_norm_1"] == rms_provider
assert selected["rms_norm_2"] == "native"
# Compiled function guards on global value, avoid recompilation
with ir.enable_torch_wrap(True):
output2 = compiled_model(x)
torch.testing.assert_close(output_unlowered, output)
torch.testing.assert_close(output_unlowered, output2)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
def test_lowering_gelu_and_mul_sparse(default_vllm_config):
torch.set_default_device("cuda")
lowering_pass = VllmIRLoweringPass(get_current_vllm_config())
backend = TestBackend(lowering_pass)
model = SparseActivationModel()
x = torch.randn(8, 32768, dtype=torch.bfloat16)
with (
ops.gelu_and_mul_sparse.set_priority(["triton", "native"]),
ir.enable_torch_wrap(True),
):
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
output = compiled_model(x)
selected = lowering_pass.selected_impls["gelu_and_mul_sparse"]
assert selected == {"gelu_and_mul_sparse": "triton"}
expected = ops.gelu_and_mul_sparse.impls["native"].impl_fn(
x, 1.6448536269514722, "tanh"
)
torch.testing.assert_close(output, expected, rtol=1.6e-2, atol=1e-3)
@@ -1,340 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import pytest
import torch
from tests.compile.backend import TestBackend
from tests.utils import TestFP8Layer
from vllm.compilation.passes.fusion.act_quant_fusion import (
ActivationQuantFusionPass,
)
from vllm.compilation.passes.fusion.rms_quant_fusion import RMSNormQuantFusionPass
from vllm.compilation.passes.fx_utils import find_auto_fn, find_auto_fn_maybe, is_func
from vllm.compilation.passes.utility.fix_functionalization import (
FixFunctionalizationPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CompilationConfig,
ModelConfig,
PassConfig,
VllmConfig,
get_current_vllm_config,
set_current_vllm_config,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8StaticTensorSym,
)
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
TEST_FP8 = current_platform.supports_fp8()
FP8_DTYPE = current_platform.fp8_dtype()
class TestSiluMul(torch.nn.Module):
quant_key = kFp8StaticTensorSym
def __init__(self, hidden_size: int = 128):
super().__init__()
self.silu_and_mul = SiluAndMul()
if TEST_FP8:
self.fp8_linear = TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
input_dtype=get_current_vllm_config().model_config.dtype,
)
def forward(self, x):
y = self.silu_and_mul(x)
if TEST_FP8:
return self.fp8_linear(y)
else:
return y
def example_inputs(self, num_tokens=32, hidden_size=128):
return (torch.rand(num_tokens, hidden_size * 2),)
def ops_in_model(self, do_fusion):
if TEST_FP8 and do_fusion:
return [torch.ops._C.silu_and_mul_quant.default]
else:
return [torch.ops._C.silu_and_mul.default]
def ops_not_in_model(self):
return []
class TestFusedAddRMSNorm(torch.nn.Module):
quant_key = kFp8StaticTensorSym
def __init__(self, hidden_size=16, intermediate_size=32):
super().__init__()
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.gate_proj = torch.nn.Parameter(
torch.empty((intermediate_size, hidden_size))
)
self.norm = RMSNorm(intermediate_size, 1e-05)
self.norm.weight = torch.nn.Parameter(torch.ones(intermediate_size))
torch.nn.init.normal_(self.gate_proj, std=0.02)
if TEST_FP8:
self.fp8_linear = TestFP8Layer(
weight_shape=(hidden_size, intermediate_size),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
input_dtype=get_current_vllm_config().model_config.dtype,
)
def forward(self, hidden_states, residual):
# Reshape input
view = hidden_states.reshape(-1, self.hidden_size)
# matrix multiplication
permute = self.gate_proj.permute(1, 0)
mm = torch.mm(view, permute)
# layer normalization
norm_output, residual_output = self.norm(mm, residual)
if TEST_FP8:
# scaled_mm with static input quantization
fp8_linear_result = self.fp8_linear(norm_output)
return fp8_linear_result, residual_output
else:
return norm_output, residual_output
def example_inputs(self, batch_size=8, seq_len=16):
hidden_states = torch.randn((batch_size * seq_len, self.hidden_size))
residual = torch.randn((batch_size * seq_len, self.intermediate_size))
return (hidden_states, residual)
def ops_in_model(self, do_fusion):
if TEST_FP8 and do_fusion:
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
else:
return []
def ops_not_in_model(self):
return []
class TestRotaryEmbedding(torch.nn.Module):
def __init__(self, head_dim=64, max_position=2048, base=10000):
super().__init__()
self.head_dim = head_dim
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position,
rope_parameters={"rope_type": "default", "rope_theta": base},
)
def forward(self, positions, q, k):
q_rotated, k_rotated = self.rotary_emb(positions, q, k)
return q_rotated, k_rotated
def example_inputs(self, num_tokens=32, head_dim=64):
positions = torch.arange(num_tokens, dtype=torch.long)
q = torch.randn(num_tokens, head_dim)
k = torch.randn(num_tokens, head_dim)
return (positions, q, k)
def ops_in_model(self, do_fusion):
return [torch.ops._C.rotary_embedding.default]
def ops_not_in_model(self):
return []
class TestRotaryEmbeddingSliceScatter(torch.nn.Module):
def __init__(self, head_dim=64, num_heads=4, max_position=2048, base=10000):
super().__init__()
self.head_dim = head_dim
self.num_heads = num_heads
self.hidden_size = head_dim * num_heads
self.qkv_proj = torch.nn.Linear(
self.hidden_size, self.hidden_size * 3, bias=False
)
self.rotary_emb = get_rope(
self.head_dim,
max_position=max_position,
rope_parameters={"rope_type": "default", "rope_theta": base},
)
def forward(self, positions, hidden_states):
# Simulate the pattern: mm -> split_with_sizes -> rotary_embedding
# -> slice_scatter -> split_with_sizes
qkv = self.qkv_proj(hidden_states)
split_sizes = [self.hidden_size, self.hidden_size, self.hidden_size]
q, k, v = torch.split(qkv, split_sizes, dim=-1)
q_rotated, k_rotated = self.rotary_emb(positions, q, k)
qkv_updated = torch.cat([q_rotated, k_rotated, v], dim=-1)
return qkv_updated
def example_inputs(self, num_tokens=32, head_dim=64, num_heads=4):
hidden_size = head_dim * num_heads
positions = torch.arange(num_tokens, dtype=torch.long)
hidden_states = torch.randn(num_tokens, hidden_size)
return (positions, hidden_states)
def ops_in_model(self, do_fusion):
return [torch.ops._C.rotary_embedding.default]
def ops_not_in_model(self):
return [torch.ops.aten.slice_scatter.default]
class TestFunctionWithMutatedArgsAndReturn(torch.nn.Module):
OP_REGISTERED = False
def __init__(self):
super().__init__()
self.register_test_custom_op()
@classmethod
def register_test_custom_op(cls):
if not cls.OP_REGISTERED:
def function_with_mutated_args_and_return_impl(
x: torch.Tensor,
) -> torch.Tensor:
ret = x + 1
x.add_(2)
return ret
def function_with_mutated_args_and_return_fake(
x: torch.Tensor,
) -> torch.Tensor:
return torch.empty_like(x)
direct_register_custom_op(
op_name="function_with_mutated_args_and_return",
op_func=function_with_mutated_args_and_return_impl,
mutates_args=["x"],
fake_impl=function_with_mutated_args_and_return_fake,
)
cls.OP_REGISTERED = True
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
# Clone x to avoid mutating the original tensor
ret = torch.ops.vllm.function_with_mutated_args_and_return(x)
return x, ret
def example_inputs(self, num_tokens=32):
hidden_states = torch.randn(num_tokens)
return (hidden_states,)
def ops_in_model(self, do_fusion):
return [torch.ops.vllm.function_with_mutated_args_and_return.default]
def ops_not_in_model(self):
return []
MODELS_AND_DO_FUSION = {
TestSiluMul: [True, False],
TestFusedAddRMSNorm: [True, False],
TestRotaryEmbedding: [False],
TestRotaryEmbeddingSliceScatter: [False],
TestFunctionWithMutatedArgsAndReturn: [False],
}
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize(
"model_class, do_fusion",
[
(model_class, do_fusion)
for model_class, fusions in MODELS_AND_DO_FUSION.items()
for do_fusion in fusions
],
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on cuda and rocm platform",
)
def test_fix_functionalization(
model_class: torch.nn.Module, do_fusion: bool, dtype: torch.dtype
):
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(0)
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
custom_ops=["all"],
pass_config=PassConfig(
fuse_norm_quant=do_fusion,
fuse_act_quant=do_fusion,
eliminate_noops=True,
),
),
)
with set_current_vllm_config(vllm_config):
assert RMSNorm.enabled()
noop_pass = NoOpEliminationPass(vllm_config)
fusion_pass = RMSNormQuantFusionPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
act_quant_fusion_pass = ActivationQuantFusionPass(vllm_config)
passes = (
[noop_pass, fusion_pass, act_quant_fusion_pass, cleanup_pass]
if do_fusion
else [noop_pass, cleanup_pass]
)
func_pass = FixFunctionalizationPass(vllm_config)
backend_func = TestBackend(*passes, func_pass)
backend_no_func = TestBackend(*passes)
model = model_class()
inputs_func = model.example_inputs()
inputs_no_func = copy.deepcopy(inputs_func)
model_func = copy.deepcopy(model)
model_no_func = copy.deepcopy(model)
model_func = torch.compile(model_func, backend=backend_func)
model_no_func = torch.compile(model_no_func, backend=backend_no_func)
# deepcopy inputs to prevent potential in place mutation
outputs_func = model_func(*copy.deepcopy(inputs_func))
outputs_no_func = model_no_func(*copy.deepcopy(inputs_no_func))
torch.testing.assert_close(outputs_func, outputs_no_func)
# check if the functionalization pass is applied
for op in model.ops_in_model(do_fusion):
find_auto_fn(backend_no_func.graph_post_pass.nodes, op)
assert find_auto_fn_maybe(backend_func.graph_post_pass.nodes, op) is None
# make sure the ops were all de-functionalized
found = dict()
for node in backend_func.graph_post_pass.nodes:
for op in model.ops_in_model(do_fusion):
if is_func(node, op):
found[op] = True
for op in model.ops_not_in_model():
if is_func(node, op):
found[op] = True
assert all(found[op] for op in model.ops_in_model(do_fusion))
assert all(not found.get(op) for op in model.ops_not_in_model())
@@ -1,418 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit test for the MLADualRMSNormFusionPass.
The pass fuses paired q/kv RMS norms in MLA attention into a single
fused_mla_dual_rms_norm op backed by AITER's fused_qk_rmsnorm kernel.
"""
import pytest
import torch
import vllm.config
from tests.compile.backend import TestBackend
from vllm._aiter_ops import (
is_aiter_found_and_supported,
rocm_aiter_ops,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.platforms import current_platform
# MLA attention geometry for DeepSeek-V3 / Kimi-K2
Q_DIM = 1536
KV_C_DIM = 512
K_PE_DIM = 64
EPS = 1e-6
FP8_DTYPE = current_platform.fp8_dtype()
class MLADualRMSNormTestModel(torch.nn.Module):
"""Minimal model reproducing the MLA dual RMS norm pattern:
linear -> split([q_dim, kv_dim])
+-- q_c (getitem 0) -> rms_norm(q_w, eps) -> linear
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
+-- kv_c (getitem 0) -> rms_norm(kv_w, eps)
+-- k_pe
"""
def __init__(
self,
hidden_size: int,
q_dim: int = Q_DIM,
kv_c_dim: int = KV_C_DIM,
k_pe_dim: int = K_PE_DIM,
eps: float = EPS,
):
super().__init__()
self.q_dim = q_dim
self.kv_dim = kv_c_dim + k_pe_dim
self.kv_c_dim = kv_c_dim
self.k_pe_dim = k_pe_dim
self.proj = torch.nn.Linear(hidden_size, q_dim + self.kv_dim, bias=False)
self.q_norm = RMSNorm(q_dim, eps=eps)
self.kv_norm = RMSNorm(kv_c_dim, eps=eps)
self.q_b_proj = torch.nn.Linear(q_dim, hidden_size, bias=False)
def forward(
self, x: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
# Avoid graph input being a direct arg to a matched pattern node
x = torch.relu(x)
projected = self.proj(x)
q_c, kv_lora = projected.split([self.q_dim, self.kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([self.kv_c_dim, self.k_pe_dim], dim=-1)
q_normed = self.q_norm(q_c)
kv_normed = self.kv_norm(kv_c)
q_out = self.q_b_proj(q_normed)
return q_out, kv_normed, k_pe
def ops_in_model_before(self):
return [torch.ops.vllm_ir.rms_norm.default]
def ops_in_model_after(self):
return [torch.ops.vllm.fused_mla_dual_rms_norm.default]
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [7168])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def test_fuse_mla_dual_rms_norm(
dtype: torch.dtype,
hidden_size: int,
monkeypatch: pytest.MonkeyPatch,
):
torch._dynamo.reset()
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm"],
pass_config=PassConfig(
fuse_mla_dual_rms_norm=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
MLADualRMSNormFusionPass,
)
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(42)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
fusion_pass = MLADualRMSNormFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
model = MLADualRMSNormTestModel(hidden_size)
x = torch.randn(1, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
outputs_unfused = model(x)
model_fused = torch.compile(model, backend=backend)
outputs_fused = model_fused(x)
torch.testing.assert_close(outputs_unfused, outputs_fused, atol=1e-2, rtol=1e-2)
assert fusion_pass.matched_count == 1, (
f"Expected 1 fused pair, got {fusion_pass.matched_count}"
)
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
class MLADualRMSNormFp8PerTokenTestModel(torch.nn.Module):
"""Minimal model reproducing the FP8 MLA attention path with *per-token* quant:
linear -> split([q_dim, kv_dim])
+-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fused_dynamic_quant -> dequant
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
+-- kv_c (getitem 0) -> rms_norm (bf16)
+-- k_pe
"""
def __init__(
self,
hidden_size: int,
q_dim: int = Q_DIM,
kv_c_dim: int = KV_C_DIM,
k_pe_dim: int = K_PE_DIM,
eps: float = EPS,
):
super().__init__()
self.q_dim = q_dim
self.kv_dim = kv_c_dim + k_pe_dim
self.kv_c_dim = kv_c_dim
self.k_pe_dim = k_pe_dim
self.eps = eps
self.proj = torch.nn.Linear(hidden_size, q_dim + self.kv_dim, bias=False)
self.q_weight = torch.nn.Parameter(torch.ones(q_dim))
self.kv_norm = RMSNorm(kv_c_dim, eps=eps)
def _dequant(self, x_fp8: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
# Per-token: a single (M, 1) scale broadcast across the row.
return (x_fp8.to(torch.float32) * scale).to(torch.bfloat16)
def forward(self, x: torch.Tensor):
# Avoid graph input being a direct arg to a matched pattern node
x = torch.relu(x)
projected = self.proj(x)
q_c, kv_lora = projected.split([self.q_dim, self.kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([self.kv_c_dim, self.k_pe_dim], dim=-1)
q_fp8, q_scale = torch.ops.vllm.rocm_aiter_rmsnorm_fused_dynamic_quant(
q_c, self.q_weight, self.eps, FP8_DTYPE
)
kv_normed = self.kv_norm(kv_c)
return self._dequant(q_fp8, q_scale), kv_normed, k_pe
def ops_in_model_before(self):
return [
torch.ops.vllm.rocm_aiter_rmsnorm_fused_dynamic_quant.default,
torch.ops.vllm_ir.rms_norm.default,
]
def ops_in_model_after(self):
return [torch.ops.vllm.fused_mla_dual_rms_norm_per_token_quant.default]
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [7168])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def test_fuse_mla_dual_rms_norm_fp8_per_token(
dtype: torch.dtype,
hidden_size: int,
monkeypatch: pytest.MonkeyPatch,
):
torch._dynamo.reset()
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm"],
pass_config=PassConfig(
fuse_mla_dual_rms_norm=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
MLADualRMSNormFusionPass,
)
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(42)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
fusion_pass = MLADualRMSNormFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
model = MLADualRMSNormFp8PerTokenTestModel(hidden_size)
x = torch.randn(4, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
with torch.inference_mode():
outputs_unfused = model(x)
model_fused = torch.compile(model, backend=backend)
outputs_fused = model_fused(x)
q_deq_u, kv_normed_u, k_pe_u = outputs_unfused
q_deq_f, kv_normed_f, k_pe_f = outputs_fused
torch.testing.assert_close(k_pe_u, k_pe_f, atol=0, rtol=0)
torch.testing.assert_close(kv_normed_u, kv_normed_f, atol=1e-2, rtol=1e-2)
E4M3_STEP = 0.125
exact_frac = (q_deq_u == q_deq_f).float().mean().item()
assert exact_frac > 0.99, (
f"q: only {exact_frac:.4f} of elements bit-exact; scales likely differ"
)
torch.testing.assert_close(q_deq_u, q_deq_f, atol=1e-2, rtol=E4M3_STEP)
assert fusion_pass.matched_count == 1, (
f"Expected 1 fused pair, got {fusion_pass.matched_count}"
)
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
class MLADualRMSNormFp8GroupTestModel(torch.nn.Module):
"""Minimal model reproducing the FP8 MLA attention path with *group* quant
(block-scaled fp8 weights, 128-wide activation groups):
linear -> split([q_dim, kv_dim])
+-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fp8_group_quant -> dequant
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
+-- kv_c (getitem 0) -> rms_norm (bf16)
+-- k_pe
"""
GROUP_SIZE = 128
def __init__(
self,
hidden_size: int,
q_dim: int = Q_DIM,
kv_c_dim: int = KV_C_DIM,
k_pe_dim: int = K_PE_DIM,
eps: float = EPS,
):
super().__init__()
self.q_dim = q_dim
self.kv_dim = kv_c_dim + k_pe_dim
self.kv_c_dim = kv_c_dim
self.k_pe_dim = k_pe_dim
self.eps = eps
self.proj = torch.nn.Linear(hidden_size, q_dim + self.kv_dim, bias=False)
self.q_weight = torch.nn.Parameter(torch.ones(q_dim))
self.kv_norm = RMSNorm(kv_c_dim, eps=eps)
def _dequant(self, x_fp8: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
# Group: one (M, N / 128) scale per 128-column group of the row.
m = x_fp8.shape[0]
x = x_fp8.to(torch.float32).view(m, -1, self.GROUP_SIZE)
return (x * scale.view(m, -1, 1)).view(m, -1).to(torch.bfloat16)
def forward(self, x: torch.Tensor):
# Avoid graph input being a direct arg to a matched pattern node
x = torch.relu(x)
projected = self.proj(x)
q_c, kv_lora = projected.split([self.q_dim, self.kv_dim], dim=-1)
kv_c, k_pe = kv_lora.split([self.kv_c_dim, self.k_pe_dim], dim=-1)
q_fp8, q_scale = torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant(
q_c, self.q_weight, self.eps, self.GROUP_SIZE
)
kv_normed = self.kv_norm(kv_c)
# Also return the group scales so the test can check them directly:
# equal scales pin any fp8 payload difference to rounding alone.
return self._dequant(q_fp8, q_scale), q_scale, kv_normed, k_pe
def ops_in_model_before(self):
return [
torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant.default,
torch.ops.vllm_ir.rms_norm.default,
]
def ops_in_model_after(self):
return [torch.ops.vllm.fused_mla_dual_rms_norm_group_quant.default]
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [7168])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def test_fuse_mla_dual_rms_norm_fp8_group(
dtype: torch.dtype,
hidden_size: int,
monkeypatch: pytest.MonkeyPatch,
):
torch._dynamo.reset()
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm"],
pass_config=PassConfig(
fuse_mla_dual_rms_norm=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
MLADualRMSNormFusionPass,
)
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(42)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
fusion_pass = MLADualRMSNormFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
model = MLADualRMSNormFp8GroupTestModel(hidden_size)
x = torch.randn(64, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
with torch.inference_mode():
outputs_unfused = model(x)
model_fused = torch.compile(model, backend=backend)
outputs_fused = model_fused(x)
q_deq_u, q_scale_u, kv_normed_u, k_pe_u = outputs_unfused
q_deq_f, q_scale_f, kv_normed_f, k_pe_f = outputs_fused
torch.testing.assert_close(k_pe_u, k_pe_f, atol=0, rtol=0)
torch.testing.assert_close(kv_normed_u, kv_normed_f, atol=1e-2, rtol=1e-2)
torch.testing.assert_close(q_scale_u, q_scale_f, atol=0, rtol=1e-5)
E4M3_STEP = 0.125
exact_frac = (q_deq_u == q_deq_f).float().mean().item()
assert exact_frac > 0.99, f"q: only {exact_frac:.4f} of elements bit-exact"
torch.testing.assert_close(q_deq_u, q_deq_f, atol=1e-2, rtol=E4M3_STEP)
assert fusion_pass.matched_count == 1, (
f"Expected 1 fused pair, got {fusion_pass.matched_count}"
)
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
-813
View File
@@ -1,813 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.config
import vllm.ir.ops
import vllm.plugins
from tests.compile.backend import TestBackend
from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor
from tests.utils import TestFP8Layer
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
from vllm.compilation.passes.fusion.matcher_utils import QUANT_OPS
from vllm.compilation.passes.fusion.rms_quant_fusion import (
FUSED_OPS,
FusedRMSQuantKey,
RMSNormQuantFusionPass,
)
from vllm.compilation.passes.fx_utils import find_auto_fn_maybe
from vllm.compilation.passes.utility.fix_functionalization import (
FixFunctionalizationPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
from vllm.model_executor.kernels.linear import (
AiterFp8BlockScaledMMKernel,
ChannelWiseTorchFP8ScaledMMLinearKernel,
CutlassFp8BlockScaledMMKernel,
CutlassFP8ScaledMMLinearKernel,
DeepGemmFp8BlockScaledMMKernel,
FlashInferFp8DeepGEMMDynamicBlockScaledKernel,
FlashInferFP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ROCmFP8ScaledMMLinearKernel,
RowWiseTorchFP8ScaledMMLinearKernel,
TritonFp8BlockScaledMMKernel,
_KernelT,
)
from vllm.model_executor.layers.layernorm import RMSNorm, RMSNormGated
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
create_fp8_quant_key,
)
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
cutlass_block_fp8_supported,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
is_deep_gemm_e8m0_used,
is_deep_gemm_supported,
)
# Kernel and group_shape combinations: (kernel, group_shape)
# CUDA kernels
CUDA_KERNEL_GROUPSHAPE_COMBINATIONS = [
# FlashInferFP8ScaledMMLinearKernel supports both per-tensor only
(FlashInferFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR),
# CutlassFP8ScaledMMLinearKernel supports both per-tensor and per-token
(CutlassFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN),
(CutlassFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR),
# PerTensorTorchFP8ScaledMMLinearKernel only supports per-tensor
(PerTensorTorchFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR),
# ChannelWiseTorchFP8ScaledMMLinearKernel only supports per-token
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN),
# Blockwise group shapes
(FlashInferFp8DeepGEMMDynamicBlockScaledKernel, GroupShape(1, 128)),
(CutlassFp8BlockScaledMMKernel, GroupShape(1, 128)),
(DeepGemmFp8BlockScaledMMKernel, GroupShape(1, 128)),
(TritonFp8BlockScaledMMKernel, GroupShape(1, 128)),
(TritonFp8BlockScaledMMKernel, GroupShape(1, 64)),
]
# ROCm kernels
ROCM_KERNEL_GROUPSHAPE_COMBINATIONS = [
# ROCmFP8ScaledMMLinearKernel supports per-tensor only
(ROCmFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR),
# RowWiseTorchFP8ScaledMMLinearKernel only supports per-token
(RowWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN),
# ChannelWiseTorchFP8ScaledMMLinearKernel only supports per-token
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN),
# Blockwise group shapes (no kernel abstraction)
(TritonFp8BlockScaledMMKernel, GroupShape(1, 128)),
(TritonFp8BlockScaledMMKernel, GroupShape(1, 64)),
]
KERNEL_GROUPSHAPE_COMBINATIONS = (
CUDA_KERNEL_GROUPSHAPE_COMBINATIONS
if current_platform.is_cuda()
else ROCM_KERNEL_GROUPSHAPE_COMBINATIONS
)
# For Aiter tests we toggle use_aiter_quant_op
AITER_KERNEL_GROUPSHAPE_COMBINATIONS = [
# Per-token with RowWiseTorchFP8ScaledMMLinearKernel
(RowWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, True),
(RowWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, False),
# Per-token with ChannelWiseTorchFP8ScaledMMLinearKernel
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, True),
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, False),
# Static per-tensor: only the aiter quant op is matched by the pass
(ROCmFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR, True),
# Blockwise
(AiterFp8BlockScaledMMKernel, GroupShape(1, 128), True),
]
class TestModel(torch.nn.Module):
def __init__(
self,
hidden_size: int,
eps: float,
force_kernel: type[_KernelT] | None,
group_shape: GroupShape,
dtype: torch.dtype,
use_aiter_fusion: bool = False,
use_aiter_quant: bool = False,
*args,
**kwargs,
):
super().__init__(*args, **kwargs)
self.fp8_linear_layers: list[torch.nn.Module]
self.group_shape = group_shape
self.use_aiter_quant_op = use_aiter_quant
self.use_aiter_fusion = use_aiter_fusion
self.norm = [RMSNorm(hidden_size, eps) for _ in range(4)]
self.enable_rms_norm_custom_op = self.norm[0].enabled()
# Determine if blockwise based on group_shape
is_blockwise = group_shape.is_per_group()
if is_blockwise:
block_size = group_shape.col
self.activation_quant_key = create_fp8_quant_key(
static=False, group_shape=group_shape
)
self.weight_quant_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(block_size, block_size)
)
else:
is_static = group_shape == GroupShape.PER_TENSOR
self.activation_quant_key = create_fp8_quant_key(
is_static, group_shape=group_shape
)
self.weight_quant_key = create_fp8_quant_key(
static=True, group_shape=group_shape
)
self.fp8_linear_layers = [
TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
force_kernel=force_kernel,
transpose_weights=use_aiter_fusion,
input_dtype=dtype,
)
for _ in range(3)
]
# Enable aiter quantization if requested
for layer in self.fp8_linear_layers:
layer.kernel.quant_fp8.use_aiter = use_aiter_quant
self.enable_quant_fp8_custom_op = self.fp8_linear_layers[
0
].is_quant_fp8_enabled()
def forward(self, x):
# avoid having graph input be an arg to a pattern directly
x = resid = torch.relu(x)
y = self.norm[0](x)
x2 = self.fp8_linear_layers[0](y)
# make sure resid is used for replacement to work
y2, resid = self.norm[1](x2, resid)
x3 = self.fp8_linear_layers[1](y2)
y3, resid = self.norm[2](x3, resid) # use resid here
x4 = self.fp8_linear_layers[2](y3)
y4, resid = self.norm[3](x4, resid) # use resid here
return y4
def ops_in_model_before(self):
if self.group_shape.is_per_group():
# Blockwise path
if self.use_aiter_fusion and self.use_aiter_quant_op:
return [rocm_aiter_ops.get_group_quant_op()]
elif self.group_shape.is_per_tensor():
if self.use_aiter_quant_op:
return [rocm_aiter_ops.get_per_tensor_quant_op()]
else:
if self.use_aiter_quant_op:
return [rocm_aiter_ops.get_per_token_quant_op()]
# Common path
return (
[QUANT_OPS[self.activation_quant_key]]
if self.enable_quant_fp8_custom_op
else [torch.ops.aten.reciprocal]
)
def ops_in_model_after(self):
if self.use_aiter_fusion:
if self.group_shape.is_per_group():
# Blockwise aiter fusion
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
AiterFusedAddRMSFp8GroupQuantPattern,
AiterRMSFp8GroupQuantPattern,
)
return [
AiterFusedAddRMSFp8GroupQuantPattern.FUSED_OP,
AiterRMSFp8GroupQuantPattern.FUSED_OP,
]
elif self.group_shape.is_per_tensor():
# Static per-tensor aiter fusion
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
AiterFusedAddRMSNormStaticQuantPattern,
AiterRMSNormStaticQuantPattern,
)
return [
AiterFusedAddRMSNormStaticQuantPattern.FUSED_OP,
AiterRMSNormStaticQuantPattern.FUSED_OP,
]
else:
# Per-token aiter fusion
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
AiterFusedAddRMSNormDynamicQuantPattern,
AiterRMSNormDynamicQuantPattern,
)
return [
AiterFusedAddRMSNormDynamicQuantPattern.FUSED_OP,
AiterRMSNormDynamicQuantPattern.FUSED_OP,
]
# Regular fusion
return [
FUSED_OPS[FusedRMSQuantKey(self.activation_quant_key, True)],
FUSED_OPS[FusedRMSQuantKey(self.activation_quant_key, False)],
]
def ops_in_model_before_partial(self):
return [
torch.ops.vllm_ir.rms_norm,
torch.ops.vllm_ir.fused_add_rms_norm.default,
]
class AddRMSNormNvfp4Model(torch.nn.Module):
def __init__(self, hidden_size: int, eps: float):
super().__init__()
self.norm = RMSNorm(hidden_size, eps)
self.activation_scale = torch.rand((1, 1), dtype=torch.float32)
weight = torch.rand((hidden_size, hidden_size))
self.weight, self.weight_scale, weight_global_scale = quant_nvfp4_tensor(weight)
self.alpha = (1 / (weight_global_scale * self.activation_scale)).reshape(1)
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
x = residual = torch.relu(x)
x, residual = self.norm(x, residual)
x, block_scale = scaled_fp4_quant(x, self.activation_scale)
x = cutlass_scaled_fp4_mm(
x,
self.weight,
block_scale,
self.weight_scale,
self.alpha,
out_dtype=torch.get_default_dtype(),
)
return x, residual
def _run_fusion_test(
model,
fusion_pass,
vllm_config,
dtype,
hidden_size,
num_tokens,
):
"""Helper function for common fusion test logic.
Must be called within vllm_config context.
"""
noop_pass = NoOpEliminationPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(noop_pass, fusion_pass, cleanup_pass)
backend2 = TestBackend(noop_pass, cleanup_pass)
x = torch.rand(num_tokens, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
model_fused = torch.compile(model, backend=backend)
result_fused = model_fused(x)
model_unfused = torch.compile(model, backend=backend2)
result_unfused = model_unfused(x)
if dtype == torch.float16:
ATOL, RTOL = (2e-3, 2e-3)
else:
ATOL, RTOL = (1e-2, 1e-2)
torch.testing.assert_close(result_fused, result_unfused, atol=ATOL, rtol=RTOL)
assert fusion_pass.matched_count == 3
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
return backend, backend2
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [256])
@pytest.mark.parametrize("num_tokens", [257])
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
@pytest.mark.parametrize("kernel_groupshape", KERNEL_GROUPSHAPE_COMBINATIONS)
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
@pytest.mark.parametrize("enable_quant_fp8_custom_op", [True, False])
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Only test on CUDA and ROCm"
)
def test_fusion_rmsnorm_quant(
dtype,
hidden_size,
num_tokens,
eps,
kernel_groupshape,
enable_rms_norm_custom_op,
enable_quant_fp8_custom_op,
):
force_kernel, group_shape = kernel_groupshape
if not enable_quant_fp8_custom_op and group_shape.is_per_group():
pytest.skip("Unsupported unwrapped quant fp8 op for blockwise quantization")
if group_shape == GroupShape(1, 64) and (
cutlass_block_fp8_supported() or is_deep_gemm_supported()
):
pytest.skip("Unsupported group shape 64 for CUTLASS/DeepGemm")
# TODO(quant-rms-fusion): DeepGEMM UE8M0 activation quant on B200 lowers
# to a packed int32-scale op (per_token_group_quant_fp8_packed_for_deepgemm),
# but the rms+quant fusion pattern only matches the fp32-scale variant, so
# the fused output gets a mismatched scale layout and produces NaN. Only
# reproduces on bf16 (DeepGEMM UE8M0 on B200 is bf16-only).
# To re-enable: make rms_norm_per_block_quant emit packed UE8M0 scales
# and extend the fusion pattern to rewrite the packed activation quant.
deepgemm_kernels = (
DeepGemmFp8BlockScaledMMKernel,
FlashInferFp8DeepGEMMDynamicBlockScaledKernel,
)
if (
dtype == torch.bfloat16
and force_kernel in deepgemm_kernels
and is_deep_gemm_e8m0_used()
):
pytest.skip(
"rms+quant fusion does not yet match the packed UE8M0 DeepGEMM path"
)
custom_ops = []
if enable_rms_norm_custom_op:
custom_ops.append("+rms_norm")
if enable_quant_fp8_custom_op:
custom_ops.append("+quant_fp8")
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops,
pass_config=PassConfig(
fuse_norm_quant=True, fuse_act_quant=True, eliminate_noops=True
),
),
)
with (
vllm.config.set_current_vllm_config(vllm_config),
vllm_config.kernel_config.ir_op_priority.set_priority(),
):
# Setup device before model creation
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(1)
fusion_pass = RMSNormQuantFusionPass(vllm_config)
model = TestModel(
hidden_size=hidden_size,
eps=eps,
force_kernel=force_kernel,
group_shape=group_shape,
dtype=dtype,
use_aiter_fusion=False,
use_aiter_quant=False,
)
if any(
type(layer.kernel) is not force_kernel for layer in model.fp8_linear_layers
):
pytest.skip(f"{force_kernel.__name__} is not supported on this platform")
backend, _ = _run_fusion_test(
model, fusion_pass, vllm_config, dtype, hidden_size, num_tokens
)
backend.check_before_ops(
model.ops_in_model_before_partial(), fully_replaced=False
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
@pytest.mark.parametrize(
("num_tokens", "hidden_size"),
[(1, 256), (32, 1024), (257, 4096)],
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
def test_fusion_add_rmsnorm_nvfp4_quant(
dtype: torch.dtype, eps: float, num_tokens: int, hidden_size: int
):
fused_op = getattr(
torch.ops.vllm,
"flashinfer_fused_add_rms_norm_nvfp4_quant",
None,
)
if not current_platform.has_device_capability(100) or fused_op is None:
pytest.skip("FlashInfer add-RMSNorm NVFP4 fusion is not available")
assert fused_op is not None
fused_op_default = fused_op.default
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(
fuse_norm_quant=True,
eliminate_noops=True,
),
),
)
with (
vllm.config.set_current_vllm_config(vllm_config),
vllm_config.kernel_config.ir_op_priority.set_priority(),
):
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(1)
fusion_pass = RMSNormQuantFusionPass(vllm_config)
fused_backend = TestBackend(
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
FixFunctionalizationPass(vllm_config),
)
unfused_backend = TestBackend(
NoOpEliminationPass(vllm_config),
PostCleanupPass(vllm_config),
FixFunctionalizationPass(vllm_config),
)
model = AddRMSNormNvfp4Model(hidden_size=hidden_size, eps=eps)
x = torch.rand(num_tokens, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
result_fused = torch.compile(model, backend=fused_backend)(x)
result_unfused = torch.compile(model, backend=unfused_backend)(x)
torch.testing.assert_close(result_fused, result_unfused, atol=2e-1, rtol=2e-1)
assert fusion_pass.matched_count == 1
fused_backend.check_before_ops([torch.ops._C.scaled_fp4_quant.out])
fused_backend.check_after_ops([fused_op_default])
assert (
find_auto_fn_maybe(
fused_backend.graph_post_pass.nodes,
fused_op_default,
)
is None
)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("hidden_size", [256])
@pytest.mark.parametrize("num_tokens", [257])
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
@pytest.mark.parametrize(
"kernel_groupshape_quant", AITER_KERNEL_GROUPSHAPE_COMBINATIONS
)
@pytest.mark.skipif(
(not current_platform.is_rocm() or not IS_AITER_FOUND),
reason="Only test on ROCm with aiter package installed",
)
def test_aiter_fusion_rmsnorm_quant(
dtype: torch.dtype,
hidden_size: int,
num_tokens: int,
eps: float,
kernel_groupshape_quant: tuple,
monkeypatch: pytest.MonkeyPatch,
):
force_kernel, group_shape, use_aiter_quant_op = kernel_groupshape_quant
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm", "+quant_fp8"],
pass_config=PassConfig(fuse_norm_quant=True, eliminate_noops=True),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
RocmAiterRMSNormQuantFusionPass,
)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(1)
fusion_pass = RocmAiterRMSNormQuantFusionPass(vllm_config)
model = TestModel(
hidden_size=hidden_size,
eps=eps,
force_kernel=force_kernel,
group_shape=group_shape,
dtype=dtype,
use_aiter_fusion=True, # Always use aiter fusion ops in aiter test
use_aiter_quant=use_aiter_quant_op, # Toggle aiter quantization
)
_run_fusion_test(
model, fusion_pass, vllm_config, dtype, hidden_size, num_tokens
)
class TestGatedModel(torch.nn.Module):
"""Model that uses RMSNormGated + reshape + group FP8 quant + linear.
Mimics GatedDeltaNetAttention's output projection path where:
- RMSNormGated operates on per-head tensors, either (N, H, D) as
QwenGatedDeltaNetAttention passes them or flattened to (N*H, D) as
OlmoHybridGatedDeltaNetAttention does
- Output is reshaped to (N, H*D) before group quantization + linear
"""
def __init__(
self,
num_heads: int,
head_dim: int,
eps: float,
force_kernel: type[_KernelT],
group_shape: GroupShape,
dtype: torch.dtype,
use_aiter_quant: bool = True,
flatten_heads: bool = True,
):
super().__init__()
self.num_heads = num_heads
self.head_dim = head_dim
self.flatten_heads = flatten_heads
hidden_dim = num_heads * head_dim
self.norm = RMSNormGated(
head_dim,
eps=eps,
group_size=None,
norm_before_gate=True,
)
self.activation_quant_key = create_fp8_quant_key(
static=False, group_shape=group_shape
)
self.weight_quant_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(group_shape.col, group_shape.col)
)
self.fp8_linear = TestFP8Layer(
weight_shape=(hidden_dim, hidden_dim),
activation_quant_key=self.activation_quant_key,
weight_quant_key=self.weight_quant_key,
force_kernel=force_kernel,
transpose_weights=True,
input_dtype=dtype,
)
self.fp8_linear.kernel.quant_fp8.use_aiter = use_aiter_quant
def forward(self, x, z):
num_heads = self.num_heads
head_dim = self.head_dim
hidden_dim = num_heads * head_dim
x = torch.relu(x)
z = torch.relu(z)
x_heads = x.reshape(-1, num_heads, head_dim)
z_heads = z.reshape(-1, num_heads, head_dim)
if self.flatten_heads:
x_heads = x_heads.reshape(-1, head_dim)
z_heads = z_heads.reshape(-1, head_dim)
normed = self.norm(x_heads, z_heads)
merged = normed.reshape(-1, hidden_dim)
out = self.fp8_linear(merged)
return out
def ops_in_model_after(self):
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
AiterRMSNormGatedFp8GroupQuantPattern,
)
return [AiterRMSNormGatedFp8GroupQuantPattern.FUSED_OP]
class _MockGDNLayer:
"""Minimal mock to populate static_forward_context for pass discovery.
Uses __class__ assignment to pass isinstance checks against
GatedDeltaNetAttention without requiring a full config-based init.
"""
def __init__(self, num_v_heads: int, head_v_dim: int, tp_size: int = 1):
self.num_v_heads = num_v_heads
self.head_v_dim = head_v_dim
self.tp_size = tp_size
from vllm.model_executor.layers.mamba.gdn.base import (
GatedDeltaNetAttention,
)
self.__class__ = GatedDeltaNetAttention
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("num_heads", [2])
@pytest.mark.parametrize("head_dim", [128])
@pytest.mark.parametrize("num_tokens", [8])
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
@pytest.mark.parametrize("flatten_heads", [True, False])
@pytest.mark.parametrize("quant_fp8_op", ["-quant_fp8", "+quant_fp8"])
@pytest.mark.skipif(
(not current_platform.is_rocm() or not IS_AITER_FOUND),
reason="Only test on ROCm with aiter package installed",
)
def test_aiter_fusion_rmsnorm_gated_quant(
dtype: torch.dtype,
num_heads: int,
head_dim: int,
num_tokens: int,
eps: float,
flatten_heads: bool,
quant_fp8_op: str,
monkeypatch: pytest.MonkeyPatch,
):
group_shape = GroupShape(1, 128)
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["-rms_norm", "-silu_and_mul", quant_fp8_op],
pass_config=PassConfig(fuse_norm_quant=True, eliminate_noops=True),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
RocmAiterRMSNormQuantFusionPass,
)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
# Register a mock GDN layer so the pass discovers num_heads/head_dim
mock_gdn = _MockGDNLayer(num_v_heads=num_heads, head_v_dim=head_dim, tp_size=1)
vllm_config.compilation_config.static_forward_context["mock_gdn_layer"] = (
mock_gdn
)
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(1)
fusion_pass = RocmAiterRMSNormQuantFusionPass(vllm_config)
model = TestGatedModel(
num_heads=num_heads,
head_dim=head_dim,
eps=eps,
force_kernel=AiterFp8BlockScaledMMKernel,
group_shape=group_shape,
dtype=dtype,
use_aiter_quant=True,
flatten_heads=flatten_heads,
)
noop_pass = NoOpEliminationPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(noop_pass, fusion_pass, cleanup_pass)
backend2 = TestBackend(noop_pass, cleanup_pass)
hidden_dim = num_heads * head_dim
x = torch.rand(num_tokens, hidden_dim)
z = torch.rand(num_tokens, hidden_dim)
torch._dynamo.mark_dynamic(x, 0)
torch._dynamo.mark_dynamic(z, 0)
model_fused = torch.compile(model, backend=backend)
result_fused = model_fused(x, z)
model_unfused = torch.compile(model, backend=backend2)
result_unfused = model_unfused(x, z)
# The opaque AITER quant op preserves a BF16 intermediate in the
# unfused path, while the fused kernel normalizes and quantizes
# directly. Allow for that extra rounding before the FP8 GEMM.
tol = 2e-2 if quant_fp8_op == "+quant_fp8" else 1e-2
torch.testing.assert_close(result_fused, result_unfused, atol=tol, rtol=tol)
assert fusion_pass.matched_count == 1
backend.check_after_ops(model.ops_in_model_after())
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("num_heads", [2])
@pytest.mark.parametrize("head_dim", [128])
@pytest.mark.parametrize("num_tokens", [8])
@pytest.mark.parametrize("eps", [1e-6])
@pytest.mark.skipif(
(not current_platform.is_rocm() or not IS_AITER_FOUND),
reason="Only test on ROCm with aiter package installed",
)
def test_aiter_fusion_rmsnorm_gated_quant_no_gdn_layers(
dtype: torch.dtype,
num_heads: int,
head_dim: int,
num_tokens: int,
eps: float,
monkeypatch: pytest.MonkeyPatch,
):
"""Verify that without GDN layers in static_forward_context,
the gated pattern is not registered and no matches occur."""
group_shape = GroupShape(1, 128)
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["-rms_norm", "-silu_and_mul", "-quant_fp8"],
pass_config=PassConfig(fuse_norm_quant=True, eliminate_noops=True),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
RocmAiterRMSNormQuantFusionPass,
)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(1)
# No mock GDN layer registered -- pass should not register gated pattern
fusion_pass = RocmAiterRMSNormQuantFusionPass(vllm_config)
model = TestGatedModel(
num_heads=num_heads,
head_dim=head_dim,
eps=eps,
force_kernel=AiterFp8BlockScaledMMKernel,
group_shape=group_shape,
dtype=dtype,
use_aiter_quant=True,
)
noop_pass = NoOpEliminationPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(noop_pass, fusion_pass, cleanup_pass)
hidden_dim = num_heads * head_dim
x = torch.rand(num_tokens, hidden_dim)
z = torch.rand(num_tokens, hidden_dim)
torch._dynamo.mark_dynamic(x, 0)
torch._dynamo.mark_dynamic(z, 0)
model_fused = torch.compile(model, backend=backend)
model_fused(x, z)
assert fusion_pass.matched_count == 0
-582
View File
@@ -1,582 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import pytest
import torch._dynamo
from tests.compile.backend import LazyInitPass, TestBackend
from tests.utils import TestFP8Layer, flat_product
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
dense_kv_cache_views,
)
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
from vllm.compilation.passes.fusion.attn_quant_fusion import (
ATTN_OP,
AttnQuantFusionPass,
)
from vllm.compilation.passes.fusion.matcher_utils import QUANT_OPS
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
AttentionConfig,
CacheConfig,
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
SchedulerConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.attention.backends.utils import (
get_supported_kv_cache_layouts,
resolve_kv_cache_layout,
)
from vllm.v1.kv_cache_interface import (
AttentionSpec,
get_kv_quant_mode,
)
DEVICE_TYPE = current_platform.device_type
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
class AttentionQuantPatternModel(torch.nn.Module):
"""Base model for AttentionQuantPattern fusion."""
def __init__(
self,
num_qo_heads: int,
num_kv_heads: int,
head_size: int,
device: torch.device,
vllm_config: VllmConfig,
block_size: int,
**kwargs,
):
super().__init__()
self.num_qo_heads = num_qo_heads
self.num_kv_heads = num_kv_heads
self.head_size = head_size
self.device = device
self.vllm_config = vllm_config
self.dtype = vllm_config.model_config.dtype
self.attn = Attention(
num_heads=self.num_qo_heads,
head_size=self.head_size,
scale=1.0 / (self.head_size**0.5),
num_kv_heads=self.num_kv_heads,
cache_config=vllm_config.cache_config,
prefix="model.layers.0.self_attn.attn",
)
self.attn._k_scale = self.attn._k_scale.to(device)
self.attn._v_scale = self.attn._v_scale.to(device)
self.block_size = block_size
# Initialize attn MetadataBuilder (match Attention.get_kv_cache_spec)
self.builder = self.attn.attn_backend.get_builder_cls()(
kv_cache_spec=AttentionSpec(
block_size=self.block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
dtype=self.attn.kv_cache_torch_dtype,
kv_quant_mode=get_kv_quant_mode(self.attn.kv_cache_dtype),
),
layer_names=[self.attn.layer_name],
vllm_config=self.vllm_config,
device=self.device,
)
def build_attn_metadata(self, batch_size: int) -> AttentionMetadata:
"""Initialize attention metadata."""
# TODO (Rohan138) reuse utils from vllm/v1/worker/gpu/attn_utils.py
# Create common attn metadata
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
common_attn_metadata = create_common_attn_metadata(
batch_spec, self.block_size, self.device, arange_block_indices=True
)
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
num_blocks = batch_size * max_blocks
spec = self.attn.attn_backend.customize_spec(
AttentionSpec(
block_size=self.block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
dtype=self.attn.kv_cache_torch_dtype,
kv_quant_mode=get_kv_quant_mode(self.attn.kv_cache_dtype),
)
)
supported = get_supported_kv_cache_layouts([self.attn.attn_backend])
layout = resolve_kv_cache_layout(
self.vllm_config, [[m.name for m in supported]]
)
raw_tensor = torch.zeros(
num_blocks * spec.page_size_bytes,
dtype=torch.int8,
device=self.device,
)
kv_cache = dense_kv_cache_views(
raw_tensor,
spec,
num_blocks,
num_layers=1,
layout=layout,
)[0]
self.attn.kv_cache = kv_cache
# Build attn metadata
self.attn_metadata = self.builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
return self.attn_metadata
class TestAttentionFp8StaticQuantPatternModel(AttentionQuantPatternModel):
"""Test model for AttentionFp8StaticQuantPattern fusion."""
quant_key = kFp8StaticTensorSym
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
hidden_size = self.num_qo_heads * self.head_size
self.fp8_linear = TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
device=self.device,
input_dtype=self.dtype,
)
w = kwargs.get("w")
if w is not None:
self.fp8_linear.weight = w["weight"]
self.fp8_linear.weight_scale = w["wscale"]
self.fp8_linear.input_scale = w["scale"]
self.w = {
"weight": self.fp8_linear.weight,
"wscale": self.fp8_linear.weight_scale,
"scale": self.fp8_linear.input_scale,
}
def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor):
"""Forward pass that creates the pattern to be fused."""
attn_output = self.attn(q, k, v)
return self.fp8_linear(attn_output)
class TestAttentionAiterFp8StaticQuantPatternModel(
TestAttentionFp8StaticQuantPatternModel
):
aiter_quant = True
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fp8_linear.kernel.quant_fp8.use_aiter = True
assert self.fp8_linear.input_scale is not None
self.fp8_linear.input_scale.fill_(0.125)
class TestAttentionNvfp4QuantPatternModel(AttentionQuantPatternModel):
"""Test model for AttentionNvfp4QuantPattern fusion."""
quant_key = kNvfp4Dynamic
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
hidden_size = self.num_qo_heads * self.head_size
self.w = kwargs.get(
"w",
{
"weight": torch.randint(
256,
(hidden_size, hidden_size // 2),
dtype=FP4_DTYPE,
device=self.device,
),
"wscale_swizzled": torch.randn(hidden_size, hidden_size // 16).to(
dtype=FP8_DTYPE, device=self.device
),
"wscale": torch.tensor([500], dtype=torch.float32, device=self.device),
"scale": torch.tensor([0.002], dtype=torch.float32, device=self.device),
},
)
def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor):
"""Forward pass that creates the pattern to be fused."""
attn_output = self.attn(q, k, v)
quant_output, output_block_scale = scaled_fp4_quant(
attn_output, 1 / self.w["scale"]
)
return cutlass_scaled_fp4_mm(
a=quant_output,
b=self.w["weight"],
block_scale_a=output_block_scale,
block_scale_b=self.w["wscale_swizzled"],
alpha=self.w["scale"] * self.w["wscale"],
out_dtype=attn_output.dtype,
)
PATTERN_TEST_MODELS_FP8: list[tuple[str, type]] = []
PATTERN_TEST_MODELS_FP4: list[tuple[str, type]] = []
HEADS: list[tuple[int, int]] = []
BACKENDS_FP8: list[AttentionBackendEnum] = []
BACKENDS_FP4: list[AttentionBackendEnum] = []
if current_platform.is_cuda():
HEADS = [(64, 8), (40, 8)]
PATTERN_TEST_MODELS_FP8 = [
(
"RedHatAI/Meta-Llama-3.1-8B-FP8",
TestAttentionFp8StaticQuantPatternModel,
)
]
PATTERN_TEST_MODELS_FP4 = [
(
"nvidia/Llama-3.1-8B-Instruct-NVFP4",
TestAttentionNvfp4QuantPatternModel,
)
]
BACKENDS_FP8 = [AttentionBackendEnum.TRITON_ATTN, AttentionBackendEnum.FLASHINFER]
BACKENDS_FP4 = [AttentionBackendEnum.FLASHINFER]
elif current_platform.is_rocm():
HEADS = [(32, 8), (40, 8)]
PATTERN_TEST_MODELS_FP8 = [
("amd/Llama-3.1-8B-Instruct-FP8-KV", TestAttentionFp8StaticQuantPatternModel)
]
BACKENDS_FP8 = [
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
AttentionBackendEnum.ROCM_ATTN,
AttentionBackendEnum.TRITON_ATTN,
]
@pytest.mark.parametrize("num_qo_heads, num_kv_heads", HEADS)
@pytest.mark.parametrize("head_size", [128])
@pytest.mark.parametrize(
"batch_size", [7, 256, 533] if current_platform.is_cuda() else [8]
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "nvfp4"])
@pytest.mark.parametrize(
"backend, model_name, model_class, custom_ops",
# Test attention+quant_fp8 fusion with custom and torch impls of QuantFP8
list(
flat_product(
BACKENDS_FP8, PATTERN_TEST_MODELS_FP8, ["+quant_fp8", "-quant_fp8"]
)
)
# quant_fp4 only has the custom impl
+ list(flat_product(BACKENDS_FP4, PATTERN_TEST_MODELS_FP4, [""])),
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Only test ROCm or CUDA"
)
@pytest.mark.skipif(not current_platform.supports_fp8(), reason="Need FP8")
def test_attention_quant_pattern(
num_qo_heads: int,
num_kv_heads: int,
head_size: int,
batch_size: int,
dtype: torch.dtype,
kv_cache_dtype: CacheDType,
custom_ops: str,
model_name: str,
model_class: type[AttentionQuantPatternModel],
backend: AttentionBackendEnum,
dist_init,
disable_vllm_compile_cache,
):
"""Test AttentionStaticQuantPattern fusion pass."""
if backend == AttentionBackendEnum.FLASHINFER and (
not current_platform.is_device_capability((10, 0)) or not has_flashinfer()
):
# This also captures the FP4 case
pytest.skip("FlashInfer attn fusion requires Blackwell and flashinfer")
if kv_cache_dtype == "nvfp4" and backend != AttentionBackendEnum.FLASHINFER:
pytest.skip("NVFP4 KV cache is only supported by FlashInfer")
# trtllm-gen has no NVFP4-KV kernel with NVFP4 output, so with an NVFP4 KV
# cache the FP4 output fusion must be rejected: the standalone
# scaled_fp4_quant stays in the graph and the result is unchanged. FP8 output
# fusion with NVFP4 KV is supported.
expect_fusion = not (
kv_cache_dtype == "nvfp4" and model_class.quant_key is kNvfp4Dynamic
)
custom_ops_list = custom_ops.split(",") if custom_ops else []
device = torch.device(f"{DEVICE_TYPE}:0")
torch.set_default_dtype(dtype)
torch.manual_seed(42)
backend_cls = backend.get_class()
block_size = backend_cls.get_preferred_block_size(16)
model_config = ModelConfig(
model=model_name,
max_model_len=2048,
dtype=dtype,
)
vllm_config = VllmConfig(
model_config=model_config,
scheduler_config=SchedulerConfig(
max_num_seqs=1024,
max_model_len=model_config.max_model_len,
is_encoder_decoder=model_config.is_encoder_decoder,
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops_list,
),
cache_config=CacheConfig(cache_dtype=kv_cache_dtype),
attention_config=AttentionConfig(backend=backend),
)
# Create test inputs
q = torch.randn(batch_size, num_qo_heads * head_size, dtype=dtype, device=device)
k = torch.randn(batch_size, num_kv_heads * head_size, dtype=dtype, device=device)
v = torch.randn(batch_size, num_kv_heads * head_size, dtype=dtype, device=device)
# Mark first dimension as dynamic for realistic testing
torch._dynamo.mark_dynamic(q, 0)
torch._dynamo.mark_dynamic(k, 0)
torch._dynamo.mark_dynamic(v, 0)
# Run model directly without compilation and fusion
vllm_config_unfused = copy.deepcopy(vllm_config)
with (
set_current_vllm_config(vllm_config_unfused),
set_forward_context(attn_metadata=None, vllm_config=vllm_config_unfused),
):
model_unfused = model_class(
num_qo_heads=num_qo_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
device=device,
vllm_config=vllm_config_unfused,
block_size=block_size,
)
model_unfused = model_unfused.to(device)
result_unfused_0 = model_unfused(q, k, v) # noqa: F841 HACK: See #131044
forward_ctx = get_forward_context()
forward_ctx.attn_metadata = model_unfused.build_attn_metadata(batch_size)
# Run model directly without fusion
# Still compile so query QuantFP8 has closer numerics
compiled_unfused = torch.compile(model_unfused, fullgraph=True)
result_unfused = compiled_unfused(q, k, v)
# Run model with attn fusion enabled
vllm_config.compilation_config.pass_config = PassConfig(
fuse_attn_quant=True, eliminate_noops=True
)
with (
set_current_vllm_config(vllm_config),
set_forward_context(attn_metadata=None, vllm_config=vllm_config),
):
model_fused = model_class(
num_qo_heads=num_qo_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
device=device,
vllm_config=vllm_config,
w=model_unfused.w,
block_size=block_size,
)
model_fused = model_fused.to(device)
forward_ctx = get_forward_context()
forward_ctx.attn_metadata = model_fused.build_attn_metadata(batch_size)
# Create test backend with fusion passes enabled
noop_pass = NoOpEliminationPass(vllm_config)
attn_pass = LazyInitPass(AttnQuantFusionPass, vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
test_backend = TestBackend(noop_pass, attn_pass, cleanup_pass)
# HACK: See https://github.com/vllm-project/vllm/issues/31044
result_fused_0 = model_fused(q, k, v) # noqa: F841
# Compile model with fusion enabled
compiled_fused = torch.compile(
model_fused, backend=test_backend, fullgraph=True
)
assert compiled_fused.attn._o_scale_float is None
result_fused = compiled_fused(q, k, v)
if backend == AttentionBackendEnum.FLASHINFER and expect_fusion:
# With the Flashinfer backend after the 1st round of the forward
# pass, output quant scale should be loaded into the attn layer's
# _o_scale_float, the 2nd round should reuse the loaded
# _o_scale_float
assert compiled_fused.attn._o_scale_float is not None
result_fused_2 = compiled_fused(q, k, v)
assert compiled_fused.attn._o_scale_float is not None
torch.testing.assert_close(
result_unfused, result_fused_2, atol=1e-2, rtol=1e-2
)
elif not expect_fusion:
assert compiled_fused.attn._o_scale_float is None
# Check attn fusion support
quant_key: QuantKey = model_class.quant_key
attn_fusion_supported = [
layer.impl.fused_output_quant_supported(quant_key)
for key, layer in vllm_config.compilation_config.static_forward_context.items()
]
if expect_fusion:
assert sum(attn_fusion_supported) == len(attn_fusion_supported), (
"All layers should support attention fusion"
)
else:
assert sum(attn_fusion_supported) == 0, (
"No layer should support attention fusion"
)
# Check quantization ops in the graph before and after fusion
if "-quant_fp8" in custom_ops_list:
quant_op = torch.ops.aten.reciprocal
elif getattr(model_class, "aiter_quant", False):
quant_op = torch.ops.vllm.rocm_aiter_per_tensor_quant.default
else:
quant_op = QUANT_OPS[quant_key]
if expect_fusion:
# Note: for fp8, fully_replaced=False because query quant ops remain in
# graph. Only output quant ops are fused into attention.
test_backend.check_before_ops(
[quant_op], fully_replaced=quant_key is kNvfp4Dynamic
)
else:
# Fusion rejected: the standalone output quant must survive untouched.
num_pre = test_backend.op_count(quant_op, before=True)
assert num_pre > 0, f"Op {quant_op} not found in pre-pass graph"
assert test_backend.op_count(quant_op) == num_pre, (
"Output quant op should remain when fusion is rejected"
)
# access the underlying `AttnQuantFusionPass` on the `LazyInitPass`
assert attn_pass.pass_.matched_count == sum(attn_fusion_supported)
# Check attention ops in the graph before and after fusion
attn_nodes_pre = list(find_op_nodes(ATTN_OP, test_backend.graph_pre_pass))
attn_nodes_post = list(find_op_nodes(ATTN_OP, test_backend.graph_post_pass))
assert len(attn_nodes_pre) > 0, "Should have attention nodes before fusion"
assert len(attn_nodes_pre) == len(attn_nodes_post), (
"Should have same number of attention nodes before and after fusion"
)
assert attn_nodes_pre[0].kwargs.get("output_scale") is None, (
"Attention should not have output_scale before fusion"
)
if expect_fusion:
assert attn_nodes_post[0].kwargs.get("output_scale") is not None, (
"Attention should have output_scale after fusion"
)
else:
assert attn_nodes_post[0].kwargs.get("output_scale") is None, (
"Attention should not have output_scale when fusion is rejected"
)
assert attn_nodes_pre[0].kwargs.get("output_block_scale") is None, (
"Attention should not have output_block_scale before fusion"
)
kv_cache_dummy_dep_pre_is_none = (
attn_nodes_pre[0].kwargs.get("kv_cache_dummy_dep") is None
)
kv_cache_dummy_dep_post_is_none = (
attn_nodes_post[0].kwargs.get("kv_cache_dummy_dep") is None
)
assert not (kv_cache_dummy_dep_pre_is_none ^ kv_cache_dummy_dep_post_is_none), (
"The kv_cache_dummy_dep should be consistent before and after fusion"
)
if quant_key.dtype == FP8_DTYPE or not expect_fusion:
assert attn_nodes_post[0].kwargs.get("output_block_scale") is None, (
"Attention should not have output_block_scale unless FP4 fusion happened"
)
elif quant_key.dtype == FP4_DTYPE:
assert attn_nodes_post[0].kwargs.get("output_block_scale") is not None, (
"Attention should have output_block_scale after FP4 fusion"
)
# Check that results are close
torch.testing.assert_close(result_unfused, result_fused, atol=1e-2, rtol=1e-2)
if getattr(model_class, "aiter_quant", False):
# Query quantization remains; exactly the output quant op is removed.
assert test_backend.op_count(quant_op, before=True) == (
test_backend.op_count(quant_op) + 1
)
for model in (model_unfused, model_fused):
torch.testing.assert_close(
model.fp8_linear.input_scale,
torch.full_like(model.fp8_linear.input_scale, 0.125),
atol=0,
rtol=0,
)
@pytest.mark.skipif(
not current_platform.is_rocm() or not is_aiter_found_and_supported(),
reason="Requires ROCm AITER attention and quantization",
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_rocm_aiter_static_quant_attention(
dtype, dist_init, disable_vllm_compile_cache, monkeypatch
):
"""Fuse the actual AITER op without changing its static scale or outputs."""
try:
with monkeypatch.context() as patch:
patch.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
test_attention_quant_pattern(
num_qo_heads=32,
num_kv_heads=8,
head_size=128,
batch_size=8,
dtype=dtype,
kv_cache_dtype="fp8",
custom_ops="+quant_fp8",
model_name="amd/Llama-3.1-8B-Instruct-FP8-KV",
model_class=TestAttentionAiterFp8StaticQuantPatternModel,
backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
dist_init=dist_init,
disable_vllm_compile_cache=disable_vllm_compile_cache,
)
finally:
rocm_aiter_ops.refresh_env_variables()
@@ -1,590 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
import pytest
import torch._dynamo
from tests.compile.backend import LazyInitPass, TestBackend
from tests.utils import TestFP8Layer, flat_product
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
from vllm.compilation.passes.fusion.matcher_utils import QUANT_OPS
from vllm.compilation.passes.fusion.mla_attn_quant_fusion import (
MLA_ATTN_OP,
MLAAttnQuantFusionPass,
)
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
AttentionConfig,
CacheConfig,
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
SchedulerConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
CutlassFp8BlockScaledMMKernel,
)
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
from vllm.model_executor.layers.quantization.modelopt import ModelOptNvFp4Config
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
create_fp8_quant_key,
kFp8Dynamic128Sym,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import MLAAttentionSpec
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
DEVICE_TYPE = current_platform.device_type
class MLAAttentionQuantPatternModel(torch.nn.Module):
"""Base model for MLA AttentionQuantPattern fusion."""
def __init__(
self,
num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
kv_lora_rank: int,
kv_cache_dtype: torch.dtype,
device: torch.device,
vllm_config: VllmConfig,
**kwargs,
):
super().__init__()
self.num_heads = num_heads
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_rope_head_dim = qk_rope_head_dim
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
self.v_head_dim = v_head_dim
self.kv_lora_rank = kv_lora_rank
self.output_dim = num_heads * v_head_dim
self.head_size = kv_lora_rank + qk_rope_head_dim
self.kv_cache_dtype = kv_cache_dtype
self.device = device
self.vllm_config = vllm_config
self.dtype = vllm_config.model_config.dtype
kv_b_proj = ColumnParallelLinear(
input_size=kv_lora_rank,
output_size=num_heads * (qk_nope_head_dim + v_head_dim),
bias=False,
prefix="model.layers.0.self_attn.kv_b_proj",
).to(device)
kv_b_proj_weight = kwargs.get("kv_b_proj_weight")
if kv_b_proj_weight is not None:
kv_b_proj.weight.data.copy_(kv_b_proj_weight)
else:
kv_b_proj.weight.data.normal_()
# Create MLAAttention
self.mla_attn = MLAAttention(
num_heads=num_heads,
scale=1.0 / (self.qk_head_dim**0.5),
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
q_lora_rank=None,
kv_lora_rank=kv_lora_rank,
kv_b_proj=kv_b_proj,
cache_config=vllm_config.cache_config,
quant_config=self.quant_config,
prefix="model.layers.0.self_attn.attn",
)
self.mla_attn._k_scale = self.mla_attn._k_scale.to(device)
self.mla_attn._v_scale = self.mla_attn._v_scale.to(device)
# Initialize W_UK_T and W_UV from kv_b_proj weights
self.mla_attn.process_weights_after_loading(torch.get_default_dtype())
self.kv_b_proj_weight = kv_b_proj.weight.data.clone()
self.block_size = 16
# Initialize MLA MetadataBuilder
self.builder = self.mla_attn.attn_backend.get_builder_cls()(
kv_cache_spec=MLAAttentionSpec(
block_size=self.block_size,
num_kv_heads=1,
head_size=self.head_size,
dtype=self.kv_cache_dtype,
),
layer_names=[self.mla_attn.layer_name],
vllm_config=self.vllm_config,
device=self.device,
)
def build_attn_metadata(self, batch_size: int) -> AttentionMetadata:
"""Initialize MLA attention metadata.
NOTE: Uses decode-only batch (query_len=1 per request). The prefill
(forward_mha) path is not separately tested here because it requires
FlashAttention availability and different input tensor shapes. The
quant logic in forward_impl is identical for both paths — it quantizes
the full output[:num_actual_toks] buffer after both forward_mha and
forward_mqa have written their results.
"""
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
common_attn_metadata = create_common_attn_metadata(
batch_spec, self.block_size, self.device, arange_block_indices=True
)
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
num_blocks = batch_size * max_blocks
# MLA KV cache is 4D: (num_blocks, num_heads=1, block_size, head_size)
kv_cache = torch.zeros(
(num_blocks, 1, self.block_size, self.head_size),
dtype=self.kv_cache_dtype,
device=self.device,
)
self.mla_attn.bind_kv_cache(kv_cache)
self.attn_metadata = self.builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
return self.attn_metadata
class TestMLAAttentionFp8StaticQuantPatternModel(MLAAttentionQuantPatternModel):
"""Test model for MLA Attention + FP8 static quant fusion."""
quant_key = kFp8StaticTensorSym
quant_config = Fp8Config()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fp8_linear = TestFP8Layer(
weight_shape=(self.output_dim, self.output_dim),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
device=self.device,
input_dtype=self.dtype,
)
w = kwargs.get("w")
if w is not None:
self.fp8_linear.weight = w["weight"]
self.fp8_linear.weight_scale = w["wscale"]
self.fp8_linear.input_scale = w["scale"]
self.w = {
"weight": self.fp8_linear.weight,
"wscale": self.fp8_linear.weight_scale,
"scale": self.fp8_linear.input_scale,
}
def forward(
self,
q: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
):
"""Forward pass that creates the MLA attention + FP8 quant pattern."""
attn_output = self.mla_attn(
q,
kv_c_normed,
k_pe,
output_shape=(q.shape[0], self.output_dim),
)
return self.fp8_linear(attn_output)
class TestMLAAttentionNvfp4QuantPatternModel(MLAAttentionQuantPatternModel):
"""Test model for MLA Attention + NVFP4 quant fusion."""
quant_key = kNvfp4Dynamic
quant_config = ModelOptNvFp4Config(
is_checkpoint_nvfp4_serialized=False,
kv_cache_quant_algo=None,
exclude_modules=[],
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.w = kwargs.get(
"w",
{
"weight": torch.randint(
256,
(self.output_dim, self.output_dim // 2),
dtype=FP4_DTYPE,
device=self.device,
),
"wscale_swizzled": torch.randn(
self.output_dim, self.output_dim // 16
).to(dtype=FP8_DTYPE, device=self.device),
"wscale": torch.tensor([500], dtype=torch.float32, device=self.device),
"scale": torch.tensor([0.002], dtype=torch.float32, device=self.device),
},
)
def forward(
self,
q: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
):
"""Forward pass that creates the MLA attention + NVFP4 quant pattern."""
attn_output = self.mla_attn(
q,
kv_c_normed,
k_pe,
output_shape=(q.shape[0], self.output_dim),
)
quant_output, output_block_scale = scaled_fp4_quant(
attn_output, 1 / self.w["scale"]
)
return cutlass_scaled_fp4_mm(
a=quant_output,
b=self.w["weight"],
block_scale_a=output_block_scale,
block_scale_b=self.w["wscale_swizzled"],
alpha=self.w["scale"] * self.w["wscale"],
out_dtype=attn_output.dtype,
)
class TestMLAAttentionFp8GroupQuantPatternModel(MLAAttentionQuantPatternModel):
"""Test model for MLA Attention + per-group FP8 (block quant) fusion."""
quant_key = kFp8Dynamic128Sym
quant_config = Fp8Config(
weight_block_size=[128, 128],
)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
weight_quant_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(128, 128)
)
device = kwargs.get("device", torch.device("cuda:0"))
# Subclass to set weight_block_size before process_weights_after_loading
class _BlockFP8Layer(TestFP8Layer):
def __init__(self, *a, **kw):
self.weight_block_size = [128, 128]
super().__init__(*a, **kw)
# Force CutlassFp8BlockScaledMMKernel to ensure the graph uses
# per_token_group_fp8_quant (not the deepgemm packed variant).
self.block_fp8_linear = _BlockFP8Layer(
weight_shape=(self.output_dim, self.output_dim),
activation_quant_key=self.quant_key,
weight_quant_key=weight_quant_key,
input_dtype=self.dtype,
device=device,
force_kernel=CutlassFp8BlockScaledMMKernel,
)
w = kwargs.get("w")
if w is not None:
self.block_fp8_linear.weight = w["weight"]
# Block-wise uses weight_scale_inv, not weight_scale
self.block_fp8_linear.weight_scale_inv = w["wscale"]
self.w = {
"weight": self.block_fp8_linear.weight,
"wscale": self.block_fp8_linear.weight_scale_inv,
}
def forward(
self,
q: torch.Tensor,
kv_c_normed: torch.Tensor,
k_pe: torch.Tensor,
):
"""Forward pass: MLA attention -> block FP8 linear (group quant)."""
attn_output = self.mla_attn(
q,
kv_c_normed,
k_pe,
output_shape=(q.shape[0], self.output_dim),
)
return self.block_fp8_linear(attn_output)
def is_nvfp4_supported():
return current_platform.has_device_capability(100)
# MLA test configuration
MLA_DIMS: list[tuple[int, int, int, int, int]] = []
PATTERN_TEST_MODELS_MLA_FP8: list[tuple[str, type]] = []
PATTERN_TEST_MODELS_MLA_GROUP_FP8: list[tuple[str, type]] = []
PATTERN_TEST_MODELS_MLA_FP4: list[tuple[str, type]] = []
BACKENDS_MLA_FP8: list[AttentionBackendEnum] = []
BACKENDS_MLA_FP4: list[AttentionBackendEnum] = []
if current_platform.is_cuda():
# (num_heads, qk_nope_head_dim, qk_rope_head_dim, v_head_dim, kv_lora_rank)
MLA_DIMS = [(16, 128, 64, 128, 512)]
PATTERN_TEST_MODELS_MLA_FP8 = [
(
"deepseek-ai/DeepSeek-V2-Lite",
TestMLAAttentionFp8StaticQuantPatternModel,
)
]
PATTERN_TEST_MODELS_MLA_GROUP_FP8 = [
(
"deepseek-ai/DeepSeek-V3",
TestMLAAttentionFp8GroupQuantPatternModel,
)
]
PATTERN_TEST_MODELS_MLA_FP4 = [
(
"deepseek-ai/DeepSeek-V2-Lite",
TestMLAAttentionNvfp4QuantPatternModel,
)
]
BACKENDS_MLA_FP8 = [AttentionBackendEnum.TRITON_MLA]
BACKENDS_MLA_FP4 = [AttentionBackendEnum.TRITON_MLA]
elif current_platform.is_rocm():
# ROCm supports the static-FP8 output-quant fusion through AITER MLA.
# Per-group FP8 still relies on the CUDA-only CUTLASS block-scaled
# kernel, and NVFP4 is NVIDIA-specific.
MLA_DIMS = [(16, 128, 64, 128, 512)]
PATTERN_TEST_MODELS_MLA_FP8 = [
(
"deepseek-ai/DeepSeek-V2-Lite",
TestMLAAttentionFp8StaticQuantPatternModel,
)
]
BACKENDS_MLA_FP8 = [AttentionBackendEnum.ROCM_AITER_MLA]
@pytest.mark.parametrize(
"num_heads, qk_nope_head_dim, qk_rope_head_dim, v_head_dim, kv_lora_rank",
MLA_DIMS,
)
@pytest.mark.parametrize(
"batch_size", [7, 256] if current_platform.is_cuda_alike() else [8]
)
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize(
"backend, model_name, model_class, custom_ops",
list(
flat_product(
BACKENDS_MLA_FP8,
PATTERN_TEST_MODELS_MLA_FP8,
["+quant_fp8", "-quant_fp8"],
)
)
+ list(
flat_product(
BACKENDS_MLA_FP8,
PATTERN_TEST_MODELS_MLA_GROUP_FP8,
["+quant_fp8"],
)
)
+ list(flat_product(BACKENDS_MLA_FP4, PATTERN_TEST_MODELS_MLA_FP4, [""])),
)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Only test ROCm or CUDA"
)
@pytest.mark.skipif(not current_platform.supports_fp8(), reason="Need FP8")
def test_mla_attention_quant_pattern(
num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
kv_lora_rank: int,
batch_size: int,
dtype: torch.dtype,
custom_ops: str,
model_name: str,
model_class: type[MLAAttentionQuantPatternModel],
backend: AttentionBackendEnum,
dist_init,
disable_vllm_compile_cache,
workspace_init,
):
"""Test MLA AttentionQuantPattern fusion pass."""
if (
model_class is TestMLAAttentionNvfp4QuantPatternModel
and not is_nvfp4_supported()
):
pytest.skip("NVFP4 is not supported on this GPU (requires SM 100+).")
custom_ops_list = custom_ops.split(",") if custom_ops else []
device = torch.device(f"{DEVICE_TYPE}:0")
torch.set_default_dtype(dtype)
torch.manual_seed(42)
model_config = ModelConfig(
model=model_name,
max_model_len=2048,
dtype=dtype,
)
vllm_config = VllmConfig(
model_config=model_config,
scheduler_config=SchedulerConfig(
max_num_seqs=1024,
max_model_len=model_config.max_model_len,
is_encoder_decoder=model_config.is_encoder_decoder,
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops_list,
),
cache_config=CacheConfig(cache_dtype="auto"),
attention_config=AttentionConfig(backend=backend),
)
# MLA inputs: q(B, N, qk_head_dim), kv_c_normed(B, L), k_pe(B, 1, R)
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
q = torch.randn(batch_size, num_heads, qk_head_dim, dtype=dtype, device=device)
kv_c_normed = torch.randn(batch_size, kv_lora_rank, dtype=dtype, device=device)
k_pe = torch.randn(batch_size, 1, qk_rope_head_dim, dtype=dtype, device=device)
# Mark first dimension as dynamic
torch._dynamo.mark_dynamic(q, 0)
torch._dynamo.mark_dynamic(kv_c_normed, 0)
torch._dynamo.mark_dynamic(k_pe, 0)
# Run model without fusion
vllm_config_unfused = copy.deepcopy(vllm_config)
with (
set_current_vllm_config(vllm_config_unfused),
set_forward_context(attn_metadata=None, vllm_config=vllm_config_unfused),
):
model_unfused = model_class(
num_heads=num_heads,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
kv_lora_rank=kv_lora_rank,
kv_cache_dtype=dtype,
device=device,
vllm_config=vllm_config_unfused,
)
if (
model_class is TestMLAAttentionFp8GroupQuantPatternModel
and type(model_unfused.block_fp8_linear.kernel)
is not CutlassFp8BlockScaledMMKernel
):
pytest.skip("CUTLASS FP8 block kernel is not supported on this platform")
model_unfused = model_unfused.to(device)
# HACK: See #131044
result_unfused_0 = model_unfused(q, kv_c_normed, k_pe) # noqa: F841
forward_ctx = get_forward_context()
forward_ctx.attn_metadata = model_unfused.build_attn_metadata(batch_size)
compiled_unfused = torch.compile(model_unfused, fullgraph=True)
result_unfused = compiled_unfused(q, kv_c_normed, k_pe)
# Run model with attn fusion enabled
vllm_config.compilation_config.pass_config = PassConfig(
fuse_attn_quant=True, eliminate_noops=True
)
with (
set_current_vllm_config(vllm_config),
set_forward_context(attn_metadata=None, vllm_config=vllm_config),
):
model_fused = model_class(
num_heads=num_heads,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
kv_lora_rank=kv_lora_rank,
kv_cache_dtype=dtype,
device=device,
vllm_config=vllm_config,
w=model_unfused.w,
kv_b_proj_weight=model_unfused.kv_b_proj_weight,
)
model_fused = model_fused.to(device)
forward_ctx = get_forward_context()
forward_ctx.attn_metadata = model_fused.build_attn_metadata(batch_size)
# Create test backend with fusion passes
noop_pass = NoOpEliminationPass(vllm_config)
attn_pass = LazyInitPass(MLAAttnQuantFusionPass, vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
test_backend = TestBackend(noop_pass, attn_pass, cleanup_pass)
# HACK: See https://github.com/vllm-project/vllm/issues/31044
result_fused_0 = model_fused(q, kv_c_normed, k_pe) # noqa: F841
compiled_fused = torch.compile(
model_fused, backend=test_backend, fullgraph=True
)
result_fused = compiled_fused(q, kv_c_normed, k_pe)
# Check attn fusion support
quant_key: QuantKey = model_class.quant_key
attn_fusion_supported = [
layer.impl.fused_output_quant_supported(quant_key)
for key, layer in vllm_config.compilation_config.static_forward_context.items()
if isinstance(layer, MLAAttention)
]
assert sum(attn_fusion_supported) == len(attn_fusion_supported), (
"All MLA layers should support attention fusion"
)
# Check quantization ops in the graph
is_per_group = quant_key.scale.group_shape.is_per_group()
quant_op = (
torch.ops.aten.reciprocal
if "-quant_fp8" in custom_ops_list
else QUANT_OPS[quant_key]
)
test_backend.check_before_ops([quant_op], fully_replaced=is_per_group)
assert attn_pass.pass_.matched_count == sum(attn_fusion_supported)
# Check MLA attention ops in the graph
attn_nodes_pre = list(find_op_nodes(MLA_ATTN_OP, test_backend.graph_pre_pass))
attn_nodes_post = list(find_op_nodes(MLA_ATTN_OP, test_backend.graph_post_pass))
assert len(attn_nodes_pre) > 0, "Should have MLA attention nodes before fusion"
assert len(attn_nodes_pre) == len(attn_nodes_post), (
"Should have same number of MLA attention nodes before and after fusion"
)
# Before fusion: neither scale should be set
assert attn_nodes_pre[0].kwargs.get("output_scale") is None
assert attn_nodes_pre[0].kwargs.get("output_block_scale") is None
# After fusion: derive expected scale presence from quant_key properties.
# - output_scale: present for static quant or non-FP8 (NVFP4 carries input_scale)
# - output_block_scale: present when quant uses per-group/block scaling
has_output_scale = attn_nodes_post[0].kwargs.get("output_scale") is not None
has_block_scale = attn_nodes_post[0].kwargs.get("output_block_scale") is not None
expects_output_scale = quant_key.scale.static or quant_key.dtype != FP8_DTYPE
assert has_output_scale == expects_output_scale, (
f"output_scale: expected present={expects_output_scale}, got {has_output_scale}"
)
assert has_block_scale == is_per_group, (
f"output_block_scale: expected present={is_per_group}, got {has_block_scale}"
)
# Check numerical correctness
torch.testing.assert_close(result_unfused, result_fused, atol=1e-2, rtol=1e-2)
@@ -1,399 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.config
from tests.compile.backend import TestBackend
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.compilation.passes.fusion.mla_rope_kvcache_cat_fusion import (
MLARoPEKVCacheCatFusionPass,
)
from vllm.compilation.passes.utility.fix_functionalization import (
FixFunctionalizationPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CacheConfig,
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import MLAAttention
from vllm.model_executor.layers.linear import ColumnParallelLinear
from vllm.model_executor.layers.rotary_embedding import (
DeepseekScalingRotaryEmbedding,
RotaryEmbedding,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import _encode_layer_name
from vllm.v1.attention.backend import (
AttentionBackend,
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
from vllm.v1.attention.backends.registry import AttentionBackendEnum
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
VLLM_UNIFIED_MLA_KV_CACHE_UPDATE_OP = torch.ops.vllm.unified_mla_kv_cache_update
FP8_DTYPE = current_platform.fp8_dtype()
class MLARoPEKVCacheCatTestModel(torch.nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
attn_backend: AttentionBackendEnum,
use_deepseek_scaling_rope: bool,
num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
q_lora_rank: int,
kv_lora_rank: int,
is_neox: bool,
dtype: torch.dtype,
device: torch.device,
prefix: str = "model.layers.0.self_attn.attn",
):
super().__init__()
self.num_heads = num_heads
self.qk_nope_head_dim = qk_nope_head_dim
self.qk_rope_head_dim = qk_rope_head_dim
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
self.v_head_dim = v_head_dim
self.q_lora_rank = q_lora_rank
self.kv_lora_rank = kv_lora_rank
self.dtype = dtype
self.device = device
self.layer_name = prefix
self.num_kv_heads = 1
self.head_size = kv_lora_rank + qk_rope_head_dim
self.block_size = vllm_config.cache_config.block_size
self.scale = self.qk_head_dim**-0.5
if use_deepseek_scaling_rope:
self.rotary_emb = DeepseekScalingRotaryEmbedding(
head_size=qk_rope_head_dim,
rotary_dim=qk_rope_head_dim,
max_position_embeddings=4096,
base=10000,
is_neox_style=is_neox,
scaling_factor=1.0,
dtype=dtype,
)
else:
self.rotary_emb = RotaryEmbedding(
head_size=qk_rope_head_dim,
rotary_dim=qk_rope_head_dim,
max_position_embeddings=4096,
base=10000,
is_neox_style=is_neox,
dtype=dtype,
)
# Initialize intermediate mm layers for unit test
self.q_b_proj = ColumnParallelLinear(
self.q_lora_rank,
self.num_heads * self.qk_head_dim,
bias=False,
prefix=f"{prefix}.q_b_proj",
).to(device)
self.kv_b_proj = ColumnParallelLinear(
self.kv_lora_rank,
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
bias=False,
prefix=f"{prefix}.kv_b_proj",
).to(device)
# ColumnParallelLinear default init in bf16 with seed 0 produces
# near-zero weights (7/4.7M nonzero), making the GEMM output almost
# entirely zero and masking correctness bugs. Reinitialize to get
# dense outputs.
with torch.no_grad():
torch.nn.init.normal_(self.q_b_proj.weight, std=0.02)
torch.nn.init.normal_(self.kv_b_proj.weight, std=0.02)
# Register layer metadata for the fusion pass via MLAAttention
self.mla_attn = MLAAttention(
num_heads=self.num_heads,
scale=self.scale,
qk_nope_head_dim=self.qk_nope_head_dim,
qk_rope_head_dim=self.qk_rope_head_dim,
v_head_dim=self.v_head_dim,
q_lora_rank=self.q_lora_rank,
kv_lora_rank=self.kv_lora_rank,
kv_b_proj=self.kv_b_proj,
cache_config=vllm_config.cache_config,
quant_config=vllm_config.quant_config,
prefix=prefix,
attn_backend=attn_backend.get_class(),
)
self.attn_backend: type[AttentionBackend] = self.mla_attn.get_attn_backend()
self.mla_attn._k_scale = self.mla_attn._k_scale.to(device)
self.mla_attn._v_scale = self.mla_attn._v_scale.to(device)
# Keep both the string dtype (for ops) and torch dtype (for tensors)
self.kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
self.kv_cache_dtype = (
FP8_DTYPE if self.kv_cache_dtype_str.startswith("fp8") else self.dtype
)
# Initialize attn MetadataBuilder
self.builder = self.attn_backend.get_builder_cls()(
kv_cache_spec=self.mla_attn.get_kv_cache_spec(vllm_config),
layer_names=[self.mla_attn.layer_name],
vllm_config=vllm_config,
device=device,
)
def build_attn_metadata(self, batch_size: int) -> CommonAttentionMetadata:
"""Initialize attention metadata."""
# Create common attn metadata
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
common_attn_metadata = create_common_attn_metadata(
batch_spec, self.block_size, self.device, arange_block_indices=True
)
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
num_blocks = batch_size * max_blocks
# MLA KV cache is 4D: (num_blocks, num_heads=1, block_size, head_size)
kv_cache = torch.zeros(
(num_blocks, 1, self.block_size, self.head_size),
dtype=self.kv_cache_dtype,
device=self.device,
)
self.mla_attn.bind_kv_cache(kv_cache)
# Build attn metadata
attn_metadata = self.builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
return attn_metadata
def forward(
self, qkv_lora: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
qkv_lora = qkv_lora.clone()
q_c, kv_lora = qkv_lora.split(
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
dim=-1,
)
q = self.q_b_proj(q_c)[0]
kv_c, k_pe = kv_lora.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
q = q.view(-1, self.num_heads, self.qk_head_dim)
k_pe = k_pe.unsqueeze(1)
q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb(
positions, q[..., self.qk_nope_head_dim :], k_pe
)
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
kv_c,
k_pe,
_encode_layer_name(self.layer_name),
self.kv_cache_dtype_str,
self.mla_attn._k_scale,
)
return q, kv_c, k_pe, dummy
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
ops = [
INDEX_SELECT_OP,
torch.ops.vllm.unified_mla_kv_cache_update.default,
]
return ops
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.vllm.fused_rope_unified_mla_kv_cache_update.default]
MLA_BACKENDS = [AttentionBackendEnum.TRITON_MLA]
if flash_attn_supports_mla():
MLA_BACKENDS += [AttentionBackendEnum.FLASH_ATTN_MLA]
if is_aiter_found_and_supported():
MLA_BACKENDS += [AttentionBackendEnum.ROCM_AITER_MLA]
@pytest.mark.parametrize("attn_backend", MLA_BACKENDS)
@pytest.mark.parametrize("use_deepseek_scaling_rope", [True])
@pytest.mark.parametrize("num_heads", [16])
@pytest.mark.parametrize("qk_nope_head_dim", [128])
@pytest.mark.parametrize("qk_rope_head_dim", [64])
@pytest.mark.parametrize("v_head_dim", [128])
@pytest.mark.parametrize("q_lora_rank", [1536])
@pytest.mark.parametrize("kv_lora_rank", [512])
@pytest.mark.parametrize("block_size", [16])
@pytest.mark.parametrize("is_neox", [True, False])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="MLA RoPE+KVCache+Cat fusion is only supported on CUDA and ROCm.",
)
def test_mla_rope_kvcache_cat_fusion(
attn_backend: AttentionBackendEnum,
use_deepseek_scaling_rope: bool,
num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
q_lora_rank: int,
kv_lora_rank: int,
block_size: int,
is_neox: bool,
dtype: torch.dtype,
kv_cache_dtype: CacheDType,
monkeypatch: pytest.MonkeyPatch,
):
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(0)
vllm_config = VllmConfig(
model_config=ModelConfig(
model="deepseek-ai/DeepSeek-V2-Lite",
dtype=dtype,
),
cache_config=CacheConfig(
block_size=block_size,
cache_dtype=kv_cache_dtype,
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(
fuse_rope_kvcache_cat_mla=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
if not torch.distributed.is_initialized():
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.utils.system_utils import update_environment_variables
update_environment_variables(
{
"RANK": "0",
"LOCAL_RANK": "0",
"WORLD_SIZE": "1",
"MASTER_ADDR": "localhost",
"MASTER_PORT": "54321",
}
)
init_distributed_environment()
initialize_model_parallel()
if attn_backend == AttentionBackendEnum.ROCM_AITER_MLA:
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
model = MLARoPEKVCacheCatTestModel(
vllm_config=vllm_config,
attn_backend=attn_backend,
use_deepseek_scaling_rope=use_deepseek_scaling_rope,
num_heads=num_heads,
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
q_lora_rank=q_lora_rank,
kv_lora_rank=kv_lora_rank,
is_neox=is_neox,
dtype=dtype,
device=torch.get_default_device(),
)
fusion_pass = MLARoPEKVCacheCatFusionPass(vllm_config)
# note: FixFunctionalizationPass is required to correctly lower
# the fused op to its inplace version with auto-functionalization v1.
# Without it, decompose_auto_functionalized calls clone_preserve_strides
# on the non-contiguous q_pe slice directly, and inductor's lowering
# of the resulting as_strided chain incorrectly drops the storage offset.
# auto-functionalization v2 avoids this: it clones the contiguous base
# tensor (_all_bases) and reconstructs the slice as a view, so the
# offset is never passed through as_strided lowering.
passes = [
NoOpEliminationPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
FixFunctionalizationPass(vllm_config),
]
backend = TestBackend(*passes)
T = 5
qkv_lora = torch.randn(
T,
q_lora_rank + kv_lora_rank + qk_rope_head_dim,
dtype=dtype,
)
pos = torch.arange(T, dtype=torch.long)
qkv_unfused = qkv_lora.clone()
pos_unfused = pos.clone()
# Run unfused version
with set_forward_context(None, vllm_config):
forward_context = get_forward_context()
attn_metadata = model.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_unfused, kv_c_unfused, k_pe_unfused, dummy = model(
qkv_unfused, pos_unfused
)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_unfused = attn_layer.kv_cache.clone()
del dummy
# Run fused version (compiled)
torch._dynamo.mark_dynamic(qkv_lora, 0)
torch._dynamo.mark_dynamic(pos, 0)
with set_forward_context(None, vllm_config):
model_fused = torch.compile(model, backend=backend)
forward_context = get_forward_context()
attn_metadata = model.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_fused, kv_c_fused, k_pe_fused, dummy = model_fused(qkv_lora, pos)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_fused = attn_layer.kv_cache
del dummy
assert fusion_pass.matched_count == 1
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
if dtype == torch.float16:
ATOL, RTOL = (2e-3, 2e-3)
else:
ATOL, RTOL = (1e-2, 1e-2)
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(kv_c_unfused, kv_c_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(k_pe_unfused, k_pe_fused, atol=ATOL, rtol=RTOL)
# Cannot compare fp8_* directly here, cast to model dtype instead
torch.testing.assert_close(
kv_cache_unfused.view(dtype),
kv_cache_fused.view(dtype),
atol=ATOL,
rtol=RTOL,
)
@@ -1,120 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm
from tests.compile.backend import TestBackend
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.config import CompilationConfig, CompilationMode, PassConfig, VllmConfig
from vllm.platforms import current_platform
DEVICE_TYPE = current_platform.device_type
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
# Important edge case is when `num_tokens == buffer_size`
@pytest.mark.parametrize(
("num_tokens", "buffer_size"), [(256, 256), (256, 512), (1024, 1024), (1024, 1025)]
)
@pytest.mark.parametrize("hidden_size", [64, 4096])
def test_noop_elimination(dtype, num_tokens, hidden_size, buffer_size):
torch.set_default_device(DEVICE_TYPE)
torch.set_default_dtype(dtype)
torch.manual_seed(1)
class Model(torch.nn.Module):
def __init__(self) -> None:
super().__init__()
# Avoid using empty, since on rocm torch.empty
# does not initialize the memory.
self.pos_embed = torch.randn(buffer_size, hidden_size, dtype=dtype)
def forward(self, x):
# Avoid += to prevent inplace addition.
x = x + self.pos_embed[: x.shape[0]]
# Chain of reshapes
y = x.reshape(-1, 128, 32)
z = y.reshape(-1, 4096)
# No-op reshape
a = z.reshape(-1, 4096)
# Final reshape that should remain
b = a.reshape(-1, 128, 32)
# No-op slice
c = b[0 : b.shape[0]]
# The pass should replace the result of this op with `c`.
d = torch.slice_scatter(
torch.ones_like(c), # Dummy tensor to be scattered into
c, # Source tensor
0, # dim
0, # start
c.shape[0], # end
)
return d
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(eliminate_noops=True),
)
)
with vllm.config.set_current_vllm_config(vllm_config):
noop_pass = NoOpEliminationPass(vllm_config)
backend = TestBackend(noop_pass)
model = Model()
# First dimension dynamic
x = torch.rand(num_tokens, hidden_size)
torch._dynamo.mark_dynamic(x, 0)
result = model(x)
model2 = torch.compile(model, backend=backend)
result2 = model2(x)
ATOL, RTOL = (2e-3, 2e-3)
torch.testing.assert_close(result, result2, atol=ATOL, rtol=RTOL)
# The no-op reshape and slice should be eliminated.
# The initial slice on the positional embedding should remain.
# The chain of reshapes should be fused into a single reshape.
assert backend.op_count(torch.ops.aten.reshape.default) == 1
assert backend.op_count(torch.ops.aten.slice.Tensor) == 1
assert backend.op_count(torch.ops.aten.slice_scatter.default) == 0
def test_non_noop_slice_preserved():
"""Ensure that a slice with end=-1 (dropping last row) is NOT eliminated.
Regression test for a bug where end=-1 was treated like an inferred
dimension (reshape semantics) leading to incorrect elimination.
"""
torch.set_default_device(DEVICE_TYPE)
x = torch.randn(16, 16)
class SliceModel(torch.nn.Module):
def forward(self, x):
base = x.clone()
src = torch.ones(15, 16)
y = torch.slice_scatter(base, src, dim=0, start=0, end=-1)
return x[0:-1, :], y
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(eliminate_noops=True),
)
)
with vllm.config.set_current_vllm_config(vllm_config):
noop_pass = NoOpEliminationPass(vllm_config)
backend = TestBackend(noop_pass)
model = SliceModel()
ref = model(x)
compiled = torch.compile(model, backend=backend)
out = compiled(x)
torch.testing.assert_close(ref, out)
# The slice should remain (not a no-op).
assert backend.op_count(torch.ops.aten.slice.Tensor) == 1
assert backend.op_count(torch.ops.aten.slice_scatter.default) == 1
@@ -1,284 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from torch._ops import OpOverload, OpOverloadPacket
from tests.compile.backend import TestBackend
from vllm.compilation.passes.fusion.matcher_utils import (
FLASHINFER_ROTARY_OP,
ROTARY_OP,
)
from vllm.compilation.passes.fusion.qk_norm_rope_fusion import (
FUSED_QK_ROPE_OP,
QKNormRoPEFusionPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
from vllm.config import (
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.platforms import current_platform
from vllm.v1.attention.backend import AttentionType
RSQRT_OP = torch.ops.aten.rsqrt.default
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
class QKNormRoPETestModel(torch.nn.Module):
def __init__(
self,
*,
num_heads: int,
num_kv_heads: int,
head_dim: int,
eps: float,
is_neox: bool,
vllm_config: VllmConfig,
dtype: torch.dtype,
test_scattered_split: bool = False,
prefix: str = "model.layers.0.self_attn.attn",
) -> None:
super().__init__()
self.num_heads = num_heads
self.num_kv_heads = num_kv_heads
self.head_dim = head_dim
self.q_size = num_heads * head_dim
self.kv_size = num_kv_heads * head_dim
self.rotary_dim = head_dim
self.eps = eps
self.dtype = dtype
# Register layer metadata for the fusion pass via Attention.
self.attn = Attention(
num_heads=self.num_heads,
head_size=self.head_dim,
scale=1.0 / self.head_dim**0.5,
num_kv_heads=self.num_kv_heads,
cache_config=vllm_config.cache_config,
prefix=prefix,
attn_type=AttentionType.DECODER,
)
self.q_norm = RMSNorm(self.head_dim, eps=self.eps)
self.k_norm = RMSNorm(self.head_dim, eps=self.eps)
self.rotary_emb = RotaryEmbedding(
self.head_dim,
rotary_dim=self.rotary_dim,
max_position_embeddings=4096,
base=10000,
is_neox_style=is_neox,
dtype=self.dtype,
)
self.test_scattered_split = test_scattered_split
self.enable_rms_norm_custom_op = self.q_norm.enabled()
self.enable_rope_custom_op = self.rotary_emb.enabled()
def forward(self, qkv: torch.Tensor, positions: torch.Tensor):
if self.test_scattered_split:
q, _, _ = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
_, k, _ = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
_, _, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
else:
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q_by_head = q.view(*q.shape[:-1], q.shape[-1] // self.head_dim, self.head_dim)
q_by_head = self.q_norm(q_by_head)
q = q_by_head.view(q.shape)
k_by_head = k.view(*k.shape[:-1], k.shape[-1] // self.head_dim, self.head_dim)
k_by_head = self.k_norm(k_by_head)
k = k_by_head.view(k.shape)
q, k = self.rotary_emb(positions, q, k)
return q, k, v
def ops_in_model_before(self) -> list[OpOverload | OpOverloadPacket]:
ops: list[OpOverload | OpOverloadPacket] = [torch.ops.vllm_ir.rms_norm]
if self.enable_rope_custom_op:
if self.rotary_emb.use_flashinfer:
ops.append(FLASHINFER_ROTARY_OP)
else:
ops.append(ROTARY_OP)
else:
ops.append(INDEX_SELECT_OP)
return ops
def ops_in_model_after(self) -> list[OpOverload | OpOverloadPacket]:
return [FUSED_QK_ROPE_OP]
@pytest.mark.parametrize("scattered_split", [True, False])
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
@pytest.mark.parametrize("is_neox", [True, False])
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
@pytest.mark.parametrize("enable_rope_custom_op", [True])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.skipif(
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
reason="Only test on cuda and rocm platform",
)
def test_qk_norm_rope_fusion(
eps,
is_neox,
enable_rms_norm_custom_op,
enable_rope_custom_op,
dtype,
scattered_split,
):
if not hasattr(torch.ops._C, "fused_qk_norm_rope"):
pytest.skip("fused_qk_norm_rope custom op not available")
torch.set_default_device(current_platform.device_type)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
custom_ops: list[str] = []
if enable_rms_norm_custom_op:
custom_ops.append("+rms_norm")
if enable_rope_custom_op:
custom_ops.append("+rotary_embedding")
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops,
pass_config=PassConfig(
enable_qk_norm_rope_fusion=True,
eliminate_noops=True,
),
),
)
num_heads, num_kv_heads, head_dim = 16, 4, 128
T = 5
with (
set_current_vllm_config(vllm_config),
vllm_config.kernel_config.ir_op_priority.set_priority(),
):
model = QKNormRoPETestModel(
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_dim=head_dim,
eps=eps,
is_neox=is_neox,
vllm_config=vllm_config,
dtype=dtype,
test_scattered_split=scattered_split,
)
noop_pass = NoOpEliminationPass(vllm_config)
coalesce_pass = SplitCoalescingPass(vllm_config)
fusion_pass = QKNormRoPEFusionPass(vllm_config)
cleanup_pass = PostCleanupPass(vllm_config)
backend = TestBackend(noop_pass, coalesce_pass, fusion_pass, cleanup_pass)
backend_baseline = TestBackend(noop_pass, cleanup_pass)
qkv = torch.randn(T, model.q_size + 2 * model.kv_size)
pos = torch.arange(T, dtype=torch.long, device=qkv.device)
qkv_unfused = qkv.clone()
pos_unfused = pos.clone()
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(pos, 0)
model_fused = torch.compile(model, backend=backend)
q_fused, k_fused, v_fused = model_fused(qkv, pos)
torch._dynamo.mark_dynamic(qkv_unfused, 0)
torch._dynamo.mark_dynamic(pos_unfused, 0)
model_unfused = torch.compile(model, backend=backend_baseline)
q_unfused, k_unfused, v_unfused = model_unfused(qkv_unfused, pos_unfused)
if dtype == torch.float16:
ATOL, RTOL = (2e-3, 2e-3)
else:
ATOL, RTOL = (1e-2, 1e-2)
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(v_unfused, v_fused, atol=ATOL, rtol=RTOL)
assert fusion_pass.matched_count == 1
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
@pytest.mark.skipif(
not current_platform.is_cuda_alike(),
reason="Only test on CUDA and ROCm platforms",
)
def test_qk_norm_rope_fusion_with_heterogeneous_draft_model():
"""The fusion pass should support target and draft attention geometries."""
if not hasattr(torch.ops._C, "fused_qk_norm_rope"):
pytest.skip("fused_qk_norm_rope custom op not available")
dtype = torch.bfloat16
torch.set_default_device(current_platform.device_type)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rms_norm", "+rotary_embedding"],
pass_config=PassConfig(
enable_qk_norm_rope_fusion=True,
eliminate_noops=True,
),
),
)
with (
set_current_vllm_config(vllm_config),
vllm_config.kernel_config.ir_op_priority.set_priority(),
):
target_model = QKNormRoPETestModel(
num_heads=32,
num_kv_heads=8,
head_dim=128,
eps=1e-6,
is_neox=True,
vllm_config=vllm_config,
dtype=dtype,
prefix="model.layers.0.self_attn.attn",
)
draft_model = QKNormRoPETestModel(
num_heads=16,
num_kv_heads=8,
head_dim=128,
eps=1e-6,
is_neox=True,
vllm_config=vllm_config,
dtype=dtype,
prefix="draft_model.layers.0.self_attn.attn",
)
fusion_pass = QKNormRoPEFusionPass(vllm_config)
backend = TestBackend(
NoOpEliminationPass(vllm_config),
SplitCoalescingPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
)
for model in (target_model, draft_model):
num_tokens = 5
qkv = torch.randn(num_tokens, model.q_size + 2 * model.kv_size)
positions = torch.arange(num_tokens, dtype=torch.long, device=qkv.device)
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(positions, 0)
torch.compile(model, backend=backend)(qkv, positions)
assert fusion_pass.matched_count == 1
@@ -1,105 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.ir.ops
from tests.compile.backend import TestBackend
from vllm.compilation.passes.fusion.add_rms_fusion import (
AddRMSNormFusionPass,
RMSNormReshapeFusionPass,
)
from vllm.compilation.passes.fx_utils import find_op_nodes, is_func
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import CompilationConfig, CompilationMode, VllmConfig
from vllm.platforms import current_platform
pytestmark = pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Requires CUDA or ROCm"
)
class RMSNormModel(torch.nn.Module):
def __init__(self, hidden_size: int) -> None:
super().__init__()
self.weight = torch.nn.Parameter(torch.randn(hidden_size))
def forward(self, x: torch.Tensor) -> torch.Tensor:
rms = vllm.ir.ops.rms_norm(x, self.weight, 1e-6)
return rms.reshape(-1, rms.shape[-1])
class AddRMSNormModel(torch.nn.Module):
def __init__(self, hidden_size: int, residual_first: bool) -> None:
super().__init__()
self.weight = torch.nn.Parameter(torch.randn(hidden_size))
self.residual_first = residual_first
def forward(
self, x: torch.Tensor, residual: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
residual = residual + x if self.residual_first else x + residual
rms = vllm.ir.ops.rms_norm(residual, self.weight, 1e-6)
return rms.reshape(-1, rms.shape[-1]), residual
def ops_in_model_before(self):
return [torch.ops.aten.add, torch.ops.vllm_ir.rms_norm]
def ops_in_model_after(self):
return [torch.ops.vllm_ir.fused_add_rms_norm]
def _run_fusion_test(model, config, passes, *inputs):
backend = TestBackend(NoOpEliminationPass(config), *passes, PostCleanupPass(config))
outputs_unfused = model(*inputs)
outputs_fused = torch.compile(model, backend=backend)(*inputs)
torch.testing.assert_close(outputs_fused, outputs_unfused)
return backend
def _is_reshape(node):
return isinstance(node, torch.fx.Node) and is_func(
node, torch.ops.aten.reshape.default
)
@pytest.fixture
def vllm_config():
config = VllmConfig(
compilation_config=CompilationConfig(mode=CompilationMode.VLLM_COMPILE)
)
with vllm.config.set_current_vllm_config(config):
torch.set_default_device("cuda")
torch.set_default_dtype(torch.bfloat16)
torch.manual_seed(0)
yield config
def test_rmsnorm_reshape_fusion(vllm_config):
fusion_pass = RMSNormReshapeFusionPass(vllm_config)
model = RMSNormModel(hidden_size=32)
x = torch.randn(2, 7, 32)
backend = _run_fusion_test(model, vllm_config, [fusion_pass], x)
assert fusion_pass.matched_count == 1
(rms_node,) = find_op_nodes(torch.ops.vllm_ir.rms_norm, backend.graph_post_pass)
assert _is_reshape(rms_node.args[0])
@pytest.mark.parametrize("residual_first", [True, False])
def test_add_rmsnorm_reshape_fusion(vllm_config, residual_first):
add_fusion = AddRMSNormFusionPass(vllm_config)
reshape_fusion = RMSNormReshapeFusionPass(vllm_config)
model = AddRMSNormModel(hidden_size=32, residual_first=residual_first)
x = torch.randn(2, 7, 32)
residual = torch.randn_like(x)
backend = _run_fusion_test(
model, vllm_config, [add_fusion, reshape_fusion], x, residual
)
assert add_fusion.matched_count == 1
assert reshape_fusion.matched_count == 1
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
@@ -1,901 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from importlib.metadata import version
import pytest
import torch
from packaging.version import Version
import vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion as fusion_module
import vllm.config
from tests.compile.backend import TestBackend
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
dense_kv_cache_views,
)
from vllm._aiter_ops import (
IS_AITER_FOUND,
is_aiter_found_and_supported,
rocm_aiter_ops,
)
from vllm.compilation.passes.fusion.matcher_utils import MROPE_OP, ROTARY_OP
from vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion import (
QkNormRopeKvCacheFusionPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.compilation.passes.utility.scatter_split_replace import (
ScatterSplitReplacementPass,
)
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
from vllm.config import (
CacheConfig,
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
from vllm.config.cache import CacheDType
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
from vllm.platforms import current_platform
from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name
from vllm.v1.attention.backend import (
AttentionBackend,
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
pytestmark = pytest.mark.skip_global_cleanup
if not is_aiter_found_and_supported():
pytest.skip(
"ROCm with supported AITER is required",
allow_module_level=True,
)
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
FP8_DTYPE = current_platform.fp8_dtype()
@pytest.fixture(scope="module", autouse=True)
def module_global_cleanup():
# Cleanup once at the end of the module
from vllm.distributed import cleanup_dist_env_and_memory
yield
cleanup_dist_env_and_memory()
class QKNormRoPEKVCacheTestModel(torch.nn.Module):
"""Minimal model that reproduces the QK-norm + RoPE + KV cache update
pattern matched by QkNormRopeKvCacheFusionPass:
q, k, v = split(qkv)
q = rms_norm(q.view(heads, dim), q_weight).view(flat)
k = rms_norm(k.view(heads, dim), k_weight).view(flat)
q, k = rotary_emb(positions, q, k)
q = q.view(num_heads, head_dim)
k = k.view(num_kv_heads, head_dim)
v = v.view(num_kv_heads, head_dim)
dummy = unified_kv_cache_update(k, v, layer_name)
"""
def __init__(
self,
vllm_config: VllmConfig,
attn_backend: AttentionBackendEnum,
num_heads: int,
num_kv_heads: int,
head_size: int,
is_neox: bool,
rms_norm_eps: float,
dtype: torch.dtype,
device: torch.device,
rotary_dim: int | None = None,
prefix: str = "model.layers.0.self_attn.attn",
expected_query_quant_group_shape: GroupShape | None = None,
):
super().__init__()
self.num_heads = num_heads
self.num_kv_heads = num_kv_heads
self.head_size = head_size
self.rotary_dim = rotary_dim if rotary_dim is not None else head_size
self.block_size = vllm_config.cache_config.block_size
self.q_size = num_heads * head_size
self.kv_size = num_kv_heads * head_size
self.is_neox = is_neox
self.dtype = dtype
self.device = device
self.layer_name = prefix
self.q_norm = RMSNorm(head_size, eps=rms_norm_eps)
self.k_norm = RMSNorm(head_size, eps=rms_norm_eps)
self.rotary_emb = RotaryEmbedding(
head_size,
rotary_dim=self.rotary_dim,
max_position_embeddings=4096,
base=10000,
is_neox_style=is_neox,
dtype=self.dtype,
)
self.enable_rope_custom_op = self.rotary_emb.enabled()
self.attn = Attention(
num_heads=num_heads,
head_size=head_size,
scale=1.0 / head_size**0.5,
num_kv_heads=num_kv_heads,
cache_config=vllm_config.cache_config,
quant_config=vllm_config.quant_config,
prefix=prefix,
attn_backend=attn_backend.get_class(),
)
self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend()
if expected_query_quant_group_shape is not None:
assert self.attn.query_quant is not None
assert self.attn.query_quant.group_shape == expected_query_quant_group_shape
assert not self.attn_backend.forward_includes_kv_cache_update, (
f"Attention backend {self.attn_backend} does not support "
"fuse_qk_norm_rope_kvcache."
)
kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
self.kv_cache_dtype = (
FP8_DTYPE if kv_cache_dtype_str.startswith("fp8") else self.dtype
)
if self.kv_cache_dtype != self.dtype:
# Non-unit scales so the fused and unfused writers must agree on them.
self.attn._k_scale = torch.tensor(0.5, dtype=torch.float32, device=device)
self.attn._v_scale = torch.tensor(0.25, dtype=torch.float32, device=device)
self.attn._k_scale_cpu = torch.tensor(
0.5, dtype=torch.float32, device="cpu"
)
self.attn._v_scale_cpu = torch.tensor(
0.25, dtype=torch.float32, device="cpu"
)
self.attn._k_scale_float = 0.5
self.attn._v_scale_float = 0.25
else:
self.attn._k_scale = self.attn._k_scale.to(device)
self.attn._v_scale = self.attn._v_scale.to(device)
# Mirror the worker spec-collection loop: the backend publishes the
# packing it needs.
self.kv_cache_spec = self.attn.attn_backend.customize_spec(
AttentionSpec(
block_size=self.block_size,
num_kv_heads=self.num_kv_heads,
head_size=head_size,
dtype=self.kv_cache_dtype,
)
)
self.builder = self.attn.attn_backend.get_builder_cls()(
kv_cache_spec=self.kv_cache_spec,
layer_names=[self.attn.layer_name],
vllm_config=vllm_config,
device=device,
)
def build_attn_metadata(
self, batch_size: int, layout: KVCacheLayout
) -> CommonAttentionMetadata:
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
common_attn_metadata = create_common_attn_metadata(
batch_spec, self.block_size, self.device, arange_block_indices=True
)
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
num_blocks = batch_size * max_blocks
raw_tensor = torch.zeros(
num_blocks * self.kv_cache_spec.page_size_bytes,
dtype=torch.int8,
device=self.device,
)
kv_cache = dense_kv_cache_views(
raw_tensor,
self.kv_cache_spec,
num_blocks,
num_layers=1,
layout=layout,
)[0]
# Store as a bare tensor (not wrapped in a list) to match production
# `bind_kv_cache` behavior. `get_attention_context` returns this
# attribute directly to the fused/unfused `do_kv_cache_update` impls,
# which call `kv_cache.unbind(0)` and therefore require a tensor.
self.attn.kv_cache = kv_cache
attn_metadata = self.builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
return attn_metadata
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
qkv = qkv.clone()
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
# QK-norm: RMSNorm on per-head Q and K
q = q.view(-1, self.num_heads, self.head_size)
q = self.q_norm(q)
q = q.view(-1, self.q_size)
k = k.view(-1, self.num_kv_heads, self.head_size)
k = self.k_norm(k)
k = k.view(-1, self.kv_size)
# RoPE
q, k = self.rotary_emb(positions, q, k)
# Mirror Attention.forward: quant-query impls consume an fp8 q.
if (
self.kv_cache_dtype != self.dtype
and self.attn.impl.supports_quant_query_input
):
assert self.attn.query_quant is not None
q, _ = self.attn.query_quant(q, self.attn._q_scale)
# Final views + KV cache update
q = q.view(-1, self.num_heads, self.head_size)
k = k.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size)
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
k, v, _encode_layer_name(self.layer_name)
)
return q, k, v, kv_cache_dummy_dep
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
ops: list[torch._ops.OpOverload] = []
# RoPE is not yet IR-migrated, so its custom op still surfaces
# directly in the graph based on `enable_rope_custom_op`.
if self.enable_rope_custom_op:
if rocm_aiter_ops.is_triton_rotary_embed_enabled():
ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default)
else:
ops.append(ROTARY_OP)
else:
ops.append(INDEX_SELECT_OP)
ops.append(torch.ops.vllm.unified_kv_cache_update.default)
return ops
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default]
class QKNormMRoPEKVCacheTestModel(QKNormRoPEKVCacheTestModel):
def __init__(
self,
*,
mrope_section: tuple[int, int, int],
mrope_interleaved: bool,
**kwargs,
):
super().__init__(**kwargs)
self.rotary_emb = MRotaryEmbedding(
self.head_size,
rotary_dim=self.rotary_dim,
max_position_embeddings=4096,
base=10000,
is_neox_style=self.is_neox,
dtype=self.dtype,
mrope_section=list(mrope_section),
mrope_interleaved=mrope_interleaved,
)
self.enable_rope_custom_op = self.rotary_emb.enabled()
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q, k, v, kv_cache_dummy_dep = super().forward(qkv, positions)
output = torch.empty(
q.shape[0],
self.num_heads * self.head_size,
device=q.device,
dtype=self.dtype,
)
torch.ops.vllm.unified_attention_with_output(
q,
k,
v,
output,
_encode_layer_name(self.layer_name),
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
# Production decoder attention consumes K from the cache rather than
# its direct K argument. Returning `output` keeps that production user
# in the graph without exposing K to an unsupported consumer.
return q, output, v, kv_cache_dummy_dep
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
return [MROPE_OP, torch.ops.vllm.unified_kv_cache_update.default]
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.vllm.fused_qk_norm_mrope_and_unified_kv_cache_update.default]
class QKNormMRoPEObservableKTestModel(QKNormMRoPEKVCacheTestModel):
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
return QKNormRoPEKVCacheTestModel.forward(self, qkv, positions)
def _run_qk_norm_rope_kvcache_fusion_test(
*,
attn_backend: AttentionBackendEnum,
enable_aiter_triton_rope: bool,
num_tokens: int,
num_heads: int,
num_kv_heads: int,
head_size: int,
rotary_dim: int,
block_size: int,
is_neox: bool,
use_shuffle_kv_layout: str,
kv_layout: KVCacheLayout,
dtype: torch.dtype,
kv_cache_dtype: CacheDType,
rms_norm_eps: float,
custom_op: str,
monkeypatch: pytest.MonkeyPatch,
mrope_section: tuple[int, int, int] | None = None,
mrope_interleaved: bool = False,
observe_mrope_k: bool = False,
expect_fusion: bool = True,
use_quark_scalar_query_scale: bool = False,
) -> None:
device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda")
torch.set_default_device(device)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
# Set AITER toggles before VllmConfig initialization enables env caching.
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
rocm_aiter_ops.refresh_env_variables()
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
cache_config=CacheConfig(
block_size=block_size,
cache_dtype=kv_cache_dtype,
),
quant_config=(
QuarkConfig(quant_config={"exclude": []})
if use_quark_scalar_query_scale
else None
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=[custom_op],
pass_config=PassConfig(
fuse_qk_norm_rope_kvcache=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
m.setenv(
"VLLM_ROCM_USE_AITER_TRITON_ROPE",
"1" if enable_aiter_triton_rope else "0",
)
m.setenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", use_shuffle_kv_layout)
rocm_aiter_ops.refresh_env_variables()
expected_query_quant_group_shape = None
if use_quark_scalar_query_scale:
expected_query_quant_group_shape = (
GroupShape(-1, num_heads * head_size)
if num_kv_heads == 1
else GroupShape.PER_TENSOR
)
model_kwargs = {
"vllm_config": vllm_config,
"attn_backend": attn_backend,
"num_heads": num_heads,
"num_kv_heads": num_kv_heads,
"head_size": head_size,
"rotary_dim": rotary_dim,
"is_neox": is_neox,
"rms_norm_eps": rms_norm_eps,
"dtype": dtype,
"device": torch.get_default_device(),
"expected_query_quant_group_shape": expected_query_quant_group_shape,
}
if mrope_section is None:
model = QKNormRoPEKVCacheTestModel(**model_kwargs)
else:
alternate_section = (8, 8, 8)
m.setattr(
fusion_module,
"_discover_mrope_configs",
lambda _: (
(mrope_section, mrope_interleaved),
(alternate_section, not mrope_interleaved),
),
)
model_cls = (
QKNormMRoPEObservableKTestModel
if observe_mrope_k
else QKNormMRoPEKVCacheTestModel
)
model = model_cls(
**model_kwargs,
mrope_section=mrope_section,
mrope_interleaved=mrope_interleaved,
)
fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
SplitCoalescingPass(vllm_config),
ScatterSplitReplacementPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
qkv = torch.randn(
num_tokens,
num_heads * head_size + 2 * num_kv_heads * head_size,
dtype=dtype,
)
if mrope_section is None:
pos = torch.arange(num_tokens, dtype=torch.long)
else:
# Real Qwen-VL position IDs can be a strided [3, N] view.
pos = torch.randint(0, 4096, (3, 2 * num_tokens), dtype=torch.long)[:, ::2]
assert not pos.is_contiguous()
qkv_unfused = qkv.clone()
pos_unfused = pos.clone()
# Run unfused (eager) forward
with set_forward_context(None, vllm_config):
forward_context = get_forward_context()
attn_metadata = model.build_attn_metadata(num_tokens, kv_layout)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_unfused = attn_layer.kv_cache
del dummy
# Run fused (compiled) forward
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(pos, 0 if mrope_section is None else 1)
with set_forward_context(None, vllm_config):
model_fused = torch.compile(model, backend=backend)
forward_context = get_forward_context()
attn_metadata = model_fused.build_attn_metadata(num_tokens, kv_layout)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_fused = attn_layer.kv_cache
del dummy
assert fusion_pass.matched_count == int(expect_fusion)
if expect_fusion:
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
else:
for op in model.ops_in_model_before():
assert backend.op_count(op, before=True) > 0
assert backend.op_count(op) > 0
if use_quark_scalar_query_scale:
expected_quant_op = (
torch.ops.vllm.rocm_aiter_per_tensor_quant.default
if num_kv_heads != 1
else torch.ops._C.static_scaled_fp8_quant.default
)
assert backend.op_count(expected_quant_op, before=True) > 0
# Query quantization remains separate after the prologue fusion.
assert backend.op_count(expected_quant_op) > 0
# Sweep-backed (18.2k pts, PR #42749): native-rope ref worst 7.7e-3 -> 1e-2;
# AITER-triton-rope ref is itself approximate (plateau 1.28e-2) -> 2e-2.
ATOL, RTOL = (
(2e-2, 2e-2)
if enable_aiter_triton_rope or mrope_section is not None
else (1e-2, 1e-2)
)
is_fp8_cache = model.kv_cache_dtype != dtype
if q_fused.dtype == FP8_DTYPE:
# Quant-query path: both q are fp8; compare dequant within 1 fp8 ULP.
torch.testing.assert_close(
q_unfused.float(), q_fused.float(), atol=1.25e-1, rtol=1.25e-1
)
else:
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
# Fused and unfused arithmetic can straddle an FP8 rounding boundary.
# Allow one E4M3 quantization step while still comparing every block.
if is_fp8_cache:
cache_atol = cache_rtol = 1.25e-1
else:
cache_atol, cache_rtol = ATOL, RTOL
# The fused k_out is the dequantized cache write, so it carries the same
# rounding; FA and ROCM_ATTN prefill consume it directly. Unified reads K
# from the cache and asks the kernel not to write k_out at all.
if attn_backend != AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN:
torch.testing.assert_close(
k_unfused, k_fused, atol=cache_atol, rtol=cache_rtol
)
# Should be bit exact since no processing had been done on v for both paths
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
# Whole cache, so the interleaved V write (ROCM_ATTN kv_cache[1]) is checked.
torch.testing.assert_close(
kv_cache_unfused.float(),
kv_cache_fused.float(),
atol=cache_atol,
rtol=cache_rtol,
)
_FUSION_CONFIGS = [
# Full rotary, both neox styles (the original coverage).
pytest.param(64, 8, 64, 64, True, id="full-neox"),
pytest.param(64, 8, 64, 64, False, id="full-non_neox"),
# GLM-4.5/4.6/4.7 (glm4_moe.py:275 partial_rotary_factor=0.5, neox-style)
pytest.param(32, 8, 128, 64, True, id="glm4_moe"),
# GLM-4 dense (glm4.py:97,124 partial_rotary_factor=0.5, non-neox)
pytest.param(32, 2, 128, 64, False, id="glm4_dense"),
# Moondream3-style small head (head_size=64, rotary_dim=32)
pytest.param(16, 2, 64, 32, True, id="partial_small_head"),
]
@pytest.mark.parametrize(
"num_heads, num_kv_heads, head_size, rotary_dim, is_neox",
_FUSION_CONFIGS,
)
@pytest.mark.parametrize(
"attn_backend, use_shuffle_kv_layout, kv_layout",
[
# ROCM_ATTN always writes the shuffled, interleaved-V layout and needs each
# K/V side contiguous per block, so head-major only.
pytest.param(
AttentionBackendEnum.ROCM_ATTN,
"0",
KVCacheLayout.LBHNC,
id="rocm_attn",
),
# Unified never reads the shuffle env; its strided views take any layout.
pytest.param(
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
"0",
KVCacheLayout.LBHNC,
id="unified-head_major",
),
pytest.param(
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
"0",
KVCacheLayout.LBNHC,
id="unified-token_major",
),
# FA takes any layout unshuffled; its shuffle writer needs each K/V side
# contiguous per block, so head-major only.
pytest.param(
AttentionBackendEnum.ROCM_AITER_FA,
"0",
KVCacheLayout.LBHNC,
id="fa-head_major",
),
pytest.param(
AttentionBackendEnum.ROCM_AITER_FA,
"0",
KVCacheLayout.LBNHC,
id="fa-token_major",
),
pytest.param(
AttentionBackendEnum.ROCM_AITER_FA,
"1",
KVCacheLayout.LBHNC,
id="fa-shuffle",
),
],
)
@pytest.mark.parametrize("num_tokens", [5, 2048])
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
@pytest.mark.parametrize("block_size", [16, 32, 64])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("rms_norm_eps", [1e-6])
@pytest.mark.parametrize("custom_op", ["+rotary_embedding", "+rms_norm"])
# The fused_qk_norm_rope_cache kernel used by this test aborts on AITER < 0.1.20
# (fused_qk_norm_rope_cache_quant.cu: "k_cache/v_cache must be contiguous within
# a block")
@pytest.mark.skipif(
not Version(version("amd_aiter")) >= Version("0.1.20"),
reason=(
"Requires AITER >= 0.1.20; older kernels abort on "
"fused_qk_norm_rope_cache (k_cache/v_cache contiguity)"
),
)
def test_qk_norm_rope_kvcache_fusion(
num_tokens: int,
num_heads: int,
num_kv_heads: int,
head_size: int,
rotary_dim: int,
is_neox: bool,
attn_backend: AttentionBackendEnum,
enable_aiter_triton_rope: bool,
kv_layout: KVCacheLayout,
block_size: int,
dtype: torch.dtype,
kv_cache_dtype: CacheDType,
rms_norm_eps: float,
custom_op: str,
use_shuffle_kv_layout: str,
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=attn_backend,
enable_aiter_triton_rope=enable_aiter_triton_rope,
num_tokens=num_tokens,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
rotary_dim=rotary_dim,
block_size=block_size,
is_neox=is_neox,
use_shuffle_kv_layout=use_shuffle_kv_layout,
kv_layout=kv_layout,
dtype=dtype,
kv_cache_dtype=kv_cache_dtype,
rms_norm_eps=rms_norm_eps,
custom_op=custom_op,
monkeypatch=monkeypatch,
)
@pytest.mark.parametrize(
"mrope_section",
[
pytest.param((16, 24, 24), id="full_rotary"),
pytest.param((4, 6, 6), id="partial_rotary"),
],
)
@pytest.mark.parametrize("mrope_interleaved", [False, True])
@pytest.mark.parametrize("is_neox", [False, True])
@pytest.mark.parametrize(
"kv_layout",
[
pytest.param(KVCacheLayout.LBHNC, id="head_major"),
pytest.param(KVCacheLayout.LBNHC, id="token_major"),
pytest.param(KVCacheLayout.LHBNC, id="block_major"),
],
)
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.skipif(
not IS_AITER_FOUND,
reason="Requires AITER",
)
def test_qk_norm_mrope_kvcache_fusion(
mrope_section: tuple[int, int, int],
mrope_interleaved: bool,
is_neox: bool,
kv_layout: KVCacheLayout,
kv_cache_dtype: CacheDType,
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
enable_aiter_triton_rope=False,
num_tokens=5,
num_heads=8,
num_kv_heads=2,
head_size=128,
rotary_dim=2 * sum(mrope_section),
block_size=16,
is_neox=is_neox,
use_shuffle_kv_layout="0",
kv_layout=kv_layout,
dtype=torch.bfloat16,
kv_cache_dtype=kv_cache_dtype,
rms_norm_eps=1e-6,
custom_op="+rotary_embedding",
monkeypatch=monkeypatch,
mrope_section=mrope_section,
mrope_interleaved=mrope_interleaved,
)
@pytest.mark.parametrize(
("num_heads", "num_kv_heads"),
[(32, 2), (16, 1), (8, 1)],
ids=["tp2", "tp4", "tp8"],
)
def test_qk_norm_mrope_kvcache_fusion_quark_scalar_query_scale(
num_heads: int,
num_kv_heads: int,
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
enable_aiter_triton_rope=False,
num_tokens=5,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=128,
rotary_dim=128,
block_size=16,
is_neox=True,
use_shuffle_kv_layout="0",
kv_layout=KVCacheLayout.LBHNC,
dtype=torch.bfloat16,
kv_cache_dtype="fp8",
rms_norm_eps=1e-6,
custom_op="+rotary_embedding",
monkeypatch=monkeypatch,
mrope_section=(24, 20, 20),
mrope_interleaved=True,
use_quark_scalar_query_scale=True,
)
def test_mrope_fusion_rejects_observable_k(monkeypatch: pytest.MonkeyPatch):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
enable_aiter_triton_rope=False,
num_tokens=5,
num_heads=8,
num_kv_heads=2,
head_size=128,
rotary_dim=128,
block_size=16,
is_neox=True,
use_shuffle_kv_layout="0",
kv_layout=KVCacheLayout.LBHNC,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
rms_norm_eps=1e-6,
custom_op="+rotary_embedding",
monkeypatch=monkeypatch,
mrope_section=(16, 24, 24),
observe_mrope_k=True,
expect_fusion=False,
)
def test_mrope_fusion_rejects_unaligned_rotary_dim(
monkeypatch: pytest.MonkeyPatch,
):
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
enable_aiter_triton_rope=False,
num_tokens=5,
num_heads=8,
num_kv_heads=2,
head_size=128,
rotary_dim=6,
block_size=16,
is_neox=True,
use_shuffle_kv_layout="0",
kv_layout=KVCacheLayout.LBHNC,
dtype=torch.bfloat16,
kv_cache_dtype="auto",
rms_norm_eps=1e-6,
custom_op="+rotary_embedding",
monkeypatch=monkeypatch,
mrope_section=(1, 1, 1),
expect_fusion=False,
)
@pytest.mark.skipif(not _USE_LAYERNAME, reason="requires opaque LayerName")
def test_opaque_layer_name_patterns_register_once(monkeypatch: pytest.MonkeyPatch):
device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda")
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=torch.bfloat16),
cache_config=CacheConfig(block_size=16, cache_dtype="fp8_e4m3"),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rotary_embedding"],
pass_config=PassConfig(fuse_qk_norm_rope_kvcache=True),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
model_kwargs = {
"vllm_config": vllm_config,
"attn_backend": AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
"num_heads": 8,
"num_kv_heads": 2,
"head_size": 128,
"is_neox": True,
"rms_norm_eps": 1e-6,
"dtype": torch.bfloat16,
"device": torch.device(device),
}
models = [
QKNormRoPEKVCacheTestModel(
**model_kwargs,
prefix=f"model.layers.{index}.self_attn.attn",
)
for index in range(2)
]
# Opaque layer names are wildcards in the pattern graph. Registering
# once per physical layer raises "Duplicate pattern" on torch >= 2.11.
fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config)
assert models
assert fusion_pass.patterns
def _call_fused_mrope_impl(q_out: torch.Tensor) -> None:
device = q_out.device
fusion_module.fused_qk_norm_mrope_and_unified_kv_cache_update_impl(
q_out=q_out,
qkv=torch.zeros(2, 16, device=device),
positions=torch.arange(2, device=device),
q_weight=torch.ones(4, device=device),
k_weight=torch.ones(4, device=device),
rms_norm_eps=1e-6,
cos_sin_cache=torch.zeros(16, 4, device=device),
is_neox=True,
mrope_section_t=1,
mrope_section_h=1,
mrope_section_w=0,
is_interleaved=False,
rotary_dim=4,
layer_name="model.layers.0.self_attn.attn",
)
def test_mrope_profiling_pass_initializes_query(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(
fusion_module,
"get_attention_context",
lambda _: (None, None, None, None),
)
q_out = torch.full((2, 8), torch.nan, device="cpu")
_call_fused_mrope_impl(q_out)
torch.testing.assert_close(q_out, torch.zeros_like(q_out))
def test_mrope_attention_context_failure_propagates(
monkeypatch: pytest.MonkeyPatch,
):
def fail_context(_):
raise RuntimeError("missing attention context")
monkeypatch.setattr(fusion_module, "get_attention_context", fail_context)
with pytest.raises(RuntimeError, match="missing attention context"):
_call_fused_mrope_impl(torch.empty(2, 8, device="cpu"))
@@ -1,635 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from torch._higher_order_ops import auto_functionalized
import vllm.config
from tests.compile.backend import TestBackend
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
dense_kv_cache_views,
)
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.compilation.passes.fusion import rope_kvcache_fusion
from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP
from vllm.compilation.passes.fusion.rope_kvcache_fusion import RopeKVCacheFusionPass
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.compilation.passes.utility.scatter_split_replace import (
ScatterSplitReplacementPass,
)
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
from vllm.config import (
CacheConfig,
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
from vllm.config.cache import CacheDType
from vllm.config.utils import Range
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.attention import attention as attention_module
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.platforms import current_platform
from vllm.utils.torch_utils import _encode_layer_name
from vllm.v1.attention.backend import (
AttentionBackend,
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheLayout,
)
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
VLLM_UNIFIED_KV_CACHE_UPDATE_OP = torch.ops.vllm.unified_kv_cache_update
FP8_DTYPE = current_platform.fp8_dtype()
def test_kv_cache_update_ops_fake_tensor_metadata(monkeypatch: pytest.MonkeyPatch):
query = torch.empty(1, dtype=torch.bfloat16)
kv_cache = torch.empty(1, dtype=torch.uint8)
context = (None, None, kv_cache, None)
monkeypatch.setattr(attention_module, "get_attention_context", lambda _: context)
monkeypatch.setattr(rope_kvcache_fusion, "get_attention_context", lambda _: context)
runtime_output = attention_module.unified_kv_cache_update(query, query, "layer")
fake_output = attention_module.unified_kv_cache_update_fake(query, query, "layer")
assert runtime_output.shape == fake_output.shape
assert runtime_output.dtype == fake_output.dtype
assert runtime_output.device == fake_output.device
args = (
query,
query,
query,
torch.empty(1, dtype=torch.int64),
torch.empty(1),
True,
"layer",
)
runtime_output = rope_kvcache_fusion.fused_rope_and_unified_kv_cache_update_impl(
*args
)
fake_output = rope_kvcache_fusion.fused_rope_and_unified_kv_cache_update_fake(*args)
assert runtime_output.shape == fake_output.shape
assert runtime_output.dtype == fake_output.dtype
assert runtime_output.device == fake_output.device
def test_rope_kvcache_fusion_default_keeps_large_ranges_unfused():
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(fuse_rope_kvcache=True),
),
)
fusion_pass = RopeKVCacheFusionPass(vllm_config)
assert fusion_pass.is_applicable_for_range(Range(1, 256))
assert not fusion_pass.is_applicable_for_range(Range(257, 11650))
assert not fusion_pass.is_applicable_for_range(Range(11651, 16384))
class QKRoPEKVCacheTestModel(torch.nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
attn_backend: AttentionBackendEnum,
num_heads: int,
num_kv_heads: int,
head_size: int,
is_neox: bool,
dtype: torch.dtype,
device: torch.device,
prefix: str = "model.layers.0.self_attn.attn",
):
super().__init__()
self.num_heads = num_heads
self.num_kv_heads = num_kv_heads
self.head_size = head_size
self.block_size = vllm_config.cache_config.block_size
self.q_size = num_heads * head_size
self.kv_size = num_kv_heads * head_size
self.is_neox = is_neox
self.dtype = dtype
self.device = device
self.layer_name = prefix
self.rotary_emb = RotaryEmbedding(
head_size,
rotary_dim=head_size,
max_position_embeddings=4096,
base=10000,
is_neox_style=is_neox,
dtype=self.dtype,
)
# Whether to check for the RoPE custom op or component index_select
self.enable_rope_custom_op = self.rotary_emb.enabled()
# Register layer metadata for the fusion pass via Attention.
self.attn = Attention(
num_heads=num_heads,
head_size=head_size,
scale=1.0 / head_size**0.5,
num_kv_heads=num_kv_heads,
cache_config=vllm_config.cache_config,
quant_config=vllm_config.quant_config,
prefix=prefix,
attn_backend=attn_backend.get_class(),
)
self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend()
assert not self.attn_backend.forward_includes_kv_cache_update, (
f"Attention backend {self.attn_backend} does not support fuse_rope_kvcache."
)
self.attn._k_scale = self.attn._k_scale.to(device)
self.attn._v_scale = self.attn._v_scale.to(device)
kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
self.kv_cache_dtype = (
FP8_DTYPE if kv_cache_dtype_str.startswith("fp8") else self.dtype
)
# Mirror the worker spec-collection loop: the backend publishes its
# packing (e.g. ROCM_ATTN's separate K/V head groups).
self.kv_cache_spec = self.attn_backend.customize_spec(
FullAttentionSpec(
block_size=self.block_size,
num_kv_heads=self.num_kv_heads,
head_size=self.head_size,
dtype=self.kv_cache_dtype,
)
)
# Initialize attn MetadataBuilder
self.builder = self.attn_backend.get_builder_cls()(
kv_cache_spec=self.kv_cache_spec,
layer_names=[self.attn.layer_name],
vllm_config=vllm_config,
device=device,
)
def build_attn_metadata(self, batch_size: int) -> CommonAttentionMetadata:
"""Initialize attention metadata."""
# Create common attn metadata
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
common_attn_metadata = create_common_attn_metadata(
batch_spec, self.block_size, self.device, arange_block_indices=True
)
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
num_blocks = batch_size * max_blocks
# A backend's preferred supported layout wins over the default, mirroring
# selector-time resolution (e.g. ROCM_ATTN's head groups force LHBNC).
layout = KVCacheLayout.LBNHC
supported = self.attn_backend.supported_kv_cache_layouts()
if supported:
layout = supported[0]
raw_tensor = torch.zeros(
num_blocks * self.kv_cache_spec.page_size_bytes,
dtype=torch.int8,
device=self.device,
)
self.attn.kv_cache = dense_kv_cache_views(
raw_tensor,
self.kv_cache_spec,
num_blocks,
num_layers=1,
layout=layout,
)[0]
# Build attn metadata
attn_metadata = self.builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
return attn_metadata
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# Create copy so inplace ops do not modify the original tensors
qkv = qkv.clone()
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
# Instead of a full forward pass, match only the KV cache update op here
q = q.view(-1, self.num_heads, self.head_size)
k = k.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size)
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
k, v, _encode_layer_name(self.layer_name)
)
return q, k, v, kv_cache_dummy_dep
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
ops = []
if self.enable_rope_custom_op:
if rocm_aiter_ops.is_triton_rotary_embed_enabled():
ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default)
else:
ops.append(ROTARY_OP)
else:
ops.append(INDEX_SELECT_OP)
ops.append(torch.ops.vllm.unified_kv_cache_update.default)
return ops
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default]
class QKRoPEStaticQKVCacheTestModel(QKRoPEKVCacheTestModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.q_scale = torch.ones((), dtype=torch.float32, device=self.device)
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
# Create copy so inplace ops do not modify the original tensors
qkv = qkv.clone()
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
q_fp8 = torch.empty(q.shape, device=q.device, dtype=FP8_DTYPE)
_, q_fp8 = auto_functionalized(
torch.ops._C.static_scaled_fp8_quant.default,
result=q_fp8,
input=q,
scale=self.q_scale,
group_shape=(-1, -1),
)
q = q_fp8.view(-1, self.num_heads, self.head_size)
k = k.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size)
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
k, v, _encode_layer_name(self.layer_name)
)
return q, k, v, kv_cache_dummy_dep
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
ops = []
if self.enable_rope_custom_op:
if rocm_aiter_ops.is_triton_rotary_embed_enabled():
ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default)
else:
ops.append(ROTARY_OP)
else:
ops.append(INDEX_SELECT_OP)
ops.append(torch.ops.vllm.unified_kv_cache_update.default)
return ops
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default]
@pytest.mark.parametrize(
"attn_backend",
[
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
AttentionBackendEnum.TRITON_ATTN,
AttentionBackendEnum.ROCM_ATTN,
AttentionBackendEnum.ROCM_AITER_FA,
],
)
@pytest.mark.parametrize("enable_rope_custom_op", [True]) # [True, False])
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
@pytest.mark.parametrize("num_heads", [64])
@pytest.mark.parametrize("num_kv_heads", [8])
@pytest.mark.parametrize("head_size", [64])
@pytest.mark.parametrize("block_size", [16])
@pytest.mark.parametrize("is_neox", [True, False])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def test_rope_kvcache_fusion(
attn_backend: AttentionBackendEnum,
enable_rope_custom_op: bool,
enable_aiter_triton_rope: bool,
num_heads: int,
num_kv_heads: int,
head_size: int,
block_size: int,
is_neox: bool,
dtype: torch.dtype,
kv_cache_dtype: CacheDType,
monkeypatch: pytest.MonkeyPatch,
):
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(0)
custom_ops: list[str] = []
if enable_rope_custom_op:
custom_ops.append("+rotary_embedding")
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
cache_config=CacheConfig(
block_size=block_size,
cache_dtype=kv_cache_dtype,
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops,
pass_config=PassConfig(
fuse_rope_kvcache=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv(
"VLLM_ROCM_USE_AITER_TRITON_ROPE", "1" if enable_aiter_triton_rope else "0"
)
rocm_aiter_ops.refresh_env_variables()
model = QKRoPEKVCacheTestModel(
vllm_config=vllm_config,
attn_backend=attn_backend,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
is_neox=is_neox,
dtype=dtype,
device=torch.get_default_device(),
)
fusion_pass = RopeKVCacheFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
SplitCoalescingPass(vllm_config),
ScatterSplitReplacementPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
T = 5
qkv = torch.randn(
T, num_heads * head_size + 2 * num_kv_heads * head_size, dtype=dtype
)
pos = torch.arange(T, dtype=torch.long)
qkv_unfused = qkv.clone()
pos_unfused = pos.clone()
with set_forward_context(None, vllm_config):
forward_context = get_forward_context()
attn_metadata = model.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_unfused = attn_layer.kv_cache
del dummy
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(pos, 0)
with set_forward_context(None, vllm_config):
model_fused = torch.compile(model, backend=backend)
forward_context = get_forward_context()
attn_metadata = model_fused.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_fused = attn_layer.kv_cache
del dummy
assert fusion_pass.matched_count == 1
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
if dtype == torch.float16:
ATOL, RTOL = (2e-3, 2e-3)
else:
ATOL, RTOL = (1e-2, 1e-2)
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(v_unfused, v_fused, atol=ATOL, rtol=RTOL)
# Cannot compare fp8_* directly here, cast to model dtype instead
# TODO(charlifu): switch back to ATOL, RTOL after aiter fix is merged.
torch.testing.assert_close(
kv_cache_unfused.to(dtype),
kv_cache_fused.to(dtype),
atol=1e-1,
rtol=1e-1,
)
@pytest.mark.parametrize(
"attn_backend",
[AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN],
)
@pytest.mark.parametrize("enable_rope_custom_op", [True])
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
@pytest.mark.parametrize("num_heads", [64])
@pytest.mark.parametrize("num_kv_heads", [8])
@pytest.mark.parametrize("head_size", [64])
@pytest.mark.parametrize("block_size", [16])
@pytest.mark.parametrize("is_neox", [True, False])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
@pytest.mark.skipif(
not hasattr(torch.ops._C, "static_scaled_fp8_quant"),
reason="static fp8 quant op not available on this build",
)
def test_rope_static_qquant_kvcache_fusion(
attn_backend: AttentionBackendEnum,
enable_rope_custom_op: bool,
enable_aiter_triton_rope: bool,
num_heads: int,
num_kv_heads: int,
head_size: int,
block_size: int,
is_neox: bool,
dtype: torch.dtype,
kv_cache_dtype: CacheDType,
monkeypatch: pytest.MonkeyPatch,
):
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
torch.manual_seed(0)
custom_ops: list[str] = []
if enable_rope_custom_op:
custom_ops.append("+rotary_embedding")
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
cache_config=CacheConfig(
block_size=block_size,
cache_dtype=kv_cache_dtype,
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops,
pass_config=PassConfig(
fuse_rope_kvcache=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv(
"VLLM_ROCM_USE_AITER_TRITON_ROPE", "1" if enable_aiter_triton_rope else "0"
)
rocm_aiter_ops.refresh_env_variables()
model = QKRoPEStaticQKVCacheTestModel(
vllm_config=vllm_config,
attn_backend=attn_backend,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
is_neox=is_neox,
dtype=dtype,
device=torch.get_default_device(),
)
fusion_pass = RopeKVCacheFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
SplitCoalescingPass(vllm_config),
ScatterSplitReplacementPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
T = 5
qkv = torch.randn(
T, num_heads * head_size + 2 * num_kv_heads * head_size, dtype=dtype
)
pos = torch.arange(T, dtype=torch.long)
qkv_unfused = qkv.clone()
pos_unfused = pos.clone()
with set_forward_context(None, vllm_config):
forward_context = get_forward_context()
attn_metadata = model.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_unfused = attn_layer.kv_cache
del dummy
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(pos, 0)
with set_forward_context(None, vllm_config):
model_fused = torch.compile(model, backend=backend)
forward_context = get_forward_context()
attn_metadata = model_fused.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_fused = attn_layer.kv_cache
del dummy
assert fusion_pass.matched_count == 1
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
static_quant_pre = backend.op_count(
torch.ops._C.static_scaled_fp8_quant.default, before=True
)
static_quant_post = backend.op_count(
torch.ops._C.static_scaled_fp8_quant.default
)
assert static_quant_pre > 0
# The replacement still emits static quant, so count is expected to
# remain non-zero after fusion.
assert static_quant_post > 0
# Negative control: without the static-Q pattern, the generic RoPE+KV
# pattern cannot match this rope -> static-quant -> kv graph, so the
# fusion above is attributable solely to RopeStaticQQuantKVCachePattern.
# This is a structural property independent of the rope/dtype/neox axes,
# so run the (extra compile) check only once on a representative combo.
if is_neox and enable_aiter_triton_rope and kv_cache_dtype == "auto":
m.setattr(
rope_kvcache_fusion,
"_supports_static_q_fp8_quant_fusion",
lambda: False,
)
generic_pass = RopeKVCacheFusionPass(vllm_config)
generic_backend = TestBackend(
NoOpEliminationPass(vllm_config),
SplitCoalescingPass(vllm_config),
ScatterSplitReplacementPass(vllm_config),
generic_pass,
PostCleanupPass(vllm_config),
)
# Reset dynamo so the model is recompiled through generic_backend
# instead of reusing the cached compilation from above.
torch._dynamo.reset()
with set_forward_context(None, vllm_config):
model_generic = torch.compile(model, backend=generic_backend)
forward_context = get_forward_context()
attn_metadata = model_generic.build_attn_metadata(T)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
model_generic(qkv, pos)
# op_count reads the post-pass graph, so it also confirms the pass ran
# (a no-op pass would raise instead of silently passing on count 0).
assert generic_pass.matched_count == 0
assert (
generic_backend.op_count(
torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default
)
== 0
)
if dtype == torch.float16:
ATOL, RTOL = (2e-3, 2e-3)
else:
ATOL, RTOL = (1e-2, 1e-2)
# TODO(charlifu): switch back to ATOL, RTOL after aiter fix is merged.
torch.testing.assert_close(
q_unfused.to(torch.float32),
q_fused.to(torch.float32),
atol=1e-1,
rtol=1e-1,
)
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
torch.testing.assert_close(v_unfused, v_fused, atol=ATOL, rtol=RTOL)
# TODO(charlifu): switch back to ATOL, RTOL after aiter fix is merged.
torch.testing.assert_close(
kv_cache_unfused.to(dtype),
kv_cache_fused.to(dtype),
atol=1e-1,
rtol=1e-1,
)
@@ -1,110 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import torch.nn as nn
import vllm
from tests.compile.backend import TestBackend
from vllm.compilation.passes.utility.scatter_split_replace import (
ScatterSplitReplacementPass,
)
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
from vllm.config import CompilationConfig, CompilationMode, VllmConfig
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.platforms import current_platform
DEVICE_TYPE = current_platform.device_type
class ScatterSplitReplacementModel(nn.Module):
"""Model with a rope+getitem+slice_scatter+split_with_sizes sequence."""
def __init__(
self,
num_heads: int,
num_kv_heads: int,
head_size: int,
dtype: torch.dtype,
):
super().__init__()
self.q_size = num_heads * head_size
self.kv_size = num_kv_heads * head_size
self.rotary_emb = RotaryEmbedding(
head_size,
rotary_dim=head_size,
max_position_embeddings=4096,
base=10000,
is_neox_style=True,
dtype=dtype,
)
def forward(self, qkv: torch.Tensor, positions: torch.Tensor):
# Create copy so inplace ops do not modify the original tensors
qkv = qkv.clone()
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
q, k = self.rotary_emb(positions, q, k)
q = q + 1
k = k + 2
v = v + 3
return q, k, v
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
return [
torch.ops.aten.slice_scatter.default,
torch.ops.aten.split_with_sizes.default,
torch.ops.aten.getitem.default,
]
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.aten.getitem.default]
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_scatter_split_replace(dtype):
torch.set_default_device(DEVICE_TYPE)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
num_heads = 8
num_kv_heads = 4
head_size = 64
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=["+rotary_embedding"],
),
)
with vllm.config.set_current_vllm_config(vllm_config):
# ScatterSplitReplacementPass requires SplitCoalescingPass to be run before it
coalesce_pass = SplitCoalescingPass(vllm_config)
replace_pass = ScatterSplitReplacementPass(vllm_config)
passes = [coalesce_pass, replace_pass]
backend = TestBackend(*passes)
model = ScatterSplitReplacementModel(num_heads, num_kv_heads, head_size, dtype)
T = 5
qkv = torch.randn(
T, num_heads * head_size + 2 * num_kv_heads * head_size, dtype=dtype
)
pos = torch.arange(T, dtype=torch.long)
qkv_eager = qkv.clone()
pos_eager = pos.clone()
result_eager = model(qkv_eager, pos_eager)
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(pos, 0)
model_compiled = torch.compile(model, backend=backend)
result_compiled = model_compiled(qkv, pos)
for eager, compiled in zip(result_eager, result_compiled):
torch.testing.assert_close(eager, compiled)
assert backend.op_count(torch.ops.aten.slice_scatter.default) == 0
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
@@ -1,373 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
from functools import partial
import pytest
import torch
import vllm.envs as envs
from tests.compile.backend import TestBackend
from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor
from tests.utils import TestFP8Layer
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
from vllm.compilation.passes.fusion.act_quant_fusion import (
FUSED_OPS,
SILU_MUL_OP,
ActivationQuantFusionPass,
)
from vllm.compilation.passes.fusion.rms_quant_fusion import QUANT_OPS
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.config import (
CompilationConfig,
CompilationMode,
PassConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.model_executor.kernels.linear import (
CutlassFP8ScaledMMLinearKernel,
FlashInferFP8ScaledMMLinearKernel,
FP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ROCmFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
create_fp8_quant_key,
kFp8Dynamic128Sym,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import is_deep_gemm_supported
FP8_DTYPE = current_platform.fp8_dtype()
FP4_DTYPE = torch.uint8
def is_nvfp4_supported():
return current_platform.has_device_capability(100)
class TestSiluMulFp8QuantModel(torch.nn.Module):
quant_key = kFp8StaticTensorSym
def __init__(
self,
hidden_size: int,
force_kernel: type[FP8ScaledMMLinearKernel],
dtype: torch.dtype,
**kwargs,
):
super().__init__()
self.silu_and_mul = SiluAndMul()
self.fp8_linear = TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=self.quant_key,
weight_quant_key=self.quant_key,
force_kernel=force_kernel,
input_dtype=dtype,
)
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
self.enable_quant_fp8_custom_op = self.fp8_linear.is_quant_fp8_enabled()
def forward(self, x):
y = self.silu_and_mul(x)
x2 = self.fp8_linear(y)
return x2
def ops_in_model_before(self):
return [
SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul,
(
QUANT_OPS[kFp8StaticTensorSym]
if self.enable_quant_fp8_custom_op
else torch.ops.aten.reciprocal
),
]
def ops_in_model_after(self):
return [FUSED_OPS[kFp8StaticTensorSym]]
class TestSiluMulNvfp4QuantModel(torch.nn.Module):
def __init__(self, hidden_size: int, x: torch.Tensor, **kwargs):
super().__init__()
from vllm.compilation.passes.fusion.act_quant_fusion import (
silu_and_mul_nvfp4_quant_supported,
)
assert silu_and_mul_nvfp4_quant_supported
self.silu_and_mul = SiluAndMul()
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
# create nvfp4 weight
w = torch.rand((hidden_size, hidden_size))
self.w, self.w_block_scale, self.w_global_scale = quant_nvfp4_tensor(w)
# get global scale offline
_, _, self.y_global_scale = quant_nvfp4_tensor(self.silu_and_mul(x))
self.alpha = 1.0 / (self.w_global_scale * self.y_global_scale)
def forward(self, x):
y = self.silu_and_mul(x)
y_quant, y_block_scale = scaled_fp4_quant(y, self.y_global_scale)
out = cutlass_scaled_fp4_mm(
a=y_quant,
b=self.w,
block_scale_a=y_block_scale,
block_scale_b=self.w_block_scale,
alpha=self.alpha,
out_dtype=y.dtype,
)
return out
def ops_in_model_before(self):
return [
SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul,
QUANT_OPS[kNvfp4Dynamic],
]
def ops_in_model_after(self):
return [FUSED_OPS[kNvfp4Dynamic]]
class TestSiluMulGroupFp8QuantModel(torch.nn.Module):
act_quant_key = kFp8Dynamic128Sym
def __init__(self, hidden_size: int, dtype: torch.dtype, **kwargs):
super().__init__()
self.silu_and_mul = SiluAndMul()
self.weight_quant_key = create_fp8_quant_key(
static=True, group_shape=GroupShape(hidden_size, hidden_size)
)
self.w8a8_block_fp8_linear = TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
weight_quant_key=self.weight_quant_key,
activation_quant_key=self.act_quant_key,
input_dtype=dtype,
)
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
def forward(self, x):
y = self.silu_and_mul(x)
x2 = self.w8a8_block_fp8_linear(y)
return x2
def ops_in_model_before(self):
return [
SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul,
rocm_aiter_ops.get_group_quant_op(),
]
def ops_in_model_after(self):
return [torch.ops.vllm.rocm_aiter_act_mul_and_fp8_group_quant]
class TestSiluMulBlockQuantModel(torch.nn.Module):
quant_key = kFp8Dynamic128Sym
def __init__(self, hidden_size: int, is_scale_transposed: bool = False, **kwargs):
super().__init__()
self.silu_and_mul = SiluAndMul()
self.is_scale_transposed = is_scale_transposed
self.quant_fp8 = QuantFP8(
static=False,
group_shape=GroupShape(1, 128),
column_major_scales=is_scale_transposed,
compile_native=False,
)
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
self.enable_quant_fp8_custom_op = self.quant_fp8.enabled()
def forward(self, x):
y = self.silu_and_mul(x)
out, scale = self.quant_fp8(y)
group_size = self.quant_key.scale.group_shape[1]
scale_expanded = scale.repeat_interleave(group_size, dim=1)
dequant = out.to(dtype=torch.float32) * scale_expanded
return (dequant,)
def ops_in_model_before(self):
ops = []
if self.enable_silu_mul_custom_op:
ops.append(SILU_MUL_OP)
# When silu custom op is disabled, aten.mul.Tensor also appears
# in dequant code, so we skip checking it to avoid false positives.
ops.append(
QUANT_OPS[self.quant_key]
if self.enable_quant_fp8_custom_op
else torch.ops.aten.reciprocal.default
)
return ops
def ops_in_model_after(self):
return [FUSED_OPS[self.quant_key]]
ROCM_KERNELS = [ROCmFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel]
CUDA_KERNELS = [
FlashInferFP8ScaledMMLinearKernel,
CutlassFP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
]
TEST_KERNELS = ROCM_KERNELS if current_platform.is_rocm() else CUDA_KERNELS
@pytest.mark.parametrize("num_tokens", [32, 64])
@pytest.mark.parametrize("hidden_size", [128, 256])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("enable_silu_mul_custom_op", [True, False])
@pytest.mark.parametrize(
"model_class, enable_quant_fp8_custom_op, force_kernel",
list(itertools.product([TestSiluMulFp8QuantModel], [True, False], TEST_KERNELS))
+ [
pytest.param(
TestSiluMulNvfp4QuantModel,
False,
None,
marks=pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUDA only"
),
),
# GroupFP8Quant fusion only works with AITER on ROCm.
# and the enable_quant_fp8_custom_op must be True.
pytest.param(
TestSiluMulGroupFp8QuantModel,
True,
None,
marks=pytest.mark.skipif(
not current_platform.is_rocm(), reason="ROCm only"
),
),
# Block quant fusion for per-group FP8 (CUDA only).
*[
pytest.param(
partial(TestSiluMulBlockQuantModel, is_scale_transposed=transposed),
True,
None,
marks=pytest.mark.skipif(
not current_platform.is_cuda(), reason="CUDA only"
),
id=f"TestSiluMulBlockQuant-transposed={transposed}",
)
for transposed in [False, True]
],
],
)
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_fusion_silu_and_mul_quant(
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
model_class: type[
TestSiluMulFp8QuantModel
| TestSiluMulNvfp4QuantModel
| TestSiluMulGroupFp8QuantModel
| TestSiluMulBlockQuantModel
],
enable_silu_mul_custom_op: bool,
enable_quant_fp8_custom_op: bool,
force_kernel: type[FP8ScaledMMLinearKernel] | None,
monkeypatch: pytest.MonkeyPatch,
):
if model_class is TestSiluMulNvfp4QuantModel and not is_nvfp4_supported():
pytest.skip("NVFP4 is not supported on this GPU.")
if model_class is TestSiluMulGroupFp8QuantModel and not IS_AITER_FOUND:
pytest.skip("AITER is not supported on this GPU.")
if (
isinstance(model_class, partial)
and model_class.func is TestSiluMulBlockQuantModel
and is_deep_gemm_supported()
):
pytest.skip("SiluMul+BlockQuant fusion not applicable with DeepGemm")
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
x = torch.rand(num_tokens, hidden_size * 2)
# Reshape pass is needed for the fusion pass to work
custom_ops = ["none"]
if enable_silu_mul_custom_op:
custom_ops.append("+silu_and_mul")
if enable_quant_fp8_custom_op:
custom_ops.append("+quant_fp8")
config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=custom_ops,
backend="eager", # avoid compilation for SiluAndMul and QuantFP8
pass_config=PassConfig(fuse_act_quant=True, eliminate_noops=True),
),
)
with set_current_vllm_config(config), monkeypatch.context() as m:
fusion_passes = [ActivationQuantFusionPass(config)]
if IS_AITER_FOUND and model_class is TestSiluMulGroupFp8QuantModel:
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
RocmAiterSiluMulFp8GroupQuantFusionPass,
)
m.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
fusion_passes += [RocmAiterSiluMulFp8GroupQuantFusionPass(config)]
passes = [NoOpEliminationPass(config), *fusion_passes, PostCleanupPass(config)]
backend = TestBackend(*passes)
model = model_class(
hidden_size=hidden_size,
force_kernel=force_kernel,
x=x,
dtype=dtype,
)
# First dimension dynamic
torch._dynamo.mark_dynamic(x, 0)
result = model(x)
model2 = torch.compile(model, backend=backend)
result2 = model2(x)
# Check that it gives the same answer
if isinstance(model, TestSiluMulFp8QuantModel):
atol, rtol = 1e-3, 1e-3
elif isinstance(model, TestSiluMulNvfp4QuantModel):
atol, rtol = 1e-1, 1e-1
elif isinstance(model, TestSiluMulGroupFp8QuantModel):
atol, rtol = 5e-2, 5e-2
elif isinstance(model, TestSiluMulBlockQuantModel):
if current_platform.is_rocm():
atol, rtol = 1e-3, 1e-3
else:
# CUDA fused kernel computes silu*mul in fp32 while the reference
# goes through bf16/fp16 storage, so group maxima (and thus scales)
# can shift by one FP8-e4m3 code (~1/8 relative step).
atol, rtol = 5e-2, 5e-2
torch.testing.assert_close(
result[0].to(dtype=dtype), result2[0].to(dtype=dtype), atol=atol, rtol=rtol
)
assert sum([p.matched_count for p in fusion_passes]) == 1
# In pre-nodes, quant op should be present and fused kernels should not
backend.check_before_ops(model.ops_in_model_before())
# In post-nodes, fused kernels should be present and quant op should not
backend.check_after_ops(model.ops_in_model_after())
@@ -1,379 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for manual fusion via maybe_fused_act_quant.
Tests all fusion paths in _FUSED_ACT_QUANT:
- kFp8StaticTensorSym: all platforms
- kFp8Dynamic128Sym: CUDA only
- kNvfp4Dynamic: CUDA SM100+ only
"""
import pytest
import torch
import vllm.envs as envs
from tests.utils import TestFP8Layer
from vllm.config import (
CompilationConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.model_executor.kernels.linear import (
CutlassFP8ScaledMMLinearKernel,
FlashInferFP8ScaledMMLinearKernel,
FP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
ROCmFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.fusion.fused_act_quant import (
_FUSED_ACT_QUANT,
maybe_fused_act_quant,
)
from vllm.model_executor.layers.fusion.quant_activation import (
QuantizedActivation,
expose_input_quant_key,
get_input_quant_key,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
kFp8Dynamic128Sym,
kFp8StaticTensorSym,
kNvfp4Dynamic,
)
from vllm.platforms import current_platform
# Mock linear layer for testing fusion paths that don't have real kernel support
class MockLinearForFusion(torch.nn.Module):
"""Mock linear layer that exposes an input key for fusion testing."""
def __init__(
self,
quant_key,
input_scale=None,
input_global_scale=None,
input_global_scale_inv=None,
):
super().__init__()
self._input_quant_key = quant_key
if input_scale is not None:
self.input_scale = input_scale
if input_global_scale is not None:
self.input_global_scale = input_global_scale
if input_global_scale_inv is not None:
self.input_global_scale_inv = input_global_scale_inv
ROCM_KERNELS = [ROCmFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel]
CUDA_KERNELS = [
FlashInferFP8ScaledMMLinearKernel,
CutlassFP8ScaledMMLinearKernel,
PerTensorTorchFP8ScaledMMLinearKernel,
]
TEST_KERNELS = ROCM_KERNELS if current_platform.is_rocm() else CUDA_KERNELS
@pytest.mark.parametrize("num_tokens", [32, 64])
@pytest.mark.parametrize("hidden_size", [128, 256])
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("force_kernel", TEST_KERNELS)
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_fp8_static_with_linear(
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
force_kernel: type[FP8ScaledMMLinearKernel],
):
"""Test manual fusion with real FP8 linear layer (kFp8StaticTensorSym).
This is an end-to-end test that verifies the full flow:
unfused (silu_and_mul -> in-kernel quant) vs fused (silu_and_mul_quant).
"""
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
x = torch.rand(num_tokens, hidden_size * 2)
config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["none"]),
)
with set_current_vllm_config(config):
silu_and_mul = SiluAndMul()
fp8_linear = TestFP8Layer(
weight_shape=(hidden_size, hidden_size),
activation_quant_key=kFp8StaticTensorSym,
weight_quant_key=kFp8StaticTensorSym,
force_kernel=force_kernel,
input_dtype=dtype,
)
# Run without fusion: silu_and_mul returns plain tensor
y_unfused = maybe_fused_act_quant(silu_and_mul, x, fp8_linear)
assert isinstance(y_unfused, torch.Tensor)
result_unfused = fp8_linear(y_unfused)
# Enable fusion
expose_input_quant_key(fp8_linear, fp8_linear.kernel)
if get_input_quant_key(fp8_linear) is None:
pytest.skip(
f"Kernel {force_kernel.__name__} doesn't support input_quant_key"
)
# Run with fusion: silu_and_mul returns QuantizedActivation
y_fused = maybe_fused_act_quant(silu_and_mul, x, fp8_linear)
assert isinstance(y_fused, QuantizedActivation)
assert y_fused.quant_key == kFp8StaticTensorSym
assert y_fused.data.dtype == current_platform.fp8_dtype()
assert y_fused.data.shape == (num_tokens, hidden_size)
assert y_fused.orig_dtype == dtype
assert y_fused.orig_shape == (num_tokens, hidden_size)
result_fused = fp8_linear(y_fused)
torch.testing.assert_close(
result_fused.to(dtype=dtype),
result_unfused.to(dtype=dtype),
atol=5e-2,
rtol=5e-2,
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="Dynamic block quant CUDA only"
)
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_fp8_dynamic_128(dtype: torch.dtype):
"""Test kFp8Dynamic128Sym fusion path (group_size=128).
Compares fused (silu_and_mul_per_block_quant) vs unfused (silu_and_mul)
by dequantizing the fused result and comparing with unfused.
"""
if (SiluAndMul, kFp8Dynamic128Sym) not in _FUSED_ACT_QUANT:
pytest.skip("kFp8Dynamic128Sym fusion not available")
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
# hidden_size must be divisible by group_size (128)
num_tokens, hidden_size = 32, 256
group_size = 128
x = torch.rand(num_tokens, hidden_size * 2)
config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["none"]),
)
with set_current_vllm_config(config):
silu_and_mul = SiluAndMul()
# Unfused path: just apply silu_and_mul
mock_linear_no_key = torch.nn.Linear(hidden_size, hidden_size)
result_unfused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_no_key)
assert isinstance(result_unfused, torch.Tensor)
# Fused path: apply silu_and_mul + per-block quantization
mock_linear_with_key = MockLinearForFusion(kFp8Dynamic128Sym)
result_fused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_with_key)
# Verify fused result structure
assert isinstance(result_fused, QuantizedActivation)
assert result_fused.quant_key == kFp8Dynamic128Sym
assert result_fused.data.dtype == current_platform.fp8_dtype()
assert result_fused.data.shape == (num_tokens, hidden_size)
assert result_fused.orig_dtype == dtype
# Check scale shape
expected_num_groups = hidden_size // group_size
assert result_fused.scale.shape == (num_tokens, expected_num_groups)
# Dequantize fused result and compare with unfused
# Per-block dequant: data * scale (broadcast scale across group)
dequant_data = result_fused.data.to(dtype).view(
num_tokens, expected_num_groups, group_size
)
scales_expanded = result_fused.scale.unsqueeze(
-1
) # (num_tokens, num_groups, 1)
dequant_result = (
(dequant_data * scales_expanded).view(num_tokens, hidden_size).to(dtype)
)
torch.testing.assert_close(
dequant_result,
result_unfused,
atol=5e-2,
rtol=5e-2,
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
# Cover M < 128 (padded swizzled SF tiles) and hidden_size that makes the
# kernel launch grid.y > 1 (num_packed_cols = hidden_size / 16 > 512).
@pytest.mark.parametrize("num_tokens", [1, 127, 128])
@pytest.mark.parametrize("hidden_size", [256, 14336])
@pytest.mark.skipif(not current_platform.is_cuda(), reason="NVFP4 CUDA only")
@pytest.mark.skipif(
not current_platform.has_device_capability(100), reason="NVFP4 requires SM100+"
)
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_nvfp4_dynamic(
dtype: torch.dtype, num_tokens: int, hidden_size: int
):
"""Test kNvfp4Dynamic fusion path.
Compares fused (silu_and_mul_nvfp4_quant) vs unfused (silu_and_mul)
by dequantizing the fused result and comparing with unfused.
"""
if (SiluAndMul, kNvfp4Dynamic) not in _FUSED_ACT_QUANT:
pytest.skip("kNvfp4Dynamic fusion not available")
from tests.kernels.quantization.nvfp4_utils import dequantize_nvfp4_to_dtype
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
# NVFP4 requires hidden_size divisible by 16 (block size) and by 2 (packing).
x = torch.rand(num_tokens, hidden_size * 2)
# Non-1.0 global scale so the test is sensitive to the scale direction:
# the fused producer must quantize with input_global_scale_inv (the GEMM's
# alpha divides by input_global_scale), matching the unfused path.
input_global_scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
input_global_scale_inv = 1.0 / input_global_scale
config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["none"]),
)
with set_current_vllm_config(config):
silu_and_mul = SiluAndMul()
# Unfused path: just apply silu_and_mul
mock_linear_no_key = torch.nn.Linear(hidden_size, hidden_size)
result_unfused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_no_key)
assert isinstance(result_unfused, torch.Tensor)
# Fused path: apply silu_and_mul + NVFP4 quantization
mock_linear_with_key = MockLinearForFusion(
kNvfp4Dynamic,
input_global_scale=input_global_scale,
input_global_scale_inv=input_global_scale_inv,
)
result_fused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_with_key)
# Verify fused result structure
assert isinstance(result_fused, QuantizedActivation)
assert result_fused.quant_key == kNvfp4Dynamic
# NVFP4 packs 2 values into 1 byte
assert result_fused.data.dtype == torch.uint8
assert result_fused.data.shape == (num_tokens, hidden_size // 2)
assert result_fused.orig_dtype == dtype
assert result_fused.orig_shape == (num_tokens, hidden_size)
# The scale tensor must use the padded 128x4-tile swizzled layout,
# same as scaled_fp4_quant's allocator; the kernel writes a full
# 128-row tile even when M is smaller.
from vllm.utils.math_utils import round_up
assert result_fused.scale.dtype == current_platform.fp8_dtype()
assert result_fused.scale.shape == (
round_up(num_tokens, 128),
round_up(hidden_size // 16, 4),
)
# Dequantize fused result and compare with unfused. The block scales
# must have been folded with input_global_scale_inv, so dequantize
# with the same inverse scale to recover the activation.
dequant_result = dequantize_nvfp4_to_dtype(
tensor_fp4=result_fused.data,
tensor_sf=result_fused.scale,
global_scale=input_global_scale_inv,
dtype=dtype,
device="cuda",
block_size=16,
is_sf_128x4_layout=True,
)
torch.testing.assert_close(
dequant_result,
result_unfused,
atol=3e-1,
rtol=3e-1,
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_fallback_no_key(dtype: torch.dtype):
"""Test that maybe_fused_act_quant falls back without an input key."""
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
x = torch.rand(32, 256)
config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["none"]),
)
with set_current_vllm_config(config):
silu_and_mul = SiluAndMul()
# Linear without an input quantization key
mock_linear = torch.nn.Linear(128, 128)
result = maybe_fused_act_quant(silu_and_mul, x, mock_linear)
# Should fall back to plain silu_and_mul
assert isinstance(result, torch.Tensor)
assert result.shape == (32, 128)
assert result.dtype == dtype
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.skipif(
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
)
def test_manual_fusion_fallback_unsupported_key(dtype: torch.dtype):
"""Test that maybe_fused_act_quant falls back for unsupported quant keys."""
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
QuantKey,
ScaleDesc,
)
torch.set_default_device("cuda")
torch.set_default_dtype(dtype)
x = torch.rand(32, 256)
config = VllmConfig(
compilation_config=CompilationConfig(custom_ops=["none"]),
)
with set_current_vllm_config(config):
# Create an unsupported quant key
unsupported_key = QuantKey(
dtype=torch.int8,
scale=ScaleDesc(
dtype=torch.float32, static=False, group_shape=GroupShape(1, 1)
),
)
silu_and_mul = SiluAndMul()
mock_linear = MockLinearForFusion(unsupported_key)
result = maybe_fused_act_quant(silu_and_mul, x, mock_linear)
# Should fall back to plain silu_and_mul since key not in _FUSED_ACT_QUANT
assert isinstance(result, torch.Tensor)
assert result.shape == (32, 128)
assert result.dtype == dtype
@@ -1,273 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import operator
import pytest
import torch
import vllm
from tests.compile.backend import TestBackend
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
from vllm.config import CompilationConfig, CompilationMode, PassConfig, VllmConfig
from vllm.platforms import current_platform
DEVICE_TYPE = current_platform.device_type
class SplitCoalescingModel(torch.nn.Module):
"""Model with 3 separate split_with_sizes calls on the same input,
simulating the B200+FP8 graph where CSE fails to merge them."""
def __init__(self, q_size: int, kv_size: int) -> None:
super().__init__()
self.q_size = q_size
self.kv_size = kv_size
def forward(self, qkv: torch.Tensor):
q, _, _ = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
_, k, _ = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
_, _, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
return q + 1, k + 2, v + 3
class SliceCoalescingModel(torch.nn.Module):
"""Model with a complete QKV partition represented as contiguous slices."""
def __init__(self, q_size: int, kv_size: int) -> None:
super().__init__()
self.q_size = q_size
self.kv_size = kv_size
def forward(self, qkv: torch.Tensor):
q_end = self.q_size
k_end = q_end + self.kv_size
v_end = k_end + self.kv_size
q = torch.ops.aten.slice.Tensor(qkv, -1, 0, q_end)
k = torch.ops.aten.slice.Tensor(qkv, -1, q_end, k_end)
v = torch.ops.aten.slice.Tensor(qkv, -1, k_end, v_end)
return q + 1, k + 2, v + 3
def make_vllm_config() -> VllmConfig:
return VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
pass_config=PassConfig(),
)
)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_split_coalescing(dtype):
torch.set_default_device(DEVICE_TYPE)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
q_size, kv_size = 2048, 512
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
coalesce_pass = SplitCoalescingPass(vllm_config)
backend = TestBackend(coalesce_pass)
model = SplitCoalescingModel(q_size, kv_size)
T = 5
qkv = torch.randn(T, q_size + 2 * kv_size)
torch._dynamo.mark_dynamic(qkv, 0)
result_eager = model(qkv)
model_compiled = torch.compile(model, backend=backend)
result_compiled = model_compiled(qkv)
ATOL, RTOL = (2e-3, 2e-3)
for eager, compiled in zip(result_eager, result_compiled):
torch.testing.assert_close(eager, compiled, atol=ATOL, rtol=RTOL)
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
def test_slice_partition_canonicalization(dtype):
torch.set_default_device(DEVICE_TYPE)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
q_size, kv_size = 2048, 512
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
coalesce_pass = SplitCoalescingPass(vllm_config)
backend = TestBackend(coalesce_pass)
model = SliceCoalescingModel(q_size, kv_size)
qkv = torch.randn(5, q_size + 2 * kv_size)
torch._dynamo.mark_dynamic(qkv, 0)
result_eager = model(qkv)
result_compiled = torch.compile(model, backend=backend, fullgraph=True)(qkv)
for eager, compiled in zip(result_eager, result_compiled):
torch.testing.assert_close(eager, compiled, atol=2e-3, rtol=2e-3)
assert backend.op_count(torch.ops.aten.slice.Tensor, before=True) == 3
assert backend.op_count(torch.ops.aten.slice.Tensor) == 0
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
split = next(
node
for node in backend.final_graph.nodes
if node.target == torch.ops.aten.split_with_sizes.default
)
assert list(split.args[1]) == [q_size, kv_size, kv_size]
assert split.args[2] == -1
getitems = [
node
for node in split.users
if node.op == "call_function" and node.target == operator.getitem
]
assert sorted(node.args[1] for node in getitems) == [0, 1, 2]
assert isinstance(split.meta["val"], list)
for getitem in getitems:
index = getitem.args[1]
assert getitem.meta["val"] is split.meta["val"][index]
def make_slice_graph(
ranges: list[tuple[int, int, int]],
) -> torch.fx.Graph:
graph = torch.fx.Graph()
source = graph.placeholder("qkv")
source.meta["val"] = torch.empty((2, 8), device="meta")
slices = []
for start, end, step in ranges:
node = graph.call_function(
torch.ops.aten.slice.Tensor,
args=(source, -1, start, end, step),
)
node.meta["val"] = torch.empty(
(2, len(range(start, min(end, 8), step))), device="meta"
)
slices.append(node)
graph.output(tuple(slices))
return graph
def test_slice_partition_accepts_safe_group():
graph = make_slice_graph([(0, 4, 1), (4, 8, 1)])
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
graph.lint()
assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0
assert (
sum(
node.target == torch.ops.aten.split_with_sizes.default
for node in graph.nodes
)
== 1
)
@pytest.mark.parametrize(
"ranges",
[
[(0, 3, 1), (4, 8, 1)], # gap
[(0, 5, 1), (4, 8, 1)], # overlap
[(1, 4, 1), (4, 8, 1)], # missing prefix
[(0, 4, 1), (4, 7, 1)], # missing suffix
[(0, 8, 1)], # single full-range slice
[(0, 8, 2)], # non-unit step
],
)
def test_slice_partition_rejects_unsafe_groups(ranges):
graph = make_slice_graph(ranges)
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
slice_count = sum(
node.target == torch.ops.aten.slice.Tensor for node in graph.nodes
)
split_count = sum(
node.target == torch.ops.aten.split_with_sizes.default for node in graph.nodes
)
assert slice_count == len(ranges)
assert split_count == 0
def test_nested_slice_partitions_use_live_source():
"""Use the rewired source when a slice partition consumes another slice."""
graph = torch.fx.Graph()
source = graph.placeholder("input")
source.meta["val"] = torch.empty((2, 8), device="meta")
outer_left = graph.call_function(
torch.ops.aten.slice.Tensor, args=(source, -1, 0, 4)
)
outer_left.meta["val"] = torch.empty((2, 4), device="meta")
outer_right = graph.call_function(
torch.ops.aten.slice.Tensor, args=(source, -1, 4, 8)
)
outer_right.meta["val"] = torch.empty((2, 4), device="meta")
inner_left = graph.call_function(
torch.ops.aten.slice.Tensor, args=(outer_left, -1, 0, 2)
)
inner_left.meta["val"] = torch.empty((2, 2), device="meta")
inner_right = graph.call_function(
torch.ops.aten.slice.Tensor, args=(outer_left, -1, 2, 4)
)
inner_right.meta["val"] = torch.empty((2, 2), device="meta")
graph.output((inner_left, inner_right, outer_right))
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
graph.lint()
assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0
splits = [
node
for node in graph.nodes
if node.target == torch.ops.aten.split_with_sizes.default
]
assert len(splits) == 2
outer_split = next(node for node in splits if list(node.args[1]) == [4, 4])
inner_split = next(node for node in splits if list(node.args[1]) == [2, 2])
assert outer_split.args[0] is source
assert inner_split.args[0].target == operator.getitem
assert inner_split.args[0].args == (outer_split, 0)
def test_split_coalescing_preserves_different_dimensions():
"""Keep equal-sized splits on different dimensions as separate nodes."""
graph = torch.fx.Graph()
source = graph.placeholder("input")
source.meta["val"] = torch.empty((8, 8), device="meta")
outputs = []
for dim in (0, 1):
split = graph.call_function(
torch.ops.aten.split_with_sizes.default,
args=(source, [4, 4], dim),
)
for index in range(2):
outputs.append(graph.call_function(operator.getitem, args=(split, index)))
graph.output(tuple(outputs))
vllm_config = make_vllm_config()
with vllm.config.set_current_vllm_config(vllm_config):
SplitCoalescingPass(vllm_config)(graph)
splits = [
node
for node in graph.nodes
if node.target == torch.ops.aten.split_with_sizes.default
]
assert len(splits) == 2
assert all(split.args[0] is source for split in splits)
assert {split.args[2] for split in splits} == {0, 1}
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.config
from tests.compile.backend import TestBackend
from vllm.compilation.passes.vllm_inductor_pass import (
VllmFusionPatternMatcherPass,
VllmPatternMatcherPass,
VllmPatternReplacement,
)
from vllm.config import CompilationConfig, CompilationMode, VllmConfig
from vllm.platforms import current_platform
class ReluToAbsPattern(VllmPatternReplacement):
"""Replaces relu(x) with abs(x) — a minimal test fixture."""
@property
def pattern(self):
def _pattern(x: torch.Tensor) -> torch.Tensor:
return torch.ops.aten.relu.default(x)
return _pattern
@property
def replacement(self):
def _replacement(x: torch.Tensor) -> torch.Tensor:
return torch.ops.aten.abs.default(x)
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
return [self.empty_fp32(4)]
class ExpToSqrtPattern(VllmPatternReplacement):
"""A second distinct pattern type — used to test uuid differentiation."""
@property
def pattern(self):
def _pattern(x: torch.Tensor) -> torch.Tensor:
return torch.ops.aten.exp.default(x)
return _pattern
@property
def replacement(self):
def _replacement(x: torch.Tensor) -> torch.Tensor:
return torch.ops.aten.sqrt.default(x)
return _replacement
def get_inputs(self) -> list[torch.Tensor]:
return [self.empty_fp32(4)]
class ReluFusionPass(VllmFusionPatternMatcherPass):
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "test_relu_fusion")
self.register(ReluToAbsPattern())
class TwoPatternFusionPass(VllmFusionPatternMatcherPass):
def __init__(self, config: VllmConfig) -> None:
super().__init__(config, "test_two_pattern_fusion")
self.register(ReluToAbsPattern())
self.register(ExpToSqrtPattern())
@pytest.fixture
def vllm_config():
return VllmConfig(
compilation_config=CompilationConfig(mode=CompilationMode.VLLM_COMPILE),
)
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Requires CUDA")
def test_register_tracks_patterns(vllm_config):
"""register() appends each VllmPatternReplacement to _pattern_replacements."""
with vllm.config.set_current_vllm_config(vllm_config):
single = ReluFusionPass(vllm_config)
two = TwoPatternFusionPass(vllm_config)
assert len(single._pattern_replacements) == 1
assert len(two._pattern_replacements) == 2
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Requires CUDA")
def test_uuid_stable(vllm_config):
"""Two instances of the same pass class produce identical uuids."""
with vllm.config.set_current_vllm_config(vllm_config):
p1 = ReluFusionPass(vllm_config)
p2 = ReluFusionPass(vllm_config)
p3 = TwoPatternFusionPass(vllm_config)
assert p1.uuid() == p2.uuid()
assert p1.uuid() != p3.uuid()
assert p2.uuid() != p3.uuid()
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Requires CUDA")
@pytest.mark.parametrize("N", [1, 2, 4])
def test_matched_count_and_match_table(vllm_config, N):
"""matched_count and match_table reflect the number of matched patterns."""
class Model(torch.nn.Module):
def forward(self, *inputs):
# N independent relus
return sum(torch.relu(x) for x in inputs)
with vllm.config.set_current_vllm_config(vllm_config):
torch.set_default_device("cuda")
torch.set_default_dtype(torch.float32)
fusion_pass = ReluFusionPass(vllm_config)
backend = TestBackend(fusion_pass)
model = torch.compile(Model(), backend=backend)
inputs = [torch.rand(8) for _ in range(N)]
model(*inputs)
assert fusion_pass.matched_count == N
assert VllmPatternMatcherPass.match_table["test_relu_fusion"] >= N
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
-752
View File
@@ -1,752 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import operator
import pytest
import torch
import torch._dynamo
import torch.fx as fx
from torch.fx.experimental.proxy_tensor import make_fx
from vllm.compilation.backends import (
_decompose_size_nodes,
_is_empty_allocation_node,
split_graph,
)
from vllm.compilation.passes.fx_utils import find_op_nodes
from vllm.platforms import current_platform
# This import automatically registers `torch.ops.silly.attention`
from . import silly_attention # noqa: F401
DEVICE_TYPE = current_platform.device_type
def test_getitem_moved_to_producer_subgraph():
"""Test that getitem operations are moved to the same subgraph as their input,
preventing tuple inputs to submodules.
"""
def model_fn(x: torch.Tensor) -> torch.Tensor:
# torch.split returns a tuple, creating real getitem operations
# Should become first submodule that produces tuple
chunks = torch.split(x, x.shape[0] // 2, dim=0)
# Following ops should become second submodule that consumes tuple
result_0 = torch.relu(chunks[0])
result_1 = torch.relu(chunks[1])
return torch.cat([result_0, result_1], dim=0)
x = torch.randn(4, 3)
gm = make_fx(model_fn)(x)
has_getitem = any(
node.op == "call_function" and node.target == operator.getitem
for node in gm.graph.nodes
)
assert has_getitem, "Test setup failed: graph should contain getitem operations"
# Split on tuple producer aten::split
split_ops = ["aten::split.Tensor"]
split_gm, split_items = split_graph(gm, split_ops)
assert len(split_items) == 2, "Graph should be split into 2 submodules"
for split_item in split_items:
submodule = split_item.graph
getitem_on_placeholder = []
for node in submodule.graph.nodes:
if (
node.op == "call_function"
and node.target == operator.getitem
and node.args[0].op == "placeholder"
):
getitem_on_placeholder.append(node)
assert len(getitem_on_placeholder) == 0, (
f"Submodule {split_item.submod_name} has getitem operations on "
f"placeholder nodes: {[n.name for n in getitem_on_placeholder]}. "
"This means tuple inputs were not properly eliminated."
)
new_x = torch.randn(4, 3)
output_original = gm(new_x)
output_split = split_gm(new_x)
assert torch.allclose(output_original, output_split), "Output mismatch"
def test_no_tuple_inputs_with_multiple_consumers():
"""Test that when a tuple is consumed by multiple split operations,
getitem operations are properly moved to avoid tuple inputs.
"""
def model_fn(x: torch.Tensor) -> torch.Tensor:
# torch.split returns a tuple, creating real getitem operations
# Should become first submodule that produces tuple
chunks = torch.split(x, x.shape[0] // 2, dim=0)
# These should become second submodule consuming tuple
result_1 = torch.relu(chunks[0])
result_2 = torch.relu(chunks[1])
# Artificial graph splitting point to create another
# independent submodule that consumes tuple later
# This would become the third submodule
result_1 = torch.sigmoid(result_1)
# Fourth submodule that consumes tuple
result = torch.cat([chunks[0], chunks[1], result_1, result_2])
return result
x = torch.randn(4, 3)
gm = make_fx(model_fn)(x)
has_getitem = any(
node.op == "call_function" and node.target == operator.getitem
for node in gm.graph.nodes
)
assert has_getitem, "Test setup failed: graph should contain getitem operations"
split_ops = ["aten::split.Tensor", "aten::sigmoid"]
split_gm, split_items = split_graph(gm, split_ops)
assert len(split_items) == 4, "Graph should be split into 4 submodules"
for split_item in split_items:
submodule = split_item.graph
for node in submodule.graph.nodes:
if (
node.op == "call_function"
and node.target == operator.getitem
and node.args[0].op == "placeholder"
):
pytest.fail(
f"Submodule {split_item.submod_name} has getitem on "
f"placeholder {node.args[0].name}, indicating it receives "
"a tuple input"
)
new_x = torch.randn(4, 3)
output_original = gm(new_x)
output_split = split_gm(new_x)
assert torch.allclose(output_original, output_split), "Output mismatch after split"
def test_consecutive_ops_in_split():
"""Test that consecutive splitting operations are grouped into the same subgraph."""
def model_fn(x: torch.Tensor) -> torch.Tensor:
"""Define a simple model where consecutive operations create opportunities
for splitting subgraphs.
"""
# Apply silly attention followed by consecutive operations
intermediate = torch.relu(x)
attn_inout = torch.sqrt(intermediate)
torch.ops.silly.attention(intermediate, intermediate, attn_inout, attn_inout)
final_result = torch.sigmoid(attn_inout)
return final_result
torch.set_default_device(DEVICE_TYPE)
# Create the traced FX graph for the model
x = torch.randn(8, 4)
gm = make_fx(model_fn)(x)
# Assert presence of the expected operations in the setup
assert (
len(list(find_op_nodes(torch.ops.aten.relu, gm.graph))) == 1
and len(list(find_op_nodes(torch.ops.aten.sqrt, gm.graph))) == 1
), "Test setup failed: Expected sqrt and relu operations in the graph."
# Configure split operations to test
splitting_ops = ["silly::attention", "aten::sqrt"]
split_gm, split_items = split_graph(gm, splitting_ops)
# Validate the number of partitions
assert len(split_items) == 3, (
"Consecutive splitting operations were not grouped correctly."
)
# Validate that correctness is preserved
new_x = torch.randn(8, 4)
output_original = gm(new_x)
output_split = split_gm(new_x)
assert torch.allclose(output_original, output_split), (
"Output mismatch after splitting."
)
# Check the splitting item has 2 nodes exactly (relu and attn)
splitting_items = list(s for s in split_items if s.is_splitting_graph)
assert len(splitting_items) == 1, "Expecting a single splitting graph"
print(splitting_items[0].graph.graph)
splitting_gm = splitting_items[0].graph
assert len(splitting_gm.graph.nodes) == 4, "Expecting 4 nodes in splitting graph"
assert [node.op for node in splitting_gm.graph.nodes] == ["placeholder"] + 2 * [
"call_function"
] + ["output"]
def _get_empty_nodes(split_item):
return [
node for node in split_item.graph.graph.nodes if _is_empty_allocation_node(node)
]
def _subgraphs_with_empty_nodes(split_items, *, is_splitting_graph):
return [
split_item
for split_item in split_items
if split_item.is_splitting_graph == is_splitting_graph
and _get_empty_nodes(split_item)
]
def test_empty_only_partition_stays_separate_after_splitting_predecessor():
"""Empty-only subgraphs should not be merged when the only predecessor is
a splitting-op subgraph.
"""
def model_fn(x: torch.Tensor) -> torch.Tensor:
y = torch.sin(x)
out = torch.empty_like(y)
torch.ops.aten.cos.out(y, out=out)
return out
x = torch.randn(4, 3)
gm = make_fx(model_fn)(x)
split_ops = ["aten::sin", "aten::cos.out"]
split_gm, split_items = split_graph(gm, split_ops)
# Graph partitioning for this pattern is:
# [sin], [empty_like], [cos.out].
assert len(split_items) == 3, (
"Empty-only partition should not merge into splitting-op subgraph"
)
splitting_with_empty = _subgraphs_with_empty_nodes(
split_items, is_splitting_graph=True
)
assert len(splitting_with_empty) == 0, (
"Splitting-op subgraphs should not contain empty allocation nodes: "
f"{[item.submod_name for item in splitting_with_empty]}"
)
output_original = gm(x)
output_split = split_gm(x)
assert torch.allclose(output_original, output_split), "Output mismatch after split"
def test_empty_only_partition_is_merged():
"""Empty-only subgraphs should still be merged when a non-splitting predecessor
exists. The merged empty node must remain outside splitting-op subgraphs.
"""
def model_fn(x: torch.Tensor) -> torch.Tensor:
base = x + 1
y = torch.sin(base)
out = torch.empty_like(base)
torch.ops.aten.cos.out(base, out=out)
return out + y
x = torch.randn(4, 3)
gm = make_fx(model_fn)(x)
split_gm, split_items = split_graph(gm, ["aten::sin", "aten::cos.out"])
# Partitioning should be:
# [add, empty_like], [sin], [cos.out], [add].
assert len(split_items) == 4, (
"Empty-only partition should be merged into non-splitting predecessor"
)
splitting_with_empty = _subgraphs_with_empty_nodes(
split_items, is_splitting_graph=True
)
assert len(splitting_with_empty) == 0, (
"Splitting-op subgraphs should not contain empty allocation nodes: "
f"{[item.submod_name for item in splitting_with_empty]}"
)
non_splitting_with_empty = _subgraphs_with_empty_nodes(
split_items, is_splitting_graph=False
)
assert len(non_splitting_with_empty) == 1, (
"Exactly one non-splitting subgraph should contain the merged empty node"
)
assert len(_get_empty_nodes(non_splitting_with_empty[0])) == 1, (
"Expected exactly one empty allocation node in merged subgraph"
)
output_original = gm(x)
output_split = split_gm(x)
assert torch.allclose(output_original, output_split), "Output mismatch after split"
def test_builtin_empty_only_partition_is_merged():
"""In Dynamo graphs, torch.empty/empty_like may appear as builtin call targets
(not aten OpOverload). Ensure empty-only partitions are still merged.
"""
def model_fn(x: torch.Tensor) -> torch.Tensor:
hidden = x + 1
out1 = torch.empty_like(hidden)
torch.ops.silly.attention(hidden, hidden, hidden, out1)
out2 = torch.empty_like(hidden)
torch.ops.silly.attention(out1, out1, hidden, out2)
return out2 + hidden
gm = torch.fx.symbolic_trace(model_fn)
split_gm, split_items = split_graph(gm, ["silly::attention"])
# Without empty-only merge, this graph would split into:
# [add, empty_like], [attention], [empty_like], [attention], [add].
assert len(split_items) == 4, "Builtin empty-only partition should be merged"
splitting_with_empty = _subgraphs_with_empty_nodes(
split_items, is_splitting_graph=True
)
assert len(splitting_with_empty) == 0, (
"Splitting-op subgraphs should not contain empty allocation nodes: "
f"{[item.submod_name for item in splitting_with_empty]}"
)
non_splitting_with_empty = _subgraphs_with_empty_nodes(
split_items, is_splitting_graph=False
)
assert len(non_splitting_with_empty) == 1, (
"Exactly one non-splitting subgraph should contain merged empty nodes"
)
assert len(_get_empty_nodes(non_splitting_with_empty[0])) == 2, (
"Expected two builtin empty_like nodes in merged non-splitting subgraph"
)
x = torch.randn(2, 3, device=DEVICE_TYPE)
output_original = gm(x)
output_split = split_gm(x)
assert torch.allclose(output_original, output_split), "Output mismatch after split"
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_sym_size_whole_shape_boundary():
"""Test that using x.size() (whole shape) across a split boundary can be
compiled by standalone_compile.
The dynamo graph looks like:
shape = x.size()
y = sigmoid(x) # split point
z = y.clone().view(shape)
Which splits into:
subgraph0(x) -> shape # returns torch.Size — problematic
subgraph1(x) -> y # sigmoid
subgraph2(y, shape) -> z # view
Two approaches to fix the torch.Size crossing:
Approach 1 — move sym_size to consumer (memory implication: x passed to
subgraph2 just for .size()):
subgraph0(x) -> # empty
subgraph1(x) -> y
subgraph2(y, x) -> z # computes shape locally from x
Approach 2 — decompose shape into individual int/SymInt values:
subgraph0(x) -> s0, val # returns individual scalars, not Size
subgraph1(x) -> y
subgraph2(y, s0, val) -> z # reconstructs view args from scalars
"""
from torch._inductor import standalone_compile
captured_graph = None
def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
nonlocal captured_graph
captured_graph = gm
return gm
def model_fn(x: torch.Tensor) -> torch.Tensor:
shape = x.size()
x = torch.ops.aten.sigmoid.default(x)
x = x.clone().view(shape)
return x
x = torch.randn(4, 8)
torch._dynamo.mark_dynamic(x, 0)
compiled_fn = torch.compile(model_fn, backend=capturing_backend)
compiled_fn(x)
split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
assert len(split_items) == 3
submod_0 = split_gm.submod_0
example_input = torch.randn(4, 8)
compiled = standalone_compile(
submod_0, [example_input, 4], dynamic_shapes="from_example_inputs"
)
assert compiled is not None
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_symint_crosses_split_boundary():
"""Test that SymInt placeholders from torch.compile + mark_dynamic
cross split boundaries safely via split_module's natural threading.
SymInt values are threaded through subgraphs by split_module and
handled correctly by inductor — no special replacement is needed.
"""
captured_graph = None
def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
nonlocal captured_graph
captured_graph = gm
return gm
def model_fn(x: torch.Tensor) -> torch.Tensor:
batch_size = x.shape[0]
hidden_size = x.shape[1]
x = torch.ops.aten.sigmoid.default(x)
x = x.clone().view(batch_size, hidden_size)
x = torch.ops.aten.sigmoid.default(x)
x = x.clone().view(batch_size, hidden_size)
x = torch.ops.aten.sigmoid.default(x)
x = x.clone().view(batch_size, hidden_size)
return x
x = torch.randn(4, 8)
torch._dynamo.mark_dynamic(x, 0)
compiled_fn = torch.compile(model_fn, backend=capturing_backend)
compiled_fn(x)
assert captured_graph is not None, "Graph should be captured by backend"
# SymInt placeholders should exist in the captured graph
symint_placeholders = [
node
for node in captured_graph.graph.nodes
if node.op == "placeholder"
and isinstance(node.meta.get("example_value"), torch.SymInt)
]
assert len(symint_placeholders) > 0, (
"Captured graph should have SymInt placeholders from mark_dynamic."
)
# split_graph should handle SymInt placeholders without error
split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
# Should have 3 splitting subgraphs (3 sigmoids)
splitting_subgraphs = [item for item in split_items if item.is_splitting_graph]
assert len(splitting_subgraphs) == 3, (
f"Expected 3 splitting subgraphs (3 sigmoids), got {len(splitting_subgraphs)}"
)
assert len(split_items) >= 6, (
f"Expected at least 6 total subgraphs, got {len(split_items)}"
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_shape_boundary_standalone_compile():
"""Repro for the original production bug:
AssertionError: out_spec mismatch
TreeSpec(tuple, None, [*, *, TreeSpec(Size, None, [*, *]), *])
vs
TreeSpec(tuple, None, [*, *, *, *])
A subgraph outputs torch.Size (e.g. torch.Size([s72, 2048])) as one of
its values when shape info crosses a split boundary. aot_autograd / inductor
expect all submodule outputs to be flat tensors or scalars, not torch.Size.
With the fix, x.size() is decomposed into individual sym_size.int calls
so only scalar SymInts cross the boundary — not the torch.Size.
"""
from torch._inductor import standalone_compile
captured_graph = None
def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
nonlocal captured_graph
captured_graph = gm
return gm
def model_fn(x: torch.Tensor) -> torch.Tensor:
shape = x.size()
x = torch.ops.aten.sigmoid.default(x)
x = x.clone().view(shape)
return x
x = torch.randn(4, 8)
torch._dynamo.mark_dynamic(x, 0)
torch.compile(model_fn, backend=capturing_backend)(x)
split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
assert len(split_items) == 3
# Verify that the consumer subgraph only has a placeholder for the dynamic
# dim (SymInt) — the static dim (8) should be inlined as a literal, not
# threaded as a placeholder.
consumer = split_items[-1] # valid since len == 3: [producer, sigmoid, consumer]
symint_placeholders = [
n
for n in consumer.graph.graph.nodes
if n.op == "placeholder"
and isinstance(n.meta.get("example_value"), torch.SymInt)
]
static_int_placeholders = [
n
for n in consumer.graph.graph.nodes
if n.op == "placeholder"
and isinstance(n.meta.get("example_value"), int)
and not isinstance(n.meta.get("example_value"), torch.SymInt)
]
assert len(symint_placeholders) >= 1, (
"Consumer should have a SymInt placeholder for the dynamic dim."
)
assert len(static_int_placeholders) == 0, (
"Static dims should be inlined as literals, not threaded as placeholders."
)
submod_0 = split_gm.submod_0
standalone_compile(
submod_0, [torch.randn(4, 8), 4], dynamic_shapes="from_example_inputs"
)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_size_used_in_multiple_consumer_subgraphs():
"""Validates that x.size() (whole shape) used by multiple downstream subgraphs
does not cause torch.Size to cross split boundaries.
Model:
shape = x.size() # whole shape — must not cross as torch.Size
z1 = sigmoid(x) # split point 1
y1 = y.view(shape) # consumer 1 uses shape
z2 = sigmoid(z1) # split point 2
y2 = y.view(shape) # consumer 2 uses shape again
Without the fix, torch.Size crosses the boundary as a submodule output,
which aot_autograd / standalone_compile rejects.
"""
captured_graph = None
captured_inputs = None
def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
nonlocal captured_graph, captured_inputs
captured_graph = gm
captured_inputs = example_inputs
return gm
def model_fn(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
shape = x.size()
z1 = torch.ops.aten.sigmoid.default(x)
y1 = y.view(shape)
z2 = torch.ops.aten.sigmoid.default(z1)
y2 = y.view(shape)
return z2 + y1 + y2
x = torch.randn(4, 8)
y = torch.randn(4, 8) # same shape as x so view(shape) doesn't specialize dim 0
torch._dynamo.mark_dynamic(x, 0)
torch._dynamo.mark_dynamic(y, 0)
torch.compile(model_fn, backend=capturing_backend)(x, y)
assert captured_graph is not None, "Graph should be captured by backend"
assert captured_inputs is not None, "Example inputs should be captured by backend"
split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
splitting_items = [item for item in split_items if item.is_splitting_graph]
assert len(splitting_items) == 2
# Verify functional correctness — fails without the fix because torch.Size
# would cross a split boundary as a submodule output
output_original = model_fn(x, y)
output_split = split_gm(*captured_inputs)
if isinstance(output_split, tuple):
output_split = next(o for o in output_split if isinstance(o, torch.Tensor))
assert torch.allclose(output_original, output_split), "Output mismatch after split"
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_sym_size_metadata_propagated():
"""Validates that new sym_size.int nodes created by the pre-pass have
example_value metadata set. Without it, placeholder metadata in consumer
subgraphs would be None, breaking any code that dynamically builds
example inputs from metadata (e.g. standalone_compile per-submodule).
"""
from torch._inductor import standalone_compile
captured_graph = None
def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
nonlocal captured_graph
captured_graph = gm
return gm
def model_fn(x: torch.Tensor) -> torch.Tensor:
shape = x.size()
x = torch.ops.aten.sigmoid.default(x)
x = x.clone().view(shape)
return x
x = torch.randn(4, 8)
torch._dynamo.mark_dynamic(x, 0)
torch.compile(model_fn, backend=capturing_backend)(x)
split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
# For each submodule, build example inputs purely from placeholder metadata.
# This fails if example_value is None on any placeholder (i.e. metadata
# was not propagated to the sym_size.int nodes we created).
for item in split_items:
submod = item.graph
example_inputs = []
for n in submod.graph.nodes:
if n.op != "placeholder":
continue
ev = n.meta.get("example_value")
assert ev is not None, (
f"Placeholder '{n.name}' in {item.submod_name} has no "
"example_value metadata. sym_size.int nodes must propagate "
"metadata so consumer subgraphs can be introspected."
)
if isinstance(ev, torch.Tensor):
example_inputs.append(torch.randn(*(int(d) for d in ev.shape)))
else:
example_inputs.append(int(ev))
standalone_compile(submod, example_inputs, dynamic_shapes="from_example_inputs")
def test_decompose_size_with_getitem_user():
"""Regression test: _decompose_size_nodes must handle getitem users of size()
correctly.
When a graph contains x.shape[i], it can appear as:
%size = call_method[target="size"](args = (%x,))
%getitem = call_function[target=operator.getitem](args = (%size, 1))
The old code spliced *all* per-dim values into every user's args
unconditionally, turning the 2-arg getitem into a malformed 3-arg node:
%getitem(args = (%sym_size_int, 5120, 1)) # TypeError at runtime
The fix detects getitem users and replaces them with dims[idx] directly.
"""
# Build a graph manually to guarantee the size() + getitem pattern.
#
# Graph:
# %x = placeholder
# %size = x.size()
# %dim1 = getitem(%size, 1) <-- the getitem branch we're testing
# %relu = relu(%x)
# %view = view(%relu, -1, %dim1)
# return %view
graph = fx.Graph()
x = graph.placeholder("x")
size_node = graph.call_method("size", args=(x,))
getitem_node = graph.call_function(operator.getitem, args=(size_node, 1))
relu_node = graph.call_function(torch.ops.aten.relu.default, args=(x,))
view_node = graph.call_function(
torch.ops.aten.view.default, args=(relu_node, [-1, getitem_node])
)
graph.output(view_node)
# Attach example_value metadata so _decompose_size_nodes can inspect dims.
# dim 0 is dynamic (SymInt), dim 1 is static (8).
from torch._dynamo.source import LocalSource
from torch._subclasses.fake_tensor import FakeTensorMode
from torch.fx.experimental.symbolic_shapes import ShapeEnv
shape_env = ShapeEnv()
src = LocalSource("batch_size")
sym_batch = shape_env.create_symintnode(shape_env.create_symbol(4, src), hint=4)
fake_mode = FakeTensorMode(shape_env=shape_env)
with fake_mode:
fake_x = torch.empty_strided((sym_batch, 8), (8, 1))
x.meta["example_value"] = fake_x
gm = fx.GraphModule(torch.nn.Module(), graph)
# Run decomposition — this would produce a 3-arg getitem without the fix
_decompose_size_nodes(gm)
# Verify no size() nodes remain
remaining_size_nodes = list(gm.graph.find_nodes(op="call_method", target="size"))
assert len(remaining_size_nodes) == 0, (
f"size() nodes should be fully decomposed, found {len(remaining_size_nodes)}"
)
# Verify no malformed getitem nodes (3+ args)
for node in gm.graph.nodes:
if node.op == "call_function" and node.target is operator.getitem:
assert len(node.args) == 2, (
f"getitem node '{node.name}' has {len(node.args)} args "
f"(expected 2): {node.args}"
)
def test_decompose_size_leaves_scalar_size_with_dim():
"""Regression test: _decompose_size_nodes must leave x.size(dim) alone.
x.size() returns a torch.Size tuple that can't cross split boundaries and
must be decomposed. x.size(dim), however, already returns a scalar
SymInt/int that crosses fine, so the pass must not touch it.
The punica LoRA path traces token_lora_mapping[:x.size(0)] under a dynamic
batch dim, so the size(0) node ends up nested inside a slice object:
%size = call_method[target="size"](args = (%x, 0))
%slice = call_function[target=getitem](
args = (%mapping, slice(None, %size, None)))
The old pass tried to decompose this scalar node too and then erase it, but
the slice still referenced it, raising "Tried to erase Node size but it
still had N users". The fix skips size calls that carry a dim argument.
"""
from torch._dynamo.source import LocalSource
from torch._subclasses.fake_tensor import FakeTensorMode
from torch.fx.experimental.symbolic_shapes import ShapeEnv
# Build graph:
# %x = placeholder
# %mapping = placeholder
# %size = x.size(0) # scalar, with a dim arg
# %sliced = mapping[slice(None, %size, None)] # size node inside a slice
graph = fx.Graph()
x = graph.placeholder("x")
mapping = graph.placeholder("token_lora_mapping")
size_node = graph.call_method("size", args=(x, 0))
sliced_node = graph.call_function(
operator.getitem,
args=(mapping, slice(None, size_node, None)),
)
graph.output((sliced_node,))
# dim 0 dynamic (SymInt) — the realistic Unsloth + LoRA case. Without the
# skip, the pass would build per-dim replacements and then crash trying to
# erase the still-referenced size node.
shape_env = ShapeEnv()
src = LocalSource("tokens")
sym_tokens = shape_env.create_symintnode(shape_env.create_symbol(4, src), hint=4)
fake_mode = FakeTensorMode(shape_env=shape_env)
with fake_mode:
fake_x = torch.empty_strided((sym_tokens, 8), (8, 1))
x.meta["example_value"] = fake_x
gm = fx.GraphModule(torch.nn.Module(), graph)
# Must not raise "Tried to erase Node ... still had N users".
_decompose_size_nodes(gm)
# The scalar x.size(0) node is left in place, untouched.
remaining = list(gm.graph.find_nodes(op="call_method", target="size"))
assert len(remaining) == 1, (
f"x.size(0) should be left untouched, found {len(remaining)} size nodes"
)
assert remaining[0].args == (x, 0), (
f"size node args changed: {remaining[0].args} (expected (x, 0))"
)
@@ -1,70 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
import vllm.envs as envs
from vllm.compilation.decorators import support_torch_compile
from vllm.config import (
CompilationConfig,
ModelConfig,
VllmConfig,
set_current_vllm_config,
)
from vllm.config.compilation import CompilationMode, CUDAGraphMode
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.platforms import current_platform
DEVICE_TYPE = current_platform.device_type
@support_torch_compile
class RotaryEmbeddingCompileModule(torch.nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
super().__init__()
self.rotary_emb = get_rope(
head_size=32,
max_position=128,
dtype=torch.float32,
rope_parameters={"rope_type": "default", "rope_theta": 10000},
is_neox_style=True,
)
def forward(
self, positions: torch.Tensor, query: torch.Tensor, key: torch.Tensor
) -> torch.Tensor:
q_rot, k_rot = self.rotary_emb(positions, query, key)
return q_rot + k_rot
@pytest.mark.skipif(current_platform.is_cpu(), reason="Requires GPU for torch.compile")
def test_rotary_embedding_torch_compile_with_custom_op(monkeypatch):
# Ensure env toggles take effect for this test only.
# The bytecode hook is required to detect buffer mutation in compiled code,
# and AOT compile bypasses that hook entirely.
envs.disable_envs_cache()
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1")
monkeypatch.setenv("VLLM_USE_AOT_COMPILE", "0")
device = DEVICE_TYPE
positions = torch.arange(16, device=device)
query = torch.randn(16, 32, device=device, dtype=torch.bfloat16)
key = torch.randn(16, 32, device=device, dtype=torch.bfloat16)
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=torch.bfloat16),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
backend="inductor",
custom_ops=["+rotary_embedding"],
cudagraph_mode=CUDAGraphMode.NONE,
cudagraph_num_of_warmups=0,
),
)
with set_current_vllm_config(vllm_config):
model = RotaryEmbeddingCompileModule(vllm_config=vllm_config)
model(positions, query, key)
assert model._compiled_bytecode is not None
assert "update" not in model._compiled_bytecode.co_names
-124
View File
@@ -1,124 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import patch
import pytest
import regex as re
import torch
from torch import nn
import tests.compile.silly_attention # noqa
from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig, set_current_vllm_config
from vllm.config.compilation import (
CompilationConfig,
CompilationMode,
CUDAGraphMode,
)
from vllm.config.scheduler import SchedulerConfig
from vllm.forward_context import set_forward_context
from vllm.platforms import current_platform
MLP_SIZE = 64
DEVICE_TYPE = current_platform.device_type
@support_torch_compile
class SimpleModel(nn.Module):
"""A simple model with a splitting op for piecewise compilation."""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "", **kwargs):
super().__init__()
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = x + x
attn_output = torch.empty_like(x)
torch.ops.silly.attention(x, x, x, attn_output)
x = attn_output * 2
return x
class TraceStructuredCapture:
"""Captures trace_structured calls for testing."""
def __init__(self):
self.calls: list[dict] = []
def __call__(self, event_type: str, metadata_fn=None, payload_fn=None, **kwargs):
"""Capture a trace_structured call."""
metadata = metadata_fn() if metadata_fn else {}
self.calls.append(
{
"event_type": event_type,
"metadata": metadata,
}
)
def get(self, event_type: str, name_pattern: str) -> list[dict]:
"""Get all calls with the given event type and name matching pattern.
Args:
event_type: The event type to filter by (e.g., "artifact", "graph_dump")
name_pattern: Regex pattern to match against the artifact name
"""
regex = re.compile(name_pattern)
return [
c
for c in self.calls
if c["event_type"] == event_type
and regex.fullmatch(c.get("metadata", {}).get("name", ""))
]
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
def test_vllm_structured_logging_artifacts(use_fresh_inductor_cache):
"""Test that all expected vLLM artifacts are logged during compilation."""
torch.set_default_device(DEVICE_TYPE)
capture = TraceStructuredCapture()
vllm_config = VllmConfig(
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
cudagraph_mode=CUDAGraphMode.PIECEWISE,
compile_sizes=[8],
splitting_ops=["silly::attention"],
),
scheduler_config=SchedulerConfig(
max_num_seqs=8,
max_model_len=8192,
is_encoder_decoder=False,
),
)
# Patch trace_structured to capture calls
with (
patch("vllm.compilation.backends.trace_structured", capture),
patch("vllm.compilation.piecewise_backend.trace_structured", capture),
set_current_vllm_config(vllm_config),
):
model = SimpleModel(vllm_config=vllm_config, prefix="test")
with set_forward_context({}, vllm_config=vllm_config):
model(torch.randn(8, MLP_SIZE))
config_artifacts = capture.get("artifact", "vllm_compilation_config")
assert len(config_artifacts) == 1, (
f"Expected 1 vllm_compilation_config, got {len(config_artifacts)}"
)
vllm_piecewise_split_graph = capture.get("graph_dump", "vllm_piecewise_split_graph")
assert len(vllm_piecewise_split_graph) == 1, (
"Expected 1 toplevel piecewise split graph, "
f"got {len(vllm_piecewise_split_graph)}"
)
compile_start_artifacts = capture.get("artifact", "vllm_piecewise_compile_start")
assert len(compile_start_artifacts) == 4, (
"Expected 4 vllm_piecewise_compile_start "
"(2 subgraphs x 2 ranges each: dynamic + compile size), "
f"got {len(compile_start_artifacts)}"
)
submod_dumps = capture.get("graph_dump", r"vllm_submod_.*")
assert len(submod_dumps) == 2, (
"Expected 2 submods (one before attention, one after attention), "
f"got {len(submod_dumps)}"
)