[Mini-vLLM] Cut dead subsystems: models, entrypoints, backends, utils, tests
- Drop dead model modules, keeping files live imports require - Remove anthropic, cohere, codegen entrypoints and batch STT modes - Delete unused attention backends, MoE experts, and NPU/XPU util leaves - Strip dead third_party trees and ROCm leaves with no CUDA path - Remove mooncake EC connectors: dependency missing since branch base - Re-audit verified by full import sweep and Qwen3.5 smoke test Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,91 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from tests.utils import compare_two_settings
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.config import CompilationMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm() or not is_aiter_found_and_supported(),
|
||||
reason="Requires ROCm AITER attention and quantization",
|
||||
)
|
||||
def test_rocm_aiter_static_attention_output_correctness(caplog_mp_spawn):
|
||||
"""Trained FP8 outputs agree when actual AITER output quantization is fused."""
|
||||
model = hf_api().snapshot_download(
|
||||
"RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
|
||||
revision="870a1177ed082c0ba5dbefd62573ab8f91803f6a",
|
||||
allow_patterns=["*.json", "*.safetensors", "*.model", "*.txt", "*.jinja"],
|
||||
)
|
||||
common_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--load-format",
|
||||
"auto",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--max-num-batched-tokens",
|
||||
"512",
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
"--kv-cache-memory-bytes",
|
||||
str(256 * 1024**2),
|
||||
"--gpu-memory-utilization",
|
||||
"0.01",
|
||||
"--attention-backend",
|
||||
"ROCM_AITER_UNIFIED_ATTN",
|
||||
]
|
||||
|
||||
def settings(enabled):
|
||||
return [
|
||||
*common_args,
|
||||
"--compilation-config",
|
||||
json.dumps(
|
||||
{
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"cudagraph_mode": "NONE",
|
||||
"splitting_ops": [],
|
||||
"inductor_compile_config": {"force_disable_caches": True},
|
||||
"custom_ops": ["+quant_fp8", "-rms_norm"],
|
||||
"pass_config": {
|
||||
"fuse_attn_quant": enabled,
|
||||
"fuse_norm_quant": False,
|
||||
"fuse_act_quant": False,
|
||||
"fuse_allreduce_rms": False,
|
||||
},
|
||||
}
|
||||
),
|
||||
]
|
||||
|
||||
env = {
|
||||
"VLLM_ROCM_USE_AITER": "1",
|
||||
"VLLM_ROCM_USE_AITER_LINEAR": "1",
|
||||
"VLLM_DISABLE_COMPILE_CACHE": "1",
|
||||
}
|
||||
with caplog_mp_spawn(logging.DEBUG) as logs:
|
||||
compare_two_settings(
|
||||
model,
|
||||
settings(False),
|
||||
settings(True),
|
||||
env1=env,
|
||||
env2=env,
|
||||
method="generate",
|
||||
# Fusing output quant drops a bf16 round-trip, so the two settings
|
||||
# agree on the argmax but not on the exact distribution. Greedy
|
||||
# decoding survives that, sampling at temperature 1.0 does not.
|
||||
include_seeded_sampling=False,
|
||||
)
|
||||
matches = [
|
||||
int(count) for count in re.findall(r"'attn_quant_fusion': (\d+)", logs.text)
|
||||
]
|
||||
print(f"Attention output quantization rewrite counts: {matches}")
|
||||
assert matches and all(count == 32 for count in matches), (
|
||||
f"Expected output quantization fused in all 32 Llama layers, found {matches}"
|
||||
)
|
||||
@@ -1,306 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import tempfile
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.models.utils import check_logprobs_close
|
||||
from vllm import SamplingParams
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import CompilationConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.config.compilation import (
|
||||
CompilationMode,
|
||||
DynamicShapesConfig,
|
||||
DynamicShapesType,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
|
||||
|
||||
def get_test_models():
|
||||
"""Get list of models to test based on PyTorch version."""
|
||||
models = [
|
||||
"Qwen/Qwen3-0.6B",
|
||||
"openai-community/gpt2",
|
||||
]
|
||||
return models
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DynamicShapesTestCase:
|
||||
model_name: str
|
||||
shapes_type: DynamicShapesType
|
||||
use_aot_compile: bool
|
||||
use_bytecode_hook: bool
|
||||
|
||||
def __str__(self) -> str:
|
||||
model_id = self.model_name.rsplit("/", 1)[-1]
|
||||
return (
|
||||
f"{model_id}-{self.shapes_type.value}-"
|
||||
f"aot{int(self.use_aot_compile)}-hook{int(self.use_bytecode_hook)}"
|
||||
)
|
||||
|
||||
|
||||
def get_dynamic_shapes_test_cases():
|
||||
"""Pairwise-cover compilation options and smoke-test model classes."""
|
||||
reference_model = "Qwen/Qwen3-0.6B"
|
||||
cases = [
|
||||
DynamicShapesTestCase(reference_model, DynamicShapesType.BACKED, False, True),
|
||||
DynamicShapesTestCase(reference_model, DynamicShapesType.BACKED, True, False),
|
||||
DynamicShapesTestCase(
|
||||
reference_model, DynamicShapesType.UNBACKED, False, False
|
||||
),
|
||||
DynamicShapesTestCase(reference_model, DynamicShapesType.UNBACKED, True, True),
|
||||
DynamicShapesTestCase(
|
||||
reference_model,
|
||||
DynamicShapesType.BACKED_SIZE_OBLIVIOUS,
|
||||
False,
|
||||
True,
|
||||
),
|
||||
DynamicShapesTestCase(
|
||||
reference_model,
|
||||
DynamicShapesType.BACKED_SIZE_OBLIVIOUS,
|
||||
True,
|
||||
False,
|
||||
),
|
||||
]
|
||||
cases.extend(
|
||||
DynamicShapesTestCase(model_name, DynamicShapesType.BACKED, False, True)
|
||||
for model_name in get_test_models()
|
||||
if model_name != reference_model
|
||||
)
|
||||
return cases
|
||||
|
||||
|
||||
TEST_PROMPTS = ["Hello, my name is", "The capital of France is"]
|
||||
|
||||
|
||||
def generate_outputs(vllm_model):
|
||||
sampling_params = SamplingParams(max_tokens=5, temperature=0, logprobs=10)
|
||||
outputs = []
|
||||
for prompt in TEST_PROMPTS:
|
||||
output = vllm_model.llm.generate(prompt, sampling_params)[0].outputs[0]
|
||||
assert len(output.text.strip()) > 0, "Model produced empty output"
|
||||
outputs.append((output.token_ids, output.text, output.logprobs))
|
||||
return outputs
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def get_eager_outputs(vllm_runner):
|
||||
cache = {}
|
||||
|
||||
def get(model_name):
|
||||
if model_name not in cache:
|
||||
with vllm_runner(
|
||||
model_name,
|
||||
enforce_eager=True,
|
||||
max_model_len=1024,
|
||||
enable_chunked_prefill=None,
|
||||
) as vllm_model:
|
||||
cache[model_name] = generate_outputs(vllm_model)
|
||||
return cache[model_name]
|
||||
|
||||
return get
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_case", get_dynamic_shapes_test_cases(), ids=str)
|
||||
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
|
||||
def test_dynamic_shapes_compilation(
|
||||
monkeypatch,
|
||||
vllm_runner,
|
||||
get_eager_outputs,
|
||||
test_case,
|
||||
):
|
||||
"""Test representative dynamic-shapes configurations end to end."""
|
||||
if (
|
||||
test_case.shapes_type == DynamicShapesType.UNBACKED
|
||||
and not is_torch_equal_or_newer("2.11.0")
|
||||
):
|
||||
# NOTE[ROCm]: shape_id (used by Qwen2/Llama to relate input dims) only
|
||||
# landed in torch 2.11, but the ROCm CI still runs torch 2.10.x. On
|
||||
# older torch there's no way to express it, so unbacked shapes go
|
||||
# data-dependent and compilation blows up -- nothing to test.
|
||||
pytest.skip("unbacked dynamic shapes with shape_id require torch>=2.11")
|
||||
|
||||
monkeypatch.setenv(
|
||||
"VLLM_USE_AOT_COMPILE", "1" if test_case.use_aot_compile else "0"
|
||||
)
|
||||
monkeypatch.setenv(
|
||||
"VLLM_USE_BYTECODE_HOOK", "1" if test_case.use_bytecode_hook else "0"
|
||||
)
|
||||
|
||||
print(f"Testing {test_case.shapes_type.name} dynamic shapes...")
|
||||
|
||||
# The compiled engine shuts down before an uncached eager baseline starts.
|
||||
with vllm_runner(
|
||||
test_case.model_name,
|
||||
compilation_config={
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"dynamic_shapes_config": {
|
||||
"type": test_case.shapes_type.value,
|
||||
},
|
||||
},
|
||||
max_model_len=1024,
|
||||
enable_chunked_prefill=None,
|
||||
) as vllm_model:
|
||||
compiled_outputs = generate_outputs(vllm_model)
|
||||
|
||||
check_logprobs_close(
|
||||
outputs_0_lst=get_eager_outputs(test_case.model_name),
|
||||
outputs_1_lst=compiled_outputs,
|
||||
name_0="eager",
|
||||
name_1="compiled",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_aot_compile", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"dynamic_shapes_type",
|
||||
[
|
||||
DynamicShapesType.BACKED,
|
||||
DynamicShapesType.BACKED_SIZE_OBLIVIOUS,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("evaluate_guards", [False, True])
|
||||
def test_model_specialization_with_evaluate_guards(
|
||||
monkeypatch, use_aot_compile, dynamic_shapes_type, evaluate_guards
|
||||
):
|
||||
"""Test that evaluate_guards correctly detects shape specialization
|
||||
violations.
|
||||
"""
|
||||
if (
|
||||
use_aot_compile
|
||||
and dynamic_shapes_type == DynamicShapesType.BACKED
|
||||
and evaluate_guards
|
||||
):
|
||||
pytest.skip("evaluate_guards for backed does not work with aot_compile=1")
|
||||
|
||||
@support_torch_compile
|
||||
class ModelWithSizeCheck(torch.nn.Module):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
# This will cause specialization - torch.compile will guard on
|
||||
# sx.shape[0]
|
||||
if x.shape[0] >= 10:
|
||||
return x * 10
|
||||
else:
|
||||
return x * 10
|
||||
|
||||
@support_torch_compile
|
||||
class ModelWithOneSizeCheck(torch.nn.Module):
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__()
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
# This will cause 0/1 specializations.
|
||||
if x.shape[0] == 0:
|
||||
return x * 10
|
||||
if x.shape[0] == 1:
|
||||
return x * 10
|
||||
else:
|
||||
return x * 10
|
||||
|
||||
@contextmanager
|
||||
def use_vllm_config(vllm_config: VllmConfig):
|
||||
with set_forward_context({}, vllm_config), set_current_vllm_config(vllm_config):
|
||||
yield
|
||||
|
||||
monkeypatch.setenv("TOKENIZERS_PARALLELISM", "true")
|
||||
monkeypatch.setenv("VLLM_USE_AOT_COMPILE", "1" if use_aot_compile else "0")
|
||||
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "0")
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
dynamic_shapes_config=DynamicShapesConfig(
|
||||
type=dynamic_shapes_type,
|
||||
evaluate_guards=evaluate_guards,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
def test(model_class, input1, input2, is_01_specialization=False):
|
||||
with (
|
||||
torch.no_grad(),
|
||||
use_vllm_config(vllm_config),
|
||||
tempfile.TemporaryDirectory() as tmpdirname,
|
||||
):
|
||||
monkeypatch.setenv("VLLM_CACHE_ROOT", tmpdirname)
|
||||
|
||||
model = model_class(vllm_config=vllm_config).cuda()
|
||||
|
||||
model(input1)
|
||||
|
||||
if evaluate_guards and (
|
||||
not (
|
||||
is_01_specialization
|
||||
and dynamic_shapes_type == DynamicShapesType.BACKED
|
||||
)
|
||||
):
|
||||
# This should fail because guards were added.
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
model(input2)
|
||||
|
||||
# Expected failure - guard was violated
|
||||
error_msg = str(excinfo.value)
|
||||
assert (
|
||||
"GuardManager check failed" in error_msg
|
||||
or "Detected recompile when torch.compile stance" in error_msg
|
||||
), error_msg
|
||||
|
||||
else:
|
||||
model(input2)
|
||||
|
||||
test(ModelWithSizeCheck, torch.randn(20, 10).cuda(), torch.randn(5, 10).cuda())
|
||||
test(ModelWithSizeCheck, torch.randn(5, 10).cuda(), torch.randn(20, 10).cuda())
|
||||
test(
|
||||
ModelWithOneSizeCheck,
|
||||
torch.randn(20, 10).cuda(),
|
||||
torch.randn(1, 10).cuda(),
|
||||
is_01_specialization=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not is_torch_equal_or_newer("2.10.0"), reason="requires torch 2.10")
|
||||
def test_piecewise_backend_empty_sym_shape_indices(vllm_runner):
|
||||
"""Test that PiecewiseBackend handles empty sym_shape_indices correctly.
|
||||
|
||||
When all inputs have static shapes (no torch.SymInt), sym_shape_indices
|
||||
will be empty. The fix in PiecewiseBackend.__call__ handles this case
|
||||
by using the first compiled range_entry.
|
||||
"""
|
||||
# Use small max_model_len and max_num_batched_tokens to encourage
|
||||
# static shape compilation with empty sym_shape_indices
|
||||
with vllm_runner(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
max_model_len=512,
|
||||
max_num_batched_tokens=1,
|
||||
enable_chunked_prefill=None,
|
||||
compilation_config={
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"dynamic_shapes_config": {
|
||||
"type": DynamicShapesType.BACKED.value,
|
||||
},
|
||||
},
|
||||
) as vllm_model:
|
||||
sampling_params = SamplingParams(temperature=0, top_p=0.95, max_tokens=10)
|
||||
|
||||
# Generate with static shape inputs
|
||||
output = vllm_model.llm.generate(
|
||||
"Hello, my name is", sampling_params=sampling_params
|
||||
)
|
||||
result = output[0].outputs[0].text
|
||||
assert len(result) > 0, "Should generate non-empty output"
|
||||
|
||||
# Generate again to verify compilation works with empty sym_shape_indices
|
||||
output = vllm_model.llm.generate(
|
||||
"The capital of France is", sampling_params=sampling_params
|
||||
)
|
||||
result = output[0].outputs[0].text
|
||||
assert len(result) > 0, "Should generate non-empty output on second run"
|
||||
@@ -1,117 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import CompilationMode
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import compare_all_settings
|
||||
|
||||
ATTN_BACKEND = "FLASH_ATTN" if not current_platform.is_rocm() else "ROCM_ATTN"
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class TestSetting:
|
||||
model: str
|
||||
model_args: list[str]
|
||||
pp_size: int
|
||||
tp_size: int
|
||||
attn_backend: str
|
||||
method: str
|
||||
|
||||
|
||||
# we cannot afford testing the full Cartesian product
|
||||
# of all models and all modes
|
||||
@pytest.mark.parametrize(
|
||||
"test_setting",
|
||||
[
|
||||
# MoE model
|
||||
TestSetting(
|
||||
model="ibm-granite/granite-3.0-1b-a400m-instruct",
|
||||
model_args=["--max-model-len", "2048"],
|
||||
pp_size=1,
|
||||
tp_size=1,
|
||||
attn_backend=ATTN_BACKEND,
|
||||
method="generate",
|
||||
),
|
||||
# embedding model
|
||||
TestSetting(
|
||||
model="BAAI/bge-multilingual-gemma2",
|
||||
model_args=[
|
||||
"--runner",
|
||||
"pooling",
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"2048",
|
||||
"--gpu-memory-utilization",
|
||||
"0.98",
|
||||
],
|
||||
pp_size=1,
|
||||
tp_size=1,
|
||||
attn_backend=ATTN_BACKEND,
|
||||
method="encode",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("use_v2_model_runner", [False, True])
|
||||
def test_compile_correctness(
|
||||
test_setting: TestSetting,
|
||||
use_v2_model_runner: bool,
|
||||
):
|
||||
model = test_setting.model
|
||||
model_args = test_setting.model_args
|
||||
pp_size = test_setting.pp_size
|
||||
tp_size = test_setting.tp_size
|
||||
attn_backend = test_setting.attn_backend
|
||||
method = test_setting.method
|
||||
final_args = [
|
||||
*model_args,
|
||||
"-pp",
|
||||
str(pp_size),
|
||||
"-tp",
|
||||
str(tp_size),
|
||||
"-cc.cudagraph_mode=none",
|
||||
f"--attention-backend={attn_backend}",
|
||||
]
|
||||
|
||||
# Pin every compared setting to the same model runner so that only the
|
||||
# compilation mode differs within a comparison.
|
||||
runner_env = {"VLLM_USE_V2_MODEL_RUNNER": str(int(use_v2_model_runner))}
|
||||
|
||||
all_args: list[list[str]] = []
|
||||
all_envs: list[dict[str, str] | None] = []
|
||||
|
||||
# Test all compilation modes with inductor backend
|
||||
for mode in [
|
||||
CompilationMode.NONE,
|
||||
CompilationMode.STOCK_TORCH_COMPILE,
|
||||
CompilationMode.DYNAMO_TRACE_ONCE,
|
||||
CompilationMode.VLLM_COMPILE,
|
||||
]:
|
||||
all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=inductor"])
|
||||
all_envs.append(dict(runner_env))
|
||||
# inductor will change the output, so we only compare if the output
|
||||
# is close, not exactly the same.
|
||||
compare_all_settings(
|
||||
model,
|
||||
all_args,
|
||||
all_envs,
|
||||
method=method if method != "generate" else "generate_close",
|
||||
)
|
||||
|
||||
all_envs.clear()
|
||||
all_args.clear()
|
||||
|
||||
# Test all compilation modes with eager backend
|
||||
for mode in [
|
||||
CompilationMode.NONE,
|
||||
CompilationMode.STOCK_TORCH_COMPILE,
|
||||
CompilationMode.DYNAMO_TRACE_ONCE,
|
||||
CompilationMode.VLLM_COMPILE,
|
||||
]:
|
||||
all_args.append(final_args + [f"-cc.mode={mode.name}", "-cc.backend=eager"])
|
||||
all_envs.append(dict(runner_env))
|
||||
compare_all_settings(model, all_args, all_envs, method=method)
|
||||
@@ -1,191 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import contextlib
|
||||
import os
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.utils import wait_for_gpu_memory_to_clear
|
||||
from tests.v1.attention.utils import full_cg_backend_configs as backend_configs
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import CompilationConfig, CUDAGraphMode
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def temporary_environ(env_vars):
|
||||
"""Temporarily set environment variables and restore them afterward.
|
||||
We have to do this vs monkeypatch because monkeypatch doesn't work
|
||||
with "module" scoped fixtures.
|
||||
"""
|
||||
original_env = {k: os.environ.get(k) for k in env_vars}
|
||||
try:
|
||||
os.environ.update(env_vars)
|
||||
yield
|
||||
finally:
|
||||
for k, v in original_env.items():
|
||||
if v is None:
|
||||
os.environ.pop(k, None)
|
||||
else:
|
||||
os.environ[k] = v
|
||||
|
||||
|
||||
model_backends_full_cudagraph = []
|
||||
|
||||
# deepseek-ai/DeepSeek-V2-Lite with MLA
|
||||
MLA_backends = ["FlashMLA", "FlashAttentionMLA", "CutlassMLA", "FlashInferMLA"]
|
||||
for mla_backend in MLA_backends:
|
||||
model_backends_full_cudagraph.append(
|
||||
("deepseek-ai/DeepSeek-V2-Lite", backend_configs[mla_backend])
|
||||
)
|
||||
|
||||
# Qwen/Qwen2-1.5B-Instruct with other backends
|
||||
other_backend_configs = [
|
||||
backend_configs[c] for c in backend_configs if c not in MLA_backends
|
||||
]
|
||||
for backend_config in other_backend_configs:
|
||||
model_backends_full_cudagraph.append(("Qwen/Qwen2-1.5B-Instruct", backend_config))
|
||||
|
||||
|
||||
@pytest.fixture(scope="class")
|
||||
def llm_pair(request):
|
||||
model, backend_config, use_inductor_graph_partition = request.param
|
||||
if use_inductor_graph_partition and not is_torch_equal_or_newer("2.9.0.dev"):
|
||||
pytest.skip("Inductor graph partition only supported in torch>=2.9")
|
||||
|
||||
backend = AttentionBackendEnum[backend_config.attention_config["backend"]]
|
||||
if current_platform.is_rocm() and backend not in (
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
AttentionBackendEnum.ROCM_ATTN,
|
||||
):
|
||||
pytest.skip(f"{backend_config.name} is a CUDA-only attention backend")
|
||||
if backend == AttentionBackendEnum.ROCM_ATTN and not current_platform.is_rocm():
|
||||
pytest.skip("ROCM_ATTN requires ROCm")
|
||||
|
||||
# Dynamically skip test if GPU capability is not met
|
||||
if backend_config.specific_gpu_arch and (
|
||||
not current_platform.is_cuda()
|
||||
or backend_config.specific_gpu_arch != current_platform.get_device_capability()
|
||||
):
|
||||
if backend_config.specific_gpu_arch == (9, 0):
|
||||
pytest.skip("Only Hopper GPUs support FA3 and FlashMLA")
|
||||
elif backend_config.specific_gpu_arch == (10, 0):
|
||||
pytest.skip("Only Blackwell GPUs support Cutlass MLA")
|
||||
|
||||
env_vars = {
|
||||
# Force native sampler to avoid potential nondeterminism in FlashInfer
|
||||
# when per-request generators are not used in V1.
|
||||
"VLLM_USE_FLASHINFER_SAMPLER": "0",
|
||||
}
|
||||
with temporary_environ(env_vars):
|
||||
full = LLM(
|
||||
model=model,
|
||||
gpu_memory_utilization=0.43,
|
||||
trust_remote_code=True,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=128,
|
||||
attention_config=backend_config.attention_config,
|
||||
compilation_config=CompilationConfig(
|
||||
**backend_config.comp_config,
|
||||
use_inductor_graph_partition=use_inductor_graph_partition,
|
||||
),
|
||||
generation_config="vllm",
|
||||
seed=42,
|
||||
)
|
||||
piecewise = LLM(
|
||||
model=model,
|
||||
gpu_memory_utilization=0.43,
|
||||
trust_remote_code=True,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=128,
|
||||
attention_config=backend_config.attention_config,
|
||||
compilation_config=CompilationConfig(
|
||||
cudagraph_mode=CUDAGraphMode.PIECEWISE,
|
||||
use_inductor_graph_partition=use_inductor_graph_partition,
|
||||
),
|
||||
generation_config="vllm",
|
||||
seed=42,
|
||||
)
|
||||
|
||||
# PyTest caches the fixture values so we use weakref.proxy to enable GC
|
||||
yield weakref.proxy(full), weakref.proxy(piecewise)
|
||||
del full
|
||||
del piecewise
|
||||
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=[0],
|
||||
threshold_ratio=0.1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"llm_pair",
|
||||
[
|
||||
pytest.param(
|
||||
(model, backend_config, use_inductor_graph_partition),
|
||||
id=f"{backend_config.name}-partition={use_inductor_graph_partition}",
|
||||
)
|
||||
for model, backend_config in model_backends_full_cudagraph
|
||||
for use_inductor_graph_partition in [True, False]
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
class TestFullCUDAGraph:
|
||||
"""Use a class such that an llm pair is constructed once for all
|
||||
batch_size/max_tokens combinations and released immediately after.
|
||||
|
||||
Module-scope fixtures would stick around the whole time,
|
||||
meaning there would be multiple LLM instances hogging memory simultaneously.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("batch_size", "max_tokens"),
|
||||
[
|
||||
(1, 10),
|
||||
(7, 10),
|
||||
(16, 10),
|
||||
(25, 10),
|
||||
(32, 10),
|
||||
(45, 10),
|
||||
(64, 10),
|
||||
(123, 10),
|
||||
(8, 5),
|
||||
(8, 30),
|
||||
],
|
||||
)
|
||||
def test_full_cudagraph(self, batch_size, max_tokens, llm_pair: tuple[LLM, LLM]):
|
||||
"""Test various batch sizes and max_tokens to ensure that the
|
||||
full cudagraph compilation works for padded cases too.
|
||||
"""
|
||||
full_cudagraph_llm, piecewise_llm = llm_pair
|
||||
|
||||
prompts = ["the quick brown fox"] * batch_size
|
||||
# Use purely greedy decoding to avoid top-p truncation sensitivity
|
||||
# that can amplify tiny numeric differences across runtimes.
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0.0, max_tokens=max_tokens, top_p=1.0
|
||||
)
|
||||
|
||||
responses = []
|
||||
for llm in (piecewise_llm, full_cudagraph_llm):
|
||||
# Queue the whole batch before scheduling so both engines exercise
|
||||
# the requested batch size instead of timing-dependent sub-batches.
|
||||
llm.sleep(level=0, mode="keep")
|
||||
try:
|
||||
llm.enqueue(prompts, sampling_params)
|
||||
finally:
|
||||
llm.wake_up(tags=["scheduling"])
|
||||
responses.append(llm.wait_for_completion(output_type=RequestOutput))
|
||||
piecewise_responses, full_responses = responses
|
||||
|
||||
# Check that all responses are the same
|
||||
assert len(piecewise_responses) == len(full_responses) == batch_size
|
||||
for piecewise_res, full_res in zip(piecewise_responses, full_responses):
|
||||
assert (
|
||||
piecewise_res.outputs[0].text.lower()
|
||||
== full_res.outputs[0].text.lower()
|
||||
)
|
||||
@@ -1,201 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.quantization.utils import is_quant_method_supported
|
||||
from vllm import LLM, SamplingParams
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
CUDAGraphMode,
|
||||
PassConfig,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
|
||||
from ...utils import create_new_process_for_each_test, wait_for_memory_to_settle
|
||||
|
||||
|
||||
def models_list(*, all: bool = True, keywords: list[str] | None = None):
|
||||
TEST_MODELS: list[tuple[str, dict[str, Any]]] = [
|
||||
("facebook/opt-125m", {}),
|
||||
(
|
||||
"RedHatAI/Llama-3.2-1B-Instruct-FP8-dynamic",
|
||||
{"dtype": torch.float16},
|
||||
),
|
||||
]
|
||||
|
||||
if all:
|
||||
if is_quant_method_supported("gptq_marlin"):
|
||||
TEST_MODELS.append(
|
||||
(
|
||||
"LnL-AI/TinyLlama-1.1B-Chat-v1.0-GPTQ-4bit",
|
||||
{"quantization": "gptq_marlin"},
|
||||
)
|
||||
)
|
||||
|
||||
if not current_platform.is_rocm() and is_quant_method_supported("awq"):
|
||||
TEST_MODELS.append(
|
||||
("TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", {"quantization": "AWQ"})
|
||||
)
|
||||
|
||||
if keywords is None:
|
||||
return TEST_MODELS
|
||||
|
||||
# filter by keywords
|
||||
pred = lambda model: any(keyword in model[0] for keyword in keywords)
|
||||
return list(filter(pred, TEST_MODELS))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"compilation_mode",
|
||||
[CompilationMode.DYNAMO_TRACE_ONCE, CompilationMode.VLLM_COMPILE],
|
||||
)
|
||||
@pytest.mark.parametrize("model, model_kwargs", models_list(all=True))
|
||||
@create_new_process_for_each_test()
|
||||
def test_full_graph(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model: str,
|
||||
model_kwargs: dict[str, Any],
|
||||
compilation_mode: CompilationMode,
|
||||
):
|
||||
with monkeypatch.context():
|
||||
print(f"MODEL={model}")
|
||||
|
||||
run_model(compilation_mode, model, **model_kwargs)
|
||||
|
||||
|
||||
# TODO(luka) add other supported compilation config scenarios here
|
||||
@pytest.mark.parametrize(
|
||||
"compilation_config, model, model_kwargs",
|
||||
[
|
||||
# additional compile sizes, only some of the models
|
||||
(
|
||||
CompilationConfig(mode=CompilationMode.VLLM_COMPILE, compile_sizes=[1, 2]),
|
||||
*model_info,
|
||||
)
|
||||
for model_info in models_list(all=False)
|
||||
]
|
||||
+ [
|
||||
# RMSNorm + quant fusion, only 8-bit quant models
|
||||
(
|
||||
CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=["+rms_norm"],
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=True, fuse_act_quant=True, eliminate_noops=True
|
||||
),
|
||||
),
|
||||
*model_info,
|
||||
)
|
||||
for model_info in models_list(keywords=["FP8-dynamic"])
|
||||
]
|
||||
+ [
|
||||
# Test depyf integration works
|
||||
(
|
||||
CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
debug_dump_path=Path(tempfile.gettempdir()),
|
||||
),
|
||||
"facebook/opt-125m",
|
||||
{},
|
||||
),
|
||||
]
|
||||
+ [
|
||||
# graph inductor partition
|
||||
(
|
||||
CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
# inductor graph partition uses
|
||||
# torch._C.Tag.cudagraph_unsafe to specify splitting ops
|
||||
use_inductor_graph_partition=True,
|
||||
cudagraph_mode=CUDAGraphMode.PIECEWISE,
|
||||
compile_sizes=[1, 2],
|
||||
),
|
||||
*model_info,
|
||||
)
|
||||
for model_info in models_list(all=False)
|
||||
if is_torch_equal_or_newer("2.9.0.dev")
|
||||
]
|
||||
+ [
|
||||
# Test get_raw_stream patch with compile_sizes
|
||||
# This tests that TorchInductor autotune works correctly with get_raw_stream
|
||||
# patch in torch 2.9 and without patch in torch 2.10+
|
||||
(
|
||||
CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
compile_sizes=[1, 2], # Triggers autotune which uses get_raw_stream
|
||||
cudagraph_mode=CUDAGraphMode.NONE,
|
||||
),
|
||||
"facebook/opt-125m",
|
||||
{},
|
||||
),
|
||||
],
|
||||
)
|
||||
# only test some of the models
|
||||
@create_new_process_for_each_test()
|
||||
def test_custom_compile_config(
|
||||
compilation_config: CompilationConfig,
|
||||
model: str,
|
||||
model_kwargs: dict[str, Any],
|
||||
):
|
||||
if compilation_config.use_inductor_graph_partition and not is_torch_equal_or_newer(
|
||||
"2.9.0.dev"
|
||||
):
|
||||
pytest.skip("inductor graph partition is only available in PyTorch 2.9+")
|
||||
|
||||
print(f"MODEL={model}")
|
||||
run_model(compilation_config, model, **model_kwargs)
|
||||
|
||||
|
||||
def run_model(
|
||||
compile_config: CompilationMode | CompilationConfig, model: str, **model_kwargs
|
||||
):
|
||||
compilation_config = (
|
||||
compile_config
|
||||
if isinstance(compile_config, CompilationConfig)
|
||||
else CompilationConfig(mode=compile_config)
|
||||
)
|
||||
|
||||
prompts = [
|
||||
"Hello, my name is",
|
||||
"The president of the United States is",
|
||||
"The capital of France is",
|
||||
"The future of AI is",
|
||||
]
|
||||
sampling_params = SamplingParams(temperature=0)
|
||||
# Allow override from model_kwargs
|
||||
model_kwargs = {"tensor_parallel_size": 1, **model_kwargs}
|
||||
model_kwargs = {"disable_custom_all_reduce": True, **model_kwargs}
|
||||
|
||||
# No cudagraphs by default
|
||||
if compilation_config.cudagraph_mode is None:
|
||||
compilation_config.cudagraph_mode = CUDAGraphMode.NONE
|
||||
|
||||
# V1 startup requires free_memory >= total * gpu_memory_utilization, so a
|
||||
# device that is not free yet fails engine init rather than waiting for it.
|
||||
# VllmRunner guards this; building LLM directly does not. No-op off ROCm.
|
||||
wait_for_memory_to_settle(
|
||||
threshold_ratio=1.0
|
||||
- model_kwargs.get("gpu_memory_utilization", CacheConfig.gpu_memory_utilization)
|
||||
)
|
||||
|
||||
llm = LLM(
|
||||
model=model,
|
||||
compilation_config=compilation_config,
|
||||
**model_kwargs,
|
||||
)
|
||||
outputs = llm.generate(prompts, sampling_params)
|
||||
|
||||
# Print the outputs.
|
||||
for output in outputs:
|
||||
prompt = output.prompt
|
||||
generated_text = output.outputs[0].text
|
||||
print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
|
||||
@@ -1,184 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.models.utils import dummy_hf_overrides
|
||||
from vllm.compilation.counter import compilation_counter
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
def test_compile():
|
||||
vllm_config = VllmConfig()
|
||||
# Default configuration does not compile mm encoder
|
||||
assert not vllm_config.compilation_config.compile_mm_encoder
|
||||
|
||||
|
||||
@pytest.mark.forked
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
@pytest.mark.parametrize(
|
||||
("model", "tokenizer", "config_format", "tokenizer_mode", "model_arch"),
|
||||
[
|
||||
pytest.param(
|
||||
"mistralai/Pixtral-12B-2409",
|
||||
None,
|
||||
"mistral",
|
||||
"mistral",
|
||||
"PixtralForConditionalGeneration",
|
||||
id="pixtral",
|
||||
),
|
||||
pytest.param(
|
||||
"mistralai/Mistral-Small-3.1-24B-Instruct-2503",
|
||||
"mistral-community/pixtral-12b",
|
||||
"hf",
|
||||
"auto",
|
||||
"Mistral3ForConditionalGeneration",
|
||||
id="mistral3-hf",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("compile_mm_encoder", "expected_models"),
|
||||
[
|
||||
# The dummy model has two compiled text components. Enabling encoder
|
||||
# compilation adds its single vision transformer block.
|
||||
pytest.param(False, 2, id="disabled"),
|
||||
pytest.param(True, 3, id="enabled"),
|
||||
],
|
||||
)
|
||||
def test_pixtral_compilation(
|
||||
vllm_runner,
|
||||
monkeypatch,
|
||||
model: str,
|
||||
tokenizer: str | None,
|
||||
config_format: str,
|
||||
tokenizer_mode: str,
|
||||
model_arch: str,
|
||||
compile_mm_encoder: bool,
|
||||
expected_models: int,
|
||||
):
|
||||
"""Test compilation for both Pixtral vision encoder implementations."""
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
monkeypatch.setenv("VLLM_USE_FLASHINFER_SAMPLER", "0")
|
||||
|
||||
with (
|
||||
compilation_counter.expect(num_models_seen=expected_models),
|
||||
vllm_runner(
|
||||
model,
|
||||
tokenizer_name=tokenizer,
|
||||
tokenizer_mode=tokenizer_mode,
|
||||
config_format=config_format,
|
||||
load_format="dummy",
|
||||
hf_overrides=partial(dummy_hf_overrides, model_arch=model_arch),
|
||||
max_model_len=4096,
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
gpu_memory_utilization=0.8,
|
||||
attention_backend="FLASH_ATTN",
|
||||
compilation_config={
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"compile_mm_encoder": compile_mm_encoder,
|
||||
"cudagraph_mode": CUDAGraphMode.NONE,
|
||||
},
|
||||
) as _,
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
|
||||
@pytest.mark.forked
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
def test_qwen2_5_vl_compilation(vllm_runner, monkeypatch):
|
||||
"""Test that Qwen2.5-VL vision submodules are compiled.
|
||||
|
||||
This test verifies that the 3 vision submodules (Qwen2_5_VisionPatchEmbed,
|
||||
Qwen2_5_VisionBlock, and Qwen2_5_VisionPatchMerger) are properly tagged
|
||||
for compilation by checking that num_models_seen increases by at least 3.
|
||||
"""
|
||||
# Disable multiprocessing so that the counter is in the same process
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
with (
|
||||
# NOTE: Qwen2.5-VL has 35 models in total - the LLM backend
|
||||
# Vision Patch Embed, Vision Patch Merger, and then 32 Vision Blocks
|
||||
# (one for each layer) - in the future, we should fix vLLM compilation
|
||||
# logic to handle this case and only compile the Vision submodules once
|
||||
# and reuse the compiled code for all layers
|
||||
# See https://github.com/vllm-project/vllm/issues/27590
|
||||
compilation_counter.expect(num_models_seen=35),
|
||||
vllm_runner(
|
||||
"Qwen/Qwen2.5-VL-3B-Instruct",
|
||||
max_model_len=2048,
|
||||
gpu_memory_utilization=0.8,
|
||||
compilation_config={
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"compile_mm_encoder": True,
|
||||
},
|
||||
) as _,
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
|
||||
@pytest.mark.forked
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Skip if not cuda")
|
||||
def test_qwen2_5_vl_no_vit_compilation(vllm_runner, monkeypatch):
|
||||
"""Test that Qwen2.5-VL vision submodules are not compiled when the
|
||||
config is passed off
|
||||
"""
|
||||
# Disable multiprocessing so that the counter is in the same process
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
with (
|
||||
compilation_counter.expect(num_models_seen=1),
|
||||
vllm_runner(
|
||||
"Qwen/Qwen2.5-VL-3B-Instruct",
|
||||
max_model_len=2048,
|
||||
gpu_memory_utilization=0.8,
|
||||
compilation_config={
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"compile_mm_encoder": False,
|
||||
},
|
||||
) as _,
|
||||
):
|
||||
pass
|
||||
|
||||
|
||||
# forked needed to workaround https://github.com/vllm-project/vllm/issues/21073
|
||||
# Requires Cuda and 8 gpus as well
|
||||
@pytest.mark.forked
|
||||
@pytest.mark.skip(reason="Skipping due to CI resource constraints")
|
||||
def test_mllama4_vit_compilation(vllm_runner, monkeypatch):
|
||||
"""Test that Mllama4 vision submodules are compiled.
|
||||
|
||||
This test verifies that the 2 vision submodules (Llama4VisionEncoder,
|
||||
Llama4VisionPixelShuffleMLP) are properly tagged
|
||||
for compilation by checking that num_models_seen increases to 3.
|
||||
|
||||
However since we are using TP=8, we compilation_counter will not
|
||||
work properly so we will just check the run succeeds rn
|
||||
"""
|
||||
# Disable multiprocessing so that the counter is in the same process
|
||||
monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
|
||||
|
||||
with (
|
||||
monkeypatch.context(),
|
||||
# TODO: Since we require TP=8, this messes with the compilation
|
||||
# counter. We should fix this in the future, but leave for now
|
||||
# to make sure that compilation runs (no crash) with llama vision encoder
|
||||
compilation_counter.expect(num_models_seen=0),
|
||||
vllm_runner(
|
||||
"meta-llama/Llama-4-Scout-17B-16E-Instruct",
|
||||
max_model_len=512,
|
||||
gpu_memory_utilization=0.8,
|
||||
tensor_parallel_size=8,
|
||||
compilation_config={
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"compile_mm_encoder": True,
|
||||
},
|
||||
),
|
||||
):
|
||||
pass
|
||||
@@ -1,144 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import itertools
|
||||
from collections.abc import Callable, Iterable
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
|
||||
class Matches(NamedTuple):
|
||||
# simple pointwise
|
||||
aiter_rms_quant_fusion: int = 0
|
||||
rms_quant_fusion: int = 0
|
||||
act_quant_fusion: int = 0
|
||||
norm_rope_fusion: int = 0
|
||||
attn_quant_fusion: int = 0
|
||||
# distributed
|
||||
ar_rms_fusion: int = 0
|
||||
aiter_ar_rms_fusion: int = 0
|
||||
|
||||
|
||||
class ModelFusionInfo(NamedTuple):
|
||||
model_name: str
|
||||
matches: Callable[[int], Matches]
|
||||
"""Given number of hidden layers, produces the matches object"""
|
||||
model_kwargs: dict[str, Any] = {}
|
||||
hf_overrides: Callable[[int], dict] = lambda n: {"num_hidden_layers": n}
|
||||
|
||||
|
||||
class AttentionBackendCase(NamedTuple):
|
||||
backend: AttentionBackendEnum
|
||||
model_kwargs: dict[str, Any] = {}
|
||||
"""Additional args required for attn+quant fusion"""
|
||||
|
||||
|
||||
is_blackwell = lambda: current_platform.is_device_capability_family(100)
|
||||
"""Are we running on Blackwell, a lot of tests depend on it"""
|
||||
|
||||
|
||||
def nvfp4_kernel_exposes_input_quant_key() -> bool:
|
||||
"""Check if the NVFP4 kernel selected on this platform exposes input_quant_key.
|
||||
|
||||
FlashInferCuteDslNvFp4LinearKernel does not expose input_quant_key() due to
|
||||
layout incompatibility between the manual fusion kernel (silu_and_mul_nvfp4_quant)
|
||||
output format and the cutedsl backend's expected input format.
|
||||
|
||||
FlashInferCutlassNvFp4LinearKernel does expose input_quant_key() and supports
|
||||
manual fusion.
|
||||
"""
|
||||
if not current_platform.is_cuda():
|
||||
return False
|
||||
|
||||
try:
|
||||
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferCuteDslNvFp4LinearKernel,
|
||||
)
|
||||
|
||||
is_supported, _ = FlashInferCuteDslNvFp4LinearKernel.is_supported()
|
||||
if is_supported:
|
||||
return False
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
try:
|
||||
from vllm.model_executor.kernels.linear.nvfp4.flashinfer import (
|
||||
FlashInferCutlassNvFp4LinearKernel,
|
||||
)
|
||||
|
||||
is_supported, _ = FlashInferCutlassNvFp4LinearKernel.is_supported()
|
||||
if is_supported:
|
||||
return True
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Fallback: assume no manual fusion support
|
||||
return False
|
||||
|
||||
|
||||
def custom_ops_combos(*custom_ops: str) -> Iterable[str]:
|
||||
"""Generate all combinations of custom ops for parametrization."""
|
||||
custom_ops_lists = [[f"-{op}", f"+{op}"] for op in custom_ops]
|
||||
for op_list in itertools.product(*custom_ops_lists):
|
||||
yield ",".join(op_list)
|
||||
|
||||
|
||||
# Quick inline validation
|
||||
assert list(custom_ops_combos("silu_and_mul")) == ["-silu_and_mul", "+silu_and_mul"]
|
||||
assert list(custom_ops_combos("quant_fp8", "rms_norm")) == [
|
||||
"-quant_fp8,-rms_norm",
|
||||
"-quant_fp8,+rms_norm",
|
||||
"+quant_fp8,-rms_norm",
|
||||
"+quant_fp8,+rms_norm",
|
||||
]
|
||||
|
||||
|
||||
def has_cuda_graph_wrapper_metadata() -> bool:
|
||||
from importlib import import_module
|
||||
|
||||
try:
|
||||
module = import_module("torch._inductor.utils")
|
||||
module.CUDAGraphWrapperMetadata # noqa B018
|
||||
except AttributeError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
INDUCTOR_GRAPH_PARTITION = [
|
||||
pytest.param(
|
||||
True,
|
||||
marks=pytest.mark.skipif(
|
||||
not has_cuda_graph_wrapper_metadata(),
|
||||
reason="torch version does not support Inductor partition",
|
||||
),
|
||||
id="inductor_partition",
|
||||
),
|
||||
pytest.param(False, id="dynamo_partition"),
|
||||
]
|
||||
|
||||
FUSION_LOG_PATTERNS: dict[str, re.Pattern] = {
|
||||
"aiter_rms_quant_fusion": re.compile(
|
||||
r"RocmAiterRMSNormQuantFusionPass Replaced (\d+) patterns"
|
||||
),
|
||||
"rms_quant_fusion": re.compile(r"rms_quant_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
"act_quant_fusion": re.compile(r"act_quant_fusion.py:\d+] Replaced (\d+) patterns"),
|
||||
# The combined pass can consume these sites before the standalone pass.
|
||||
"norm_rope_fusion": re.compile(
|
||||
r"(?:qk_norm_rope_fusion.py:\d+] Fused QK Norm\+RoPE on|"
|
||||
r"qk_norm_rope_kvcache_fusion.py:\d+] "
|
||||
r"QK-Norm\+RoPE(?:/MRoPE)?\+KVCache fusion: replaced) (\d+)"
|
||||
),
|
||||
"attn_quant_fusion": re.compile(
|
||||
r"attn_quant_fusion.py:\d+] Fused quant onto (\d+) attention nodes"
|
||||
),
|
||||
"ar_rms_fusion": re.compile(
|
||||
r"allreduce_rms_fusion.py:\d+] Replaced (\d+) patterns"
|
||||
),
|
||||
"aiter_ar_rms_fusion": re.compile(
|
||||
r"RocmAiterAllReduceFusionPass Replaced (\d+) patterns"
|
||||
),
|
||||
}
|
||||
@@ -1,205 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
from .common import AttentionBackendCase, Matches, ModelFusionInfo, is_blackwell
|
||||
|
||||
# Attn backends
|
||||
FLASHINFER_ATTN = pytest.param(
|
||||
AttentionBackendCase(
|
||||
backend=AttentionBackendEnum.FLASHINFER,
|
||||
model_kwargs=dict(kv_cache_dtype="fp8"),
|
||||
),
|
||||
id="FLASHINFER",
|
||||
marks=pytest.mark.skipif(
|
||||
not is_blackwell() or not has_flashinfer(),
|
||||
reason="FI backend requires Blackwell and FlashInfer",
|
||||
),
|
||||
)
|
||||
|
||||
TRITON_ATTN = pytest.param(
|
||||
AttentionBackendCase(backend=AttentionBackendEnum.TRITON_ATTN), id="TRITON_ATTN"
|
||||
)
|
||||
|
||||
ROCM_ATTN = pytest.param(
|
||||
AttentionBackendCase(backend=AttentionBackendEnum.ROCM_ATTN),
|
||||
id="ROCM_ATTN",
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm attention only for AMD",
|
||||
),
|
||||
)
|
||||
|
||||
ROCM_AITER_UNIFIED_ATTN = pytest.param(
|
||||
AttentionBackendCase(backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN),
|
||||
id="ROCM_AITER_UNIFIED_ATTN",
|
||||
marks=pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="ROCM_AITER_UNIFIED_ATTN only for AMD when AITER is installed",
|
||||
),
|
||||
)
|
||||
|
||||
ROCM_AITER_MLA_ATTN = pytest.param(
|
||||
AttentionBackendCase(backend=AttentionBackendEnum.ROCM_AITER_MLA),
|
||||
id="ROCM_AITER_MLA",
|
||||
marks=pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="ROCM_AITER_MLA only for AMD when AITER is installed",
|
||||
),
|
||||
)
|
||||
|
||||
FLASHINFER_MLA_ATTN = pytest.param(
|
||||
AttentionBackendCase(backend=AttentionBackendEnum.FLASHINFER_MLA),
|
||||
id="FLASHINFER_MLA",
|
||||
marks=pytest.mark.skipif(
|
||||
not is_blackwell() or not has_flashinfer(),
|
||||
reason="FI backend requires Blackwell and FlashInfer",
|
||||
),
|
||||
)
|
||||
|
||||
TRITON_MLA_ATTN = pytest.param(
|
||||
AttentionBackendCase(backend=AttentionBackendEnum.TRITON_MLA),
|
||||
id="TRITON_MLA",
|
||||
)
|
||||
|
||||
# Models
|
||||
llama3_8b = ModelFusionInfo(
|
||||
model_name="meta-llama/Llama-3.1-8B-Instruct",
|
||||
matches=lambda n_layers: Matches(
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
aiter_ar_rms_fusion=n_layers * 2 + 1,
|
||||
),
|
||||
)
|
||||
|
||||
llama3_8b_fp8 = ModelFusionInfo(
|
||||
model_name="RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
|
||||
matches=lambda n_layers: Matches(
|
||||
rms_quant_fusion=n_layers * 2,
|
||||
act_quant_fusion=n_layers,
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
),
|
||||
)
|
||||
|
||||
llama3_8b_fp4 = ModelFusionInfo(
|
||||
model_name="nvidia/Llama-3.1-8B-Instruct-FP4",
|
||||
matches=lambda n_layers: Matches(
|
||||
act_quant_fusion=n_layers,
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
),
|
||||
)
|
||||
|
||||
# ModelOpt MIXED_PRECISION: FP8 attention projections, NVFP4 MLPs. It resolves to
|
||||
# modelopt_mixed, not modelopt_fp4, and its MLP has no manual act+quant fusion, so
|
||||
# every MLP boundary is left to ActivationQuantFusionPass.
|
||||
qwen3_8_27b_mixed_fp4 = ModelFusionInfo(
|
||||
model_name="nvidia/Qwen3.8-27B-NVFP4",
|
||||
hf_overrides=lambda n_layers: {
|
||||
"text_config": {
|
||||
"num_hidden_layers": n_layers,
|
||||
"layer_types": ["linear_attention"] * (n_layers - 1) + ["full_attention"],
|
||||
}
|
||||
},
|
||||
matches=lambda n_layers: Matches(act_quant_fusion=n_layers),
|
||||
)
|
||||
|
||||
# MoEs cannot do act+quant fusion because those ops are hidden from torch.compile.
|
||||
# MoEs also only expose 1 rms+quant fusion because the quant for up_proj is hidden.
|
||||
# TODO(luka): https://github.com/vllm-project/vllm/issues/31985
|
||||
# Also, for MoEs, gemm+collective fusion only happens for dense GEMMs (o_proj/qkv proj)
|
||||
|
||||
llama4_scout_fp8 = ModelFusionInfo(
|
||||
model_name="nvidia/Llama-4-Scout-17B-16E-Instruct-FP8",
|
||||
hf_overrides=lambda n_layers: {"text_config": {"num_hidden_layers": n_layers}},
|
||||
matches=lambda n_layers: Matches(
|
||||
rms_quant_fusion=n_layers,
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2,
|
||||
),
|
||||
)
|
||||
|
||||
llama4_scout_fp4 = ModelFusionInfo(
|
||||
model_name="nvidia/Llama-4-Scout-17B-16E-Instruct-NVFP4",
|
||||
hf_overrides=lambda n_layers: {"text_config": {"num_hidden_layers": n_layers}},
|
||||
matches=lambda n_layers: Matches(
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2,
|
||||
),
|
||||
)
|
||||
|
||||
qwen3_a3b = ModelFusionInfo(
|
||||
model_name="Qwen/Qwen3-30B-A3B",
|
||||
matches=lambda n_layers: Matches(
|
||||
norm_rope_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
aiter_ar_rms_fusion=n_layers * 2 + 1,
|
||||
),
|
||||
)
|
||||
|
||||
qwen3_a3b_fp8 = ModelFusionInfo(
|
||||
model_name="Qwen/Qwen3-30B-A3B-FP8",
|
||||
matches=lambda n_layers: Matches(
|
||||
rms_quant_fusion=n_layers,
|
||||
norm_rope_fusion=n_layers,
|
||||
attn_quant_fusion=0, # attn + group quant not supported
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
),
|
||||
)
|
||||
|
||||
deepseek_coder_v2_lite_fp8 = ModelFusionInfo(
|
||||
model_name="RedHatAI/DeepSeek-Coder-V2-Lite-Instruct-FP8",
|
||||
matches=lambda n_layers: Matches(
|
||||
# first_k_dense_replace=1; MoE hides most rms+quant sites
|
||||
rms_quant_fusion=1,
|
||||
act_quant_fusion=min(1, n_layers), # dense layers only
|
||||
# MLA attn + static FP8 quant
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
),
|
||||
)
|
||||
|
||||
deepseek_v3_fp8 = ModelFusionInfo(
|
||||
model_name="deepseek-ai/DeepSeek-V3",
|
||||
matches=lambda n_layers: Matches(
|
||||
# 3 per dense layer (first 3):
|
||||
# - input_rms + qkv_proj
|
||||
# - q_a_layernorm + q_b_proj (inside MLA wrapper)
|
||||
# - post_attn_layernorm + MLP
|
||||
# 2 per MoE layer (remaining) due to MoE wrapping
|
||||
rms_quant_fusion=n_layers * 2 + min(3, n_layers), # add for 3 dense layers
|
||||
# silu+block quant
|
||||
act_quant_fusion=min(3, n_layers), # dense layers only
|
||||
# MLA attn + per-group FP8 quant
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
),
|
||||
)
|
||||
|
||||
deepseek_r1_fp4 = ModelFusionInfo(
|
||||
model_name="nvidia/DeepSeek-R1-0528-NVFP4-v2",
|
||||
matches=lambda n_layers: Matches(
|
||||
rms_quant_fusion=0,
|
||||
act_quant_fusion=min(3, n_layers),
|
||||
attn_quant_fusion=n_layers,
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
),
|
||||
)
|
||||
|
||||
gpt_oss_20b = ModelFusionInfo(
|
||||
model_name="openai/gpt-oss-20b",
|
||||
matches=lambda n_layers: Matches(
|
||||
ar_rms_fusion=n_layers * 2 + 1,
|
||||
aiter_ar_rms_fusion=n_layers + 1,
|
||||
),
|
||||
model_kwargs=(
|
||||
{"quantization_config": {"moe": {"activation": "mxfp8"}}}
|
||||
if is_blackwell()
|
||||
else {}
|
||||
),
|
||||
)
|
||||
@@ -1,248 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import PassConfig
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import is_flashinfer_fp8_blockscale_gemm_supported
|
||||
|
||||
from .common import (
|
||||
INDUCTOR_GRAPH_PARTITION,
|
||||
AttentionBackendCase,
|
||||
Matches,
|
||||
custom_ops_combos,
|
||||
is_blackwell,
|
||||
nvfp4_kernel_exposes_input_quant_key,
|
||||
)
|
||||
from .models import (
|
||||
FLASHINFER_ATTN,
|
||||
FLASHINFER_MLA_ATTN,
|
||||
ROCM_AITER_UNIFIED_ATTN,
|
||||
ROCM_ATTN,
|
||||
TRITON_ATTN,
|
||||
TRITON_MLA_ATTN,
|
||||
deepseek_coder_v2_lite_fp8,
|
||||
deepseek_r1_fp4,
|
||||
deepseek_v3_fp8,
|
||||
llama3_8b_fp4,
|
||||
llama3_8b_fp8,
|
||||
llama4_scout_fp4,
|
||||
llama4_scout_fp8,
|
||||
qwen3_8_27b_mixed_fp4,
|
||||
qwen3_a3b_fp8,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides, use_deepgemm",
|
||||
[
|
||||
(*llama3_8b_fp8, False),
|
||||
(*qwen3_a3b_fp8, False),
|
||||
(*qwen3_a3b_fp8, True),
|
||||
(*deepseek_coder_v2_lite_fp8, False),
|
||||
(*deepseek_v3_fp8, False),
|
||||
(*deepseek_v3_fp8, True),
|
||||
pytest.param(
|
||||
*llama4_scout_fp8,
|
||||
False,
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(),
|
||||
reason="Llama4 Scout FP8 only supported on CUDA",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
[
|
||||
TRITON_ATTN,
|
||||
FLASHINFER_ATTN,
|
||||
ROCM_ATTN,
|
||||
ROCM_AITER_UNIFIED_ATTN,
|
||||
FLASHINFER_MLA_ATTN,
|
||||
TRITON_MLA_ATTN,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("n_layers", [6])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
def test_tp1_fp8_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
custom_ops: str,
|
||||
inductor_graph_partition: bool,
|
||||
use_deepgemm: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
if use_deepgemm and not current_platform.is_cuda():
|
||||
pytest.skip("DeepGemm only supported on CUDA")
|
||||
|
||||
if use_deepgemm and is_flashinfer_fp8_blockscale_gemm_supported():
|
||||
# Flashinfer block FP8 GEMM has internal quantization, so it can't
|
||||
# be fused with other ops.
|
||||
pytest.skip("FlashInfer block FP8 GEMM not supported")
|
||||
if use_deepgemm and is_blackwell():
|
||||
# TODO(luka) DeepGEMM uses different quants, matching not supported
|
||||
# - on Blackwell, uses a special quant fp8, currently not supported
|
||||
pytest.skip("DeepGEMM & quant matching not currently supported")
|
||||
|
||||
matches = matches_fn(n_layers)
|
||||
|
||||
block_fp8 = "qwen" in model_name.lower() or "deepseek" in model_name.lower()
|
||||
if block_fp8 and "-quant_fp8" in custom_ops:
|
||||
# This is why config forces +quant_fp8 by default
|
||||
pytest.skip("native QuantFP8 matching not supported for group quant")
|
||||
|
||||
# Reduce size of model and skip weight loading time
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=custom_ops.split(","),
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=True,
|
||||
fuse_act_quant=False,
|
||||
fuse_attn_quant=True,
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
),
|
||||
)
|
||||
|
||||
use_aiter = current_platform.is_rocm() and ("qwen" in model_name.lower())
|
||||
|
||||
matches_check = [
|
||||
"rms_quant_fusion",
|
||||
"norm_rope_fusion",
|
||||
"attn_quant_fusion",
|
||||
]
|
||||
|
||||
if use_aiter:
|
||||
matches_check[0] = "aiter_rms_quant_fusion"
|
||||
|
||||
matches = matches._replace(aiter_rms_quant_fusion=matches.rms_quant_fusion)
|
||||
# TODO: enable the `norm_rope_fusion` test,
|
||||
# On ROCm norm_rope_fusion is only supported without
|
||||
# enabling AITER.
|
||||
matches_check.remove("norm_rope_fusion")
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches,
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
matches_check,
|
||||
use_deepgemm=use_deepgemm,
|
||||
use_aiter=use_aiter,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
[llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
[FLASHINFER_ATTN, FLASHINFER_MLA_ATTN],
|
||||
)
|
||||
@pytest.mark.parametrize("n_layers", [6])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
|
||||
def test_tp1_fp4_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
custom_ops: str,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
if nvfp4_kernel_exposes_input_quant_key():
|
||||
pytest.skip(
|
||||
"NVFP4 kernel exposes input_quant_key; manual fusion fires "
|
||||
"instead of compiler pass-based fusion"
|
||||
)
|
||||
|
||||
matches = matches_fn(n_layers)
|
||||
|
||||
# Reduce size of model and skip weight loading time
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=custom_ops.split(","),
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=True,
|
||||
fuse_act_quant=False,
|
||||
fuse_attn_quant=True,
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
),
|
||||
)
|
||||
|
||||
matches_check = ["attn_quant_fusion", "norm_rope_fusion"]
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches,
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
matches_check,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides", [qwen3_8_27b_mixed_fp4]
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", [FLASHINFER_ATTN])
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
|
||||
def test_tp1_fp4_mixed_precision_act_fusion(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
"""Act+quant fusion matches on a MIXED_PRECISION NVFP4 checkpoint."""
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=False,
|
||||
fuse_act_quant=True,
|
||||
fuse_attn_quant=False,
|
||||
enable_qk_norm_rope_fusion=False,
|
||||
),
|
||||
)
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches_fn(n_layers),
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
["act_quant_fusion"],
|
||||
)
|
||||
@@ -1,313 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.config import PassConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...utils import multi_gpu_test
|
||||
from .common import (
|
||||
INDUCTOR_GRAPH_PARTITION,
|
||||
AttentionBackendCase,
|
||||
Matches,
|
||||
custom_ops_combos,
|
||||
is_blackwell,
|
||||
nvfp4_kernel_exposes_input_quant_key,
|
||||
)
|
||||
from .models import (
|
||||
FLASHINFER_ATTN,
|
||||
FLASHINFER_MLA_ATTN,
|
||||
ROCM_AITER_MLA_ATTN,
|
||||
ROCM_AITER_UNIFIED_ATTN,
|
||||
ROCM_ATTN,
|
||||
TRITON_ATTN,
|
||||
deepseek_coder_v2_lite_fp8,
|
||||
deepseek_r1_fp4,
|
||||
deepseek_v3_fp8,
|
||||
gpt_oss_20b,
|
||||
llama3_8b,
|
||||
llama3_8b_fp4,
|
||||
llama3_8b_fp8,
|
||||
llama4_scout_fp4,
|
||||
llama4_scout_fp8,
|
||||
qwen3_a3b,
|
||||
qwen3_a3b_fp8,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Only test CUDA/ROCm"
|
||||
)
|
||||
|
||||
|
||||
# qwen3 & dsv3 should still fuse AR+rms even though group quant is not yet
|
||||
# supported. ROCm carries only the payloads its two backends support.
|
||||
TP2_FP8_MODELS = (
|
||||
[
|
||||
llama3_8b_fp8,
|
||||
llama4_scout_fp8,
|
||||
qwen3_a3b_fp8,
|
||||
deepseek_coder_v2_lite_fp8,
|
||||
deepseek_v3_fp8,
|
||||
]
|
||||
if current_platform.is_cuda()
|
||||
else [llama3_8b_fp8, qwen3_a3b_fp8, deepseek_coder_v2_lite_fp8]
|
||||
)
|
||||
|
||||
TP2_FP8_ATTN_BACKENDS = (
|
||||
[TRITON_ATTN, FLASHINFER_ATTN, FLASHINFER_MLA_ATTN]
|
||||
if current_platform.is_cuda()
|
||||
else [ROCM_AITER_UNIFIED_ATTN, ROCM_AITER_MLA_ATTN]
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides", TP2_FP8_MODELS
|
||||
)
|
||||
@pytest.mark.parametrize("attn_backend", TP2_FP8_ATTN_BACKENDS)
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("quant_fp8", "rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
def test_tp2_ar_rms_fp8_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
custom_ops: str,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
monkeypatch,
|
||||
):
|
||||
matches = matches_fn(n_layers)
|
||||
|
||||
model_name_lower = model_name.lower()
|
||||
# DeepSeek-Coder on AITER MLA is the one static-FP8 payload here, validated
|
||||
# against the native QuantFP8 pattern; every other DeepSeek stays group
|
||||
# quantized as on CUDA.
|
||||
rocm_static_fp8_mla = (
|
||||
current_platform.is_rocm()
|
||||
and "deepseek-coder" in model_name_lower
|
||||
and attn_backend.backend.name == "ROCM_AITER_MLA"
|
||||
)
|
||||
block_fp8 = "qwen" in model_name_lower or "deepseek" in model_name_lower
|
||||
if rocm_static_fp8_mla:
|
||||
block_fp8 = False
|
||||
if custom_ops != "-quant_fp8,-rms_norm":
|
||||
pytest.skip("ROCm static-FP8 MLA parity uses native quant and RMSNorm")
|
||||
# AITER's MoE GEMM rejects this reduced dummy model shape; keep AITER
|
||||
# for MLA and the all-reduce, and let the MoE run on Triton.
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "0")
|
||||
if block_fp8 and "-quant_fp8" in custom_ops:
|
||||
# This is why config forces +quant_fp8 by default
|
||||
pytest.skip("native QuantFP8 matching not supported for group quant")
|
||||
|
||||
# Reduce size of model and skip weight loading time
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
model_kwargs["disable_custom_all_reduce"] = False
|
||||
if current_platform.is_rocm() and (
|
||||
"llama-3" in model_name_lower or "qwen" in model_name_lower
|
||||
):
|
||||
model_kwargs.update(
|
||||
gpu_memory_utilization=0.01,
|
||||
kv_cache_memory_bytes=64 * 1024**2,
|
||||
max_num_batched_tokens=1024,
|
||||
max_num_seqs=32,
|
||||
)
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=custom_ops.split(","),
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=True,
|
||||
fuse_act_quant=False,
|
||||
fuse_attn_quant=True,
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
fuse_allreduce_rms=True,
|
||||
),
|
||||
)
|
||||
|
||||
matches_check = [
|
||||
"rms_quant_fusion",
|
||||
"norm_rope_fusion",
|
||||
"attn_quant_fusion",
|
||||
"ar_rms_fusion",
|
||||
]
|
||||
|
||||
use_aiter = current_platform.is_rocm()
|
||||
if use_aiter:
|
||||
matches = matches._replace(aiter_ar_rms_fusion=matches.ar_rms_fusion)
|
||||
# The AITER all-reduce pass registers the AR+RMS(+group-quant) patterns
|
||||
# before its AR+RMS-only ones, so the standalone AITER RMS-quant pass
|
||||
# does not own the post-AR sites here. ROCm's contract is native
|
||||
# QK-norm+RoPE and attention-output quant fusion plus AITER AR+RMS.
|
||||
matches_check = [
|
||||
"norm_rope_fusion",
|
||||
"attn_quant_fusion",
|
||||
"aiter_ar_rms_fusion",
|
||||
]
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches,
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
matches_check,
|
||||
tp_size=2,
|
||||
use_aiter=use_aiter,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides",
|
||||
[llama3_8b_fp4, llama4_scout_fp4, deepseek_r1_fp4],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
[FLASHINFER_ATTN, FLASHINFER_MLA_ATTN],
|
||||
)
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("custom_ops", custom_ops_combos("rms_norm"))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@pytest.mark.skipif(not is_blackwell(), reason="Blackwell required for fp4")
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test CUDA")
|
||||
def test_tp2_ar_rms_fp4_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
custom_ops: str,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
if nvfp4_kernel_exposes_input_quant_key():
|
||||
pytest.skip(
|
||||
"NVFP4 kernel exposes input_quant_key; manual fusion fires "
|
||||
"instead of compiler pass-based fusion"
|
||||
)
|
||||
|
||||
matches = matches_fn(n_layers)
|
||||
|
||||
# Reduce size of model and skip weight loading time
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
model_kwargs["disable_custom_all_reduce"] = False
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=custom_ops.split(","),
|
||||
pass_config=PassConfig(
|
||||
fuse_act_quant=False,
|
||||
fuse_attn_quant=True,
|
||||
fuse_allreduce_rms=True,
|
||||
),
|
||||
)
|
||||
|
||||
matches_check = [
|
||||
"attn_quant_fusion",
|
||||
"ar_rms_fusion",
|
||||
]
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches,
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
matches_check,
|
||||
tp_size=2,
|
||||
)
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"model_name, matches_fn, model_kwargs, hf_overrides, model_impl",
|
||||
[
|
||||
(*llama3_8b, "auto"),
|
||||
(*llama3_8b, "transformers"),
|
||||
(*qwen3_a3b, "auto"),
|
||||
(*gpt_oss_20b, "auto"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
[
|
||||
TRITON_ATTN,
|
||||
FLASHINFER_ATTN,
|
||||
ROCM_ATTN,
|
||||
ROCM_AITER_UNIFIED_ATTN,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("n_layers", [4])
|
||||
@pytest.mark.parametrize("custom_ops", tuple(custom_ops_combos("rms_norm")))
|
||||
@pytest.mark.parametrize("inductor_graph_partition", INDUCTOR_GRAPH_PARTITION)
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Only test CUDA/ROCm")
|
||||
def test_tp2_ar_rms_fusions(
|
||||
model_name: str,
|
||||
matches_fn: Callable[[int], Matches],
|
||||
model_kwargs: dict,
|
||||
hf_overrides: Callable[[int], dict],
|
||||
model_impl: str,
|
||||
attn_backend: AttentionBackendCase,
|
||||
n_layers: int,
|
||||
custom_ops: str,
|
||||
inductor_graph_partition: bool,
|
||||
run_e2e_fusion_test,
|
||||
):
|
||||
if model_impl == "transformers" and not current_platform.is_rocm():
|
||||
pytest.skip("Transformers 3D AR+RMS regression is ROCm-only")
|
||||
|
||||
matches = matches_fn(n_layers)
|
||||
if model_impl == "transformers":
|
||||
# Transformers add+RMSNorm canonicalization exposes every generic
|
||||
# AR+RMS fusion site, including the final norm.
|
||||
matches = matches._replace(aiter_ar_rms_fusion=matches.ar_rms_fusion)
|
||||
|
||||
# Reduce size of model and skip weight loading time
|
||||
model_kwargs["hf_overrides"] = hf_overrides(n_layers)
|
||||
model_kwargs["load_format"] = "dummy"
|
||||
model_kwargs["model_impl"] = model_impl
|
||||
model_kwargs["max_model_len"] = 1024
|
||||
model_kwargs["kernel_config"] = {"enable_flashinfer_autotune": False}
|
||||
model_kwargs["disable_custom_all_reduce"] = False
|
||||
|
||||
compilation_config = dict(
|
||||
use_inductor_graph_partition=inductor_graph_partition,
|
||||
custom_ops=custom_ops.split(","),
|
||||
pass_config=PassConfig(
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
fuse_allreduce_rms=True,
|
||||
),
|
||||
)
|
||||
|
||||
matches_check = [
|
||||
"norm_rope_fusion",
|
||||
]
|
||||
|
||||
if current_platform.is_rocm():
|
||||
matches_check.append("aiter_ar_rms_fusion")
|
||||
else:
|
||||
matches_check.append("ar_rms_fusion")
|
||||
|
||||
run_e2e_fusion_test(
|
||||
model_name,
|
||||
matches,
|
||||
model_kwargs,
|
||||
attn_backend,
|
||||
compilation_config,
|
||||
matches_check,
|
||||
tp_size=2,
|
||||
use_aiter=current_platform.is_rocm(),
|
||||
)
|
||||
@@ -1,898 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from importlib.util import find_spec
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.utils import TestFP8Layer, has_module_attribute, multi_gpu_test
|
||||
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
|
||||
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
from vllm.compilation.passes.fusion.allreduce_rms_fusion import (
|
||||
AllReduceFusionPass,
|
||||
RocmAiterAllReduceFusionPass,
|
||||
_fused_ar_workspace_hidden_dim,
|
||||
_select_flashinfer_allreduce_use_oneshot,
|
||||
)
|
||||
from vllm.compilation.passes.fx_utils import find_op_nodes
|
||||
from vllm.compilation.passes.utility.fix_functionalization import (
|
||||
FixFunctionalizationPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
DeviceConfig,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.distributed import tensor_model_parallel_all_reduce
|
||||
from vllm.distributed.device_communicators.aiter_custom_all_reduce import (
|
||||
AiterCustomAllreduce,
|
||||
)
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("workspace_backend", "device_capability", "world_size", "tensor_size", "expected"),
|
||||
[
|
||||
("mnnvl", 103, 8, 2 * 1024 * 1024, None),
|
||||
("trtllm", 103, 8, 2 * 1024 * 1024, True),
|
||||
("trtllm", 103, 8, 2 * 1024 * 1024 + 1, False),
|
||||
("trtllm", 100, 4, 4 * 1024 * 1024, True),
|
||||
("trtllm", 100, 4, 4 * 1024 * 1024 + 1, False),
|
||||
("trtllm", None, 8, 128 * 1024 * 1024, True),
|
||||
],
|
||||
)
|
||||
def test_select_flashinfer_allreduce_use_oneshot(
|
||||
workspace_backend: str,
|
||||
device_capability: int | None,
|
||||
world_size: int,
|
||||
tensor_size: int,
|
||||
expected: bool | None,
|
||||
):
|
||||
assert (
|
||||
_select_flashinfer_allreduce_use_oneshot(
|
||||
workspace_backend,
|
||||
device_capability,
|
||||
world_size,
|
||||
tensor_size,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
def _make_workspace_hidden_config(
|
||||
target_hidden: int, draft_hidden: int | None
|
||||
) -> SimpleNamespace:
|
||||
"""Minimal stand-in exposing only what _fused_ar_workspace_hidden_dim reads."""
|
||||
speculative_config = None
|
||||
if draft_hidden is not None:
|
||||
speculative_config = SimpleNamespace(
|
||||
draft_model_config=SimpleNamespace(
|
||||
get_hidden_size=lambda: draft_hidden,
|
||||
)
|
||||
)
|
||||
return SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_hidden_size=lambda: target_hidden),
|
||||
speculative_config=speculative_config,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("target_hidden", "draft_hidden", "expected"),
|
||||
[
|
||||
(2048, None, 2048), # no speculative decoding -> target only
|
||||
(2048, 2560, 2560), # draft wider than target -> draft (issue #52023)
|
||||
(2048, 1024, 2048), # draft narrower than target -> target
|
||||
(2048, 2048, 2048), # equal -> target
|
||||
],
|
||||
)
|
||||
def test_fused_ar_workspace_hidden_dim(
|
||||
target_hidden: int, draft_hidden: int | None, expected: int
|
||||
):
|
||||
config = _make_workspace_hidden_config(target_hidden, draft_hidden)
|
||||
assert _fused_ar_workspace_hidden_dim(config) == expected
|
||||
|
||||
|
||||
def test_fused_ar_workspace_hidden_dim_spec_without_draft():
|
||||
# speculative_config present but no draft model (e.g. ngram) -> target only.
|
||||
config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_hidden_size=lambda: 2048),
|
||||
speculative_config=SimpleNamespace(draft_model_config=None),
|
||||
)
|
||||
assert _fused_ar_workspace_hidden_dim(config) == 2048
|
||||
|
||||
|
||||
class TestAllReduceRMSNormModel(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size=16,
|
||||
token_num=16,
|
||||
eps=1e-6,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
use_aiter: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.eps = eps
|
||||
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
|
||||
self.w = [torch.rand(hidden_size, hidden_size) for _ in range(3)]
|
||||
self.use_aiter = use_aiter
|
||||
|
||||
def forward(self, x):
|
||||
# avoid having graph input be an arg to a pattern directly
|
||||
z = torch.relu(x)
|
||||
x = resid = tensor_model_parallel_all_reduce(z)
|
||||
y = self.norm[0](x)
|
||||
|
||||
z2 = torch.mm(y, self.w[0])
|
||||
x2 = tensor_model_parallel_all_reduce(z2)
|
||||
|
||||
y2, resid = self.norm[1](x2, resid)
|
||||
|
||||
z3 = torch.mm(y2, self.w[1])
|
||||
x3 = tensor_model_parallel_all_reduce(z3)
|
||||
|
||||
y3, resid = self.norm[2](x3, resid)
|
||||
|
||||
z4 = torch.mm(y3, self.w[2])
|
||||
x4 = tensor_model_parallel_all_reduce(z4)
|
||||
|
||||
y4, resid = self.norm[3](x4, resid)
|
||||
return y4
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.vllm.all_reduce.default]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
if self.use_aiter:
|
||||
return [rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()]
|
||||
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
|
||||
|
||||
|
||||
class TestAllReduceGemmaRMSNormModel(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size=16,
|
||||
token_num=16,
|
||||
eps=1e-6,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
use_aiter: bool = False,
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.eps = eps
|
||||
self.use_aiter = use_aiter
|
||||
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
|
||||
# Non-trivial weight (~Gemma range) so (1 + w) exercises the scale path.
|
||||
for n in self.norm:
|
||||
n.weight.data.normal_(mean=0.0, std=0.1)
|
||||
self.w = [torch.rand(hidden_size, hidden_size) for _ in range(3)]
|
||||
|
||||
def forward(self, x):
|
||||
# avoid having graph input be an arg to a pattern directly
|
||||
z = torch.relu(x)
|
||||
x = resid = tensor_model_parallel_all_reduce(z)
|
||||
y = self.norm[0](x)
|
||||
|
||||
z2 = torch.mm(y, self.w[0])
|
||||
x2 = tensor_model_parallel_all_reduce(z2)
|
||||
y2, resid = self.norm[1](x2, resid)
|
||||
|
||||
z3 = torch.mm(y2, self.w[1])
|
||||
x3 = tensor_model_parallel_all_reduce(z3)
|
||||
y3, resid = self.norm[2](x3, resid)
|
||||
|
||||
z4 = torch.mm(y3, self.w[2])
|
||||
x4 = tensor_model_parallel_all_reduce(z4)
|
||||
y4, resid = self.norm[3](x4, resid)
|
||||
return y4
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.vllm.all_reduce.default]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
if self.use_aiter:
|
||||
return [rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()]
|
||||
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
|
||||
|
||||
|
||||
class TestAllReduceRMSNormStaticQuantFP8Model(torch.nn.Module):
|
||||
quant_key = kFp8StaticTensorSym
|
||||
|
||||
def __init__(
|
||||
self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.eps = eps
|
||||
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
|
||||
self.fp8_linear_layers = [
|
||||
TestFP8Layer(
|
||||
weight_shape=(hidden_size, hidden_size),
|
||||
activation_quant_key=self.quant_key,
|
||||
weight_quant_key=self.quant_key,
|
||||
input_dtype=dtype,
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
|
||||
def forward(self, hidden_states):
|
||||
# avoid having graph input be an arg to a pattern directly
|
||||
z = torch.relu(hidden_states)
|
||||
x = resid = tensor_model_parallel_all_reduce(z)
|
||||
y = self.norm[0](x)
|
||||
|
||||
z2 = self.fp8_linear_layers[0](y)
|
||||
|
||||
x2 = tensor_model_parallel_all_reduce(z2)
|
||||
y2, resid = self.norm[1](x2, resid)
|
||||
|
||||
z3 = self.fp8_linear_layers[1](y2)
|
||||
|
||||
x3 = tensor_model_parallel_all_reduce(z3)
|
||||
y3, resid = self.norm[2](x3, resid) # use resid here
|
||||
|
||||
z4 = self.fp8_linear_layers[2](y3)
|
||||
|
||||
x4 = tensor_model_parallel_all_reduce(z4)
|
||||
y4, resid = self.norm[3](x4, resid) # use resid here
|
||||
return y4
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
torch.ops.vllm.all_reduce.default,
|
||||
torch.ops._C.static_scaled_fp8_quant.default
|
||||
if self.fp8_linear_layers[0].is_quant_fp8_enabled()
|
||||
else torch.ops.aten.reciprocal.default,
|
||||
]
|
||||
|
||||
|
||||
class TestAllReduceGemmaRMSNormStaticQuantFP8Model(
|
||||
TestAllReduceRMSNormStaticQuantFP8Model
|
||||
):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size=16,
|
||||
token_num=16,
|
||||
eps=1e-6,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
):
|
||||
super().__init__(hidden_size, token_num, eps, dtype)
|
||||
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
|
||||
for norm in self.norm:
|
||||
norm.weight.requires_grad_(False)
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.vllm.all_reduce.default]
|
||||
|
||||
|
||||
class TestAiterAllReduceRMSNormGroupQuantFP8Model(torch.nn.Module):
|
||||
"""Exercises the new ROCm AITER AR+RMS+per-group-FP8-quant patterns.
|
||||
|
||||
Four ``rms_norm`` sites that together hit every pattern registered by
|
||||
``RocmAiterAllReduceFusionPass`` for the per-group FP8 quant path:
|
||||
|
||||
* ``norm[0]``: ``all_reduce -> rms_norm -> group_fp8_quant`` (no residual)
|
||||
-> ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern``
|
||||
* ``norm[1]``: ``all_reduce -> fused_add_rms_norm -> group_fp8_quant``
|
||||
(single ``rms`` consumer)
|
||||
-> ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern``
|
||||
* ``norm[2..3]``: ``all_reduce -> fused_add_rms_norm
|
||||
-> (group_fp8_quant + rocm_unquantized_gemm)`` (two ``rms`` consumers,
|
||||
modeling the DSv3.2 indexer fan-out)
|
||||
-> ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern``
|
||||
|
||||
Each site starts from the same bf16 activation and is returned separately.
|
||||
This avoids compounding four quantize/dequantize errors while still testing
|
||||
all four pattern firings without depending on a real FP8 block-scaled GEMM.
|
||||
"""
|
||||
|
||||
quant_group_size = 128
|
||||
indexer_out_dim = 8
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size=128,
|
||||
token_num=16,
|
||||
eps=1e-6,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.eps = eps
|
||||
self.dtype = dtype
|
||||
assert hidden_size % self.quant_group_size == 0, (
|
||||
f"hidden_size ({hidden_size}) must be a multiple of "
|
||||
f"quant_group_size ({self.quant_group_size}) for per-group FP8 quant"
|
||||
)
|
||||
self.norm = [RMSNorm(hidden_size, eps) for _ in range(4)]
|
||||
self.w = [torch.rand(hidden_size, hidden_size, dtype=dtype) for _ in range(3)]
|
||||
self.indexer_w = [
|
||||
torch.rand(self.indexer_out_dim, hidden_size, dtype=dtype) for _ in range(2)
|
||||
]
|
||||
|
||||
def _group_quant(self, rms: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return torch.ops.vllm.rocm_aiter_group_fp8_quant.default(
|
||||
rms, self.quant_group_size
|
||||
)
|
||||
|
||||
def _dequantize_to_bf16(self, q: torch.Tensor, s: torch.Tensor) -> torch.Tensor:
|
||||
# Broadcast the per-group scale across each group of `quant_group_size`
|
||||
# to validate the quantized result without depending on a real FP8
|
||||
# block-scaled GEMM kernel in the test.
|
||||
s_full = s.repeat_interleave(self.quant_group_size, dim=-1).to(self.dtype)
|
||||
return q.to(self.dtype) * s_full
|
||||
|
||||
def forward(self, hidden_states):
|
||||
z = torch.relu(hidden_states)
|
||||
x0 = tensor_model_parallel_all_reduce(z)
|
||||
rms0 = self.norm[0](x0)
|
||||
q0, s0 = self._group_quant(rms0)
|
||||
y0 = self._dequantize_to_bf16(q0, s0)
|
||||
|
||||
x1 = tensor_model_parallel_all_reduce(torch.mm(z, self.w[0]))
|
||||
rms1, resid1 = self.norm[1](x1, z.clone())
|
||||
q1, s1 = self._group_quant(rms1)
|
||||
y1 = self._dequantize_to_bf16(q1, s1)
|
||||
|
||||
x2 = tensor_model_parallel_all_reduce(torch.mm(z, self.w[1]))
|
||||
rms2, resid2 = self.norm[2](x2, z.clone())
|
||||
q2, s2 = self._group_quant(rms2)
|
||||
# Second consumer of ``rms2``: forces the with-indexer pattern.
|
||||
idx2 = torch.ops.vllm.rocm_unquantized_gemm(rms2, self.indexer_w[0], None)
|
||||
y2 = self._dequantize_to_bf16(q2, s2)
|
||||
|
||||
x3 = tensor_model_parallel_all_reduce(torch.mm(z, self.w[2]))
|
||||
rms3, resid3 = self.norm[3](x3, z.clone())
|
||||
q3, s3 = self._group_quant(rms3)
|
||||
# Second consumer of ``rms3``: forces the with-indexer pattern.
|
||||
idx3 = torch.ops.vllm.rocm_unquantized_gemm(rms3, self.indexer_w[1], None)
|
||||
y3 = self._dequantize_to_bf16(q3, s3)
|
||||
return y0, y1, y2, y3, resid1, resid2, resid3, idx2, idx3
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
torch.ops.vllm.all_reduce.default,
|
||||
torch.ops.vllm.rocm_aiter_group_fp8_quant.default,
|
||||
]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [
|
||||
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_op(),
|
||||
rocm_aiter_ops.get_fused_allreduce_rmsnorm_quant_per_group_with_bf16_norm_op(), # noqa: E501
|
||||
]
|
||||
|
||||
|
||||
class TestAiterAllReduceGemmaRMSNormGroupQuantFP8Model(
|
||||
TestAiterAllReduceRMSNormGroupQuantFP8Model
|
||||
):
|
||||
"""Gemma norms feeding the per-group FP8 quant sites.
|
||||
|
||||
The fused AR+RMS+quant op has no ``gemma_norm`` mode, so these sites must
|
||||
keep matching the quant-fused patterns with the pre-added ``1 + weight``.
|
||||
The Gemma AR+RMS-only patterns match a prefix of the same subgraph, so if
|
||||
they were registered ahead of the quant-fused ones they would consume the
|
||||
``all_reduce`` and strand ``rocm_aiter_group_fp8_quant`` as a separate
|
||||
kernel -- which the ``fully_replaced=True`` check below would catch.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size=128,
|
||||
token_num=16,
|
||||
eps=1e-6,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
):
|
||||
super().__init__(hidden_size, token_num, eps, dtype)
|
||||
self.norm = [GemmaRMSNorm(hidden_size, eps) for _ in range(4)]
|
||||
for norm in self.norm:
|
||||
norm.weight.requires_grad_(False)
|
||||
|
||||
|
||||
class TestAllReduceFusedAddRMSNormStaticQuantFP4Model(torch.nn.Module):
|
||||
def __init__(
|
||||
self, hidden_size=16, token_num=16, eps=1e-6, dtype: torch.dtype = torch.float16
|
||||
):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.eps = eps
|
||||
self.norm = [RMSNorm(hidden_size, eps) for i in range(4)]
|
||||
|
||||
self.w = [torch.rand(hidden_size, hidden_size) for _ in range(3)]
|
||||
self.agscale = [torch.rand(1, dtype=torch.float32) for _ in range(3)]
|
||||
wgscale = [torch.rand(1, dtype=torch.float32) for _ in range(3)]
|
||||
self.alpha = [1 / (w * a) for w, a in zip(wgscale, self.agscale)]
|
||||
|
||||
wq_gen, wscale_gen = zip(
|
||||
*(scaled_fp4_quant(w, wg) for w, wg in zip(self.w, wgscale))
|
||||
)
|
||||
self.wq, self.wscale = list(wq_gen), list(wscale_gen)
|
||||
|
||||
def forward(self, hidden_states):
|
||||
# avoid having graph input be an arg to a pattern directly
|
||||
z = torch.relu(hidden_states)
|
||||
x = resid = tensor_model_parallel_all_reduce(z)
|
||||
y = self.norm[0](x)
|
||||
|
||||
yq, y_scale = scaled_fp4_quant(y, self.agscale[0])
|
||||
z2 = cutlass_scaled_fp4_mm(
|
||||
yq, self.wq[0], y_scale, self.wscale[0], self.alpha[0], out_dtype=y.dtype
|
||||
)
|
||||
|
||||
x2 = tensor_model_parallel_all_reduce(z2)
|
||||
y2, resid = self.norm[1](x2, resid)
|
||||
|
||||
yq2, y_scale2 = scaled_fp4_quant(y2, self.agscale[1])
|
||||
z3 = cutlass_scaled_fp4_mm(
|
||||
yq2, self.wq[1], y_scale2, self.wscale[1], self.alpha[1], out_dtype=y2.dtype
|
||||
)
|
||||
|
||||
x3 = tensor_model_parallel_all_reduce(z3)
|
||||
y3, resid = self.norm[2](x3, resid) # use resid here
|
||||
|
||||
yq3, y_scale3 = scaled_fp4_quant(y3, self.agscale[2])
|
||||
z4 = cutlass_scaled_fp4_mm(
|
||||
yq3, self.wq[2], y_scale3, self.wscale[2], self.alpha[2], out_dtype=y3.dtype
|
||||
)
|
||||
x4 = tensor_model_parallel_all_reduce(z4)
|
||||
y4, resid = self.norm[3](x4, resid) # use resid here
|
||||
return y4
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default]
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
torch.ops.vllm.all_reduce.default,
|
||||
torch.ops._C.scaled_fp4_quant.out,
|
||||
]
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize(
|
||||
"test_model, enable_quant_fp8_custom_op, use_aiter",
|
||||
[
|
||||
(TestAllReduceRMSNormModel, False, IS_AITER_FOUND),
|
||||
pytest.param(
|
||||
TestAllReduceGemmaRMSNormModel,
|
||||
False,
|
||||
IS_AITER_FOUND,
|
||||
marks=pytest.mark.skipif(
|
||||
current_platform.is_rocm() and not IS_AITER_FOUND,
|
||||
reason="AITER is required on ROCm",
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
TestAllReduceRMSNormStaticQuantFP8Model,
|
||||
True,
|
||||
False,
|
||||
marks=pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="Not supported on ROCm platform",
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
TestAllReduceGemmaRMSNormStaticQuantFP8Model,
|
||||
True,
|
||||
False,
|
||||
marks=pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="Not supported on ROCm platform",
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
TestAllReduceRMSNormStaticQuantFP8Model,
|
||||
False,
|
||||
False,
|
||||
marks=pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="Not supported on ROCm platform",
|
||||
),
|
||||
),
|
||||
pytest.param(
|
||||
TestAllReduceFusedAddRMSNormStaticQuantFP4Model,
|
||||
False,
|
||||
False,
|
||||
marks=pytest.mark.skipif(
|
||||
current_platform.is_rocm(),
|
||||
reason="Not supported on ROCm platform",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("batch_size", [8])
|
||||
@pytest.mark.parametrize("seq_len", [8])
|
||||
@pytest.mark.parametrize("hidden_size", [64])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
|
||||
@pytest.mark.parametrize("flashinfer_allreduce_backend", ["trtllm", "mnnvl"])
|
||||
@pytest.mark.skipif(envs.VLLM_TARGET_DEVICE not in ["cuda"], reason="Only test on CUDA")
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_rocm() and not IS_AITER_FOUND,
|
||||
reason="aiter is not found",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
current_platform.is_cuda()
|
||||
and (
|
||||
not find_spec("flashinfer")
|
||||
or not has_module_attribute("flashinfer.comm", "allreduce_fusion")
|
||||
or not has_module_attribute(
|
||||
"flashinfer.comm", "create_allreduce_fusion_workspace"
|
||||
)
|
||||
),
|
||||
reason="flashinfer is not found or flashinfer "
|
||||
"is not compiled with allreduce_fusion",
|
||||
)
|
||||
def test_all_reduce_fusion_pass_replace(
|
||||
test_model: torch.nn.Module,
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
enable_rms_norm_custom_op,
|
||||
enable_quant_fp8_custom_op,
|
||||
flashinfer_allreduce_backend,
|
||||
use_aiter: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
if use_aiter:
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", str(use_aiter))
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
num_processes = 2
|
||||
if (
|
||||
test_model == TestAllReduceFusedAddRMSNormStaticQuantFP4Model
|
||||
and not current_platform.has_device_capability(100)
|
||||
):
|
||||
pytest.skip(
|
||||
"Skip as nvfp4 is only supported on "
|
||||
"devices with compute capability 10.0 (Blackwell)"
|
||||
)
|
||||
|
||||
def run_torch_spawn(fn, nprocs):
|
||||
master_port = get_open_port()
|
||||
torch.multiprocessing.spawn(
|
||||
fn,
|
||||
args=(
|
||||
num_processes,
|
||||
master_port,
|
||||
test_model,
|
||||
batch_size,
|
||||
seq_len,
|
||||
hidden_size,
|
||||
dtype,
|
||||
enable_rms_norm_custom_op,
|
||||
enable_quant_fp8_custom_op,
|
||||
flashinfer_allreduce_backend,
|
||||
use_aiter,
|
||||
monkeypatch,
|
||||
),
|
||||
nprocs=nprocs,
|
||||
)
|
||||
|
||||
run_torch_spawn(all_reduce_fusion_pass_on_test_model, num_processes)
|
||||
|
||||
|
||||
def all_reduce_fusion_pass_on_test_model(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
master_port: int,
|
||||
test_model_cls: torch.nn.Module,
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
enable_rms_norm_custom_op,
|
||||
enable_quant_fp8_custom_op,
|
||||
flashinfer_allreduce_backend,
|
||||
use_aiter: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
set_random_seed(0)
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": str(master_port),
|
||||
"VLLM_FLASHINFER_ALLREDUCE_BACKEND": flashinfer_allreduce_backend,
|
||||
"VLLM_ROCM_USE_AITER": str(int(use_aiter)),
|
||||
"VLLM_ROCM_USE_AITER_CUSTOM_AR": str(int(use_aiter)),
|
||||
}
|
||||
)
|
||||
if use_aiter:
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
init_distributed_environment()
|
||||
|
||||
custom_ops = []
|
||||
if enable_rms_norm_custom_op:
|
||||
custom_ops.append("+rms_norm")
|
||||
if enable_quant_fp8_custom_op:
|
||||
custom_ops.append("+quant_fp8")
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE, custom_ops=custom_ops
|
||||
)
|
||||
)
|
||||
vllm_config.compilation_config.pass_config = PassConfig(
|
||||
fuse_allreduce_rms=True, eliminate_noops=True
|
||||
)
|
||||
vllm_config.device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
|
||||
vllm_config.parallel_config.rank = local_rank # Setup rank for debug path
|
||||
|
||||
# this is a fake model name to construct the model config
|
||||
# in the vllm_config, it's not really used.
|
||||
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
|
||||
vllm_config.model_config = ModelConfig(
|
||||
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
all_reduce_fusion_pass = (
|
||||
RocmAiterAllReduceFusionPass(vllm_config)
|
||||
if use_aiter
|
||||
else AllReduceFusionPass(vllm_config)
|
||||
)
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
func_pass = FixFunctionalizationPass(vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
|
||||
backend = TestBackend(
|
||||
noop_pass, all_reduce_fusion_pass, func_pass, cleanup_pass
|
||||
)
|
||||
|
||||
token_num = batch_size * seq_len
|
||||
if test_model_cls in (
|
||||
TestAllReduceRMSNormModel,
|
||||
TestAllReduceGemmaRMSNormModel,
|
||||
):
|
||||
model = test_model_cls(
|
||||
hidden_size, token_num, dtype=dtype, use_aiter=use_aiter
|
||||
)
|
||||
else:
|
||||
model = test_model_cls(hidden_size, token_num, dtype=dtype)
|
||||
|
||||
hidden_states = torch.randn((token_num, hidden_size), requires_grad=False)
|
||||
|
||||
compiled_model = torch.compile(model, backend=backend)
|
||||
compiled_model(hidden_states)
|
||||
|
||||
results_unfused = model(hidden_states)
|
||||
results_fused = compiled_model(hidden_states)
|
||||
torch.testing.assert_close(results_unfused, results_fused, atol=1e-2, rtol=1e-2)
|
||||
|
||||
assert all_reduce_fusion_pass.matched_count == 4, (
|
||||
f"{all_reduce_fusion_pass.matched_count=}"
|
||||
)
|
||||
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=False)
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
if (
|
||||
test_model_cls
|
||||
in (
|
||||
TestAllReduceGemmaRMSNormModel,
|
||||
TestAllReduceGemmaRMSNormStaticQuantFP8Model,
|
||||
)
|
||||
and not use_aiter
|
||||
):
|
||||
fused_op = torch.ops.vllm.flashinfer_trtllm_fused_allreduce_norm.default
|
||||
fused_nodes = list(find_op_nodes(fused_op, backend.graph_post_pass))
|
||||
assert fused_nodes
|
||||
assert all(n.kwargs.get("weight_bias") == 1.0 for n in fused_nodes)
|
||||
del all_reduce_fusion_pass
|
||||
|
||||
|
||||
@multi_gpu_test(num_gpus=2)
|
||||
@pytest.mark.parametrize("batch_size", [8])
|
||||
@pytest.mark.parametrize("seq_len", [8])
|
||||
@pytest.mark.parametrize("hidden_size", [128])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"test_model_cls",
|
||||
[
|
||||
TestAiterAllReduceRMSNormGroupQuantFP8Model,
|
||||
TestAiterAllReduceGemmaRMSNormGroupQuantFP8Model,
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm AITER AR+RMS+per-group-FP8-quant fusion is ROCm-only",
|
||||
)
|
||||
@pytest.mark.skipif(not IS_AITER_FOUND, reason="aiter is not found")
|
||||
def test_rocm_aiter_all_reduce_rmsnorm_group_quant_fp8_fusion_pass_replace(
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
enable_rms_norm_custom_op: bool,
|
||||
test_model_cls: type[torch.nn.Module],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Sibling of ``test_all_reduce_fusion_pass_replace`` for the new
|
||||
ROCm AITER AR+RMS+per-group-FP8-quant fusion patterns.
|
||||
|
||||
Validates the three new ``VllmPatternReplacement`` patterns added to
|
||||
``RocmAiterAllReduceFusionPass``:
|
||||
|
||||
* ``AiterAllreduceFusedRMSNormGroupQuantFP8Pattern`` (no-residual)
|
||||
* ``AiterAllreduceFusedAddRMSNormGroupQuantFP8Pattern`` (with-residual,
|
||||
single ``rms`` consumer)
|
||||
* ``AiterAllreduceFusedAddRMSNormGroupQuantWithIndexerPattern`` (with-
|
||||
residual, DSv3.2 indexer fan-out; parametrized over
|
||||
``rocm_aiter_group_fp8_quant``
|
||||
producer).
|
||||
|
||||
Parametrized over plain and Gemma norms: the Gemma AR+RMS-only patterns
|
||||
match a prefix of these subgraphs, so this also pins their registration
|
||||
order behind the quant-fused ones.
|
||||
"""
|
||||
with monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
if not AiterCustomAllreduce.build_supports_per_group_quant():
|
||||
pytest.skip(
|
||||
"aiter build is missing 'fused_ar_rms_per_group_quant' (needs "
|
||||
"ROCm/aiter PR #2823); the new patterns aren't registered."
|
||||
)
|
||||
|
||||
num_processes = 2
|
||||
|
||||
def run_torch_spawn(fn, nprocs):
|
||||
master_port = get_open_port()
|
||||
torch.multiprocessing.spawn(
|
||||
fn,
|
||||
args=(
|
||||
num_processes,
|
||||
master_port,
|
||||
test_model_cls,
|
||||
batch_size,
|
||||
seq_len,
|
||||
hidden_size,
|
||||
dtype,
|
||||
enable_rms_norm_custom_op,
|
||||
monkeypatch,
|
||||
),
|
||||
nprocs=nprocs,
|
||||
)
|
||||
|
||||
run_torch_spawn(rocm_aiter_group_quant_fusion_pass_on_test_model, num_processes)
|
||||
|
||||
|
||||
def rocm_aiter_group_quant_fusion_pass_on_test_model(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
master_port: int,
|
||||
test_model_cls: torch.nn.Module,
|
||||
batch_size: int,
|
||||
seq_len: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
enable_rms_norm_custom_op: bool,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
set_random_seed(0)
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:{local_rank}")
|
||||
torch.accelerator.set_device_index(device)
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": str(local_rank),
|
||||
"LOCAL_RANK": str(local_rank),
|
||||
"WORLD_SIZE": str(world_size),
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": str(master_port),
|
||||
"VLLM_ROCM_USE_AITER": "1",
|
||||
"VLLM_ROCM_USE_AITER_CUSTOM_AR": "1",
|
||||
}
|
||||
)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
init_distributed_environment()
|
||||
|
||||
custom_ops = []
|
||||
if enable_rms_norm_custom_op:
|
||||
custom_ops.append("+rms_norm")
|
||||
# We always enable +quant_fp8 so the matcher's example
|
||||
# trace finds the same form the test model uses.
|
||||
custom_ops.append("+quant_fp8")
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE, custom_ops=custom_ops
|
||||
)
|
||||
)
|
||||
vllm_config.compilation_config.pass_config = PassConfig(
|
||||
fuse_allreduce_rms=True, eliminate_noops=True
|
||||
)
|
||||
vllm_config.device_config = DeviceConfig(device=torch.device(DEVICE_TYPE))
|
||||
vllm_config.parallel_config.rank = local_rank
|
||||
|
||||
model_name = "RedHatAI/Llama-3.2-1B-Instruct-FP8"
|
||||
vllm_config.model_config = ModelConfig(
|
||||
model=model_name, trust_remote_code=True, dtype=dtype, seed=42
|
||||
)
|
||||
with set_current_vllm_config(vllm_config):
|
||||
initialize_model_parallel(tensor_model_parallel_size=world_size)
|
||||
all_reduce_fusion_pass = RocmAiterAllReduceFusionPass(vllm_config)
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
func_pass = FixFunctionalizationPass(vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
|
||||
backend = TestBackend(
|
||||
noop_pass, all_reduce_fusion_pass, func_pass, cleanup_pass
|
||||
)
|
||||
|
||||
token_num = batch_size * seq_len
|
||||
model = test_model_cls(hidden_size, token_num, dtype=dtype)
|
||||
|
||||
hidden_states = torch.randn((token_num, hidden_size), requires_grad=False)
|
||||
|
||||
compiled_model = torch.compile(model, backend=backend)
|
||||
compiled_model(hidden_states)
|
||||
|
||||
results_unfused = model(hidden_states)
|
||||
results_fused = compiled_model(hidden_states)
|
||||
# AITER's kernel-level oracle permits up to 5% isolated FP8 values to
|
||||
# fall outside a 5% allclose envelope after collective reordering.
|
||||
for site, (reference, actual) in enumerate(
|
||||
zip(results_unfused[:4], results_fused[:4])
|
||||
):
|
||||
close = torch.isclose(reference, actual, atol=5e-2, rtol=5e-2)
|
||||
mismatch_ratio = float((~close).float().mean().item())
|
||||
assert mismatch_ratio <= 0.05, (
|
||||
f"FP8 site {site} mismatch ratio {mismatch_ratio:.2%} exceeds 5%"
|
||||
)
|
||||
|
||||
# Residual and bf16 indexer outputs do not incur FP8 quantization noise.
|
||||
torch.testing.assert_close(
|
||||
results_unfused[4:], results_fused[4:], atol=1e-2, rtol=1e-2
|
||||
)
|
||||
|
||||
# Four pattern firings: norm[0] (no-add quant), norm[1] (add quant,
|
||||
# single ``rms`` consumer), norm[2..3] (add quant + indexer fan-out).
|
||||
assert all_reduce_fusion_pass.matched_count == 4, (
|
||||
f"{all_reduce_fusion_pass.matched_count=}"
|
||||
)
|
||||
backend.check_before_ops(model.ops_in_model_before(), fully_replaced=True)
|
||||
fused_quant_op, fused_indexer_op = model.ops_in_model_after()
|
||||
assert backend.op_count(fused_quant_op) == 2
|
||||
assert backend.op_count(fused_indexer_op) == 2
|
||||
del all_reduce_fusion_pass
|
||||
@@ -1,462 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for IR inplace functionalization pass integration.
|
||||
|
||||
This test suite verifies that the inplace functionalization pass, lowering pass,
|
||||
and clone cleanup pass work together correctly with donated buffer tracking.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch._dynamo.exc
|
||||
from torch import nn
|
||||
|
||||
import vllm.kernels # noqa: F401 to register kernels
|
||||
from vllm.compilation.passes.inductor_pass import InductorPass, get_pass_context
|
||||
from vllm.compilation.passes.ir.clone_elimination import (
|
||||
UnsafeCloneEliminationPass,
|
||||
)
|
||||
from vllm.compilation.passes.ir.inplace_functionalization import (
|
||||
VllmIRInplaceFunctionalizationPass,
|
||||
)
|
||||
from vllm.compilation.passes.ir.lowering_pass import VllmIRLoweringPass
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.ir import ops
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
|
||||
from ...backend import TestBackend
|
||||
|
||||
|
||||
class StoreDonationInfoPass(InductorPass):
|
||||
def __init__(self):
|
||||
self.donated_input_ids_sets: list[set[int]] = []
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
ctx = get_pass_context()
|
||||
self.donated_input_ids_sets += [ctx.donated_input_ids]
|
||||
|
||||
|
||||
class MaybeInplaceModel(nn.Module):
|
||||
"""Model using only maybe_inplace variants."""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
|
||||
):
|
||||
# First maybe_inplace - x & residual1 are donated
|
||||
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x, residual1, self.weight1, 1e-5
|
||||
)
|
||||
# Second maybe_inplace - residual2 is donated
|
||||
x_normed2, residual_out2 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x_normed1, residual2, self.weight2, 1e-5
|
||||
)
|
||||
return x_normed2, residual_out1, residual_out2
|
||||
|
||||
|
||||
class FunctionalModel(nn.Module):
|
||||
"""Model using only functional (default) variants."""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
|
||||
):
|
||||
# First functional - no donation
|
||||
x_normed1, residual_out1 = ops.fused_add_rms_norm(
|
||||
x, residual1, self.weight1, 1e-5
|
||||
)
|
||||
# Second functional - no donation
|
||||
x_normed2, residual_out2 = ops.fused_add_rms_norm(
|
||||
x_normed1, residual2, self.weight2, 1e-5
|
||||
)
|
||||
return x_normed2, residual_out1, residual_out2
|
||||
|
||||
|
||||
class MixedModel(nn.Module):
|
||||
"""Model mixing maybe_inplace and functional variants."""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight1 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
self.weight2 = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual1: torch.Tensor, residual2: torch.Tensor
|
||||
):
|
||||
# First maybe_inplace - x & residual1 are donated
|
||||
x_normed1, residual_out1 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x, residual1, self.weight1, 1e-5
|
||||
)
|
||||
# Second functional - no donation, x_normed1 must be preserved as it's returned
|
||||
x_normed2, residual_out2 = ops.fused_add_rms_norm(
|
||||
x_normed1, residual2, self.weight2, 1e-5
|
||||
)
|
||||
# Return both to prevent x_normed1 from being optimized away
|
||||
return x_normed1, x_normed2, residual_out1, residual_out2
|
||||
|
||||
|
||||
class ModelWithTritonAfterMaybeInplace(nn.Module):
|
||||
"""Model using maybe_inplace followed by a Triton kernel.
|
||||
Test clone elimination can handle Triton in the graph
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
@triton.jit
|
||||
def _triton_add_kernel(
|
||||
x_ptr,
|
||||
y_ptr,
|
||||
n_elements,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
):
|
||||
pid = tl.program_id(axis=0)
|
||||
block_start = pid * BLOCK_SIZE
|
||||
offsets = block_start + tl.arange(0, BLOCK_SIZE)
|
||||
mask = offsets < n_elements
|
||||
x = tl.load(x_ptr + offsets, mask=mask)
|
||||
y = x + 0.1
|
||||
tl.store(y_ptr + offsets, y, mask=mask)
|
||||
|
||||
def triton_add(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Simple Triton add kernel."""
|
||||
y = torch.empty_like(x)
|
||||
n_elements = x.numel()
|
||||
grid = (triton.cdiv(n_elements, 256),)
|
||||
_triton_add_kernel[grid](x, y, n_elements, BLOCK_SIZE=256)
|
||||
return y
|
||||
|
||||
self.triton_add = triton_add
|
||||
|
||||
def forward(self, x: torch.Tensor, residual: torch.Tensor, residual2: torch.Tensor):
|
||||
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x, residual, self.weight, 1e-5
|
||||
)
|
||||
|
||||
x_processed = self.triton_add(x_normed)
|
||||
|
||||
# x_processed does not need to be cloned, residual2 does
|
||||
x_normed2, residual_out2 = ops.fused_add_rms_norm(
|
||||
x_processed, residual2, self.weight, 1e-5
|
||||
)
|
||||
return x_normed2, residual_out2
|
||||
|
||||
|
||||
skipif_no_triton = pytest.mark.skipif(not HAS_TRITON, reason="Requires Triton")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"model_class,expected_functionalized,expected_donated,expected_clones",
|
||||
[
|
||||
# 2 inplace calls, all activations donated, all clones eliminated
|
||||
(MaybeInplaceModel, 2, 3, 0),
|
||||
# No inplace calls, no donations, 3 clones (one eliminated)
|
||||
(FunctionalModel, 0, 0, 3),
|
||||
# One inplace call, two donated activations, 2 clones
|
||||
(MixedModel, 1, 2, 2),
|
||||
# One inplace call, two donated, 1 clone remaining
|
||||
pytest.param(ModelWithTritonAfterMaybeInplace, 1, 2, 1, marks=skipif_no_triton),
|
||||
],
|
||||
)
|
||||
def test_inplace_functionalization(
|
||||
default_vllm_config: VllmConfig,
|
||||
model_class,
|
||||
expected_functionalized: int,
|
||||
expected_clones: int,
|
||||
expected_donated: int,
|
||||
):
|
||||
"""Test inplace functionalization, lowering, and clone cleanup."""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
# Use vllm_c so inplace path is triggered
|
||||
default_vllm_config.kernel_config.ir_op_priority.fused_add_rms_norm = [
|
||||
"vllm_c",
|
||||
"native",
|
||||
]
|
||||
|
||||
# Create passes in order they run during compilation
|
||||
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
|
||||
lowering_pass = VllmIRLoweringPass(default_vllm_config)
|
||||
donated_info_pass = StoreDonationInfoPass()
|
||||
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
|
||||
|
||||
# Set up backend with pre-grad pass
|
||||
backend = TestBackend(lowering_pass, donated_info_pass, cleanup_pass)
|
||||
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
|
||||
|
||||
model = model_class()
|
||||
x = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
|
||||
with default_vllm_config.kernel_config.ir_op_priority.set_priority():
|
||||
# Reference output without optimization
|
||||
ref_output = model(x.clone(), residual1.clone(), residual2.clone())
|
||||
|
||||
# Compile with inplace optimization
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
output = compiled_model(x.clone(), residual1.clone(), residual2.clone())
|
||||
|
||||
# Verify correctness (relaxed tolerance for bfloat16)
|
||||
for i in range(len(ref_output)):
|
||||
torch.testing.assert_close(output[i], ref_output[i], rtol=1e-2, atol=1e-2)
|
||||
|
||||
# Verify expected number of ops were functionalized
|
||||
func_ops = functionalization_pass.functionalized_ops
|
||||
assert len(func_ops) == int(bool(expected_functionalized))
|
||||
if expected_functionalized > 0:
|
||||
assert "fused_add_rms_norm" in func_ops
|
||||
assert func_ops["fused_add_rms_norm"] == expected_functionalized
|
||||
|
||||
# Verify lowering happened (2 ops in all cases)
|
||||
assert "fused_add_rms_norm" in lowering_pass.selected_impls
|
||||
assert len(lowering_pass.selected_impls["fused_add_rms_norm"]) == 2
|
||||
assert all(
|
||||
provider == "vllm_c"
|
||||
for node, provider in lowering_pass.selected_impls["fused_add_rms_norm"].items()
|
||||
), lowering_pass.selected_impls
|
||||
|
||||
# Verify correct number of donated IDs
|
||||
assert len(donated_info_pass.donated_input_ids_sets) == 1
|
||||
assert len(donated_info_pass.donated_input_ids_sets[0]) == expected_donated
|
||||
|
||||
# Verify expected number of clones after cleanup
|
||||
actual_clones = backend.op_count(torch.ops.aten.clone.default, before=False)
|
||||
assert actual_clones == expected_clones, (
|
||||
f"Expected {expected_clones} clones, got {actual_clones}:"
|
||||
f"{backend.print_graphs()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
def test_donated_buffer_context_propagation(default_vllm_config):
|
||||
"""Test that donated_input_ids propagates correctly through pass_context."""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
# Create a custom backend that inspects pass_context in cleanup pass
|
||||
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
|
||||
lowering_pass = VllmIRLoweringPass(default_vllm_config)
|
||||
|
||||
donation_info_pass = StoreDonationInfoPass()
|
||||
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
|
||||
|
||||
backend = TestBackend(lowering_pass, donation_info_pass, cleanup_pass)
|
||||
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
|
||||
|
||||
model = MaybeInplaceModel()
|
||||
x = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual1 = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual2 = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
compiled_model(x.clone(), residual1.clone(), residual2.clone())
|
||||
|
||||
donated_ids_seen = donation_info_pass.donated_input_ids_sets
|
||||
# Verify donated_input_ids was set and propagated
|
||||
assert len(donated_ids_seen) == 1
|
||||
# Should have donated inputs (exact indices depend on AOTAutograd)
|
||||
assert len(donated_ids_seen[0]) == 3
|
||||
# All donated ids should be valid non-negative integers
|
||||
for idx in donated_ids_seen[0]:
|
||||
assert isinstance(idx, int) and idx >= 0, f"Invalid donated index: {idx}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
def test_maybe_inplace_reuse_error(default_vllm_config):
|
||||
"""Test that reusing a donated activation input raises ValueError."""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
class ReuseModel(nn.Module):
|
||||
"""Model that incorrectly reuses a donated activation input."""
|
||||
|
||||
def __init__(self, hidden_size=16):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(self, x: torch.Tensor, residual: torch.Tensor):
|
||||
# x is donated to maybe_inplace
|
||||
x_normed, residual_out = ops.fused_add_rms_norm.maybe_inplace(
|
||||
x, residual, self.weight, 1e-5
|
||||
)
|
||||
# ERROR: x is used again after being donated
|
||||
return x_normed + x # This should raise ValueError
|
||||
|
||||
functionalization_pass = VllmIRInplaceFunctionalizationPass(default_vllm_config)
|
||||
lowering_pass = VllmIRLoweringPass(default_vllm_config)
|
||||
cleanup_pass = UnsafeCloneEliminationPass(default_vllm_config)
|
||||
|
||||
backend = TestBackend(lowering_pass, cleanup_pass)
|
||||
backend.inductor_config["pre_grad_custom_pass"] = functionalization_pass
|
||||
|
||||
model = ReuseModel()
|
||||
x = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
residual = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
|
||||
# Compilation should raise BackendCompilerFailed wrapping ValueError
|
||||
with pytest.raises(
|
||||
torch._dynamo.exc.BackendCompilerFailed,
|
||||
match="is used again after the node",
|
||||
):
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
compiled_model(x.clone(), residual.clone())
|
||||
|
||||
|
||||
# Piecewise compilation tests with graph splitting
|
||||
|
||||
|
||||
@torch.library.custom_op("vllm::test_split_marker", mutates_args=())
|
||||
def test_split_marker(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Identity op that marks a split point for piecewise compilation."""
|
||||
return x.clone()
|
||||
|
||||
|
||||
@test_split_marker.register_fake
|
||||
def _fake_split_marker(x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.empty_like(x)
|
||||
|
||||
|
||||
class TransformerBlockWithSplits(nn.Module):
|
||||
"""Transformer block with explicit split points for piecewise compilation."""
|
||||
|
||||
def __init__(self, hidden_size=32, intermediate_size=128):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
|
||||
# Attention-like projection
|
||||
self.attn_proj = nn.Linear(
|
||||
hidden_size, hidden_size, bias=False, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
# Post-attention norm
|
||||
self.post_attn_norm = nn.Parameter(
|
||||
torch.ones(hidden_size, dtype=torch.bfloat16)
|
||||
)
|
||||
|
||||
# MLP
|
||||
self.gate_proj = nn.Linear(
|
||||
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
|
||||
)
|
||||
self.up_proj = nn.Linear(
|
||||
hidden_size, intermediate_size, bias=False, dtype=torch.bfloat16
|
||||
)
|
||||
self.down_proj = nn.Linear(
|
||||
intermediate_size, hidden_size, bias=False, dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
# Post-MLP norm
|
||||
self.post_mlp_norm = nn.Parameter(torch.ones(hidden_size, dtype=torch.bfloat16))
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
# Attention block with residual
|
||||
residual1 = x
|
||||
attn_out = self.attn_proj(x)
|
||||
|
||||
# Fused add + norm (maybe_inplace: residual1 is donated)
|
||||
normed1, residual1 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
attn_out, residual1, self.post_attn_norm, 1e-5
|
||||
)
|
||||
|
||||
# Force a graph split here
|
||||
normed1 = torch.ops.vllm.test_split_marker(normed1)
|
||||
|
||||
# MLP block
|
||||
gate = self.gate_proj(normed1)
|
||||
up = self.up_proj(normed1)
|
||||
mlp_out = self.down_proj(gate * torch.nn.functional.silu(up))
|
||||
|
||||
# Fused add + norm (maybe_inplace: residual1 is donated)
|
||||
normed2, residual2 = ops.fused_add_rms_norm.maybe_inplace(
|
||||
mlp_out, residual1, self.post_mlp_norm, 1e-5
|
||||
)
|
||||
|
||||
return normed2, residual2
|
||||
|
||||
|
||||
def with_dyn_arg(fn: Callable, arg_index: int, dim_index: int):
|
||||
def inner(*args):
|
||||
torch._dynamo.mark_dynamic(args[arg_index], dim_index)
|
||||
return fn(*args)
|
||||
|
||||
return inner
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
def test_piecewise_compilation_with_donated_buffers(monkeypatch, fresh_vllm_cache):
|
||||
"""Test piecewise compilation with donated buffers across graph splits.
|
||||
Utilizes a custom splitting op. Uses fresh cache to avoid compilation caching.
|
||||
"""
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
# Disable compilation cache to avoid serialization issues
|
||||
monkeypatch.setenv("VLLM_DISABLE_COMPILE_CACHE", "1")
|
||||
|
||||
from vllm.compilation.backends import VllmBackend
|
||||
from vllm.config import CompilationConfig, VllmConfig
|
||||
|
||||
# Create config with custom splitting op
|
||||
store_donation_info = StoreDonationInfoPass()
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
custom_ops=["all"],
|
||||
splitting_ops=["vllm::test_split_marker"],
|
||||
inductor_compile_config={"post_grad_custom_post_pass": store_donation_info},
|
||||
)
|
||||
)
|
||||
|
||||
backend = VllmBackend(vllm_config)
|
||||
|
||||
model = TransformerBlockWithSplits()
|
||||
x = torch.randn(8, 32, dtype=torch.bfloat16)
|
||||
|
||||
# Reference output
|
||||
ref_output = with_dyn_arg(model, 0, 0)(x.clone())
|
||||
|
||||
# Compile with piecewise compilation (graph will split at split_marker)
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=False)
|
||||
output = with_dyn_arg(compiled_model, 0, 0)(x.clone())
|
||||
|
||||
# Verify correctness (relaxed tolerance for bfloat16)
|
||||
torch.testing.assert_close(output[0], ref_output[0], rtol=1e-2, atol=1e-2)
|
||||
torch.testing.assert_close(output[1], ref_output[1], rtol=1e-2, atol=1e-2)
|
||||
|
||||
# Verify the model was split into multiple submodules
|
||||
assert hasattr(backend, "split_gm"), "Backend should have split graph module"
|
||||
|
||||
# Should have at least 2 submodules (split by test_split_marker op)
|
||||
submodules = list(backend.split_gm.named_children())
|
||||
num_submodules = len(submodules)
|
||||
assert num_submodules >= 2, (
|
||||
f"Expected at least 2 submodules (split), got {num_submodules}"
|
||||
)
|
||||
|
||||
# Check that donation info was propagated correctly
|
||||
donated_inputs_sets = store_donation_info.donated_input_ids_sets
|
||||
assert len(donated_inputs_sets) == 2
|
||||
assert len(donated_inputs_sets[0]) == 1
|
||||
assert len(donated_inputs_sets[1]) == 1
|
||||
@@ -1,103 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
import vllm.kernels # noqa: F401 to register kernels
|
||||
from vllm import ir
|
||||
from vllm.compilation.passes.ir.lowering_pass import (
|
||||
VllmIRLoweringPass,
|
||||
)
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.ir import ops
|
||||
from vllm.model_executor.layers.activation import GeluAndMulSparse
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from ...backend import TestBackend
|
||||
|
||||
|
||||
class Model(nn.Module):
|
||||
def __init__(self, hidden_size=16, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.hidden_size = hidden_size
|
||||
self.weight = torch.ones(hidden_size, dtype=torch.bfloat16)
|
||||
|
||||
def forward(self, x):
|
||||
x1 = x + 4.0
|
||||
x2 = ops.rms_norm(x1, self.weight, 1e-5)
|
||||
x3 = x2 * 5.0
|
||||
# no weight
|
||||
x4 = ops.rms_norm(x3, None, 1e-5)
|
||||
x5 = x4 / 2.0
|
||||
# dispatch to native due to variance_size parameter
|
||||
x6 = ops.rms_norm(x5, self.weight, 1e-5, self.hidden_size // 2)
|
||||
return x6 + 3.0
|
||||
|
||||
|
||||
class SparseActivationModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.activation = GeluAndMulSparse(0.95, "tanh")
|
||||
|
||||
def forward(self, x):
|
||||
return self.activation(x)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rms_provider", ops.rms_norm.supported_providers())
|
||||
def test_lowering_rms_norm(rms_provider, default_vllm_config):
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
|
||||
lowering_pass = VllmIRLoweringPass(get_current_vllm_config())
|
||||
backend = TestBackend(lowering_pass)
|
||||
backend_unlowered = TestBackend()
|
||||
|
||||
model = Model()
|
||||
x = torch.randn(8, 16, dtype=torch.bfloat16)
|
||||
with (
|
||||
ops.rms_norm.set_priority([rms_provider, "native"]),
|
||||
ir.enable_torch_wrap(True),
|
||||
):
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
compiled_unlowered_model = torch.compile(
|
||||
model, backend=backend_unlowered, fullgraph=True
|
||||
)
|
||||
output = compiled_model(x)
|
||||
output_unlowered = compiled_unlowered_model(x)
|
||||
|
||||
selected = lowering_pass.selected_impls["rms_norm"]
|
||||
assert len(selected) == 3
|
||||
assert selected["rms_norm"] == rms_provider
|
||||
assert selected["rms_norm_1"] == rms_provider
|
||||
assert selected["rms_norm_2"] == "native"
|
||||
|
||||
# Compiled function guards on global value, avoid recompilation
|
||||
with ir.enable_torch_wrap(True):
|
||||
output2 = compiled_model(x)
|
||||
|
||||
torch.testing.assert_close(output_unlowered, output)
|
||||
torch.testing.assert_close(output_unlowered, output2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA-only provider")
|
||||
def test_lowering_gelu_and_mul_sparse(default_vllm_config):
|
||||
torch.set_default_device("cuda")
|
||||
lowering_pass = VllmIRLoweringPass(get_current_vllm_config())
|
||||
backend = TestBackend(lowering_pass)
|
||||
model = SparseActivationModel()
|
||||
x = torch.randn(8, 32768, dtype=torch.bfloat16)
|
||||
|
||||
with (
|
||||
ops.gelu_and_mul_sparse.set_priority(["triton", "native"]),
|
||||
ir.enable_torch_wrap(True),
|
||||
):
|
||||
compiled_model = torch.compile(model, backend=backend, fullgraph=True)
|
||||
output = compiled_model(x)
|
||||
|
||||
selected = lowering_pass.selected_impls["gelu_and_mul_sparse"]
|
||||
assert selected == {"gelu_and_mul_sparse": "triton"}
|
||||
|
||||
expected = ops.gelu_and_mul_sparse.impls["native"].impl_fn(
|
||||
x, 1.6448536269514722, "tanh"
|
||||
)
|
||||
torch.testing.assert_close(output, expected, rtol=1.6e-2, atol=1e-3)
|
||||
@@ -1,340 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.utils import TestFP8Layer
|
||||
from vllm.compilation.passes.fusion.act_quant_fusion import (
|
||||
ActivationQuantFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.fusion.rms_quant_fusion import RMSNormQuantFusionPass
|
||||
from vllm.compilation.passes.fx_utils import find_auto_fn, find_auto_fn_maybe, is_func
|
||||
from vllm.compilation.passes.utility.fix_functionalization import (
|
||||
FixFunctionalizationPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
get_current_vllm_config,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8StaticTensorSym,
|
||||
)
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
TEST_FP8 = current_platform.supports_fp8()
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
|
||||
class TestSiluMul(torch.nn.Module):
|
||||
quant_key = kFp8StaticTensorSym
|
||||
|
||||
def __init__(self, hidden_size: int = 128):
|
||||
super().__init__()
|
||||
self.silu_and_mul = SiluAndMul()
|
||||
if TEST_FP8:
|
||||
self.fp8_linear = TestFP8Layer(
|
||||
weight_shape=(hidden_size, hidden_size),
|
||||
activation_quant_key=self.quant_key,
|
||||
weight_quant_key=self.quant_key,
|
||||
input_dtype=get_current_vllm_config().model_config.dtype,
|
||||
)
|
||||
|
||||
def forward(self, x):
|
||||
y = self.silu_and_mul(x)
|
||||
if TEST_FP8:
|
||||
return self.fp8_linear(y)
|
||||
else:
|
||||
return y
|
||||
|
||||
def example_inputs(self, num_tokens=32, hidden_size=128):
|
||||
return (torch.rand(num_tokens, hidden_size * 2),)
|
||||
|
||||
def ops_in_model(self, do_fusion):
|
||||
if TEST_FP8 and do_fusion:
|
||||
return [torch.ops._C.silu_and_mul_quant.default]
|
||||
else:
|
||||
return [torch.ops._C.silu_and_mul.default]
|
||||
|
||||
def ops_not_in_model(self):
|
||||
return []
|
||||
|
||||
|
||||
class TestFusedAddRMSNorm(torch.nn.Module):
|
||||
quant_key = kFp8StaticTensorSym
|
||||
|
||||
def __init__(self, hidden_size=16, intermediate_size=32):
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
|
||||
self.gate_proj = torch.nn.Parameter(
|
||||
torch.empty((intermediate_size, hidden_size))
|
||||
)
|
||||
self.norm = RMSNorm(intermediate_size, 1e-05)
|
||||
self.norm.weight = torch.nn.Parameter(torch.ones(intermediate_size))
|
||||
|
||||
torch.nn.init.normal_(self.gate_proj, std=0.02)
|
||||
|
||||
if TEST_FP8:
|
||||
self.fp8_linear = TestFP8Layer(
|
||||
weight_shape=(hidden_size, intermediate_size),
|
||||
activation_quant_key=self.quant_key,
|
||||
weight_quant_key=self.quant_key,
|
||||
input_dtype=get_current_vllm_config().model_config.dtype,
|
||||
)
|
||||
|
||||
def forward(self, hidden_states, residual):
|
||||
# Reshape input
|
||||
view = hidden_states.reshape(-1, self.hidden_size)
|
||||
|
||||
# matrix multiplication
|
||||
permute = self.gate_proj.permute(1, 0)
|
||||
mm = torch.mm(view, permute)
|
||||
|
||||
# layer normalization
|
||||
norm_output, residual_output = self.norm(mm, residual)
|
||||
|
||||
if TEST_FP8:
|
||||
# scaled_mm with static input quantization
|
||||
fp8_linear_result = self.fp8_linear(norm_output)
|
||||
|
||||
return fp8_linear_result, residual_output
|
||||
|
||||
else:
|
||||
return norm_output, residual_output
|
||||
|
||||
def example_inputs(self, batch_size=8, seq_len=16):
|
||||
hidden_states = torch.randn((batch_size * seq_len, self.hidden_size))
|
||||
residual = torch.randn((batch_size * seq_len, self.intermediate_size))
|
||||
return (hidden_states, residual)
|
||||
|
||||
def ops_in_model(self, do_fusion):
|
||||
if TEST_FP8 and do_fusion:
|
||||
return [torch.ops._C.fused_add_rms_norm_static_fp8_quant.default]
|
||||
else:
|
||||
return []
|
||||
|
||||
def ops_not_in_model(self):
|
||||
return []
|
||||
|
||||
|
||||
class TestRotaryEmbedding(torch.nn.Module):
|
||||
def __init__(self, head_dim=64, max_position=2048, base=10000):
|
||||
super().__init__()
|
||||
self.head_dim = head_dim
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position,
|
||||
rope_parameters={"rope_type": "default", "rope_theta": base},
|
||||
)
|
||||
|
||||
def forward(self, positions, q, k):
|
||||
q_rotated, k_rotated = self.rotary_emb(positions, q, k)
|
||||
return q_rotated, k_rotated
|
||||
|
||||
def example_inputs(self, num_tokens=32, head_dim=64):
|
||||
positions = torch.arange(num_tokens, dtype=torch.long)
|
||||
q = torch.randn(num_tokens, head_dim)
|
||||
k = torch.randn(num_tokens, head_dim)
|
||||
return (positions, q, k)
|
||||
|
||||
def ops_in_model(self, do_fusion):
|
||||
return [torch.ops._C.rotary_embedding.default]
|
||||
|
||||
def ops_not_in_model(self):
|
||||
return []
|
||||
|
||||
|
||||
class TestRotaryEmbeddingSliceScatter(torch.nn.Module):
|
||||
def __init__(self, head_dim=64, num_heads=4, max_position=2048, base=10000):
|
||||
super().__init__()
|
||||
self.head_dim = head_dim
|
||||
self.num_heads = num_heads
|
||||
self.hidden_size = head_dim * num_heads
|
||||
|
||||
self.qkv_proj = torch.nn.Linear(
|
||||
self.hidden_size, self.hidden_size * 3, bias=False
|
||||
)
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=max_position,
|
||||
rope_parameters={"rope_type": "default", "rope_theta": base},
|
||||
)
|
||||
|
||||
def forward(self, positions, hidden_states):
|
||||
# Simulate the pattern: mm -> split_with_sizes -> rotary_embedding
|
||||
# -> slice_scatter -> split_with_sizes
|
||||
|
||||
qkv = self.qkv_proj(hidden_states)
|
||||
split_sizes = [self.hidden_size, self.hidden_size, self.hidden_size]
|
||||
q, k, v = torch.split(qkv, split_sizes, dim=-1)
|
||||
|
||||
q_rotated, k_rotated = self.rotary_emb(positions, q, k)
|
||||
|
||||
qkv_updated = torch.cat([q_rotated, k_rotated, v], dim=-1)
|
||||
return qkv_updated
|
||||
|
||||
def example_inputs(self, num_tokens=32, head_dim=64, num_heads=4):
|
||||
hidden_size = head_dim * num_heads
|
||||
positions = torch.arange(num_tokens, dtype=torch.long)
|
||||
hidden_states = torch.randn(num_tokens, hidden_size)
|
||||
return (positions, hidden_states)
|
||||
|
||||
def ops_in_model(self, do_fusion):
|
||||
return [torch.ops._C.rotary_embedding.default]
|
||||
|
||||
def ops_not_in_model(self):
|
||||
return [torch.ops.aten.slice_scatter.default]
|
||||
|
||||
|
||||
class TestFunctionWithMutatedArgsAndReturn(torch.nn.Module):
|
||||
OP_REGISTERED = False
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.register_test_custom_op()
|
||||
|
||||
@classmethod
|
||||
def register_test_custom_op(cls):
|
||||
if not cls.OP_REGISTERED:
|
||||
|
||||
def function_with_mutated_args_and_return_impl(
|
||||
x: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
ret = x + 1
|
||||
x.add_(2)
|
||||
return ret
|
||||
|
||||
def function_with_mutated_args_and_return_fake(
|
||||
x: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(x)
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="function_with_mutated_args_and_return",
|
||||
op_func=function_with_mutated_args_and_return_impl,
|
||||
mutates_args=["x"],
|
||||
fake_impl=function_with_mutated_args_and_return_fake,
|
||||
)
|
||||
|
||||
cls.OP_REGISTERED = True
|
||||
|
||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
# Clone x to avoid mutating the original tensor
|
||||
ret = torch.ops.vllm.function_with_mutated_args_and_return(x)
|
||||
return x, ret
|
||||
|
||||
def example_inputs(self, num_tokens=32):
|
||||
hidden_states = torch.randn(num_tokens)
|
||||
return (hidden_states,)
|
||||
|
||||
def ops_in_model(self, do_fusion):
|
||||
return [torch.ops.vllm.function_with_mutated_args_and_return.default]
|
||||
|
||||
def ops_not_in_model(self):
|
||||
return []
|
||||
|
||||
|
||||
MODELS_AND_DO_FUSION = {
|
||||
TestSiluMul: [True, False],
|
||||
TestFusedAddRMSNorm: [True, False],
|
||||
TestRotaryEmbedding: [False],
|
||||
TestRotaryEmbeddingSliceScatter: [False],
|
||||
TestFunctionWithMutatedArgsAndReturn: [False],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"model_class, do_fusion",
|
||||
[
|
||||
(model_class, do_fusion)
|
||||
for model_class, fusions in MODELS_AND_DO_FUSION.items()
|
||||
for do_fusion in fusions
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
def test_fix_functionalization(
|
||||
model_class: torch.nn.Module, do_fusion: bool, dtype: torch.dtype
|
||||
):
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
compilation_config=CompilationConfig(
|
||||
custom_ops=["all"],
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=do_fusion,
|
||||
fuse_act_quant=do_fusion,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
assert RMSNorm.enabled()
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
fusion_pass = RMSNormQuantFusionPass(vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
act_quant_fusion_pass = ActivationQuantFusionPass(vllm_config)
|
||||
|
||||
passes = (
|
||||
[noop_pass, fusion_pass, act_quant_fusion_pass, cleanup_pass]
|
||||
if do_fusion
|
||||
else [noop_pass, cleanup_pass]
|
||||
)
|
||||
func_pass = FixFunctionalizationPass(vllm_config)
|
||||
|
||||
backend_func = TestBackend(*passes, func_pass)
|
||||
backend_no_func = TestBackend(*passes)
|
||||
|
||||
model = model_class()
|
||||
inputs_func = model.example_inputs()
|
||||
inputs_no_func = copy.deepcopy(inputs_func)
|
||||
model_func = copy.deepcopy(model)
|
||||
model_no_func = copy.deepcopy(model)
|
||||
model_func = torch.compile(model_func, backend=backend_func)
|
||||
model_no_func = torch.compile(model_no_func, backend=backend_no_func)
|
||||
|
||||
# deepcopy inputs to prevent potential in place mutation
|
||||
outputs_func = model_func(*copy.deepcopy(inputs_func))
|
||||
outputs_no_func = model_no_func(*copy.deepcopy(inputs_no_func))
|
||||
torch.testing.assert_close(outputs_func, outputs_no_func)
|
||||
|
||||
# check if the functionalization pass is applied
|
||||
for op in model.ops_in_model(do_fusion):
|
||||
find_auto_fn(backend_no_func.graph_post_pass.nodes, op)
|
||||
assert find_auto_fn_maybe(backend_func.graph_post_pass.nodes, op) is None
|
||||
|
||||
# make sure the ops were all de-functionalized
|
||||
found = dict()
|
||||
for node in backend_func.graph_post_pass.nodes:
|
||||
for op in model.ops_in_model(do_fusion):
|
||||
if is_func(node, op):
|
||||
found[op] = True
|
||||
for op in model.ops_not_in_model():
|
||||
if is_func(node, op):
|
||||
found[op] = True
|
||||
assert all(found[op] for op in model.ops_in_model(do_fusion))
|
||||
assert all(not found.get(op) for op in model.ops_not_in_model())
|
||||
@@ -1,418 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit test for the MLADualRMSNormFusionPass.
|
||||
|
||||
The pass fuses paired q/kv RMS norms in MLA attention into a single
|
||||
fused_mla_dual_rms_norm op backed by AITER's fused_qk_rmsnorm kernel.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.config
|
||||
from tests.compile.backend import TestBackend
|
||||
from vllm._aiter_ops import (
|
||||
is_aiter_found_and_supported,
|
||||
rocm_aiter_ops,
|
||||
)
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# MLA attention geometry for DeepSeek-V3 / Kimi-K2
|
||||
Q_DIM = 1536
|
||||
KV_C_DIM = 512
|
||||
K_PE_DIM = 64
|
||||
EPS = 1e-6
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
|
||||
class MLADualRMSNormTestModel(torch.nn.Module):
|
||||
"""Minimal model reproducing the MLA dual RMS norm pattern:
|
||||
linear -> split([q_dim, kv_dim])
|
||||
+-- q_c (getitem 0) -> rms_norm(q_w, eps) -> linear
|
||||
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
|
||||
+-- kv_c (getitem 0) -> rms_norm(kv_w, eps)
|
||||
+-- k_pe
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
q_dim: int = Q_DIM,
|
||||
kv_c_dim: int = KV_C_DIM,
|
||||
k_pe_dim: int = K_PE_DIM,
|
||||
eps: float = EPS,
|
||||
):
|
||||
super().__init__()
|
||||
self.q_dim = q_dim
|
||||
self.kv_dim = kv_c_dim + k_pe_dim
|
||||
self.kv_c_dim = kv_c_dim
|
||||
self.k_pe_dim = k_pe_dim
|
||||
|
||||
self.proj = torch.nn.Linear(hidden_size, q_dim + self.kv_dim, bias=False)
|
||||
self.q_norm = RMSNorm(q_dim, eps=eps)
|
||||
self.kv_norm = RMSNorm(kv_c_dim, eps=eps)
|
||||
self.q_b_proj = torch.nn.Linear(q_dim, hidden_size, bias=False)
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Avoid graph input being a direct arg to a matched pattern node
|
||||
x = torch.relu(x)
|
||||
|
||||
projected = self.proj(x)
|
||||
|
||||
q_c, kv_lora = projected.split([self.q_dim, self.kv_dim], dim=-1)
|
||||
kv_c, k_pe = kv_lora.split([self.kv_c_dim, self.k_pe_dim], dim=-1)
|
||||
|
||||
q_normed = self.q_norm(q_c)
|
||||
kv_normed = self.kv_norm(kv_c)
|
||||
|
||||
q_out = self.q_b_proj(q_normed)
|
||||
return q_out, kv_normed, k_pe
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.vllm_ir.rms_norm.default]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.vllm.fused_mla_dual_rms_norm.default]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("hidden_size", [7168])
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="Only test on ROCm with AITER installed and supported",
|
||||
)
|
||||
def test_fuse_mla_dual_rms_norm(
|
||||
dtype: torch.dtype,
|
||||
hidden_size: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
torch._dynamo.reset()
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=["+rms_norm"],
|
||||
pass_config=PassConfig(
|
||||
fuse_mla_dual_rms_norm=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
MLADualRMSNormFusionPass,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(42)
|
||||
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
fusion_pass = MLADualRMSNormFusionPass(vllm_config)
|
||||
passes = [
|
||||
NoOpEliminationPass(vllm_config),
|
||||
fusion_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
]
|
||||
backend = TestBackend(*passes)
|
||||
model = MLADualRMSNormTestModel(hidden_size)
|
||||
|
||||
x = torch.randn(1, hidden_size)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
|
||||
outputs_unfused = model(x)
|
||||
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
outputs_fused = model_fused(x)
|
||||
|
||||
torch.testing.assert_close(outputs_unfused, outputs_fused, atol=1e-2, rtol=1e-2)
|
||||
|
||||
assert fusion_pass.matched_count == 1, (
|
||||
f"Expected 1 fused pair, got {fusion_pass.matched_count}"
|
||||
)
|
||||
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
|
||||
|
||||
class MLADualRMSNormFp8PerTokenTestModel(torch.nn.Module):
|
||||
"""Minimal model reproducing the FP8 MLA attention path with *per-token* quant:
|
||||
linear -> split([q_dim, kv_dim])
|
||||
+-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fused_dynamic_quant -> dequant
|
||||
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
|
||||
+-- kv_c (getitem 0) -> rms_norm (bf16)
|
||||
+-- k_pe
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
q_dim: int = Q_DIM,
|
||||
kv_c_dim: int = KV_C_DIM,
|
||||
k_pe_dim: int = K_PE_DIM,
|
||||
eps: float = EPS,
|
||||
):
|
||||
super().__init__()
|
||||
self.q_dim = q_dim
|
||||
self.kv_dim = kv_c_dim + k_pe_dim
|
||||
self.kv_c_dim = kv_c_dim
|
||||
self.k_pe_dim = k_pe_dim
|
||||
self.eps = eps
|
||||
|
||||
self.proj = torch.nn.Linear(hidden_size, q_dim + self.kv_dim, bias=False)
|
||||
self.q_weight = torch.nn.Parameter(torch.ones(q_dim))
|
||||
self.kv_norm = RMSNorm(kv_c_dim, eps=eps)
|
||||
|
||||
def _dequant(self, x_fp8: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
||||
# Per-token: a single (M, 1) scale broadcast across the row.
|
||||
return (x_fp8.to(torch.float32) * scale).to(torch.bfloat16)
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
# Avoid graph input being a direct arg to a matched pattern node
|
||||
x = torch.relu(x)
|
||||
|
||||
projected = self.proj(x)
|
||||
|
||||
q_c, kv_lora = projected.split([self.q_dim, self.kv_dim], dim=-1)
|
||||
kv_c, k_pe = kv_lora.split([self.kv_c_dim, self.k_pe_dim], dim=-1)
|
||||
|
||||
q_fp8, q_scale = torch.ops.vllm.rocm_aiter_rmsnorm_fused_dynamic_quant(
|
||||
q_c, self.q_weight, self.eps, FP8_DTYPE
|
||||
)
|
||||
kv_normed = self.kv_norm(kv_c)
|
||||
|
||||
return self._dequant(q_fp8, q_scale), kv_normed, k_pe
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
torch.ops.vllm.rocm_aiter_rmsnorm_fused_dynamic_quant.default,
|
||||
torch.ops.vllm_ir.rms_norm.default,
|
||||
]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.vllm.fused_mla_dual_rms_norm_per_token_quant.default]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("hidden_size", [7168])
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="Only test on ROCm with AITER installed and supported",
|
||||
)
|
||||
def test_fuse_mla_dual_rms_norm_fp8_per_token(
|
||||
dtype: torch.dtype,
|
||||
hidden_size: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
torch._dynamo.reset()
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=["+rms_norm"],
|
||||
pass_config=PassConfig(
|
||||
fuse_mla_dual_rms_norm=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
MLADualRMSNormFusionPass,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(42)
|
||||
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
fusion_pass = MLADualRMSNormFusionPass(vllm_config)
|
||||
passes = [
|
||||
NoOpEliminationPass(vllm_config),
|
||||
fusion_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
]
|
||||
backend = TestBackend(*passes)
|
||||
model = MLADualRMSNormFp8PerTokenTestModel(hidden_size)
|
||||
|
||||
x = torch.randn(4, hidden_size)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
|
||||
with torch.inference_mode():
|
||||
outputs_unfused = model(x)
|
||||
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
outputs_fused = model_fused(x)
|
||||
|
||||
q_deq_u, kv_normed_u, k_pe_u = outputs_unfused
|
||||
q_deq_f, kv_normed_f, k_pe_f = outputs_fused
|
||||
|
||||
torch.testing.assert_close(k_pe_u, k_pe_f, atol=0, rtol=0)
|
||||
|
||||
torch.testing.assert_close(kv_normed_u, kv_normed_f, atol=1e-2, rtol=1e-2)
|
||||
|
||||
E4M3_STEP = 0.125
|
||||
exact_frac = (q_deq_u == q_deq_f).float().mean().item()
|
||||
assert exact_frac > 0.99, (
|
||||
f"q: only {exact_frac:.4f} of elements bit-exact; scales likely differ"
|
||||
)
|
||||
torch.testing.assert_close(q_deq_u, q_deq_f, atol=1e-2, rtol=E4M3_STEP)
|
||||
|
||||
assert fusion_pass.matched_count == 1, (
|
||||
f"Expected 1 fused pair, got {fusion_pass.matched_count}"
|
||||
)
|
||||
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
|
||||
|
||||
class MLADualRMSNormFp8GroupTestModel(torch.nn.Module):
|
||||
"""Minimal model reproducing the FP8 MLA attention path with *group* quant
|
||||
(block-scaled fp8 weights, 128-wide activation groups):
|
||||
|
||||
linear -> split([q_dim, kv_dim])
|
||||
+-- q_c (getitem 0) -> rocm_aiter_rmsnorm_fp8_group_quant -> dequant
|
||||
+-- kv_lora (getitem 1) -> split([kv_c_dim, k_pe_dim])
|
||||
+-- kv_c (getitem 0) -> rms_norm (bf16)
|
||||
+-- k_pe
|
||||
"""
|
||||
|
||||
GROUP_SIZE = 128
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
q_dim: int = Q_DIM,
|
||||
kv_c_dim: int = KV_C_DIM,
|
||||
k_pe_dim: int = K_PE_DIM,
|
||||
eps: float = EPS,
|
||||
):
|
||||
super().__init__()
|
||||
self.q_dim = q_dim
|
||||
self.kv_dim = kv_c_dim + k_pe_dim
|
||||
self.kv_c_dim = kv_c_dim
|
||||
self.k_pe_dim = k_pe_dim
|
||||
self.eps = eps
|
||||
self.proj = torch.nn.Linear(hidden_size, q_dim + self.kv_dim, bias=False)
|
||||
self.q_weight = torch.nn.Parameter(torch.ones(q_dim))
|
||||
self.kv_norm = RMSNorm(kv_c_dim, eps=eps)
|
||||
|
||||
def _dequant(self, x_fp8: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
||||
# Group: one (M, N / 128) scale per 128-column group of the row.
|
||||
m = x_fp8.shape[0]
|
||||
x = x_fp8.to(torch.float32).view(m, -1, self.GROUP_SIZE)
|
||||
return (x * scale.view(m, -1, 1)).view(m, -1).to(torch.bfloat16)
|
||||
|
||||
def forward(self, x: torch.Tensor):
|
||||
# Avoid graph input being a direct arg to a matched pattern node
|
||||
x = torch.relu(x)
|
||||
projected = self.proj(x)
|
||||
q_c, kv_lora = projected.split([self.q_dim, self.kv_dim], dim=-1)
|
||||
kv_c, k_pe = kv_lora.split([self.kv_c_dim, self.k_pe_dim], dim=-1)
|
||||
q_fp8, q_scale = torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant(
|
||||
q_c, self.q_weight, self.eps, self.GROUP_SIZE
|
||||
)
|
||||
kv_normed = self.kv_norm(kv_c)
|
||||
# Also return the group scales so the test can check them directly:
|
||||
# equal scales pin any fp8 payload difference to rounding alone.
|
||||
return self._dequant(q_fp8, q_scale), q_scale, kv_normed, k_pe
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
torch.ops.vllm.rocm_aiter_rmsnorm_fp8_group_quant.default,
|
||||
torch.ops.vllm_ir.rms_norm.default,
|
||||
]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.vllm.fused_mla_dual_rms_norm_group_quant.default]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("hidden_size", [7168])
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="Only test on ROCm with AITER installed and supported",
|
||||
)
|
||||
def test_fuse_mla_dual_rms_norm_fp8_group(
|
||||
dtype: torch.dtype,
|
||||
hidden_size: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
torch._dynamo.reset()
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=["+rms_norm"],
|
||||
pass_config=PassConfig(
|
||||
fuse_mla_dual_rms_norm=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
MLADualRMSNormFusionPass,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(42)
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
fusion_pass = MLADualRMSNormFusionPass(vllm_config)
|
||||
passes = [
|
||||
NoOpEliminationPass(vllm_config),
|
||||
fusion_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
]
|
||||
backend = TestBackend(*passes)
|
||||
model = MLADualRMSNormFp8GroupTestModel(hidden_size)
|
||||
|
||||
x = torch.randn(64, hidden_size)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
|
||||
with torch.inference_mode():
|
||||
outputs_unfused = model(x)
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
outputs_fused = model_fused(x)
|
||||
|
||||
q_deq_u, q_scale_u, kv_normed_u, k_pe_u = outputs_unfused
|
||||
q_deq_f, q_scale_f, kv_normed_f, k_pe_f = outputs_fused
|
||||
|
||||
torch.testing.assert_close(k_pe_u, k_pe_f, atol=0, rtol=0)
|
||||
torch.testing.assert_close(kv_normed_u, kv_normed_f, atol=1e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(q_scale_u, q_scale_f, atol=0, rtol=1e-5)
|
||||
|
||||
E4M3_STEP = 0.125
|
||||
exact_frac = (q_deq_u == q_deq_f).float().mean().item()
|
||||
assert exact_frac > 0.99, f"q: only {exact_frac:.4f} of elements bit-exact"
|
||||
torch.testing.assert_close(q_deq_u, q_deq_f, atol=1e-2, rtol=E4M3_STEP)
|
||||
|
||||
assert fusion_pass.matched_count == 1, (
|
||||
f"Expected 1 fused pair, got {fusion_pass.matched_count}"
|
||||
)
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
@@ -1,813 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.config
|
||||
import vllm.ir.ops
|
||||
import vllm.plugins
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor
|
||||
from tests.utils import TestFP8Layer
|
||||
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
|
||||
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
from vllm.compilation.passes.fusion.matcher_utils import QUANT_OPS
|
||||
from vllm.compilation.passes.fusion.rms_quant_fusion import (
|
||||
FUSED_OPS,
|
||||
FusedRMSQuantKey,
|
||||
RMSNormQuantFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.fx_utils import find_auto_fn_maybe
|
||||
from vllm.compilation.passes.utility.fix_functionalization import (
|
||||
FixFunctionalizationPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
AiterFp8BlockScaledMMKernel,
|
||||
ChannelWiseTorchFP8ScaledMMLinearKernel,
|
||||
CutlassFp8BlockScaledMMKernel,
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
DeepGemmFp8BlockScaledMMKernel,
|
||||
FlashInferFp8DeepGEMMDynamicBlockScaledKernel,
|
||||
FlashInferFP8ScaledMMLinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ROCmFP8ScaledMMLinearKernel,
|
||||
RowWiseTorchFP8ScaledMMLinearKernel,
|
||||
TritonFp8BlockScaledMMKernel,
|
||||
_KernelT,
|
||||
)
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm, RMSNormGated
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
create_fp8_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.w8a8_utils import (
|
||||
cutlass_block_fp8_supported,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
is_deep_gemm_e8m0_used,
|
||||
is_deep_gemm_supported,
|
||||
)
|
||||
|
||||
# Kernel and group_shape combinations: (kernel, group_shape)
|
||||
# CUDA kernels
|
||||
CUDA_KERNEL_GROUPSHAPE_COMBINATIONS = [
|
||||
# FlashInferFP8ScaledMMLinearKernel supports both per-tensor only
|
||||
(FlashInferFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR),
|
||||
# CutlassFP8ScaledMMLinearKernel supports both per-tensor and per-token
|
||||
(CutlassFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN),
|
||||
(CutlassFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR),
|
||||
# PerTensorTorchFP8ScaledMMLinearKernel only supports per-tensor
|
||||
(PerTensorTorchFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR),
|
||||
# ChannelWiseTorchFP8ScaledMMLinearKernel only supports per-token
|
||||
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN),
|
||||
# Blockwise group shapes
|
||||
(FlashInferFp8DeepGEMMDynamicBlockScaledKernel, GroupShape(1, 128)),
|
||||
(CutlassFp8BlockScaledMMKernel, GroupShape(1, 128)),
|
||||
(DeepGemmFp8BlockScaledMMKernel, GroupShape(1, 128)),
|
||||
(TritonFp8BlockScaledMMKernel, GroupShape(1, 128)),
|
||||
(TritonFp8BlockScaledMMKernel, GroupShape(1, 64)),
|
||||
]
|
||||
|
||||
# ROCm kernels
|
||||
ROCM_KERNEL_GROUPSHAPE_COMBINATIONS = [
|
||||
# ROCmFP8ScaledMMLinearKernel supports per-tensor only
|
||||
(ROCmFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR),
|
||||
# RowWiseTorchFP8ScaledMMLinearKernel only supports per-token
|
||||
(RowWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN),
|
||||
# ChannelWiseTorchFP8ScaledMMLinearKernel only supports per-token
|
||||
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN),
|
||||
# Blockwise group shapes (no kernel abstraction)
|
||||
(TritonFp8BlockScaledMMKernel, GroupShape(1, 128)),
|
||||
(TritonFp8BlockScaledMMKernel, GroupShape(1, 64)),
|
||||
]
|
||||
|
||||
KERNEL_GROUPSHAPE_COMBINATIONS = (
|
||||
CUDA_KERNEL_GROUPSHAPE_COMBINATIONS
|
||||
if current_platform.is_cuda()
|
||||
else ROCM_KERNEL_GROUPSHAPE_COMBINATIONS
|
||||
)
|
||||
|
||||
# For Aiter tests we toggle use_aiter_quant_op
|
||||
AITER_KERNEL_GROUPSHAPE_COMBINATIONS = [
|
||||
# Per-token with RowWiseTorchFP8ScaledMMLinearKernel
|
||||
(RowWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, True),
|
||||
(RowWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, False),
|
||||
# Per-token with ChannelWiseTorchFP8ScaledMMLinearKernel
|
||||
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, True),
|
||||
(ChannelWiseTorchFP8ScaledMMLinearKernel, GroupShape.PER_TOKEN, False),
|
||||
# Static per-tensor: only the aiter quant op is matched by the pass
|
||||
(ROCmFP8ScaledMMLinearKernel, GroupShape.PER_TENSOR, True),
|
||||
# Blockwise
|
||||
(AiterFp8BlockScaledMMKernel, GroupShape(1, 128), True),
|
||||
]
|
||||
|
||||
|
||||
class TestModel(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
eps: float,
|
||||
force_kernel: type[_KernelT] | None,
|
||||
group_shape: GroupShape,
|
||||
dtype: torch.dtype,
|
||||
use_aiter_fusion: bool = False,
|
||||
use_aiter_quant: bool = False,
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fp8_linear_layers: list[torch.nn.Module]
|
||||
self.group_shape = group_shape
|
||||
self.use_aiter_quant_op = use_aiter_quant
|
||||
self.use_aiter_fusion = use_aiter_fusion
|
||||
self.norm = [RMSNorm(hidden_size, eps) for _ in range(4)]
|
||||
self.enable_rms_norm_custom_op = self.norm[0].enabled()
|
||||
|
||||
# Determine if blockwise based on group_shape
|
||||
is_blockwise = group_shape.is_per_group()
|
||||
|
||||
if is_blockwise:
|
||||
block_size = group_shape.col
|
||||
self.activation_quant_key = create_fp8_quant_key(
|
||||
static=False, group_shape=group_shape
|
||||
)
|
||||
self.weight_quant_key = create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(block_size, block_size)
|
||||
)
|
||||
|
||||
else:
|
||||
is_static = group_shape == GroupShape.PER_TENSOR
|
||||
self.activation_quant_key = create_fp8_quant_key(
|
||||
is_static, group_shape=group_shape
|
||||
)
|
||||
self.weight_quant_key = create_fp8_quant_key(
|
||||
static=True, group_shape=group_shape
|
||||
)
|
||||
|
||||
self.fp8_linear_layers = [
|
||||
TestFP8Layer(
|
||||
weight_shape=(hidden_size, hidden_size),
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
force_kernel=force_kernel,
|
||||
transpose_weights=use_aiter_fusion,
|
||||
input_dtype=dtype,
|
||||
)
|
||||
for _ in range(3)
|
||||
]
|
||||
|
||||
# Enable aiter quantization if requested
|
||||
for layer in self.fp8_linear_layers:
|
||||
layer.kernel.quant_fp8.use_aiter = use_aiter_quant
|
||||
|
||||
self.enable_quant_fp8_custom_op = self.fp8_linear_layers[
|
||||
0
|
||||
].is_quant_fp8_enabled()
|
||||
|
||||
def forward(self, x):
|
||||
# avoid having graph input be an arg to a pattern directly
|
||||
x = resid = torch.relu(x)
|
||||
y = self.norm[0](x)
|
||||
|
||||
x2 = self.fp8_linear_layers[0](y)
|
||||
# make sure resid is used for replacement to work
|
||||
y2, resid = self.norm[1](x2, resid)
|
||||
|
||||
x3 = self.fp8_linear_layers[1](y2)
|
||||
|
||||
y3, resid = self.norm[2](x3, resid) # use resid here
|
||||
|
||||
x4 = self.fp8_linear_layers[2](y3)
|
||||
|
||||
y4, resid = self.norm[3](x4, resid) # use resid here
|
||||
return y4
|
||||
|
||||
def ops_in_model_before(self):
|
||||
if self.group_shape.is_per_group():
|
||||
# Blockwise path
|
||||
if self.use_aiter_fusion and self.use_aiter_quant_op:
|
||||
return [rocm_aiter_ops.get_group_quant_op()]
|
||||
elif self.group_shape.is_per_tensor():
|
||||
if self.use_aiter_quant_op:
|
||||
return [rocm_aiter_ops.get_per_tensor_quant_op()]
|
||||
else:
|
||||
if self.use_aiter_quant_op:
|
||||
return [rocm_aiter_ops.get_per_token_quant_op()]
|
||||
|
||||
# Common path
|
||||
return (
|
||||
[QUANT_OPS[self.activation_quant_key]]
|
||||
if self.enable_quant_fp8_custom_op
|
||||
else [torch.ops.aten.reciprocal]
|
||||
)
|
||||
|
||||
def ops_in_model_after(self):
|
||||
if self.use_aiter_fusion:
|
||||
if self.group_shape.is_per_group():
|
||||
# Blockwise aiter fusion
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
AiterFusedAddRMSFp8GroupQuantPattern,
|
||||
AiterRMSFp8GroupQuantPattern,
|
||||
)
|
||||
|
||||
return [
|
||||
AiterFusedAddRMSFp8GroupQuantPattern.FUSED_OP,
|
||||
AiterRMSFp8GroupQuantPattern.FUSED_OP,
|
||||
]
|
||||
elif self.group_shape.is_per_tensor():
|
||||
# Static per-tensor aiter fusion
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
AiterFusedAddRMSNormStaticQuantPattern,
|
||||
AiterRMSNormStaticQuantPattern,
|
||||
)
|
||||
|
||||
return [
|
||||
AiterFusedAddRMSNormStaticQuantPattern.FUSED_OP,
|
||||
AiterRMSNormStaticQuantPattern.FUSED_OP,
|
||||
]
|
||||
else:
|
||||
# Per-token aiter fusion
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
AiterFusedAddRMSNormDynamicQuantPattern,
|
||||
AiterRMSNormDynamicQuantPattern,
|
||||
)
|
||||
|
||||
return [
|
||||
AiterFusedAddRMSNormDynamicQuantPattern.FUSED_OP,
|
||||
AiterRMSNormDynamicQuantPattern.FUSED_OP,
|
||||
]
|
||||
|
||||
# Regular fusion
|
||||
return [
|
||||
FUSED_OPS[FusedRMSQuantKey(self.activation_quant_key, True)],
|
||||
FUSED_OPS[FusedRMSQuantKey(self.activation_quant_key, False)],
|
||||
]
|
||||
|
||||
def ops_in_model_before_partial(self):
|
||||
return [
|
||||
torch.ops.vllm_ir.rms_norm,
|
||||
torch.ops.vllm_ir.fused_add_rms_norm.default,
|
||||
]
|
||||
|
||||
|
||||
class AddRMSNormNvfp4Model(torch.nn.Module):
|
||||
def __init__(self, hidden_size: int, eps: float):
|
||||
super().__init__()
|
||||
self.norm = RMSNorm(hidden_size, eps)
|
||||
self.activation_scale = torch.rand((1, 1), dtype=torch.float32)
|
||||
|
||||
weight = torch.rand((hidden_size, hidden_size))
|
||||
self.weight, self.weight_scale, weight_global_scale = quant_nvfp4_tensor(weight)
|
||||
self.alpha = (1 / (weight_global_scale * self.activation_scale)).reshape(1)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
x = residual = torch.relu(x)
|
||||
x, residual = self.norm(x, residual)
|
||||
x, block_scale = scaled_fp4_quant(x, self.activation_scale)
|
||||
x = cutlass_scaled_fp4_mm(
|
||||
x,
|
||||
self.weight,
|
||||
block_scale,
|
||||
self.weight_scale,
|
||||
self.alpha,
|
||||
out_dtype=torch.get_default_dtype(),
|
||||
)
|
||||
return x, residual
|
||||
|
||||
|
||||
def _run_fusion_test(
|
||||
model,
|
||||
fusion_pass,
|
||||
vllm_config,
|
||||
dtype,
|
||||
hidden_size,
|
||||
num_tokens,
|
||||
):
|
||||
"""Helper function for common fusion test logic.
|
||||
|
||||
Must be called within vllm_config context.
|
||||
"""
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
|
||||
backend = TestBackend(noop_pass, fusion_pass, cleanup_pass)
|
||||
backend2 = TestBackend(noop_pass, cleanup_pass)
|
||||
|
||||
x = torch.rand(num_tokens, hidden_size)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
result_fused = model_fused(x)
|
||||
|
||||
model_unfused = torch.compile(model, backend=backend2)
|
||||
result_unfused = model_unfused(x)
|
||||
|
||||
if dtype == torch.float16:
|
||||
ATOL, RTOL = (2e-3, 2e-3)
|
||||
else:
|
||||
ATOL, RTOL = (1e-2, 1e-2)
|
||||
|
||||
torch.testing.assert_close(result_fused, result_unfused, atol=ATOL, rtol=RTOL)
|
||||
|
||||
assert fusion_pass.matched_count == 3
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
|
||||
return backend, backend2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("hidden_size", [256])
|
||||
@pytest.mark.parametrize("num_tokens", [257])
|
||||
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
|
||||
@pytest.mark.parametrize("kernel_groupshape", KERNEL_GROUPSHAPE_COMBINATIONS)
|
||||
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
|
||||
@pytest.mark.parametrize("enable_quant_fp8_custom_op", [True, False])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Only test on CUDA and ROCm"
|
||||
)
|
||||
def test_fusion_rmsnorm_quant(
|
||||
dtype,
|
||||
hidden_size,
|
||||
num_tokens,
|
||||
eps,
|
||||
kernel_groupshape,
|
||||
enable_rms_norm_custom_op,
|
||||
enable_quant_fp8_custom_op,
|
||||
):
|
||||
force_kernel, group_shape = kernel_groupshape
|
||||
|
||||
if not enable_quant_fp8_custom_op and group_shape.is_per_group():
|
||||
pytest.skip("Unsupported unwrapped quant fp8 op for blockwise quantization")
|
||||
|
||||
if group_shape == GroupShape(1, 64) and (
|
||||
cutlass_block_fp8_supported() or is_deep_gemm_supported()
|
||||
):
|
||||
pytest.skip("Unsupported group shape 64 for CUTLASS/DeepGemm")
|
||||
|
||||
# TODO(quant-rms-fusion): DeepGEMM UE8M0 activation quant on B200 lowers
|
||||
# to a packed int32-scale op (per_token_group_quant_fp8_packed_for_deepgemm),
|
||||
# but the rms+quant fusion pattern only matches the fp32-scale variant, so
|
||||
# the fused output gets a mismatched scale layout and produces NaN. Only
|
||||
# reproduces on bf16 (DeepGEMM UE8M0 on B200 is bf16-only).
|
||||
# To re-enable: make rms_norm_per_block_quant emit packed UE8M0 scales
|
||||
# and extend the fusion pattern to rewrite the packed activation quant.
|
||||
deepgemm_kernels = (
|
||||
DeepGemmFp8BlockScaledMMKernel,
|
||||
FlashInferFp8DeepGEMMDynamicBlockScaledKernel,
|
||||
)
|
||||
if (
|
||||
dtype == torch.bfloat16
|
||||
and force_kernel in deepgemm_kernels
|
||||
and is_deep_gemm_e8m0_used()
|
||||
):
|
||||
pytest.skip(
|
||||
"rms+quant fusion does not yet match the packed UE8M0 DeepGEMM path"
|
||||
)
|
||||
|
||||
custom_ops = []
|
||||
if enable_rms_norm_custom_op:
|
||||
custom_ops.append("+rms_norm")
|
||||
if enable_quant_fp8_custom_op:
|
||||
custom_ops.append("+quant_fp8")
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=custom_ops,
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=True, fuse_act_quant=True, eliminate_noops=True
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with (
|
||||
vllm.config.set_current_vllm_config(vllm_config),
|
||||
vllm_config.kernel_config.ir_op_priority.set_priority(),
|
||||
):
|
||||
# Setup device before model creation
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(1)
|
||||
|
||||
fusion_pass = RMSNormQuantFusionPass(vllm_config)
|
||||
|
||||
model = TestModel(
|
||||
hidden_size=hidden_size,
|
||||
eps=eps,
|
||||
force_kernel=force_kernel,
|
||||
group_shape=group_shape,
|
||||
dtype=dtype,
|
||||
use_aiter_fusion=False,
|
||||
use_aiter_quant=False,
|
||||
)
|
||||
if any(
|
||||
type(layer.kernel) is not force_kernel for layer in model.fp8_linear_layers
|
||||
):
|
||||
pytest.skip(f"{force_kernel.__name__} is not supported on this platform")
|
||||
|
||||
backend, _ = _run_fusion_test(
|
||||
model, fusion_pass, vllm_config, dtype, hidden_size, num_tokens
|
||||
)
|
||||
backend.check_before_ops(
|
||||
model.ops_in_model_before_partial(), fully_replaced=False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "hidden_size"),
|
||||
[(1, 256), (32, 1024), (257, 4096)],
|
||||
)
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
def test_fusion_add_rmsnorm_nvfp4_quant(
|
||||
dtype: torch.dtype, eps: float, num_tokens: int, hidden_size: int
|
||||
):
|
||||
fused_op = getattr(
|
||||
torch.ops.vllm,
|
||||
"flashinfer_fused_add_rms_norm_nvfp4_quant",
|
||||
None,
|
||||
)
|
||||
if not current_platform.has_device_capability(100) or fused_op is None:
|
||||
pytest.skip("FlashInfer add-RMSNorm NVFP4 fusion is not available")
|
||||
assert fused_op is not None
|
||||
fused_op_default = fused_op.default
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
pass_config=PassConfig(
|
||||
fuse_norm_quant=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with (
|
||||
vllm.config.set_current_vllm_config(vllm_config),
|
||||
vllm_config.kernel_config.ir_op_priority.set_priority(),
|
||||
):
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(1)
|
||||
|
||||
fusion_pass = RMSNormQuantFusionPass(vllm_config)
|
||||
fused_backend = TestBackend(
|
||||
NoOpEliminationPass(vllm_config),
|
||||
fusion_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
FixFunctionalizationPass(vllm_config),
|
||||
)
|
||||
unfused_backend = TestBackend(
|
||||
NoOpEliminationPass(vllm_config),
|
||||
PostCleanupPass(vllm_config),
|
||||
FixFunctionalizationPass(vllm_config),
|
||||
)
|
||||
|
||||
model = AddRMSNormNvfp4Model(hidden_size=hidden_size, eps=eps)
|
||||
x = torch.rand(num_tokens, hidden_size)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
|
||||
result_fused = torch.compile(model, backend=fused_backend)(x)
|
||||
result_unfused = torch.compile(model, backend=unfused_backend)(x)
|
||||
|
||||
torch.testing.assert_close(result_fused, result_unfused, atol=2e-1, rtol=2e-1)
|
||||
assert fusion_pass.matched_count == 1
|
||||
fused_backend.check_before_ops([torch.ops._C.scaled_fp4_quant.out])
|
||||
fused_backend.check_after_ops([fused_op_default])
|
||||
assert (
|
||||
find_auto_fn_maybe(
|
||||
fused_backend.graph_post_pass.nodes,
|
||||
fused_op_default,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("hidden_size", [256])
|
||||
@pytest.mark.parametrize("num_tokens", [257])
|
||||
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
|
||||
@pytest.mark.parametrize(
|
||||
"kernel_groupshape_quant", AITER_KERNEL_GROUPSHAPE_COMBINATIONS
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
(not current_platform.is_rocm() or not IS_AITER_FOUND),
|
||||
reason="Only test on ROCm with aiter package installed",
|
||||
)
|
||||
def test_aiter_fusion_rmsnorm_quant(
|
||||
dtype: torch.dtype,
|
||||
hidden_size: int,
|
||||
num_tokens: int,
|
||||
eps: float,
|
||||
kernel_groupshape_quant: tuple,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
force_kernel, group_shape, use_aiter_quant_op = kernel_groupshape_quant
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=["+rms_norm", "+quant_fp8"],
|
||||
pass_config=PassConfig(fuse_norm_quant=True, eliminate_noops=True),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
RocmAiterRMSNormQuantFusionPass,
|
||||
)
|
||||
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(1)
|
||||
|
||||
fusion_pass = RocmAiterRMSNormQuantFusionPass(vllm_config)
|
||||
|
||||
model = TestModel(
|
||||
hidden_size=hidden_size,
|
||||
eps=eps,
|
||||
force_kernel=force_kernel,
|
||||
group_shape=group_shape,
|
||||
dtype=dtype,
|
||||
use_aiter_fusion=True, # Always use aiter fusion ops in aiter test
|
||||
use_aiter_quant=use_aiter_quant_op, # Toggle aiter quantization
|
||||
)
|
||||
|
||||
_run_fusion_test(
|
||||
model, fusion_pass, vllm_config, dtype, hidden_size, num_tokens
|
||||
)
|
||||
|
||||
|
||||
class TestGatedModel(torch.nn.Module):
|
||||
"""Model that uses RMSNormGated + reshape + group FP8 quant + linear.
|
||||
|
||||
Mimics GatedDeltaNetAttention's output projection path where:
|
||||
- RMSNormGated operates on per-head tensors, either (N, H, D) as
|
||||
QwenGatedDeltaNetAttention passes them or flattened to (N*H, D) as
|
||||
OlmoHybridGatedDeltaNetAttention does
|
||||
- Output is reshaped to (N, H*D) before group quantization + linear
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
eps: float,
|
||||
force_kernel: type[_KernelT],
|
||||
group_shape: GroupShape,
|
||||
dtype: torch.dtype,
|
||||
use_aiter_quant: bool = True,
|
||||
flatten_heads: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = head_dim
|
||||
self.flatten_heads = flatten_heads
|
||||
hidden_dim = num_heads * head_dim
|
||||
|
||||
self.norm = RMSNormGated(
|
||||
head_dim,
|
||||
eps=eps,
|
||||
group_size=None,
|
||||
norm_before_gate=True,
|
||||
)
|
||||
|
||||
self.activation_quant_key = create_fp8_quant_key(
|
||||
static=False, group_shape=group_shape
|
||||
)
|
||||
self.weight_quant_key = create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(group_shape.col, group_shape.col)
|
||||
)
|
||||
|
||||
self.fp8_linear = TestFP8Layer(
|
||||
weight_shape=(hidden_dim, hidden_dim),
|
||||
activation_quant_key=self.activation_quant_key,
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
force_kernel=force_kernel,
|
||||
transpose_weights=True,
|
||||
input_dtype=dtype,
|
||||
)
|
||||
self.fp8_linear.kernel.quant_fp8.use_aiter = use_aiter_quant
|
||||
|
||||
def forward(self, x, z):
|
||||
num_heads = self.num_heads
|
||||
head_dim = self.head_dim
|
||||
hidden_dim = num_heads * head_dim
|
||||
x = torch.relu(x)
|
||||
z = torch.relu(z)
|
||||
x_heads = x.reshape(-1, num_heads, head_dim)
|
||||
z_heads = z.reshape(-1, num_heads, head_dim)
|
||||
if self.flatten_heads:
|
||||
x_heads = x_heads.reshape(-1, head_dim)
|
||||
z_heads = z_heads.reshape(-1, head_dim)
|
||||
normed = self.norm(x_heads, z_heads)
|
||||
merged = normed.reshape(-1, hidden_dim)
|
||||
out = self.fp8_linear(merged)
|
||||
return out
|
||||
|
||||
def ops_in_model_after(self):
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
AiterRMSNormGatedFp8GroupQuantPattern,
|
||||
)
|
||||
|
||||
return [AiterRMSNormGatedFp8GroupQuantPattern.FUSED_OP]
|
||||
|
||||
|
||||
class _MockGDNLayer:
|
||||
"""Minimal mock to populate static_forward_context for pass discovery.
|
||||
|
||||
Uses __class__ assignment to pass isinstance checks against
|
||||
GatedDeltaNetAttention without requiring a full config-based init.
|
||||
"""
|
||||
|
||||
def __init__(self, num_v_heads: int, head_v_dim: int, tp_size: int = 1):
|
||||
self.num_v_heads = num_v_heads
|
||||
self.head_v_dim = head_v_dim
|
||||
self.tp_size = tp_size
|
||||
|
||||
from vllm.model_executor.layers.mamba.gdn.base import (
|
||||
GatedDeltaNetAttention,
|
||||
)
|
||||
|
||||
self.__class__ = GatedDeltaNetAttention
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("num_heads", [2])
|
||||
@pytest.mark.parametrize("head_dim", [128])
|
||||
@pytest.mark.parametrize("num_tokens", [8])
|
||||
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
|
||||
@pytest.mark.parametrize("flatten_heads", [True, False])
|
||||
@pytest.mark.parametrize("quant_fp8_op", ["-quant_fp8", "+quant_fp8"])
|
||||
@pytest.mark.skipif(
|
||||
(not current_platform.is_rocm() or not IS_AITER_FOUND),
|
||||
reason="Only test on ROCm with aiter package installed",
|
||||
)
|
||||
def test_aiter_fusion_rmsnorm_gated_quant(
|
||||
dtype: torch.dtype,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
num_tokens: int,
|
||||
eps: float,
|
||||
flatten_heads: bool,
|
||||
quant_fp8_op: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
group_shape = GroupShape(1, 128)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=["-rms_norm", "-silu_and_mul", quant_fp8_op],
|
||||
pass_config=PassConfig(fuse_norm_quant=True, eliminate_noops=True),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
RocmAiterRMSNormQuantFusionPass,
|
||||
)
|
||||
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
# Register a mock GDN layer so the pass discovers num_heads/head_dim
|
||||
mock_gdn = _MockGDNLayer(num_v_heads=num_heads, head_v_dim=head_dim, tp_size=1)
|
||||
vllm_config.compilation_config.static_forward_context["mock_gdn_layer"] = (
|
||||
mock_gdn
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(1)
|
||||
|
||||
fusion_pass = RocmAiterRMSNormQuantFusionPass(vllm_config)
|
||||
|
||||
model = TestGatedModel(
|
||||
num_heads=num_heads,
|
||||
head_dim=head_dim,
|
||||
eps=eps,
|
||||
force_kernel=AiterFp8BlockScaledMMKernel,
|
||||
group_shape=group_shape,
|
||||
dtype=dtype,
|
||||
use_aiter_quant=True,
|
||||
flatten_heads=flatten_heads,
|
||||
)
|
||||
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
|
||||
backend = TestBackend(noop_pass, fusion_pass, cleanup_pass)
|
||||
backend2 = TestBackend(noop_pass, cleanup_pass)
|
||||
|
||||
hidden_dim = num_heads * head_dim
|
||||
x = torch.rand(num_tokens, hidden_dim)
|
||||
z = torch.rand(num_tokens, hidden_dim)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
torch._dynamo.mark_dynamic(z, 0)
|
||||
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
result_fused = model_fused(x, z)
|
||||
|
||||
model_unfused = torch.compile(model, backend=backend2)
|
||||
result_unfused = model_unfused(x, z)
|
||||
|
||||
# The opaque AITER quant op preserves a BF16 intermediate in the
|
||||
# unfused path, while the fused kernel normalizes and quantizes
|
||||
# directly. Allow for that extra rounding before the FP8 GEMM.
|
||||
tol = 2e-2 if quant_fp8_op == "+quant_fp8" else 1e-2
|
||||
torch.testing.assert_close(result_fused, result_unfused, atol=tol, rtol=tol)
|
||||
|
||||
assert fusion_pass.matched_count == 1
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("num_heads", [2])
|
||||
@pytest.mark.parametrize("head_dim", [128])
|
||||
@pytest.mark.parametrize("num_tokens", [8])
|
||||
@pytest.mark.parametrize("eps", [1e-6])
|
||||
@pytest.mark.skipif(
|
||||
(not current_platform.is_rocm() or not IS_AITER_FOUND),
|
||||
reason="Only test on ROCm with aiter package installed",
|
||||
)
|
||||
def test_aiter_fusion_rmsnorm_gated_quant_no_gdn_layers(
|
||||
dtype: torch.dtype,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
num_tokens: int,
|
||||
eps: float,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
"""Verify that without GDN layers in static_forward_context,
|
||||
the gated pattern is not registered and no matches occur."""
|
||||
group_shape = GroupShape(1, 128)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=["-rms_norm", "-silu_and_mul", "-quant_fp8"],
|
||||
pass_config=PassConfig(fuse_norm_quant=True, eliminate_noops=True),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
RocmAiterRMSNormQuantFusionPass,
|
||||
)
|
||||
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(1)
|
||||
|
||||
# No mock GDN layer registered -- pass should not register gated pattern
|
||||
fusion_pass = RocmAiterRMSNormQuantFusionPass(vllm_config)
|
||||
|
||||
model = TestGatedModel(
|
||||
num_heads=num_heads,
|
||||
head_dim=head_dim,
|
||||
eps=eps,
|
||||
force_kernel=AiterFp8BlockScaledMMKernel,
|
||||
group_shape=group_shape,
|
||||
dtype=dtype,
|
||||
use_aiter_quant=True,
|
||||
)
|
||||
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
|
||||
backend = TestBackend(noop_pass, fusion_pass, cleanup_pass)
|
||||
|
||||
hidden_dim = num_heads * head_dim
|
||||
x = torch.rand(num_tokens, hidden_dim)
|
||||
z = torch.rand(num_tokens, hidden_dim)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
torch._dynamo.mark_dynamic(z, 0)
|
||||
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
model_fused(x, z)
|
||||
|
||||
assert fusion_pass.matched_count == 0
|
||||
@@ -1,582 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
import torch._dynamo
|
||||
|
||||
from tests.compile.backend import LazyInitPass, TestBackend
|
||||
from tests.utils import TestFP8Layer, flat_product
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
dense_kv_cache_views,
|
||||
)
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
from vllm.compilation.passes.fusion.attn_quant_fusion import (
|
||||
ATTN_OP,
|
||||
AttnQuantFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.fusion.matcher_utils import QUANT_OPS
|
||||
from vllm.compilation.passes.fx_utils import find_op_nodes
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.attention.backends.utils import (
|
||||
get_supported_kv_cache_layouts,
|
||||
resolve_kv_cache_layout,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
get_kv_quant_mode,
|
||||
)
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
FP4_DTYPE = torch.uint8
|
||||
|
||||
|
||||
class AttentionQuantPatternModel(torch.nn.Module):
|
||||
"""Base model for AttentionQuantPattern fusion."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_qo_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
device: torch.device,
|
||||
vllm_config: VllmConfig,
|
||||
block_size: int,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
self.num_qo_heads = num_qo_heads
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.head_size = head_size
|
||||
self.device = device
|
||||
self.vllm_config = vllm_config
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
|
||||
self.attn = Attention(
|
||||
num_heads=self.num_qo_heads,
|
||||
head_size=self.head_size,
|
||||
scale=1.0 / (self.head_size**0.5),
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=vllm_config.cache_config,
|
||||
prefix="model.layers.0.self_attn.attn",
|
||||
)
|
||||
self.attn._k_scale = self.attn._k_scale.to(device)
|
||||
self.attn._v_scale = self.attn._v_scale.to(device)
|
||||
|
||||
self.block_size = block_size
|
||||
|
||||
# Initialize attn MetadataBuilder (match Attention.get_kv_cache_spec)
|
||||
self.builder = self.attn.attn_backend.get_builder_cls()(
|
||||
kv_cache_spec=AttentionSpec(
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
dtype=self.attn.kv_cache_torch_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.attn.kv_cache_dtype),
|
||||
),
|
||||
layer_names=[self.attn.layer_name],
|
||||
vllm_config=self.vllm_config,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def build_attn_metadata(self, batch_size: int) -> AttentionMetadata:
|
||||
"""Initialize attention metadata."""
|
||||
# TODO (Rohan138) reuse utils from vllm/v1/worker/gpu/attn_utils.py
|
||||
|
||||
# Create common attn metadata
|
||||
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, self.block_size, self.device, arange_block_indices=True
|
||||
)
|
||||
|
||||
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
|
||||
num_blocks = batch_size * max_blocks
|
||||
|
||||
spec = self.attn.attn_backend.customize_spec(
|
||||
AttentionSpec(
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
dtype=self.attn.kv_cache_torch_dtype,
|
||||
kv_quant_mode=get_kv_quant_mode(self.attn.kv_cache_dtype),
|
||||
)
|
||||
)
|
||||
supported = get_supported_kv_cache_layouts([self.attn.attn_backend])
|
||||
layout = resolve_kv_cache_layout(
|
||||
self.vllm_config, [[m.name for m in supported]]
|
||||
)
|
||||
raw_tensor = torch.zeros(
|
||||
num_blocks * spec.page_size_bytes,
|
||||
dtype=torch.int8,
|
||||
device=self.device,
|
||||
)
|
||||
kv_cache = dense_kv_cache_views(
|
||||
raw_tensor,
|
||||
spec,
|
||||
num_blocks,
|
||||
num_layers=1,
|
||||
layout=layout,
|
||||
)[0]
|
||||
|
||||
self.attn.kv_cache = kv_cache
|
||||
|
||||
# Build attn metadata
|
||||
self.attn_metadata = self.builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
|
||||
return self.attn_metadata
|
||||
|
||||
|
||||
class TestAttentionFp8StaticQuantPatternModel(AttentionQuantPatternModel):
|
||||
"""Test model for AttentionFp8StaticQuantPattern fusion."""
|
||||
|
||||
quant_key = kFp8StaticTensorSym
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
hidden_size = self.num_qo_heads * self.head_size
|
||||
self.fp8_linear = TestFP8Layer(
|
||||
weight_shape=(hidden_size, hidden_size),
|
||||
activation_quant_key=self.quant_key,
|
||||
weight_quant_key=self.quant_key,
|
||||
device=self.device,
|
||||
input_dtype=self.dtype,
|
||||
)
|
||||
|
||||
w = kwargs.get("w")
|
||||
if w is not None:
|
||||
self.fp8_linear.weight = w["weight"]
|
||||
self.fp8_linear.weight_scale = w["wscale"]
|
||||
self.fp8_linear.input_scale = w["scale"]
|
||||
|
||||
self.w = {
|
||||
"weight": self.fp8_linear.weight,
|
||||
"wscale": self.fp8_linear.weight_scale,
|
||||
"scale": self.fp8_linear.input_scale,
|
||||
}
|
||||
|
||||
def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor):
|
||||
"""Forward pass that creates the pattern to be fused."""
|
||||
attn_output = self.attn(q, k, v)
|
||||
return self.fp8_linear(attn_output)
|
||||
|
||||
|
||||
class TestAttentionAiterFp8StaticQuantPatternModel(
|
||||
TestAttentionFp8StaticQuantPatternModel
|
||||
):
|
||||
aiter_quant = True
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fp8_linear.kernel.quant_fp8.use_aiter = True
|
||||
assert self.fp8_linear.input_scale is not None
|
||||
self.fp8_linear.input_scale.fill_(0.125)
|
||||
|
||||
|
||||
class TestAttentionNvfp4QuantPatternModel(AttentionQuantPatternModel):
|
||||
"""Test model for AttentionNvfp4QuantPattern fusion."""
|
||||
|
||||
quant_key = kNvfp4Dynamic
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
hidden_size = self.num_qo_heads * self.head_size
|
||||
self.w = kwargs.get(
|
||||
"w",
|
||||
{
|
||||
"weight": torch.randint(
|
||||
256,
|
||||
(hidden_size, hidden_size // 2),
|
||||
dtype=FP4_DTYPE,
|
||||
device=self.device,
|
||||
),
|
||||
"wscale_swizzled": torch.randn(hidden_size, hidden_size // 16).to(
|
||||
dtype=FP8_DTYPE, device=self.device
|
||||
),
|
||||
"wscale": torch.tensor([500], dtype=torch.float32, device=self.device),
|
||||
"scale": torch.tensor([0.002], dtype=torch.float32, device=self.device),
|
||||
},
|
||||
)
|
||||
|
||||
def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor):
|
||||
"""Forward pass that creates the pattern to be fused."""
|
||||
attn_output = self.attn(q, k, v)
|
||||
quant_output, output_block_scale = scaled_fp4_quant(
|
||||
attn_output, 1 / self.w["scale"]
|
||||
)
|
||||
return cutlass_scaled_fp4_mm(
|
||||
a=quant_output,
|
||||
b=self.w["weight"],
|
||||
block_scale_a=output_block_scale,
|
||||
block_scale_b=self.w["wscale_swizzled"],
|
||||
alpha=self.w["scale"] * self.w["wscale"],
|
||||
out_dtype=attn_output.dtype,
|
||||
)
|
||||
|
||||
|
||||
PATTERN_TEST_MODELS_FP8: list[tuple[str, type]] = []
|
||||
PATTERN_TEST_MODELS_FP4: list[tuple[str, type]] = []
|
||||
HEADS: list[tuple[int, int]] = []
|
||||
BACKENDS_FP8: list[AttentionBackendEnum] = []
|
||||
BACKENDS_FP4: list[AttentionBackendEnum] = []
|
||||
|
||||
if current_platform.is_cuda():
|
||||
HEADS = [(64, 8), (40, 8)]
|
||||
PATTERN_TEST_MODELS_FP8 = [
|
||||
(
|
||||
"RedHatAI/Meta-Llama-3.1-8B-FP8",
|
||||
TestAttentionFp8StaticQuantPatternModel,
|
||||
)
|
||||
]
|
||||
PATTERN_TEST_MODELS_FP4 = [
|
||||
(
|
||||
"nvidia/Llama-3.1-8B-Instruct-NVFP4",
|
||||
TestAttentionNvfp4QuantPatternModel,
|
||||
)
|
||||
]
|
||||
BACKENDS_FP8 = [AttentionBackendEnum.TRITON_ATTN, AttentionBackendEnum.FLASHINFER]
|
||||
BACKENDS_FP4 = [AttentionBackendEnum.FLASHINFER]
|
||||
|
||||
elif current_platform.is_rocm():
|
||||
HEADS = [(32, 8), (40, 8)]
|
||||
PATTERN_TEST_MODELS_FP8 = [
|
||||
("amd/Llama-3.1-8B-Instruct-FP8-KV", TestAttentionFp8StaticQuantPatternModel)
|
||||
]
|
||||
BACKENDS_FP8 = [
|
||||
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
AttentionBackendEnum.ROCM_ATTN,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_qo_heads, num_kv_heads", HEADS)
|
||||
@pytest.mark.parametrize("head_size", [128])
|
||||
@pytest.mark.parametrize(
|
||||
"batch_size", [7, 256, 533] if current_platform.is_cuda() else [8]
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["fp8", "nvfp4"])
|
||||
@pytest.mark.parametrize(
|
||||
"backend, model_name, model_class, custom_ops",
|
||||
# Test attention+quant_fp8 fusion with custom and torch impls of QuantFP8
|
||||
list(
|
||||
flat_product(
|
||||
BACKENDS_FP8, PATTERN_TEST_MODELS_FP8, ["+quant_fp8", "-quant_fp8"]
|
||||
)
|
||||
)
|
||||
# quant_fp4 only has the custom impl
|
||||
+ list(flat_product(BACKENDS_FP4, PATTERN_TEST_MODELS_FP4, [""])),
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Only test ROCm or CUDA"
|
||||
)
|
||||
@pytest.mark.skipif(not current_platform.supports_fp8(), reason="Need FP8")
|
||||
def test_attention_quant_pattern(
|
||||
num_qo_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
batch_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType,
|
||||
custom_ops: str,
|
||||
model_name: str,
|
||||
model_class: type[AttentionQuantPatternModel],
|
||||
backend: AttentionBackendEnum,
|
||||
dist_init,
|
||||
disable_vllm_compile_cache,
|
||||
):
|
||||
"""Test AttentionStaticQuantPattern fusion pass."""
|
||||
if backend == AttentionBackendEnum.FLASHINFER and (
|
||||
not current_platform.is_device_capability((10, 0)) or not has_flashinfer()
|
||||
):
|
||||
# This also captures the FP4 case
|
||||
pytest.skip("FlashInfer attn fusion requires Blackwell and flashinfer")
|
||||
if kv_cache_dtype == "nvfp4" and backend != AttentionBackendEnum.FLASHINFER:
|
||||
pytest.skip("NVFP4 KV cache is only supported by FlashInfer")
|
||||
|
||||
# trtllm-gen has no NVFP4-KV kernel with NVFP4 output, so with an NVFP4 KV
|
||||
# cache the FP4 output fusion must be rejected: the standalone
|
||||
# scaled_fp4_quant stays in the graph and the result is unchanged. FP8 output
|
||||
# fusion with NVFP4 KV is supported.
|
||||
expect_fusion = not (
|
||||
kv_cache_dtype == "nvfp4" and model_class.quant_key is kNvfp4Dynamic
|
||||
)
|
||||
|
||||
custom_ops_list = custom_ops.split(",") if custom_ops else []
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(42)
|
||||
|
||||
backend_cls = backend.get_class()
|
||||
block_size = backend_cls.get_preferred_block_size(16)
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=model_name,
|
||||
max_model_len=2048,
|
||||
dtype=dtype,
|
||||
)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_num_seqs=1024,
|
||||
max_model_len=model_config.max_model_len,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=custom_ops_list,
|
||||
),
|
||||
cache_config=CacheConfig(cache_dtype=kv_cache_dtype),
|
||||
attention_config=AttentionConfig(backend=backend),
|
||||
)
|
||||
|
||||
# Create test inputs
|
||||
q = torch.randn(batch_size, num_qo_heads * head_size, dtype=dtype, device=device)
|
||||
k = torch.randn(batch_size, num_kv_heads * head_size, dtype=dtype, device=device)
|
||||
v = torch.randn(batch_size, num_kv_heads * head_size, dtype=dtype, device=device)
|
||||
|
||||
# Mark first dimension as dynamic for realistic testing
|
||||
torch._dynamo.mark_dynamic(q, 0)
|
||||
torch._dynamo.mark_dynamic(k, 0)
|
||||
torch._dynamo.mark_dynamic(v, 0)
|
||||
|
||||
# Run model directly without compilation and fusion
|
||||
vllm_config_unfused = copy.deepcopy(vllm_config)
|
||||
with (
|
||||
set_current_vllm_config(vllm_config_unfused),
|
||||
set_forward_context(attn_metadata=None, vllm_config=vllm_config_unfused),
|
||||
):
|
||||
model_unfused = model_class(
|
||||
num_qo_heads=num_qo_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
device=device,
|
||||
vllm_config=vllm_config_unfused,
|
||||
block_size=block_size,
|
||||
)
|
||||
model_unfused = model_unfused.to(device)
|
||||
result_unfused_0 = model_unfused(q, k, v) # noqa: F841 HACK: See #131044
|
||||
|
||||
forward_ctx = get_forward_context()
|
||||
forward_ctx.attn_metadata = model_unfused.build_attn_metadata(batch_size)
|
||||
|
||||
# Run model directly without fusion
|
||||
# Still compile so query QuantFP8 has closer numerics
|
||||
compiled_unfused = torch.compile(model_unfused, fullgraph=True)
|
||||
result_unfused = compiled_unfused(q, k, v)
|
||||
|
||||
# Run model with attn fusion enabled
|
||||
vllm_config.compilation_config.pass_config = PassConfig(
|
||||
fuse_attn_quant=True, eliminate_noops=True
|
||||
)
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
set_forward_context(attn_metadata=None, vllm_config=vllm_config),
|
||||
):
|
||||
model_fused = model_class(
|
||||
num_qo_heads=num_qo_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
device=device,
|
||||
vllm_config=vllm_config,
|
||||
w=model_unfused.w,
|
||||
block_size=block_size,
|
||||
)
|
||||
model_fused = model_fused.to(device)
|
||||
|
||||
forward_ctx = get_forward_context()
|
||||
forward_ctx.attn_metadata = model_fused.build_attn_metadata(batch_size)
|
||||
|
||||
# Create test backend with fusion passes enabled
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
attn_pass = LazyInitPass(AttnQuantFusionPass, vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
|
||||
test_backend = TestBackend(noop_pass, attn_pass, cleanup_pass)
|
||||
# HACK: See https://github.com/vllm-project/vllm/issues/31044
|
||||
result_fused_0 = model_fused(q, k, v) # noqa: F841
|
||||
|
||||
# Compile model with fusion enabled
|
||||
compiled_fused = torch.compile(
|
||||
model_fused, backend=test_backend, fullgraph=True
|
||||
)
|
||||
assert compiled_fused.attn._o_scale_float is None
|
||||
|
||||
result_fused = compiled_fused(q, k, v)
|
||||
|
||||
if backend == AttentionBackendEnum.FLASHINFER and expect_fusion:
|
||||
# With the Flashinfer backend after the 1st round of the forward
|
||||
# pass, output quant scale should be loaded into the attn layer's
|
||||
# _o_scale_float, the 2nd round should reuse the loaded
|
||||
# _o_scale_float
|
||||
assert compiled_fused.attn._o_scale_float is not None
|
||||
result_fused_2 = compiled_fused(q, k, v)
|
||||
|
||||
assert compiled_fused.attn._o_scale_float is not None
|
||||
|
||||
torch.testing.assert_close(
|
||||
result_unfused, result_fused_2, atol=1e-2, rtol=1e-2
|
||||
)
|
||||
elif not expect_fusion:
|
||||
assert compiled_fused.attn._o_scale_float is None
|
||||
|
||||
# Check attn fusion support
|
||||
quant_key: QuantKey = model_class.quant_key
|
||||
attn_fusion_supported = [
|
||||
layer.impl.fused_output_quant_supported(quant_key)
|
||||
for key, layer in vllm_config.compilation_config.static_forward_context.items()
|
||||
]
|
||||
if expect_fusion:
|
||||
assert sum(attn_fusion_supported) == len(attn_fusion_supported), (
|
||||
"All layers should support attention fusion"
|
||||
)
|
||||
else:
|
||||
assert sum(attn_fusion_supported) == 0, (
|
||||
"No layer should support attention fusion"
|
||||
)
|
||||
|
||||
# Check quantization ops in the graph before and after fusion
|
||||
if "-quant_fp8" in custom_ops_list:
|
||||
quant_op = torch.ops.aten.reciprocal
|
||||
elif getattr(model_class, "aiter_quant", False):
|
||||
quant_op = torch.ops.vllm.rocm_aiter_per_tensor_quant.default
|
||||
else:
|
||||
quant_op = QUANT_OPS[quant_key]
|
||||
|
||||
if expect_fusion:
|
||||
# Note: for fp8, fully_replaced=False because query quant ops remain in
|
||||
# graph. Only output quant ops are fused into attention.
|
||||
test_backend.check_before_ops(
|
||||
[quant_op], fully_replaced=quant_key is kNvfp4Dynamic
|
||||
)
|
||||
else:
|
||||
# Fusion rejected: the standalone output quant must survive untouched.
|
||||
num_pre = test_backend.op_count(quant_op, before=True)
|
||||
assert num_pre > 0, f"Op {quant_op} not found in pre-pass graph"
|
||||
assert test_backend.op_count(quant_op) == num_pre, (
|
||||
"Output quant op should remain when fusion is rejected"
|
||||
)
|
||||
|
||||
# access the underlying `AttnQuantFusionPass` on the `LazyInitPass`
|
||||
assert attn_pass.pass_.matched_count == sum(attn_fusion_supported)
|
||||
|
||||
# Check attention ops in the graph before and after fusion
|
||||
attn_nodes_pre = list(find_op_nodes(ATTN_OP, test_backend.graph_pre_pass))
|
||||
attn_nodes_post = list(find_op_nodes(ATTN_OP, test_backend.graph_post_pass))
|
||||
|
||||
assert len(attn_nodes_pre) > 0, "Should have attention nodes before fusion"
|
||||
assert len(attn_nodes_pre) == len(attn_nodes_post), (
|
||||
"Should have same number of attention nodes before and after fusion"
|
||||
)
|
||||
assert attn_nodes_pre[0].kwargs.get("output_scale") is None, (
|
||||
"Attention should not have output_scale before fusion"
|
||||
)
|
||||
if expect_fusion:
|
||||
assert attn_nodes_post[0].kwargs.get("output_scale") is not None, (
|
||||
"Attention should have output_scale after fusion"
|
||||
)
|
||||
else:
|
||||
assert attn_nodes_post[0].kwargs.get("output_scale") is None, (
|
||||
"Attention should not have output_scale when fusion is rejected"
|
||||
)
|
||||
|
||||
assert attn_nodes_pre[0].kwargs.get("output_block_scale") is None, (
|
||||
"Attention should not have output_block_scale before fusion"
|
||||
)
|
||||
|
||||
kv_cache_dummy_dep_pre_is_none = (
|
||||
attn_nodes_pre[0].kwargs.get("kv_cache_dummy_dep") is None
|
||||
)
|
||||
kv_cache_dummy_dep_post_is_none = (
|
||||
attn_nodes_post[0].kwargs.get("kv_cache_dummy_dep") is None
|
||||
)
|
||||
assert not (kv_cache_dummy_dep_pre_is_none ^ kv_cache_dummy_dep_post_is_none), (
|
||||
"The kv_cache_dummy_dep should be consistent before and after fusion"
|
||||
)
|
||||
|
||||
if quant_key.dtype == FP8_DTYPE or not expect_fusion:
|
||||
assert attn_nodes_post[0].kwargs.get("output_block_scale") is None, (
|
||||
"Attention should not have output_block_scale unless FP4 fusion happened"
|
||||
)
|
||||
elif quant_key.dtype == FP4_DTYPE:
|
||||
assert attn_nodes_post[0].kwargs.get("output_block_scale") is not None, (
|
||||
"Attention should have output_block_scale after FP4 fusion"
|
||||
)
|
||||
|
||||
# Check that results are close
|
||||
torch.testing.assert_close(result_unfused, result_fused, atol=1e-2, rtol=1e-2)
|
||||
if getattr(model_class, "aiter_quant", False):
|
||||
# Query quantization remains; exactly the output quant op is removed.
|
||||
assert test_backend.op_count(quant_op, before=True) == (
|
||||
test_backend.op_count(quant_op) + 1
|
||||
)
|
||||
for model in (model_unfused, model_fused):
|
||||
torch.testing.assert_close(
|
||||
model.fp8_linear.input_scale,
|
||||
torch.full_like(model.fp8_linear.input_scale, 0.125),
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm() or not is_aiter_found_and_supported(),
|
||||
reason="Requires ROCm AITER attention and quantization",
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
def test_rocm_aiter_static_quant_attention(
|
||||
dtype, dist_init, disable_vllm_compile_cache, monkeypatch
|
||||
):
|
||||
"""Fuse the actual AITER op without changing its static scale or outputs."""
|
||||
try:
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
test_attention_quant_pattern(
|
||||
num_qo_heads=32,
|
||||
num_kv_heads=8,
|
||||
head_size=128,
|
||||
batch_size=8,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype="fp8",
|
||||
custom_ops="+quant_fp8",
|
||||
model_name="amd/Llama-3.1-8B-Instruct-FP8-KV",
|
||||
model_class=TestAttentionAiterFp8StaticQuantPatternModel,
|
||||
backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
dist_init=dist_init,
|
||||
disable_vllm_compile_cache=disable_vllm_compile_cache,
|
||||
)
|
||||
finally:
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
@@ -1,590 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
import torch._dynamo
|
||||
|
||||
from tests.compile.backend import LazyInitPass, TestBackend
|
||||
from tests.utils import TestFP8Layer, flat_product
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
from vllm.compilation.passes.fusion.matcher_utils import QUANT_OPS
|
||||
from vllm.compilation.passes.fusion.mla_attn_quant_fusion import (
|
||||
MLA_ATTN_OP,
|
||||
MLAAttnQuantFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.fx_utils import find_op_nodes
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import (
|
||||
AttentionConfig,
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.kernels.linear.scaled_mm.cutlass import (
|
||||
CutlassFp8BlockScaledMMKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.quantization.fp8 import Fp8Config
|
||||
from vllm.model_executor.layers.quantization.modelopt import ModelOptNvFp4Config
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
create_fp8_quant_key,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import MLAAttentionSpec
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
FP4_DTYPE = torch.uint8
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
class MLAAttentionQuantPatternModel(torch.nn.Module):
|
||||
"""Base model for MLA AttentionQuantPattern fusion."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
kv_lora_rank: int,
|
||||
kv_cache_dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
vllm_config: VllmConfig,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.output_dim = num_heads * v_head_dim
|
||||
self.head_size = kv_lora_rank + qk_rope_head_dim
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
self.device = device
|
||||
self.vllm_config = vllm_config
|
||||
self.dtype = vllm_config.model_config.dtype
|
||||
|
||||
kv_b_proj = ColumnParallelLinear(
|
||||
input_size=kv_lora_rank,
|
||||
output_size=num_heads * (qk_nope_head_dim + v_head_dim),
|
||||
bias=False,
|
||||
prefix="model.layers.0.self_attn.kv_b_proj",
|
||||
).to(device)
|
||||
kv_b_proj_weight = kwargs.get("kv_b_proj_weight")
|
||||
if kv_b_proj_weight is not None:
|
||||
kv_b_proj.weight.data.copy_(kv_b_proj_weight)
|
||||
else:
|
||||
kv_b_proj.weight.data.normal_()
|
||||
|
||||
# Create MLAAttention
|
||||
self.mla_attn = MLAAttention(
|
||||
num_heads=num_heads,
|
||||
scale=1.0 / (self.qk_head_dim**0.5),
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
q_lora_rank=None,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
kv_b_proj=kv_b_proj,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
prefix="model.layers.0.self_attn.attn",
|
||||
)
|
||||
self.mla_attn._k_scale = self.mla_attn._k_scale.to(device)
|
||||
self.mla_attn._v_scale = self.mla_attn._v_scale.to(device)
|
||||
|
||||
# Initialize W_UK_T and W_UV from kv_b_proj weights
|
||||
self.mla_attn.process_weights_after_loading(torch.get_default_dtype())
|
||||
self.kv_b_proj_weight = kv_b_proj.weight.data.clone()
|
||||
|
||||
self.block_size = 16
|
||||
|
||||
# Initialize MLA MetadataBuilder
|
||||
self.builder = self.mla_attn.attn_backend.get_builder_cls()(
|
||||
kv_cache_spec=MLAAttentionSpec(
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=self.head_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
),
|
||||
layer_names=[self.mla_attn.layer_name],
|
||||
vllm_config=self.vllm_config,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
def build_attn_metadata(self, batch_size: int) -> AttentionMetadata:
|
||||
"""Initialize MLA attention metadata.
|
||||
|
||||
NOTE: Uses decode-only batch (query_len=1 per request). The prefill
|
||||
(forward_mha) path is not separately tested here because it requires
|
||||
FlashAttention availability and different input tensor shapes. The
|
||||
quant logic in forward_impl is identical for both paths — it quantizes
|
||||
the full output[:num_actual_toks] buffer after both forward_mha and
|
||||
forward_mqa have written their results.
|
||||
"""
|
||||
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, self.block_size, self.device, arange_block_indices=True
|
||||
)
|
||||
|
||||
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
|
||||
num_blocks = batch_size * max_blocks
|
||||
|
||||
# MLA KV cache is 4D: (num_blocks, num_heads=1, block_size, head_size)
|
||||
kv_cache = torch.zeros(
|
||||
(num_blocks, 1, self.block_size, self.head_size),
|
||||
dtype=self.kv_cache_dtype,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
self.mla_attn.bind_kv_cache(kv_cache)
|
||||
|
||||
self.attn_metadata = self.builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
|
||||
return self.attn_metadata
|
||||
|
||||
|
||||
class TestMLAAttentionFp8StaticQuantPatternModel(MLAAttentionQuantPatternModel):
|
||||
"""Test model for MLA Attention + FP8 static quant fusion."""
|
||||
|
||||
quant_key = kFp8StaticTensorSym
|
||||
quant_config = Fp8Config()
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.fp8_linear = TestFP8Layer(
|
||||
weight_shape=(self.output_dim, self.output_dim),
|
||||
activation_quant_key=self.quant_key,
|
||||
weight_quant_key=self.quant_key,
|
||||
device=self.device,
|
||||
input_dtype=self.dtype,
|
||||
)
|
||||
|
||||
w = kwargs.get("w")
|
||||
if w is not None:
|
||||
self.fp8_linear.weight = w["weight"]
|
||||
self.fp8_linear.weight_scale = w["wscale"]
|
||||
self.fp8_linear.input_scale = w["scale"]
|
||||
|
||||
self.w = {
|
||||
"weight": self.fp8_linear.weight,
|
||||
"wscale": self.fp8_linear.weight_scale,
|
||||
"scale": self.fp8_linear.input_scale,
|
||||
}
|
||||
|
||||
def forward(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
):
|
||||
"""Forward pass that creates the MLA attention + FP8 quant pattern."""
|
||||
attn_output = self.mla_attn(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_shape=(q.shape[0], self.output_dim),
|
||||
)
|
||||
return self.fp8_linear(attn_output)
|
||||
|
||||
|
||||
class TestMLAAttentionNvfp4QuantPatternModel(MLAAttentionQuantPatternModel):
|
||||
"""Test model for MLA Attention + NVFP4 quant fusion."""
|
||||
|
||||
quant_key = kNvfp4Dynamic
|
||||
quant_config = ModelOptNvFp4Config(
|
||||
is_checkpoint_nvfp4_serialized=False,
|
||||
kv_cache_quant_algo=None,
|
||||
exclude_modules=[],
|
||||
)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.w = kwargs.get(
|
||||
"w",
|
||||
{
|
||||
"weight": torch.randint(
|
||||
256,
|
||||
(self.output_dim, self.output_dim // 2),
|
||||
dtype=FP4_DTYPE,
|
||||
device=self.device,
|
||||
),
|
||||
"wscale_swizzled": torch.randn(
|
||||
self.output_dim, self.output_dim // 16
|
||||
).to(dtype=FP8_DTYPE, device=self.device),
|
||||
"wscale": torch.tensor([500], dtype=torch.float32, device=self.device),
|
||||
"scale": torch.tensor([0.002], dtype=torch.float32, device=self.device),
|
||||
},
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
):
|
||||
"""Forward pass that creates the MLA attention + NVFP4 quant pattern."""
|
||||
attn_output = self.mla_attn(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_shape=(q.shape[0], self.output_dim),
|
||||
)
|
||||
quant_output, output_block_scale = scaled_fp4_quant(
|
||||
attn_output, 1 / self.w["scale"]
|
||||
)
|
||||
return cutlass_scaled_fp4_mm(
|
||||
a=quant_output,
|
||||
b=self.w["weight"],
|
||||
block_scale_a=output_block_scale,
|
||||
block_scale_b=self.w["wscale_swizzled"],
|
||||
alpha=self.w["scale"] * self.w["wscale"],
|
||||
out_dtype=attn_output.dtype,
|
||||
)
|
||||
|
||||
|
||||
class TestMLAAttentionFp8GroupQuantPatternModel(MLAAttentionQuantPatternModel):
|
||||
"""Test model for MLA Attention + per-group FP8 (block quant) fusion."""
|
||||
|
||||
quant_key = kFp8Dynamic128Sym
|
||||
quant_config = Fp8Config(
|
||||
weight_block_size=[128, 128],
|
||||
)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
weight_quant_key = create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(128, 128)
|
||||
)
|
||||
device = kwargs.get("device", torch.device("cuda:0"))
|
||||
|
||||
# Subclass to set weight_block_size before process_weights_after_loading
|
||||
class _BlockFP8Layer(TestFP8Layer):
|
||||
def __init__(self, *a, **kw):
|
||||
self.weight_block_size = [128, 128]
|
||||
super().__init__(*a, **kw)
|
||||
|
||||
# Force CutlassFp8BlockScaledMMKernel to ensure the graph uses
|
||||
# per_token_group_fp8_quant (not the deepgemm packed variant).
|
||||
self.block_fp8_linear = _BlockFP8Layer(
|
||||
weight_shape=(self.output_dim, self.output_dim),
|
||||
activation_quant_key=self.quant_key,
|
||||
weight_quant_key=weight_quant_key,
|
||||
input_dtype=self.dtype,
|
||||
device=device,
|
||||
force_kernel=CutlassFp8BlockScaledMMKernel,
|
||||
)
|
||||
|
||||
w = kwargs.get("w")
|
||||
if w is not None:
|
||||
self.block_fp8_linear.weight = w["weight"]
|
||||
# Block-wise uses weight_scale_inv, not weight_scale
|
||||
self.block_fp8_linear.weight_scale_inv = w["wscale"]
|
||||
|
||||
self.w = {
|
||||
"weight": self.block_fp8_linear.weight,
|
||||
"wscale": self.block_fp8_linear.weight_scale_inv,
|
||||
}
|
||||
|
||||
def forward(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
kv_c_normed: torch.Tensor,
|
||||
k_pe: torch.Tensor,
|
||||
):
|
||||
"""Forward pass: MLA attention -> block FP8 linear (group quant)."""
|
||||
attn_output = self.mla_attn(
|
||||
q,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_shape=(q.shape[0], self.output_dim),
|
||||
)
|
||||
return self.block_fp8_linear(attn_output)
|
||||
|
||||
|
||||
def is_nvfp4_supported():
|
||||
return current_platform.has_device_capability(100)
|
||||
|
||||
|
||||
# MLA test configuration
|
||||
MLA_DIMS: list[tuple[int, int, int, int, int]] = []
|
||||
PATTERN_TEST_MODELS_MLA_FP8: list[tuple[str, type]] = []
|
||||
PATTERN_TEST_MODELS_MLA_GROUP_FP8: list[tuple[str, type]] = []
|
||||
PATTERN_TEST_MODELS_MLA_FP4: list[tuple[str, type]] = []
|
||||
BACKENDS_MLA_FP8: list[AttentionBackendEnum] = []
|
||||
BACKENDS_MLA_FP4: list[AttentionBackendEnum] = []
|
||||
|
||||
if current_platform.is_cuda():
|
||||
# (num_heads, qk_nope_head_dim, qk_rope_head_dim, v_head_dim, kv_lora_rank)
|
||||
MLA_DIMS = [(16, 128, 64, 128, 512)]
|
||||
PATTERN_TEST_MODELS_MLA_FP8 = [
|
||||
(
|
||||
"deepseek-ai/DeepSeek-V2-Lite",
|
||||
TestMLAAttentionFp8StaticQuantPatternModel,
|
||||
)
|
||||
]
|
||||
PATTERN_TEST_MODELS_MLA_GROUP_FP8 = [
|
||||
(
|
||||
"deepseek-ai/DeepSeek-V3",
|
||||
TestMLAAttentionFp8GroupQuantPatternModel,
|
||||
)
|
||||
]
|
||||
PATTERN_TEST_MODELS_MLA_FP4 = [
|
||||
(
|
||||
"deepseek-ai/DeepSeek-V2-Lite",
|
||||
TestMLAAttentionNvfp4QuantPatternModel,
|
||||
)
|
||||
]
|
||||
BACKENDS_MLA_FP8 = [AttentionBackendEnum.TRITON_MLA]
|
||||
BACKENDS_MLA_FP4 = [AttentionBackendEnum.TRITON_MLA]
|
||||
elif current_platform.is_rocm():
|
||||
# ROCm supports the static-FP8 output-quant fusion through AITER MLA.
|
||||
# Per-group FP8 still relies on the CUDA-only CUTLASS block-scaled
|
||||
# kernel, and NVFP4 is NVIDIA-specific.
|
||||
MLA_DIMS = [(16, 128, 64, 128, 512)]
|
||||
PATTERN_TEST_MODELS_MLA_FP8 = [
|
||||
(
|
||||
"deepseek-ai/DeepSeek-V2-Lite",
|
||||
TestMLAAttentionFp8StaticQuantPatternModel,
|
||||
)
|
||||
]
|
||||
BACKENDS_MLA_FP8 = [AttentionBackendEnum.ROCM_AITER_MLA]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_heads, qk_nope_head_dim, qk_rope_head_dim, v_head_dim, kv_lora_rank",
|
||||
MLA_DIMS,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"batch_size", [7, 256] if current_platform.is_cuda_alike() else [8]
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize(
|
||||
"backend, model_name, model_class, custom_ops",
|
||||
list(
|
||||
flat_product(
|
||||
BACKENDS_MLA_FP8,
|
||||
PATTERN_TEST_MODELS_MLA_FP8,
|
||||
["+quant_fp8", "-quant_fp8"],
|
||||
)
|
||||
)
|
||||
+ list(
|
||||
flat_product(
|
||||
BACKENDS_MLA_FP8,
|
||||
PATTERN_TEST_MODELS_MLA_GROUP_FP8,
|
||||
["+quant_fp8"],
|
||||
)
|
||||
)
|
||||
+ list(flat_product(BACKENDS_MLA_FP4, PATTERN_TEST_MODELS_MLA_FP4, [""])),
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Only test ROCm or CUDA"
|
||||
)
|
||||
@pytest.mark.skipif(not current_platform.supports_fp8(), reason="Need FP8")
|
||||
def test_mla_attention_quant_pattern(
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
kv_lora_rank: int,
|
||||
batch_size: int,
|
||||
dtype: torch.dtype,
|
||||
custom_ops: str,
|
||||
model_name: str,
|
||||
model_class: type[MLAAttentionQuantPatternModel],
|
||||
backend: AttentionBackendEnum,
|
||||
dist_init,
|
||||
disable_vllm_compile_cache,
|
||||
workspace_init,
|
||||
):
|
||||
"""Test MLA AttentionQuantPattern fusion pass."""
|
||||
if (
|
||||
model_class is TestMLAAttentionNvfp4QuantPatternModel
|
||||
and not is_nvfp4_supported()
|
||||
):
|
||||
pytest.skip("NVFP4 is not supported on this GPU (requires SM 100+).")
|
||||
|
||||
custom_ops_list = custom_ops.split(",") if custom_ops else []
|
||||
|
||||
device = torch.device(f"{DEVICE_TYPE}:0")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(42)
|
||||
|
||||
model_config = ModelConfig(
|
||||
model=model_name,
|
||||
max_model_len=2048,
|
||||
dtype=dtype,
|
||||
)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config,
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_num_seqs=1024,
|
||||
max_model_len=model_config.max_model_len,
|
||||
is_encoder_decoder=model_config.is_encoder_decoder,
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=custom_ops_list,
|
||||
),
|
||||
cache_config=CacheConfig(cache_dtype="auto"),
|
||||
attention_config=AttentionConfig(backend=backend),
|
||||
)
|
||||
|
||||
# MLA inputs: q(B, N, qk_head_dim), kv_c_normed(B, L), k_pe(B, 1, R)
|
||||
qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
q = torch.randn(batch_size, num_heads, qk_head_dim, dtype=dtype, device=device)
|
||||
kv_c_normed = torch.randn(batch_size, kv_lora_rank, dtype=dtype, device=device)
|
||||
k_pe = torch.randn(batch_size, 1, qk_rope_head_dim, dtype=dtype, device=device)
|
||||
|
||||
# Mark first dimension as dynamic
|
||||
torch._dynamo.mark_dynamic(q, 0)
|
||||
torch._dynamo.mark_dynamic(kv_c_normed, 0)
|
||||
torch._dynamo.mark_dynamic(k_pe, 0)
|
||||
|
||||
# Run model without fusion
|
||||
vllm_config_unfused = copy.deepcopy(vllm_config)
|
||||
with (
|
||||
set_current_vllm_config(vllm_config_unfused),
|
||||
set_forward_context(attn_metadata=None, vllm_config=vllm_config_unfused),
|
||||
):
|
||||
model_unfused = model_class(
|
||||
num_heads=num_heads,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
kv_cache_dtype=dtype,
|
||||
device=device,
|
||||
vllm_config=vllm_config_unfused,
|
||||
)
|
||||
if (
|
||||
model_class is TestMLAAttentionFp8GroupQuantPatternModel
|
||||
and type(model_unfused.block_fp8_linear.kernel)
|
||||
is not CutlassFp8BlockScaledMMKernel
|
||||
):
|
||||
pytest.skip("CUTLASS FP8 block kernel is not supported on this platform")
|
||||
model_unfused = model_unfused.to(device)
|
||||
# HACK: See #131044
|
||||
result_unfused_0 = model_unfused(q, kv_c_normed, k_pe) # noqa: F841
|
||||
|
||||
forward_ctx = get_forward_context()
|
||||
forward_ctx.attn_metadata = model_unfused.build_attn_metadata(batch_size)
|
||||
|
||||
compiled_unfused = torch.compile(model_unfused, fullgraph=True)
|
||||
result_unfused = compiled_unfused(q, kv_c_normed, k_pe)
|
||||
|
||||
# Run model with attn fusion enabled
|
||||
vllm_config.compilation_config.pass_config = PassConfig(
|
||||
fuse_attn_quant=True, eliminate_noops=True
|
||||
)
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
set_forward_context(attn_metadata=None, vllm_config=vllm_config),
|
||||
):
|
||||
model_fused = model_class(
|
||||
num_heads=num_heads,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
kv_cache_dtype=dtype,
|
||||
device=device,
|
||||
vllm_config=vllm_config,
|
||||
w=model_unfused.w,
|
||||
kv_b_proj_weight=model_unfused.kv_b_proj_weight,
|
||||
)
|
||||
model_fused = model_fused.to(device)
|
||||
|
||||
forward_ctx = get_forward_context()
|
||||
forward_ctx.attn_metadata = model_fused.build_attn_metadata(batch_size)
|
||||
|
||||
# Create test backend with fusion passes
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
attn_pass = LazyInitPass(MLAAttnQuantFusionPass, vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
|
||||
test_backend = TestBackend(noop_pass, attn_pass, cleanup_pass)
|
||||
# HACK: See https://github.com/vllm-project/vllm/issues/31044
|
||||
result_fused_0 = model_fused(q, kv_c_normed, k_pe) # noqa: F841
|
||||
|
||||
compiled_fused = torch.compile(
|
||||
model_fused, backend=test_backend, fullgraph=True
|
||||
)
|
||||
|
||||
result_fused = compiled_fused(q, kv_c_normed, k_pe)
|
||||
|
||||
# Check attn fusion support
|
||||
quant_key: QuantKey = model_class.quant_key
|
||||
attn_fusion_supported = [
|
||||
layer.impl.fused_output_quant_supported(quant_key)
|
||||
for key, layer in vllm_config.compilation_config.static_forward_context.items()
|
||||
if isinstance(layer, MLAAttention)
|
||||
]
|
||||
assert sum(attn_fusion_supported) == len(attn_fusion_supported), (
|
||||
"All MLA layers should support attention fusion"
|
||||
)
|
||||
|
||||
# Check quantization ops in the graph
|
||||
is_per_group = quant_key.scale.group_shape.is_per_group()
|
||||
quant_op = (
|
||||
torch.ops.aten.reciprocal
|
||||
if "-quant_fp8" in custom_ops_list
|
||||
else QUANT_OPS[quant_key]
|
||||
)
|
||||
test_backend.check_before_ops([quant_op], fully_replaced=is_per_group)
|
||||
|
||||
assert attn_pass.pass_.matched_count == sum(attn_fusion_supported)
|
||||
|
||||
# Check MLA attention ops in the graph
|
||||
attn_nodes_pre = list(find_op_nodes(MLA_ATTN_OP, test_backend.graph_pre_pass))
|
||||
attn_nodes_post = list(find_op_nodes(MLA_ATTN_OP, test_backend.graph_post_pass))
|
||||
|
||||
assert len(attn_nodes_pre) > 0, "Should have MLA attention nodes before fusion"
|
||||
assert len(attn_nodes_pre) == len(attn_nodes_post), (
|
||||
"Should have same number of MLA attention nodes before and after fusion"
|
||||
)
|
||||
|
||||
# Before fusion: neither scale should be set
|
||||
assert attn_nodes_pre[0].kwargs.get("output_scale") is None
|
||||
assert attn_nodes_pre[0].kwargs.get("output_block_scale") is None
|
||||
|
||||
# After fusion: derive expected scale presence from quant_key properties.
|
||||
# - output_scale: present for static quant or non-FP8 (NVFP4 carries input_scale)
|
||||
# - output_block_scale: present when quant uses per-group/block scaling
|
||||
has_output_scale = attn_nodes_post[0].kwargs.get("output_scale") is not None
|
||||
has_block_scale = attn_nodes_post[0].kwargs.get("output_block_scale") is not None
|
||||
|
||||
expects_output_scale = quant_key.scale.static or quant_key.dtype != FP8_DTYPE
|
||||
assert has_output_scale == expects_output_scale, (
|
||||
f"output_scale: expected present={expects_output_scale}, got {has_output_scale}"
|
||||
)
|
||||
assert has_block_scale == is_per_group, (
|
||||
f"output_block_scale: expected present={is_per_group}, got {has_block_scale}"
|
||||
)
|
||||
|
||||
# Check numerical correctness
|
||||
torch.testing.assert_close(result_unfused, result_fused, atol=1e-2, rtol=1e-2)
|
||||
@@ -1,399 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.config
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.compilation.passes.fusion.mla_rope_kvcache_cat_fusion import (
|
||||
MLARoPEKVCacheCatFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.fix_functionalization import (
|
||||
FixFunctionalizationPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.attention import MLAAttention
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear
|
||||
from vllm.model_executor.layers.rotary_embedding import (
|
||||
DeepseekScalingRotaryEmbedding,
|
||||
RotaryEmbedding,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import _encode_layer_name
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_supports_mla
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
|
||||
VLLM_UNIFIED_MLA_KV_CACHE_UPDATE_OP = torch.ops.vllm.unified_mla_kv_cache_update
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
|
||||
class MLARoPEKVCacheCatTestModel(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
attn_backend: AttentionBackendEnum,
|
||||
use_deepseek_scaling_rope: bool,
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
q_lora_rank: int,
|
||||
kv_lora_rank: int,
|
||||
is_neox: bool,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
prefix: str = "model.layers.0.self_attn.attn",
|
||||
):
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
self.qk_nope_head_dim = qk_nope_head_dim
|
||||
self.qk_rope_head_dim = qk_rope_head_dim
|
||||
self.qk_head_dim = qk_nope_head_dim + qk_rope_head_dim
|
||||
self.v_head_dim = v_head_dim
|
||||
self.q_lora_rank = q_lora_rank
|
||||
self.kv_lora_rank = kv_lora_rank
|
||||
self.dtype = dtype
|
||||
self.device = device
|
||||
self.layer_name = prefix
|
||||
|
||||
self.num_kv_heads = 1
|
||||
self.head_size = kv_lora_rank + qk_rope_head_dim
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.scale = self.qk_head_dim**-0.5
|
||||
|
||||
if use_deepseek_scaling_rope:
|
||||
self.rotary_emb = DeepseekScalingRotaryEmbedding(
|
||||
head_size=qk_rope_head_dim,
|
||||
rotary_dim=qk_rope_head_dim,
|
||||
max_position_embeddings=4096,
|
||||
base=10000,
|
||||
is_neox_style=is_neox,
|
||||
scaling_factor=1.0,
|
||||
dtype=dtype,
|
||||
)
|
||||
else:
|
||||
self.rotary_emb = RotaryEmbedding(
|
||||
head_size=qk_rope_head_dim,
|
||||
rotary_dim=qk_rope_head_dim,
|
||||
max_position_embeddings=4096,
|
||||
base=10000,
|
||||
is_neox_style=is_neox,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
# Initialize intermediate mm layers for unit test
|
||||
self.q_b_proj = ColumnParallelLinear(
|
||||
self.q_lora_rank,
|
||||
self.num_heads * self.qk_head_dim,
|
||||
bias=False,
|
||||
prefix=f"{prefix}.q_b_proj",
|
||||
).to(device)
|
||||
self.kv_b_proj = ColumnParallelLinear(
|
||||
self.kv_lora_rank,
|
||||
self.num_heads * (self.qk_nope_head_dim + self.v_head_dim),
|
||||
bias=False,
|
||||
prefix=f"{prefix}.kv_b_proj",
|
||||
).to(device)
|
||||
|
||||
# ColumnParallelLinear default init in bf16 with seed 0 produces
|
||||
# near-zero weights (7/4.7M nonzero), making the GEMM output almost
|
||||
# entirely zero and masking correctness bugs. Reinitialize to get
|
||||
# dense outputs.
|
||||
with torch.no_grad():
|
||||
torch.nn.init.normal_(self.q_b_proj.weight, std=0.02)
|
||||
torch.nn.init.normal_(self.kv_b_proj.weight, std=0.02)
|
||||
|
||||
# Register layer metadata for the fusion pass via MLAAttention
|
||||
self.mla_attn = MLAAttention(
|
||||
num_heads=self.num_heads,
|
||||
scale=self.scale,
|
||||
qk_nope_head_dim=self.qk_nope_head_dim,
|
||||
qk_rope_head_dim=self.qk_rope_head_dim,
|
||||
v_head_dim=self.v_head_dim,
|
||||
q_lora_rank=self.q_lora_rank,
|
||||
kv_lora_rank=self.kv_lora_rank,
|
||||
kv_b_proj=self.kv_b_proj,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=prefix,
|
||||
attn_backend=attn_backend.get_class(),
|
||||
)
|
||||
self.attn_backend: type[AttentionBackend] = self.mla_attn.get_attn_backend()
|
||||
self.mla_attn._k_scale = self.mla_attn._k_scale.to(device)
|
||||
self.mla_attn._v_scale = self.mla_attn._v_scale.to(device)
|
||||
|
||||
# Keep both the string dtype (for ops) and torch dtype (for tensors)
|
||||
self.kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
|
||||
self.kv_cache_dtype = (
|
||||
FP8_DTYPE if self.kv_cache_dtype_str.startswith("fp8") else self.dtype
|
||||
)
|
||||
|
||||
# Initialize attn MetadataBuilder
|
||||
self.builder = self.attn_backend.get_builder_cls()(
|
||||
kv_cache_spec=self.mla_attn.get_kv_cache_spec(vllm_config),
|
||||
layer_names=[self.mla_attn.layer_name],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def build_attn_metadata(self, batch_size: int) -> CommonAttentionMetadata:
|
||||
"""Initialize attention metadata."""
|
||||
# Create common attn metadata
|
||||
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, self.block_size, self.device, arange_block_indices=True
|
||||
)
|
||||
|
||||
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
|
||||
num_blocks = batch_size * max_blocks
|
||||
|
||||
# MLA KV cache is 4D: (num_blocks, num_heads=1, block_size, head_size)
|
||||
kv_cache = torch.zeros(
|
||||
(num_blocks, 1, self.block_size, self.head_size),
|
||||
dtype=self.kv_cache_dtype,
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
self.mla_attn.bind_kv_cache(kv_cache)
|
||||
|
||||
# Build attn metadata
|
||||
attn_metadata = self.builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
|
||||
return attn_metadata
|
||||
|
||||
def forward(
|
||||
self, qkv_lora: torch.Tensor, positions: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
qkv_lora = qkv_lora.clone()
|
||||
q_c, kv_lora = qkv_lora.split(
|
||||
[self.q_lora_rank, self.kv_lora_rank + self.qk_rope_head_dim],
|
||||
dim=-1,
|
||||
)
|
||||
q = self.q_b_proj(q_c)[0]
|
||||
kv_c, k_pe = kv_lora.split([self.kv_lora_rank, self.qk_rope_head_dim], dim=-1)
|
||||
|
||||
q = q.view(-1, self.num_heads, self.qk_head_dim)
|
||||
k_pe = k_pe.unsqueeze(1)
|
||||
|
||||
q[..., self.qk_nope_head_dim :], k_pe = self.rotary_emb(
|
||||
positions, q[..., self.qk_nope_head_dim :], k_pe
|
||||
)
|
||||
|
||||
dummy = torch.ops.vllm.unified_mla_kv_cache_update(
|
||||
kv_c,
|
||||
k_pe,
|
||||
_encode_layer_name(self.layer_name),
|
||||
self.kv_cache_dtype_str,
|
||||
self.mla_attn._k_scale,
|
||||
)
|
||||
return q, kv_c, k_pe, dummy
|
||||
|
||||
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
|
||||
ops = [
|
||||
INDEX_SELECT_OP,
|
||||
torch.ops.vllm.unified_mla_kv_cache_update.default,
|
||||
]
|
||||
return ops
|
||||
|
||||
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
|
||||
return [torch.ops.vllm.fused_rope_unified_mla_kv_cache_update.default]
|
||||
|
||||
|
||||
MLA_BACKENDS = [AttentionBackendEnum.TRITON_MLA]
|
||||
if flash_attn_supports_mla():
|
||||
MLA_BACKENDS += [AttentionBackendEnum.FLASH_ATTN_MLA]
|
||||
if is_aiter_found_and_supported():
|
||||
MLA_BACKENDS += [AttentionBackendEnum.ROCM_AITER_MLA]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("attn_backend", MLA_BACKENDS)
|
||||
@pytest.mark.parametrize("use_deepseek_scaling_rope", [True])
|
||||
@pytest.mark.parametrize("num_heads", [16])
|
||||
@pytest.mark.parametrize("qk_nope_head_dim", [128])
|
||||
@pytest.mark.parametrize("qk_rope_head_dim", [64])
|
||||
@pytest.mark.parametrize("v_head_dim", [128])
|
||||
@pytest.mark.parametrize("q_lora_rank", [1536])
|
||||
@pytest.mark.parametrize("kv_lora_rank", [512])
|
||||
@pytest.mark.parametrize("block_size", [16])
|
||||
@pytest.mark.parametrize("is_neox", [True, False])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="MLA RoPE+KVCache+Cat fusion is only supported on CUDA and ROCm.",
|
||||
)
|
||||
def test_mla_rope_kvcache_cat_fusion(
|
||||
attn_backend: AttentionBackendEnum,
|
||||
use_deepseek_scaling_rope: bool,
|
||||
num_heads: int,
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
q_lora_rank: int,
|
||||
kv_lora_rank: int,
|
||||
block_size: int,
|
||||
is_neox: bool,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(
|
||||
model="deepseek-ai/DeepSeek-V2-Lite",
|
||||
dtype=dtype,
|
||||
),
|
||||
cache_config=CacheConfig(
|
||||
block_size=block_size,
|
||||
cache_dtype=kv_cache_dtype,
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
pass_config=PassConfig(
|
||||
fuse_rope_kvcache_cat_mla=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
if not torch.distributed.is_initialized():
|
||||
from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.utils.system_utils import update_environment_variables
|
||||
|
||||
update_environment_variables(
|
||||
{
|
||||
"RANK": "0",
|
||||
"LOCAL_RANK": "0",
|
||||
"WORLD_SIZE": "1",
|
||||
"MASTER_ADDR": "localhost",
|
||||
"MASTER_PORT": "54321",
|
||||
}
|
||||
)
|
||||
init_distributed_environment()
|
||||
initialize_model_parallel()
|
||||
|
||||
if attn_backend == AttentionBackendEnum.ROCM_AITER_MLA:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
model = MLARoPEKVCacheCatTestModel(
|
||||
vllm_config=vllm_config,
|
||||
attn_backend=attn_backend,
|
||||
use_deepseek_scaling_rope=use_deepseek_scaling_rope,
|
||||
num_heads=num_heads,
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
q_lora_rank=q_lora_rank,
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
is_neox=is_neox,
|
||||
dtype=dtype,
|
||||
device=torch.get_default_device(),
|
||||
)
|
||||
|
||||
fusion_pass = MLARoPEKVCacheCatFusionPass(vllm_config)
|
||||
# note: FixFunctionalizationPass is required to correctly lower
|
||||
# the fused op to its inplace version with auto-functionalization v1.
|
||||
# Without it, decompose_auto_functionalized calls clone_preserve_strides
|
||||
# on the non-contiguous q_pe slice directly, and inductor's lowering
|
||||
# of the resulting as_strided chain incorrectly drops the storage offset.
|
||||
# auto-functionalization v2 avoids this: it clones the contiguous base
|
||||
# tensor (_all_bases) and reconstructs the slice as a view, so the
|
||||
# offset is never passed through as_strided lowering.
|
||||
passes = [
|
||||
NoOpEliminationPass(vllm_config),
|
||||
fusion_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
FixFunctionalizationPass(vllm_config),
|
||||
]
|
||||
backend = TestBackend(*passes)
|
||||
|
||||
T = 5
|
||||
|
||||
qkv_lora = torch.randn(
|
||||
T,
|
||||
q_lora_rank + kv_lora_rank + qk_rope_head_dim,
|
||||
dtype=dtype,
|
||||
)
|
||||
pos = torch.arange(T, dtype=torch.long)
|
||||
|
||||
qkv_unfused = qkv_lora.clone()
|
||||
pos_unfused = pos.clone()
|
||||
|
||||
# Run unfused version
|
||||
with set_forward_context(None, vllm_config):
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model.build_attn_metadata(T)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
q_unfused, kv_c_unfused, k_pe_unfused, dummy = model(
|
||||
qkv_unfused, pos_unfused
|
||||
)
|
||||
attn_layer = forward_context.no_compile_layers[model.layer_name]
|
||||
kv_cache_unfused = attn_layer.kv_cache.clone()
|
||||
del dummy
|
||||
|
||||
# Run fused version (compiled)
|
||||
torch._dynamo.mark_dynamic(qkv_lora, 0)
|
||||
torch._dynamo.mark_dynamic(pos, 0)
|
||||
with set_forward_context(None, vllm_config):
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model.build_attn_metadata(T)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
q_fused, kv_c_fused, k_pe_fused, dummy = model_fused(qkv_lora, pos)
|
||||
attn_layer = forward_context.no_compile_layers[model.layer_name]
|
||||
kv_cache_fused = attn_layer.kv_cache
|
||||
del dummy
|
||||
|
||||
assert fusion_pass.matched_count == 1
|
||||
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
|
||||
if dtype == torch.float16:
|
||||
ATOL, RTOL = (2e-3, 2e-3)
|
||||
else:
|
||||
ATOL, RTOL = (1e-2, 1e-2)
|
||||
|
||||
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
|
||||
torch.testing.assert_close(kv_c_unfused, kv_c_fused, atol=ATOL, rtol=RTOL)
|
||||
torch.testing.assert_close(k_pe_unfused, k_pe_fused, atol=ATOL, rtol=RTOL)
|
||||
# Cannot compare fp8_* directly here, cast to model dtype instead
|
||||
torch.testing.assert_close(
|
||||
kv_cache_unfused.view(dtype),
|
||||
kv_cache_fused.view(dtype),
|
||||
atol=ATOL,
|
||||
rtol=RTOL,
|
||||
)
|
||||
@@ -1,120 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm
|
||||
from tests.compile.backend import TestBackend
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.config import CompilationConfig, CompilationMode, PassConfig, VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
# Important edge case is when `num_tokens == buffer_size`
|
||||
@pytest.mark.parametrize(
|
||||
("num_tokens", "buffer_size"), [(256, 256), (256, 512), (1024, 1024), (1024, 1025)]
|
||||
)
|
||||
@pytest.mark.parametrize("hidden_size", [64, 4096])
|
||||
def test_noop_elimination(dtype, num_tokens, hidden_size, buffer_size):
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(1)
|
||||
|
||||
class Model(torch.nn.Module):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
# Avoid using empty, since on rocm torch.empty
|
||||
# does not initialize the memory.
|
||||
self.pos_embed = torch.randn(buffer_size, hidden_size, dtype=dtype)
|
||||
|
||||
def forward(self, x):
|
||||
# Avoid += to prevent inplace addition.
|
||||
x = x + self.pos_embed[: x.shape[0]]
|
||||
# Chain of reshapes
|
||||
y = x.reshape(-1, 128, 32)
|
||||
z = y.reshape(-1, 4096)
|
||||
# No-op reshape
|
||||
a = z.reshape(-1, 4096)
|
||||
# Final reshape that should remain
|
||||
b = a.reshape(-1, 128, 32)
|
||||
# No-op slice
|
||||
c = b[0 : b.shape[0]]
|
||||
# The pass should replace the result of this op with `c`.
|
||||
d = torch.slice_scatter(
|
||||
torch.ones_like(c), # Dummy tensor to be scattered into
|
||||
c, # Source tensor
|
||||
0, # dim
|
||||
0, # start
|
||||
c.shape[0], # end
|
||||
)
|
||||
return d
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
pass_config=PassConfig(eliminate_noops=True),
|
||||
)
|
||||
)
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
|
||||
backend = TestBackend(noop_pass)
|
||||
|
||||
model = Model()
|
||||
# First dimension dynamic
|
||||
x = torch.rand(num_tokens, hidden_size)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
|
||||
result = model(x)
|
||||
|
||||
model2 = torch.compile(model, backend=backend)
|
||||
result2 = model2(x)
|
||||
|
||||
ATOL, RTOL = (2e-3, 2e-3)
|
||||
torch.testing.assert_close(result, result2, atol=ATOL, rtol=RTOL)
|
||||
|
||||
# The no-op reshape and slice should be eliminated.
|
||||
# The initial slice on the positional embedding should remain.
|
||||
# The chain of reshapes should be fused into a single reshape.
|
||||
assert backend.op_count(torch.ops.aten.reshape.default) == 1
|
||||
assert backend.op_count(torch.ops.aten.slice.Tensor) == 1
|
||||
assert backend.op_count(torch.ops.aten.slice_scatter.default) == 0
|
||||
|
||||
|
||||
def test_non_noop_slice_preserved():
|
||||
"""Ensure that a slice with end=-1 (dropping last row) is NOT eliminated.
|
||||
|
||||
Regression test for a bug where end=-1 was treated like an inferred
|
||||
dimension (reshape semantics) leading to incorrect elimination.
|
||||
"""
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
x = torch.randn(16, 16)
|
||||
|
||||
class SliceModel(torch.nn.Module):
|
||||
def forward(self, x):
|
||||
base = x.clone()
|
||||
src = torch.ones(15, 16)
|
||||
y = torch.slice_scatter(base, src, dim=0, start=0, end=-1)
|
||||
return x[0:-1, :], y
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
pass_config=PassConfig(eliminate_noops=True),
|
||||
)
|
||||
)
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
backend = TestBackend(noop_pass)
|
||||
model = SliceModel()
|
||||
ref = model(x)
|
||||
compiled = torch.compile(model, backend=backend)
|
||||
out = compiled(x)
|
||||
torch.testing.assert_close(ref, out)
|
||||
# The slice should remain (not a no-op).
|
||||
assert backend.op_count(torch.ops.aten.slice.Tensor) == 1
|
||||
assert backend.op_count(torch.ops.aten.slice_scatter.default) == 1
|
||||
@@ -1,284 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch._ops import OpOverload, OpOverloadPacket
|
||||
|
||||
from tests.compile.backend import TestBackend
|
||||
from vllm.compilation.passes.fusion.matcher_utils import (
|
||||
FLASHINFER_ROTARY_OP,
|
||||
ROTARY_OP,
|
||||
)
|
||||
from vllm.compilation.passes.fusion.qk_norm_rope_fusion import (
|
||||
FUSED_QK_ROPE_OP,
|
||||
QKNormRoPEFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
RSQRT_OP = torch.ops.aten.rsqrt.default
|
||||
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
|
||||
|
||||
|
||||
class QKNormRoPETestModel(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_dim: int,
|
||||
eps: float,
|
||||
is_neox: bool,
|
||||
vllm_config: VllmConfig,
|
||||
dtype: torch.dtype,
|
||||
test_scattered_split: bool = False,
|
||||
prefix: str = "model.layers.0.self_attn.attn",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.head_dim = head_dim
|
||||
self.q_size = num_heads * head_dim
|
||||
self.kv_size = num_kv_heads * head_dim
|
||||
self.rotary_dim = head_dim
|
||||
self.eps = eps
|
||||
self.dtype = dtype
|
||||
|
||||
# Register layer metadata for the fusion pass via Attention.
|
||||
self.attn = Attention(
|
||||
num_heads=self.num_heads,
|
||||
head_size=self.head_dim,
|
||||
scale=1.0 / self.head_dim**0.5,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
cache_config=vllm_config.cache_config,
|
||||
prefix=prefix,
|
||||
attn_type=AttentionType.DECODER,
|
||||
)
|
||||
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=self.eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=self.eps)
|
||||
self.rotary_emb = RotaryEmbedding(
|
||||
self.head_dim,
|
||||
rotary_dim=self.rotary_dim,
|
||||
max_position_embeddings=4096,
|
||||
base=10000,
|
||||
is_neox_style=is_neox,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
self.test_scattered_split = test_scattered_split
|
||||
self.enable_rms_norm_custom_op = self.q_norm.enabled()
|
||||
self.enable_rope_custom_op = self.rotary_emb.enabled()
|
||||
|
||||
def forward(self, qkv: torch.Tensor, positions: torch.Tensor):
|
||||
if self.test_scattered_split:
|
||||
q, _, _ = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
_, k, _ = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
_, _, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
else:
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q_by_head = q.view(*q.shape[:-1], q.shape[-1] // self.head_dim, self.head_dim)
|
||||
q_by_head = self.q_norm(q_by_head)
|
||||
q = q_by_head.view(q.shape)
|
||||
k_by_head = k.view(*k.shape[:-1], k.shape[-1] // self.head_dim, self.head_dim)
|
||||
k_by_head = self.k_norm(k_by_head)
|
||||
k = k_by_head.view(k.shape)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
return q, k, v
|
||||
|
||||
def ops_in_model_before(self) -> list[OpOverload | OpOverloadPacket]:
|
||||
ops: list[OpOverload | OpOverloadPacket] = [torch.ops.vllm_ir.rms_norm]
|
||||
if self.enable_rope_custom_op:
|
||||
if self.rotary_emb.use_flashinfer:
|
||||
ops.append(FLASHINFER_ROTARY_OP)
|
||||
else:
|
||||
ops.append(ROTARY_OP)
|
||||
else:
|
||||
ops.append(INDEX_SELECT_OP)
|
||||
return ops
|
||||
|
||||
def ops_in_model_after(self) -> list[OpOverload | OpOverloadPacket]:
|
||||
return [FUSED_QK_ROPE_OP]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scattered_split", [True, False])
|
||||
@pytest.mark.parametrize("eps", [1e-5, 1e-6])
|
||||
@pytest.mark.parametrize("is_neox", [True, False])
|
||||
@pytest.mark.parametrize("enable_rms_norm_custom_op", [True, False])
|
||||
@pytest.mark.parametrize("enable_rope_custom_op", [True])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda_alike() or current_platform.is_xpu()),
|
||||
reason="Only test on cuda and rocm platform",
|
||||
)
|
||||
def test_qk_norm_rope_fusion(
|
||||
eps,
|
||||
is_neox,
|
||||
enable_rms_norm_custom_op,
|
||||
enable_rope_custom_op,
|
||||
dtype,
|
||||
scattered_split,
|
||||
):
|
||||
if not hasattr(torch.ops._C, "fused_qk_norm_rope"):
|
||||
pytest.skip("fused_qk_norm_rope custom op not available")
|
||||
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
custom_ops: list[str] = []
|
||||
if enable_rms_norm_custom_op:
|
||||
custom_ops.append("+rms_norm")
|
||||
if enable_rope_custom_op:
|
||||
custom_ops.append("+rotary_embedding")
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=custom_ops,
|
||||
pass_config=PassConfig(
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
num_heads, num_kv_heads, head_dim = 16, 4, 128
|
||||
T = 5
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
vllm_config.kernel_config.ir_op_priority.set_priority(),
|
||||
):
|
||||
model = QKNormRoPETestModel(
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_dim=head_dim,
|
||||
eps=eps,
|
||||
is_neox=is_neox,
|
||||
vllm_config=vllm_config,
|
||||
dtype=dtype,
|
||||
test_scattered_split=scattered_split,
|
||||
)
|
||||
|
||||
noop_pass = NoOpEliminationPass(vllm_config)
|
||||
coalesce_pass = SplitCoalescingPass(vllm_config)
|
||||
fusion_pass = QKNormRoPEFusionPass(vllm_config)
|
||||
cleanup_pass = PostCleanupPass(vllm_config)
|
||||
|
||||
backend = TestBackend(noop_pass, coalesce_pass, fusion_pass, cleanup_pass)
|
||||
backend_baseline = TestBackend(noop_pass, cleanup_pass)
|
||||
|
||||
qkv = torch.randn(T, model.q_size + 2 * model.kv_size)
|
||||
pos = torch.arange(T, dtype=torch.long, device=qkv.device)
|
||||
qkv_unfused = qkv.clone()
|
||||
pos_unfused = pos.clone()
|
||||
|
||||
torch._dynamo.mark_dynamic(qkv, 0)
|
||||
torch._dynamo.mark_dynamic(pos, 0)
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
q_fused, k_fused, v_fused = model_fused(qkv, pos)
|
||||
|
||||
torch._dynamo.mark_dynamic(qkv_unfused, 0)
|
||||
torch._dynamo.mark_dynamic(pos_unfused, 0)
|
||||
model_unfused = torch.compile(model, backend=backend_baseline)
|
||||
q_unfused, k_unfused, v_unfused = model_unfused(qkv_unfused, pos_unfused)
|
||||
|
||||
if dtype == torch.float16:
|
||||
ATOL, RTOL = (2e-3, 2e-3)
|
||||
else:
|
||||
ATOL, RTOL = (1e-2, 1e-2)
|
||||
|
||||
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
|
||||
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
|
||||
torch.testing.assert_close(v_unfused, v_fused, atol=ATOL, rtol=RTOL)
|
||||
|
||||
assert fusion_pass.matched_count == 1
|
||||
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(),
|
||||
reason="Only test on CUDA and ROCm platforms",
|
||||
)
|
||||
def test_qk_norm_rope_fusion_with_heterogeneous_draft_model():
|
||||
"""The fusion pass should support target and draft attention geometries."""
|
||||
if not hasattr(torch.ops._C, "fused_qk_norm_rope"):
|
||||
pytest.skip("fused_qk_norm_rope custom op not available")
|
||||
|
||||
dtype = torch.bfloat16
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=["+rms_norm", "+rotary_embedding"],
|
||||
pass_config=PassConfig(
|
||||
enable_qk_norm_rope_fusion=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
vllm_config.kernel_config.ir_op_priority.set_priority(),
|
||||
):
|
||||
target_model = QKNormRoPETestModel(
|
||||
num_heads=32,
|
||||
num_kv_heads=8,
|
||||
head_dim=128,
|
||||
eps=1e-6,
|
||||
is_neox=True,
|
||||
vllm_config=vllm_config,
|
||||
dtype=dtype,
|
||||
prefix="model.layers.0.self_attn.attn",
|
||||
)
|
||||
draft_model = QKNormRoPETestModel(
|
||||
num_heads=16,
|
||||
num_kv_heads=8,
|
||||
head_dim=128,
|
||||
eps=1e-6,
|
||||
is_neox=True,
|
||||
vllm_config=vllm_config,
|
||||
dtype=dtype,
|
||||
prefix="draft_model.layers.0.self_attn.attn",
|
||||
)
|
||||
|
||||
fusion_pass = QKNormRoPEFusionPass(vllm_config)
|
||||
backend = TestBackend(
|
||||
NoOpEliminationPass(vllm_config),
|
||||
SplitCoalescingPass(vllm_config),
|
||||
fusion_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
)
|
||||
|
||||
for model in (target_model, draft_model):
|
||||
num_tokens = 5
|
||||
qkv = torch.randn(num_tokens, model.q_size + 2 * model.kv_size)
|
||||
positions = torch.arange(num_tokens, dtype=torch.long, device=qkv.device)
|
||||
torch._dynamo.mark_dynamic(qkv, 0)
|
||||
torch._dynamo.mark_dynamic(positions, 0)
|
||||
torch.compile(model, backend=backend)(qkv, positions)
|
||||
|
||||
assert fusion_pass.matched_count == 1
|
||||
@@ -1,105 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.ir.ops
|
||||
from tests.compile.backend import TestBackend
|
||||
from vllm.compilation.passes.fusion.add_rms_fusion import (
|
||||
AddRMSNormFusionPass,
|
||||
RMSNormReshapeFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.fx_utils import find_op_nodes, is_func
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import CompilationConfig, CompilationMode, VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Requires CUDA or ROCm"
|
||||
)
|
||||
|
||||
|
||||
class RMSNormModel(torch.nn.Module):
|
||||
def __init__(self, hidden_size: int) -> None:
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.randn(hidden_size))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
rms = vllm.ir.ops.rms_norm(x, self.weight, 1e-6)
|
||||
return rms.reshape(-1, rms.shape[-1])
|
||||
|
||||
|
||||
class AddRMSNormModel(torch.nn.Module):
|
||||
def __init__(self, hidden_size: int, residual_first: bool) -> None:
|
||||
super().__init__()
|
||||
self.weight = torch.nn.Parameter(torch.randn(hidden_size))
|
||||
self.residual_first = residual_first
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
residual = residual + x if self.residual_first else x + residual
|
||||
rms = vllm.ir.ops.rms_norm(residual, self.weight, 1e-6)
|
||||
return rms.reshape(-1, rms.shape[-1]), residual
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [torch.ops.aten.add, torch.ops.vllm_ir.rms_norm]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.vllm_ir.fused_add_rms_norm]
|
||||
|
||||
|
||||
def _run_fusion_test(model, config, passes, *inputs):
|
||||
backend = TestBackend(NoOpEliminationPass(config), *passes, PostCleanupPass(config))
|
||||
outputs_unfused = model(*inputs)
|
||||
outputs_fused = torch.compile(model, backend=backend)(*inputs)
|
||||
torch.testing.assert_close(outputs_fused, outputs_unfused)
|
||||
return backend
|
||||
|
||||
|
||||
def _is_reshape(node):
|
||||
return isinstance(node, torch.fx.Node) and is_func(
|
||||
node, torch.ops.aten.reshape.default
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vllm_config():
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.VLLM_COMPILE)
|
||||
)
|
||||
with vllm.config.set_current_vllm_config(config):
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(torch.bfloat16)
|
||||
torch.manual_seed(0)
|
||||
yield config
|
||||
|
||||
|
||||
def test_rmsnorm_reshape_fusion(vllm_config):
|
||||
fusion_pass = RMSNormReshapeFusionPass(vllm_config)
|
||||
model = RMSNormModel(hidden_size=32)
|
||||
x = torch.randn(2, 7, 32)
|
||||
backend = _run_fusion_test(model, vllm_config, [fusion_pass], x)
|
||||
|
||||
assert fusion_pass.matched_count == 1
|
||||
(rms_node,) = find_op_nodes(torch.ops.vllm_ir.rms_norm, backend.graph_post_pass)
|
||||
assert _is_reshape(rms_node.args[0])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("residual_first", [True, False])
|
||||
def test_add_rmsnorm_reshape_fusion(vllm_config, residual_first):
|
||||
add_fusion = AddRMSNormFusionPass(vllm_config)
|
||||
reshape_fusion = RMSNormReshapeFusionPass(vllm_config)
|
||||
model = AddRMSNormModel(hidden_size=32, residual_first=residual_first)
|
||||
x = torch.randn(2, 7, 32)
|
||||
residual = torch.randn_like(x)
|
||||
backend = _run_fusion_test(
|
||||
model, vllm_config, [add_fusion, reshape_fusion], x, residual
|
||||
)
|
||||
|
||||
assert add_fusion.matched_count == 1
|
||||
assert reshape_fusion.matched_count == 1
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
@@ -1,901 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
from importlib.metadata import version
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from packaging.version import Version
|
||||
|
||||
import vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion as fusion_module
|
||||
import vllm.config
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
dense_kv_cache_views,
|
||||
)
|
||||
from vllm._aiter_ops import (
|
||||
IS_AITER_FOUND,
|
||||
is_aiter_found_and_supported,
|
||||
rocm_aiter_ops,
|
||||
)
|
||||
from vllm.compilation.passes.fusion.matcher_utils import MROPE_OP, ROTARY_OP
|
||||
from vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion import (
|
||||
QkNormRopeKvCacheFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.compilation.passes.utility.scatter_split_replace import (
|
||||
ScatterSplitReplacementPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.quantization.quark.quark import QuarkConfig
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.model_executor.layers.rotary_embedding.mrope import MRotaryEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import _USE_LAYERNAME, _encode_layer_name
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheLayout
|
||||
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
if not is_aiter_found_and_supported():
|
||||
pytest.skip(
|
||||
"ROCm with supported AITER is required",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module", autouse=True)
|
||||
def module_global_cleanup():
|
||||
# Cleanup once at the end of the module
|
||||
from vllm.distributed import cleanup_dist_env_and_memory
|
||||
|
||||
yield
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
|
||||
class QKNormRoPEKVCacheTestModel(torch.nn.Module):
|
||||
"""Minimal model that reproduces the QK-norm + RoPE + KV cache update
|
||||
pattern matched by QkNormRopeKvCacheFusionPass:
|
||||
|
||||
q, k, v = split(qkv)
|
||||
q = rms_norm(q.view(heads, dim), q_weight).view(flat)
|
||||
k = rms_norm(k.view(heads, dim), k_weight).view(flat)
|
||||
q, k = rotary_emb(positions, q, k)
|
||||
q = q.view(num_heads, head_dim)
|
||||
k = k.view(num_kv_heads, head_dim)
|
||||
v = v.view(num_kv_heads, head_dim)
|
||||
dummy = unified_kv_cache_update(k, v, layer_name)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
attn_backend: AttentionBackendEnum,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
is_neox: bool,
|
||||
rms_norm_eps: float,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
rotary_dim: int | None = None,
|
||||
prefix: str = "model.layers.0.self_attn.attn",
|
||||
expected_query_quant_group_shape: GroupShape | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.head_size = head_size
|
||||
self.rotary_dim = rotary_dim if rotary_dim is not None else head_size
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.q_size = num_heads * head_size
|
||||
self.kv_size = num_kv_heads * head_size
|
||||
self.is_neox = is_neox
|
||||
self.dtype = dtype
|
||||
self.device = device
|
||||
self.layer_name = prefix
|
||||
|
||||
self.q_norm = RMSNorm(head_size, eps=rms_norm_eps)
|
||||
self.k_norm = RMSNorm(head_size, eps=rms_norm_eps)
|
||||
|
||||
self.rotary_emb = RotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim=self.rotary_dim,
|
||||
max_position_embeddings=4096,
|
||||
base=10000,
|
||||
is_neox_style=is_neox,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
|
||||
self.enable_rope_custom_op = self.rotary_emb.enabled()
|
||||
|
||||
self.attn = Attention(
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
scale=1.0 / head_size**0.5,
|
||||
num_kv_heads=num_kv_heads,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=prefix,
|
||||
attn_backend=attn_backend.get_class(),
|
||||
)
|
||||
self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend()
|
||||
if expected_query_quant_group_shape is not None:
|
||||
assert self.attn.query_quant is not None
|
||||
assert self.attn.query_quant.group_shape == expected_query_quant_group_shape
|
||||
assert not self.attn_backend.forward_includes_kv_cache_update, (
|
||||
f"Attention backend {self.attn_backend} does not support "
|
||||
"fuse_qk_norm_rope_kvcache."
|
||||
)
|
||||
kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
|
||||
self.kv_cache_dtype = (
|
||||
FP8_DTYPE if kv_cache_dtype_str.startswith("fp8") else self.dtype
|
||||
)
|
||||
|
||||
if self.kv_cache_dtype != self.dtype:
|
||||
# Non-unit scales so the fused and unfused writers must agree on them.
|
||||
self.attn._k_scale = torch.tensor(0.5, dtype=torch.float32, device=device)
|
||||
self.attn._v_scale = torch.tensor(0.25, dtype=torch.float32, device=device)
|
||||
self.attn._k_scale_cpu = torch.tensor(
|
||||
0.5, dtype=torch.float32, device="cpu"
|
||||
)
|
||||
self.attn._v_scale_cpu = torch.tensor(
|
||||
0.25, dtype=torch.float32, device="cpu"
|
||||
)
|
||||
self.attn._k_scale_float = 0.5
|
||||
self.attn._v_scale_float = 0.25
|
||||
else:
|
||||
self.attn._k_scale = self.attn._k_scale.to(device)
|
||||
self.attn._v_scale = self.attn._v_scale.to(device)
|
||||
|
||||
# Mirror the worker spec-collection loop: the backend publishes the
|
||||
# packing it needs.
|
||||
self.kv_cache_spec = self.attn.attn_backend.customize_spec(
|
||||
AttentionSpec(
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=head_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
)
|
||||
)
|
||||
self.builder = self.attn.attn_backend.get_builder_cls()(
|
||||
kv_cache_spec=self.kv_cache_spec,
|
||||
layer_names=[self.attn.layer_name],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def build_attn_metadata(
|
||||
self, batch_size: int, layout: KVCacheLayout
|
||||
) -> CommonAttentionMetadata:
|
||||
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, self.block_size, self.device, arange_block_indices=True
|
||||
)
|
||||
|
||||
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
|
||||
num_blocks = batch_size * max_blocks
|
||||
|
||||
raw_tensor = torch.zeros(
|
||||
num_blocks * self.kv_cache_spec.page_size_bytes,
|
||||
dtype=torch.int8,
|
||||
device=self.device,
|
||||
)
|
||||
kv_cache = dense_kv_cache_views(
|
||||
raw_tensor,
|
||||
self.kv_cache_spec,
|
||||
num_blocks,
|
||||
num_layers=1,
|
||||
layout=layout,
|
||||
)[0]
|
||||
|
||||
# Store as a bare tensor (not wrapped in a list) to match production
|
||||
# `bind_kv_cache` behavior. `get_attention_context` returns this
|
||||
# attribute directly to the fused/unfused `do_kv_cache_update` impls,
|
||||
# which call `kv_cache.unbind(0)` and therefore require a tensor.
|
||||
self.attn.kv_cache = kv_cache
|
||||
|
||||
attn_metadata = self.builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
|
||||
return attn_metadata
|
||||
|
||||
def forward(
|
||||
self, qkv: torch.Tensor, positions: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
qkv = qkv.clone()
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
# QK-norm: RMSNorm on per-head Q and K
|
||||
q = q.view(-1, self.num_heads, self.head_size)
|
||||
q = self.q_norm(q)
|
||||
q = q.view(-1, self.q_size)
|
||||
|
||||
k = k.view(-1, self.num_kv_heads, self.head_size)
|
||||
k = self.k_norm(k)
|
||||
k = k.view(-1, self.kv_size)
|
||||
|
||||
# RoPE
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
# Mirror Attention.forward: quant-query impls consume an fp8 q.
|
||||
if (
|
||||
self.kv_cache_dtype != self.dtype
|
||||
and self.attn.impl.supports_quant_query_input
|
||||
):
|
||||
assert self.attn.query_quant is not None
|
||||
q, _ = self.attn.query_quant(q, self.attn._q_scale)
|
||||
|
||||
# Final views + KV cache update
|
||||
q = q.view(-1, self.num_heads, self.head_size)
|
||||
k = k.view(-1, self.num_kv_heads, self.head_size)
|
||||
v = v.view(-1, self.num_kv_heads, self.head_size)
|
||||
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
|
||||
k, v, _encode_layer_name(self.layer_name)
|
||||
)
|
||||
return q, k, v, kv_cache_dummy_dep
|
||||
|
||||
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
|
||||
ops: list[torch._ops.OpOverload] = []
|
||||
# RoPE is not yet IR-migrated, so its custom op still surfaces
|
||||
# directly in the graph based on `enable_rope_custom_op`.
|
||||
if self.enable_rope_custom_op:
|
||||
if rocm_aiter_ops.is_triton_rotary_embed_enabled():
|
||||
ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default)
|
||||
else:
|
||||
ops.append(ROTARY_OP)
|
||||
else:
|
||||
ops.append(INDEX_SELECT_OP)
|
||||
ops.append(torch.ops.vllm.unified_kv_cache_update.default)
|
||||
return ops
|
||||
|
||||
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
|
||||
return [torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default]
|
||||
|
||||
|
||||
class QKNormMRoPEKVCacheTestModel(QKNormRoPEKVCacheTestModel):
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
mrope_section: tuple[int, int, int],
|
||||
mrope_interleaved: bool,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.rotary_emb = MRotaryEmbedding(
|
||||
self.head_size,
|
||||
rotary_dim=self.rotary_dim,
|
||||
max_position_embeddings=4096,
|
||||
base=10000,
|
||||
is_neox_style=self.is_neox,
|
||||
dtype=self.dtype,
|
||||
mrope_section=list(mrope_section),
|
||||
mrope_interleaved=mrope_interleaved,
|
||||
)
|
||||
self.enable_rope_custom_op = self.rotary_emb.enabled()
|
||||
|
||||
def forward(
|
||||
self, qkv: torch.Tensor, positions: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
q, k, v, kv_cache_dummy_dep = super().forward(qkv, positions)
|
||||
output = torch.empty(
|
||||
q.shape[0],
|
||||
self.num_heads * self.head_size,
|
||||
device=q.device,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
torch.ops.vllm.unified_attention_with_output(
|
||||
q,
|
||||
k,
|
||||
v,
|
||||
output,
|
||||
_encode_layer_name(self.layer_name),
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
# Production decoder attention consumes K from the cache rather than
|
||||
# its direct K argument. Returning `output` keeps that production user
|
||||
# in the graph without exposing K to an unsupported consumer.
|
||||
return q, output, v, kv_cache_dummy_dep
|
||||
|
||||
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
|
||||
return [MROPE_OP, torch.ops.vllm.unified_kv_cache_update.default]
|
||||
|
||||
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
|
||||
return [torch.ops.vllm.fused_qk_norm_mrope_and_unified_kv_cache_update.default]
|
||||
|
||||
|
||||
class QKNormMRoPEObservableKTestModel(QKNormMRoPEKVCacheTestModel):
|
||||
def forward(
|
||||
self, qkv: torch.Tensor, positions: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
return QKNormRoPEKVCacheTestModel.forward(self, qkv, positions)
|
||||
|
||||
|
||||
def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
*,
|
||||
attn_backend: AttentionBackendEnum,
|
||||
enable_aiter_triton_rope: bool,
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
block_size: int,
|
||||
is_neox: bool,
|
||||
use_shuffle_kv_layout: str,
|
||||
kv_layout: KVCacheLayout,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType,
|
||||
rms_norm_eps: float,
|
||||
custom_op: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mrope_section: tuple[int, int, int] | None = None,
|
||||
mrope_interleaved: bool = False,
|
||||
observe_mrope_k: bool = False,
|
||||
expect_fusion: bool = True,
|
||||
use_quark_scalar_query_scale: bool = False,
|
||||
) -> None:
|
||||
device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda")
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
# Set AITER toggles before VllmConfig initialization enables env caching.
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
monkeypatch.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
cache_config=CacheConfig(
|
||||
block_size=block_size,
|
||||
cache_dtype=kv_cache_dtype,
|
||||
),
|
||||
quant_config=(
|
||||
QuarkConfig(quant_config={"exclude": []})
|
||||
if use_quark_scalar_query_scale
|
||||
else None
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=[custom_op],
|
||||
pass_config=PassConfig(
|
||||
fuse_qk_norm_rope_kvcache=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
m.setenv("VLLM_ROCM_USE_AITER_LINEAR", "1")
|
||||
m.setenv(
|
||||
"VLLM_ROCM_USE_AITER_TRITON_ROPE",
|
||||
"1" if enable_aiter_triton_rope else "0",
|
||||
)
|
||||
m.setenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", use_shuffle_kv_layout)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
expected_query_quant_group_shape = None
|
||||
if use_quark_scalar_query_scale:
|
||||
expected_query_quant_group_shape = (
|
||||
GroupShape(-1, num_heads * head_size)
|
||||
if num_kv_heads == 1
|
||||
else GroupShape.PER_TENSOR
|
||||
)
|
||||
|
||||
model_kwargs = {
|
||||
"vllm_config": vllm_config,
|
||||
"attn_backend": attn_backend,
|
||||
"num_heads": num_heads,
|
||||
"num_kv_heads": num_kv_heads,
|
||||
"head_size": head_size,
|
||||
"rotary_dim": rotary_dim,
|
||||
"is_neox": is_neox,
|
||||
"rms_norm_eps": rms_norm_eps,
|
||||
"dtype": dtype,
|
||||
"device": torch.get_default_device(),
|
||||
"expected_query_quant_group_shape": expected_query_quant_group_shape,
|
||||
}
|
||||
if mrope_section is None:
|
||||
model = QKNormRoPEKVCacheTestModel(**model_kwargs)
|
||||
else:
|
||||
alternate_section = (8, 8, 8)
|
||||
m.setattr(
|
||||
fusion_module,
|
||||
"_discover_mrope_configs",
|
||||
lambda _: (
|
||||
(mrope_section, mrope_interleaved),
|
||||
(alternate_section, not mrope_interleaved),
|
||||
),
|
||||
)
|
||||
model_cls = (
|
||||
QKNormMRoPEObservableKTestModel
|
||||
if observe_mrope_k
|
||||
else QKNormMRoPEKVCacheTestModel
|
||||
)
|
||||
model = model_cls(
|
||||
**model_kwargs,
|
||||
mrope_section=mrope_section,
|
||||
mrope_interleaved=mrope_interleaved,
|
||||
)
|
||||
|
||||
fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config)
|
||||
passes = [
|
||||
NoOpEliminationPass(vllm_config),
|
||||
SplitCoalescingPass(vllm_config),
|
||||
ScatterSplitReplacementPass(vllm_config),
|
||||
fusion_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
]
|
||||
backend = TestBackend(*passes)
|
||||
|
||||
qkv = torch.randn(
|
||||
num_tokens,
|
||||
num_heads * head_size + 2 * num_kv_heads * head_size,
|
||||
dtype=dtype,
|
||||
)
|
||||
if mrope_section is None:
|
||||
pos = torch.arange(num_tokens, dtype=torch.long)
|
||||
else:
|
||||
# Real Qwen-VL position IDs can be a strided [3, N] view.
|
||||
pos = torch.randint(0, 4096, (3, 2 * num_tokens), dtype=torch.long)[:, ::2]
|
||||
assert not pos.is_contiguous()
|
||||
|
||||
qkv_unfused = qkv.clone()
|
||||
pos_unfused = pos.clone()
|
||||
|
||||
# Run unfused (eager) forward
|
||||
with set_forward_context(None, vllm_config):
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model.build_attn_metadata(num_tokens, kv_layout)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused)
|
||||
attn_layer = forward_context.no_compile_layers[model.layer_name]
|
||||
kv_cache_unfused = attn_layer.kv_cache
|
||||
del dummy
|
||||
|
||||
# Run fused (compiled) forward
|
||||
torch._dynamo.mark_dynamic(qkv, 0)
|
||||
torch._dynamo.mark_dynamic(pos, 0 if mrope_section is None else 1)
|
||||
with set_forward_context(None, vllm_config):
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model_fused.build_attn_metadata(num_tokens, kv_layout)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos)
|
||||
attn_layer = forward_context.no_compile_layers[model.layer_name]
|
||||
kv_cache_fused = attn_layer.kv_cache
|
||||
del dummy
|
||||
|
||||
assert fusion_pass.matched_count == int(expect_fusion)
|
||||
|
||||
if expect_fusion:
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
else:
|
||||
for op in model.ops_in_model_before():
|
||||
assert backend.op_count(op, before=True) > 0
|
||||
assert backend.op_count(op) > 0
|
||||
|
||||
if use_quark_scalar_query_scale:
|
||||
expected_quant_op = (
|
||||
torch.ops.vllm.rocm_aiter_per_tensor_quant.default
|
||||
if num_kv_heads != 1
|
||||
else torch.ops._C.static_scaled_fp8_quant.default
|
||||
)
|
||||
assert backend.op_count(expected_quant_op, before=True) > 0
|
||||
# Query quantization remains separate after the prologue fusion.
|
||||
assert backend.op_count(expected_quant_op) > 0
|
||||
|
||||
# Sweep-backed (18.2k pts, PR #42749): native-rope ref worst 7.7e-3 -> 1e-2;
|
||||
# AITER-triton-rope ref is itself approximate (plateau 1.28e-2) -> 2e-2.
|
||||
ATOL, RTOL = (
|
||||
(2e-2, 2e-2)
|
||||
if enable_aiter_triton_rope or mrope_section is not None
|
||||
else (1e-2, 1e-2)
|
||||
)
|
||||
is_fp8_cache = model.kv_cache_dtype != dtype
|
||||
|
||||
if q_fused.dtype == FP8_DTYPE:
|
||||
# Quant-query path: both q are fp8; compare dequant within 1 fp8 ULP.
|
||||
torch.testing.assert_close(
|
||||
q_unfused.float(), q_fused.float(), atol=1.25e-1, rtol=1.25e-1
|
||||
)
|
||||
else:
|
||||
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
|
||||
|
||||
# Fused and unfused arithmetic can straddle an FP8 rounding boundary.
|
||||
# Allow one E4M3 quantization step while still comparing every block.
|
||||
if is_fp8_cache:
|
||||
cache_atol = cache_rtol = 1.25e-1
|
||||
else:
|
||||
cache_atol, cache_rtol = ATOL, RTOL
|
||||
|
||||
# The fused k_out is the dequantized cache write, so it carries the same
|
||||
# rounding; FA and ROCM_ATTN prefill consume it directly. Unified reads K
|
||||
# from the cache and asks the kernel not to write k_out at all.
|
||||
if attn_backend != AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN:
|
||||
torch.testing.assert_close(
|
||||
k_unfused, k_fused, atol=cache_atol, rtol=cache_rtol
|
||||
)
|
||||
|
||||
# Should be bit exact since no processing had been done on v for both paths
|
||||
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
|
||||
|
||||
# Whole cache, so the interleaved V write (ROCM_ATTN kv_cache[1]) is checked.
|
||||
torch.testing.assert_close(
|
||||
kv_cache_unfused.float(),
|
||||
kv_cache_fused.float(),
|
||||
atol=cache_atol,
|
||||
rtol=cache_rtol,
|
||||
)
|
||||
|
||||
|
||||
_FUSION_CONFIGS = [
|
||||
# Full rotary, both neox styles (the original coverage).
|
||||
pytest.param(64, 8, 64, 64, True, id="full-neox"),
|
||||
pytest.param(64, 8, 64, 64, False, id="full-non_neox"),
|
||||
# GLM-4.5/4.6/4.7 (glm4_moe.py:275 partial_rotary_factor=0.5, neox-style)
|
||||
pytest.param(32, 8, 128, 64, True, id="glm4_moe"),
|
||||
# GLM-4 dense (glm4.py:97,124 partial_rotary_factor=0.5, non-neox)
|
||||
pytest.param(32, 2, 128, 64, False, id="glm4_dense"),
|
||||
# Moondream3-style small head (head_size=64, rotary_dim=32)
|
||||
pytest.param(16, 2, 64, 32, True, id="partial_small_head"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_heads, num_kv_heads, head_size, rotary_dim, is_neox",
|
||||
_FUSION_CONFIGS,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend, use_shuffle_kv_layout, kv_layout",
|
||||
[
|
||||
# ROCM_ATTN always writes the shuffled, interleaved-V layout and needs each
|
||||
# K/V side contiguous per block, so head-major only.
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_ATTN,
|
||||
"0",
|
||||
KVCacheLayout.LBHNC,
|
||||
id="rocm_attn",
|
||||
),
|
||||
# Unified never reads the shuffle env; its strided views take any layout.
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
"0",
|
||||
KVCacheLayout.LBHNC,
|
||||
id="unified-head_major",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
"0",
|
||||
KVCacheLayout.LBNHC,
|
||||
id="unified-token_major",
|
||||
),
|
||||
# FA takes any layout unshuffled; its shuffle writer needs each K/V side
|
||||
# contiguous per block, so head-major only.
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
"0",
|
||||
KVCacheLayout.LBHNC,
|
||||
id="fa-head_major",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
"0",
|
||||
KVCacheLayout.LBNHC,
|
||||
id="fa-token_major",
|
||||
),
|
||||
pytest.param(
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
"1",
|
||||
KVCacheLayout.LBHNC,
|
||||
id="fa-shuffle",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [5, 2048])
|
||||
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
|
||||
@pytest.mark.parametrize("block_size", [16, 32, 64])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("rms_norm_eps", [1e-6])
|
||||
@pytest.mark.parametrize("custom_op", ["+rotary_embedding", "+rms_norm"])
|
||||
# The fused_qk_norm_rope_cache kernel used by this test aborts on AITER < 0.1.20
|
||||
# (fused_qk_norm_rope_cache_quant.cu: "k_cache/v_cache must be contiguous within
|
||||
# a block")
|
||||
@pytest.mark.skipif(
|
||||
not Version(version("amd_aiter")) >= Version("0.1.20"),
|
||||
reason=(
|
||||
"Requires AITER >= 0.1.20; older kernels abort on "
|
||||
"fused_qk_norm_rope_cache (k_cache/v_cache contiguity)"
|
||||
),
|
||||
)
|
||||
def test_qk_norm_rope_kvcache_fusion(
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
is_neox: bool,
|
||||
attn_backend: AttentionBackendEnum,
|
||||
enable_aiter_triton_rope: bool,
|
||||
kv_layout: KVCacheLayout,
|
||||
block_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType,
|
||||
rms_norm_eps: float,
|
||||
custom_op: str,
|
||||
use_shuffle_kv_layout: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
attn_backend=attn_backend,
|
||||
enable_aiter_triton_rope=enable_aiter_triton_rope,
|
||||
num_tokens=num_tokens,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
rotary_dim=rotary_dim,
|
||||
block_size=block_size,
|
||||
is_neox=is_neox,
|
||||
use_shuffle_kv_layout=use_shuffle_kv_layout,
|
||||
kv_layout=kv_layout,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
custom_op=custom_op,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mrope_section",
|
||||
[
|
||||
pytest.param((16, 24, 24), id="full_rotary"),
|
||||
pytest.param((4, 6, 6), id="partial_rotary"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("mrope_interleaved", [False, True])
|
||||
@pytest.mark.parametrize("is_neox", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"kv_layout",
|
||||
[
|
||||
pytest.param(KVCacheLayout.LBHNC, id="head_major"),
|
||||
pytest.param(KVCacheLayout.LBNHC, id="token_major"),
|
||||
pytest.param(KVCacheLayout.LHBNC, id="block_major"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.skipif(
|
||||
not IS_AITER_FOUND,
|
||||
reason="Requires AITER",
|
||||
)
|
||||
def test_qk_norm_mrope_kvcache_fusion(
|
||||
mrope_section: tuple[int, int, int],
|
||||
mrope_interleaved: bool,
|
||||
is_neox: bool,
|
||||
kv_layout: KVCacheLayout,
|
||||
kv_cache_dtype: CacheDType,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
enable_aiter_triton_rope=False,
|
||||
num_tokens=5,
|
||||
num_heads=8,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
rotary_dim=2 * sum(mrope_section),
|
||||
block_size=16,
|
||||
is_neox=is_neox,
|
||||
use_shuffle_kv_layout="0",
|
||||
kv_layout=kv_layout,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
rms_norm_eps=1e-6,
|
||||
custom_op="+rotary_embedding",
|
||||
monkeypatch=monkeypatch,
|
||||
mrope_section=mrope_section,
|
||||
mrope_interleaved=mrope_interleaved,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("num_heads", "num_kv_heads"),
|
||||
[(32, 2), (16, 1), (8, 1)],
|
||||
ids=["tp2", "tp4", "tp8"],
|
||||
)
|
||||
def test_qk_norm_mrope_kvcache_fusion_quark_scalar_query_scale(
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
enable_aiter_triton_rope=False,
|
||||
num_tokens=5,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=128,
|
||||
rotary_dim=128,
|
||||
block_size=16,
|
||||
is_neox=True,
|
||||
use_shuffle_kv_layout="0",
|
||||
kv_layout=KVCacheLayout.LBHNC,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8",
|
||||
rms_norm_eps=1e-6,
|
||||
custom_op="+rotary_embedding",
|
||||
monkeypatch=monkeypatch,
|
||||
mrope_section=(24, 20, 20),
|
||||
mrope_interleaved=True,
|
||||
use_quark_scalar_query_scale=True,
|
||||
)
|
||||
|
||||
|
||||
def test_mrope_fusion_rejects_observable_k(monkeypatch: pytest.MonkeyPatch):
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
enable_aiter_triton_rope=False,
|
||||
num_tokens=5,
|
||||
num_heads=8,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
rotary_dim=128,
|
||||
block_size=16,
|
||||
is_neox=True,
|
||||
use_shuffle_kv_layout="0",
|
||||
kv_layout=KVCacheLayout.LBHNC,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
rms_norm_eps=1e-6,
|
||||
custom_op="+rotary_embedding",
|
||||
monkeypatch=monkeypatch,
|
||||
mrope_section=(16, 24, 24),
|
||||
observe_mrope_k=True,
|
||||
expect_fusion=False,
|
||||
)
|
||||
|
||||
|
||||
def test_mrope_fusion_rejects_unaligned_rotary_dim(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
attn_backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
enable_aiter_triton_rope=False,
|
||||
num_tokens=5,
|
||||
num_heads=8,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
rotary_dim=6,
|
||||
block_size=16,
|
||||
is_neox=True,
|
||||
use_shuffle_kv_layout="0",
|
||||
kv_layout=KVCacheLayout.LBHNC,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="auto",
|
||||
rms_norm_eps=1e-6,
|
||||
custom_op="+rotary_embedding",
|
||||
monkeypatch=monkeypatch,
|
||||
mrope_section=(1, 1, 1),
|
||||
expect_fusion=False,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _USE_LAYERNAME, reason="requires opaque LayerName")
|
||||
def test_opaque_layer_name_patterns_register_once(monkeypatch: pytest.MonkeyPatch):
|
||||
device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda")
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=torch.bfloat16),
|
||||
cache_config=CacheConfig(block_size=16, cache_dtype="fp8_e4m3"),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=["+rotary_embedding"],
|
||||
pass_config=PassConfig(fuse_qk_norm_rope_kvcache=True),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
model_kwargs = {
|
||||
"vllm_config": vllm_config,
|
||||
"attn_backend": AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
"num_heads": 8,
|
||||
"num_kv_heads": 2,
|
||||
"head_size": 128,
|
||||
"is_neox": True,
|
||||
"rms_norm_eps": 1e-6,
|
||||
"dtype": torch.bfloat16,
|
||||
"device": torch.device(device),
|
||||
}
|
||||
models = [
|
||||
QKNormRoPEKVCacheTestModel(
|
||||
**model_kwargs,
|
||||
prefix=f"model.layers.{index}.self_attn.attn",
|
||||
)
|
||||
for index in range(2)
|
||||
]
|
||||
|
||||
# Opaque layer names are wildcards in the pattern graph. Registering
|
||||
# once per physical layer raises "Duplicate pattern" on torch >= 2.11.
|
||||
fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config)
|
||||
|
||||
assert models
|
||||
assert fusion_pass.patterns
|
||||
|
||||
|
||||
def _call_fused_mrope_impl(q_out: torch.Tensor) -> None:
|
||||
device = q_out.device
|
||||
fusion_module.fused_qk_norm_mrope_and_unified_kv_cache_update_impl(
|
||||
q_out=q_out,
|
||||
qkv=torch.zeros(2, 16, device=device),
|
||||
positions=torch.arange(2, device=device),
|
||||
q_weight=torch.ones(4, device=device),
|
||||
k_weight=torch.ones(4, device=device),
|
||||
rms_norm_eps=1e-6,
|
||||
cos_sin_cache=torch.zeros(16, 4, device=device),
|
||||
is_neox=True,
|
||||
mrope_section_t=1,
|
||||
mrope_section_h=1,
|
||||
mrope_section_w=0,
|
||||
is_interleaved=False,
|
||||
rotary_dim=4,
|
||||
layer_name="model.layers.0.self_attn.attn",
|
||||
)
|
||||
|
||||
|
||||
def test_mrope_profiling_pass_initializes_query(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(
|
||||
fusion_module,
|
||||
"get_attention_context",
|
||||
lambda _: (None, None, None, None),
|
||||
)
|
||||
q_out = torch.full((2, 8), torch.nan, device="cpu")
|
||||
|
||||
_call_fused_mrope_impl(q_out)
|
||||
|
||||
torch.testing.assert_close(q_out, torch.zeros_like(q_out))
|
||||
|
||||
|
||||
def test_mrope_attention_context_failure_propagates(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
def fail_context(_):
|
||||
raise RuntimeError("missing attention context")
|
||||
|
||||
monkeypatch.setattr(fusion_module, "get_attention_context", fail_context)
|
||||
|
||||
with pytest.raises(RuntimeError, match="missing attention context"):
|
||||
_call_fused_mrope_impl(torch.empty(2, 8, device="cpu"))
|
||||
@@ -1,635 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch._higher_order_ops import auto_functionalized
|
||||
|
||||
import vllm.config
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.v1.attention.utils import (
|
||||
BatchSpec,
|
||||
create_common_attn_metadata,
|
||||
dense_kv_cache_views,
|
||||
)
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.compilation.passes.fusion import rope_kvcache_fusion
|
||||
from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP
|
||||
from vllm.compilation.passes.fusion.rope_kvcache_fusion import RopeKVCacheFusionPass
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.compilation.passes.utility.scatter_split_replace import (
|
||||
ScatterSplitReplacementPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.config.utils import Range
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.attention import attention as attention_module
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import _encode_layer_name
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheLayout,
|
||||
)
|
||||
|
||||
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
|
||||
VLLM_UNIFIED_KV_CACHE_UPDATE_OP = torch.ops.vllm.unified_kv_cache_update
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
|
||||
def test_kv_cache_update_ops_fake_tensor_metadata(monkeypatch: pytest.MonkeyPatch):
|
||||
query = torch.empty(1, dtype=torch.bfloat16)
|
||||
kv_cache = torch.empty(1, dtype=torch.uint8)
|
||||
context = (None, None, kv_cache, None)
|
||||
monkeypatch.setattr(attention_module, "get_attention_context", lambda _: context)
|
||||
monkeypatch.setattr(rope_kvcache_fusion, "get_attention_context", lambda _: context)
|
||||
|
||||
runtime_output = attention_module.unified_kv_cache_update(query, query, "layer")
|
||||
fake_output = attention_module.unified_kv_cache_update_fake(query, query, "layer")
|
||||
assert runtime_output.shape == fake_output.shape
|
||||
assert runtime_output.dtype == fake_output.dtype
|
||||
assert runtime_output.device == fake_output.device
|
||||
|
||||
args = (
|
||||
query,
|
||||
query,
|
||||
query,
|
||||
torch.empty(1, dtype=torch.int64),
|
||||
torch.empty(1),
|
||||
True,
|
||||
"layer",
|
||||
)
|
||||
runtime_output = rope_kvcache_fusion.fused_rope_and_unified_kv_cache_update_impl(
|
||||
*args
|
||||
)
|
||||
fake_output = rope_kvcache_fusion.fused_rope_and_unified_kv_cache_update_fake(*args)
|
||||
assert runtime_output.shape == fake_output.shape
|
||||
assert runtime_output.dtype == fake_output.dtype
|
||||
assert runtime_output.device == fake_output.device
|
||||
|
||||
|
||||
def test_rope_kvcache_fusion_default_keeps_large_ranges_unfused():
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
pass_config=PassConfig(fuse_rope_kvcache=True),
|
||||
),
|
||||
)
|
||||
fusion_pass = RopeKVCacheFusionPass(vllm_config)
|
||||
|
||||
assert fusion_pass.is_applicable_for_range(Range(1, 256))
|
||||
assert not fusion_pass.is_applicable_for_range(Range(257, 11650))
|
||||
assert not fusion_pass.is_applicable_for_range(Range(11651, 16384))
|
||||
|
||||
|
||||
class QKRoPEKVCacheTestModel(torch.nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
attn_backend: AttentionBackendEnum,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
is_neox: bool,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
prefix: str = "model.layers.0.self_attn.attn",
|
||||
):
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.head_size = head_size
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.q_size = num_heads * head_size
|
||||
self.kv_size = num_kv_heads * head_size
|
||||
self.is_neox = is_neox
|
||||
self.dtype = dtype
|
||||
self.device = device
|
||||
self.layer_name = prefix
|
||||
|
||||
self.rotary_emb = RotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim=head_size,
|
||||
max_position_embeddings=4096,
|
||||
base=10000,
|
||||
is_neox_style=is_neox,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
|
||||
# Whether to check for the RoPE custom op or component index_select
|
||||
self.enable_rope_custom_op = self.rotary_emb.enabled()
|
||||
|
||||
# Register layer metadata for the fusion pass via Attention.
|
||||
self.attn = Attention(
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
scale=1.0 / head_size**0.5,
|
||||
num_kv_heads=num_kv_heads,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=prefix,
|
||||
attn_backend=attn_backend.get_class(),
|
||||
)
|
||||
self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend()
|
||||
assert not self.attn_backend.forward_includes_kv_cache_update, (
|
||||
f"Attention backend {self.attn_backend} does not support fuse_rope_kvcache."
|
||||
)
|
||||
self.attn._k_scale = self.attn._k_scale.to(device)
|
||||
self.attn._v_scale = self.attn._v_scale.to(device)
|
||||
|
||||
kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
|
||||
self.kv_cache_dtype = (
|
||||
FP8_DTYPE if kv_cache_dtype_str.startswith("fp8") else self.dtype
|
||||
)
|
||||
|
||||
# Mirror the worker spec-collection loop: the backend publishes its
|
||||
# packing (e.g. ROCM_ATTN's separate K/V head groups).
|
||||
self.kv_cache_spec = self.attn_backend.customize_spec(
|
||||
FullAttentionSpec(
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=self.head_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
)
|
||||
)
|
||||
|
||||
# Initialize attn MetadataBuilder
|
||||
self.builder = self.attn_backend.get_builder_cls()(
|
||||
kv_cache_spec=self.kv_cache_spec,
|
||||
layer_names=[self.attn.layer_name],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def build_attn_metadata(self, batch_size: int) -> CommonAttentionMetadata:
|
||||
"""Initialize attention metadata."""
|
||||
# Create common attn metadata
|
||||
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, self.block_size, self.device, arange_block_indices=True
|
||||
)
|
||||
|
||||
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
|
||||
num_blocks = batch_size * max_blocks
|
||||
|
||||
# A backend's preferred supported layout wins over the default, mirroring
|
||||
# selector-time resolution (e.g. ROCM_ATTN's head groups force LHBNC).
|
||||
layout = KVCacheLayout.LBNHC
|
||||
supported = self.attn_backend.supported_kv_cache_layouts()
|
||||
if supported:
|
||||
layout = supported[0]
|
||||
|
||||
raw_tensor = torch.zeros(
|
||||
num_blocks * self.kv_cache_spec.page_size_bytes,
|
||||
dtype=torch.int8,
|
||||
device=self.device,
|
||||
)
|
||||
self.attn.kv_cache = dense_kv_cache_views(
|
||||
raw_tensor,
|
||||
self.kv_cache_spec,
|
||||
num_blocks,
|
||||
num_layers=1,
|
||||
layout=layout,
|
||||
)[0]
|
||||
|
||||
# Build attn metadata
|
||||
attn_metadata = self.builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
|
||||
return attn_metadata
|
||||
|
||||
def forward(
|
||||
self, qkv: torch.Tensor, positions: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Create copy so inplace ops do not modify the original tensors
|
||||
qkv = qkv.clone()
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
# Instead of a full forward pass, match only the KV cache update op here
|
||||
q = q.view(-1, self.num_heads, self.head_size)
|
||||
k = k.view(-1, self.num_kv_heads, self.head_size)
|
||||
v = v.view(-1, self.num_kv_heads, self.head_size)
|
||||
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
|
||||
k, v, _encode_layer_name(self.layer_name)
|
||||
)
|
||||
return q, k, v, kv_cache_dummy_dep
|
||||
|
||||
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
|
||||
ops = []
|
||||
if self.enable_rope_custom_op:
|
||||
if rocm_aiter_ops.is_triton_rotary_embed_enabled():
|
||||
ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default)
|
||||
else:
|
||||
ops.append(ROTARY_OP)
|
||||
else:
|
||||
ops.append(INDEX_SELECT_OP)
|
||||
ops.append(torch.ops.vllm.unified_kv_cache_update.default)
|
||||
return ops
|
||||
|
||||
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
|
||||
return [torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default]
|
||||
|
||||
|
||||
class QKRoPEStaticQKVCacheTestModel(QKRoPEKVCacheTestModel):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.q_scale = torch.ones((), dtype=torch.float32, device=self.device)
|
||||
|
||||
def forward(
|
||||
self, qkv: torch.Tensor, positions: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
# Create copy so inplace ops do not modify the original tensors
|
||||
qkv = qkv.clone()
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
q_fp8 = torch.empty(q.shape, device=q.device, dtype=FP8_DTYPE)
|
||||
_, q_fp8 = auto_functionalized(
|
||||
torch.ops._C.static_scaled_fp8_quant.default,
|
||||
result=q_fp8,
|
||||
input=q,
|
||||
scale=self.q_scale,
|
||||
group_shape=(-1, -1),
|
||||
)
|
||||
q = q_fp8.view(-1, self.num_heads, self.head_size)
|
||||
k = k.view(-1, self.num_kv_heads, self.head_size)
|
||||
v = v.view(-1, self.num_kv_heads, self.head_size)
|
||||
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
|
||||
k, v, _encode_layer_name(self.layer_name)
|
||||
)
|
||||
return q, k, v, kv_cache_dummy_dep
|
||||
|
||||
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
|
||||
ops = []
|
||||
if self.enable_rope_custom_op:
|
||||
if rocm_aiter_ops.is_triton_rotary_embed_enabled():
|
||||
ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default)
|
||||
else:
|
||||
ops.append(ROTARY_OP)
|
||||
else:
|
||||
ops.append(INDEX_SELECT_OP)
|
||||
ops.append(torch.ops.vllm.unified_kv_cache_update.default)
|
||||
return ops
|
||||
|
||||
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
|
||||
return [torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
[
|
||||
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
AttentionBackendEnum.TRITON_ATTN,
|
||||
AttentionBackendEnum.ROCM_ATTN,
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("enable_rope_custom_op", [True]) # [True, False])
|
||||
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
|
||||
@pytest.mark.parametrize("num_heads", [64])
|
||||
@pytest.mark.parametrize("num_kv_heads", [8])
|
||||
@pytest.mark.parametrize("head_size", [64])
|
||||
@pytest.mark.parametrize("block_size", [16])
|
||||
@pytest.mark.parametrize("is_neox", [True, False])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="Only test on ROCm with AITER installed and supported",
|
||||
)
|
||||
def test_rope_kvcache_fusion(
|
||||
attn_backend: AttentionBackendEnum,
|
||||
enable_rope_custom_op: bool,
|
||||
enable_aiter_triton_rope: bool,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
is_neox: bool,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
custom_ops: list[str] = []
|
||||
if enable_rope_custom_op:
|
||||
custom_ops.append("+rotary_embedding")
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
cache_config=CacheConfig(
|
||||
block_size=block_size,
|
||||
cache_dtype=kv_cache_dtype,
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=custom_ops,
|
||||
pass_config=PassConfig(
|
||||
fuse_rope_kvcache=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
m.setenv(
|
||||
"VLLM_ROCM_USE_AITER_TRITON_ROPE", "1" if enable_aiter_triton_rope else "0"
|
||||
)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
model = QKRoPEKVCacheTestModel(
|
||||
vllm_config=vllm_config,
|
||||
attn_backend=attn_backend,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
is_neox=is_neox,
|
||||
dtype=dtype,
|
||||
device=torch.get_default_device(),
|
||||
)
|
||||
|
||||
fusion_pass = RopeKVCacheFusionPass(vllm_config)
|
||||
passes = [
|
||||
NoOpEliminationPass(vllm_config),
|
||||
SplitCoalescingPass(vllm_config),
|
||||
ScatterSplitReplacementPass(vllm_config),
|
||||
fusion_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
]
|
||||
backend = TestBackend(*passes)
|
||||
|
||||
T = 5
|
||||
|
||||
qkv = torch.randn(
|
||||
T, num_heads * head_size + 2 * num_kv_heads * head_size, dtype=dtype
|
||||
)
|
||||
pos = torch.arange(T, dtype=torch.long)
|
||||
|
||||
qkv_unfused = qkv.clone()
|
||||
pos_unfused = pos.clone()
|
||||
|
||||
with set_forward_context(None, vllm_config):
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model.build_attn_metadata(T)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused)
|
||||
attn_layer = forward_context.no_compile_layers[model.layer_name]
|
||||
kv_cache_unfused = attn_layer.kv_cache
|
||||
del dummy
|
||||
|
||||
torch._dynamo.mark_dynamic(qkv, 0)
|
||||
torch._dynamo.mark_dynamic(pos, 0)
|
||||
with set_forward_context(None, vllm_config):
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model_fused.build_attn_metadata(T)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos)
|
||||
attn_layer = forward_context.no_compile_layers[model.layer_name]
|
||||
kv_cache_fused = attn_layer.kv_cache
|
||||
del dummy
|
||||
|
||||
assert fusion_pass.matched_count == 1
|
||||
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
|
||||
if dtype == torch.float16:
|
||||
ATOL, RTOL = (2e-3, 2e-3)
|
||||
else:
|
||||
ATOL, RTOL = (1e-2, 1e-2)
|
||||
|
||||
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
|
||||
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
|
||||
torch.testing.assert_close(v_unfused, v_fused, atol=ATOL, rtol=RTOL)
|
||||
# Cannot compare fp8_* directly here, cast to model dtype instead
|
||||
# TODO(charlifu): switch back to ATOL, RTOL after aiter fix is merged.
|
||||
torch.testing.assert_close(
|
||||
kv_cache_unfused.to(dtype),
|
||||
kv_cache_fused.to(dtype),
|
||||
atol=1e-1,
|
||||
rtol=1e-1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
[AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN],
|
||||
)
|
||||
@pytest.mark.parametrize("enable_rope_custom_op", [True])
|
||||
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
|
||||
@pytest.mark.parametrize("num_heads", [64])
|
||||
@pytest.mark.parametrize("num_kv_heads", [8])
|
||||
@pytest.mark.parametrize("head_size", [64])
|
||||
@pytest.mark.parametrize("block_size", [16])
|
||||
@pytest.mark.parametrize("is_neox", [True, False])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="Only test on ROCm with AITER installed and supported",
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not hasattr(torch.ops._C, "static_scaled_fp8_quant"),
|
||||
reason="static fp8 quant op not available on this build",
|
||||
)
|
||||
def test_rope_static_qquant_kvcache_fusion(
|
||||
attn_backend: AttentionBackendEnum,
|
||||
enable_rope_custom_op: bool,
|
||||
enable_aiter_triton_rope: bool,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
is_neox: bool,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: CacheDType,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
custom_ops: list[str] = []
|
||||
if enable_rope_custom_op:
|
||||
custom_ops.append("+rotary_embedding")
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
cache_config=CacheConfig(
|
||||
block_size=block_size,
|
||||
cache_dtype=kv_cache_dtype,
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=custom_ops,
|
||||
pass_config=PassConfig(
|
||||
fuse_rope_kvcache=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
m.setenv(
|
||||
"VLLM_ROCM_USE_AITER_TRITON_ROPE", "1" if enable_aiter_triton_rope else "0"
|
||||
)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
model = QKRoPEStaticQKVCacheTestModel(
|
||||
vllm_config=vllm_config,
|
||||
attn_backend=attn_backend,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
is_neox=is_neox,
|
||||
dtype=dtype,
|
||||
device=torch.get_default_device(),
|
||||
)
|
||||
|
||||
fusion_pass = RopeKVCacheFusionPass(vllm_config)
|
||||
passes = [
|
||||
NoOpEliminationPass(vllm_config),
|
||||
SplitCoalescingPass(vllm_config),
|
||||
ScatterSplitReplacementPass(vllm_config),
|
||||
fusion_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
]
|
||||
backend = TestBackend(*passes)
|
||||
|
||||
T = 5
|
||||
qkv = torch.randn(
|
||||
T, num_heads * head_size + 2 * num_kv_heads * head_size, dtype=dtype
|
||||
)
|
||||
pos = torch.arange(T, dtype=torch.long)
|
||||
|
||||
qkv_unfused = qkv.clone()
|
||||
pos_unfused = pos.clone()
|
||||
|
||||
with set_forward_context(None, vllm_config):
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model.build_attn_metadata(T)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused)
|
||||
attn_layer = forward_context.no_compile_layers[model.layer_name]
|
||||
kv_cache_unfused = attn_layer.kv_cache
|
||||
del dummy
|
||||
|
||||
torch._dynamo.mark_dynamic(qkv, 0)
|
||||
torch._dynamo.mark_dynamic(pos, 0)
|
||||
with set_forward_context(None, vllm_config):
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model_fused.build_attn_metadata(T)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos)
|
||||
attn_layer = forward_context.no_compile_layers[model.layer_name]
|
||||
kv_cache_fused = attn_layer.kv_cache
|
||||
del dummy
|
||||
|
||||
assert fusion_pass.matched_count == 1
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
static_quant_pre = backend.op_count(
|
||||
torch.ops._C.static_scaled_fp8_quant.default, before=True
|
||||
)
|
||||
static_quant_post = backend.op_count(
|
||||
torch.ops._C.static_scaled_fp8_quant.default
|
||||
)
|
||||
assert static_quant_pre > 0
|
||||
# The replacement still emits static quant, so count is expected to
|
||||
# remain non-zero after fusion.
|
||||
assert static_quant_post > 0
|
||||
|
||||
# Negative control: without the static-Q pattern, the generic RoPE+KV
|
||||
# pattern cannot match this rope -> static-quant -> kv graph, so the
|
||||
# fusion above is attributable solely to RopeStaticQQuantKVCachePattern.
|
||||
# This is a structural property independent of the rope/dtype/neox axes,
|
||||
# so run the (extra compile) check only once on a representative combo.
|
||||
if is_neox and enable_aiter_triton_rope and kv_cache_dtype == "auto":
|
||||
m.setattr(
|
||||
rope_kvcache_fusion,
|
||||
"_supports_static_q_fp8_quant_fusion",
|
||||
lambda: False,
|
||||
)
|
||||
generic_pass = RopeKVCacheFusionPass(vllm_config)
|
||||
generic_backend = TestBackend(
|
||||
NoOpEliminationPass(vllm_config),
|
||||
SplitCoalescingPass(vllm_config),
|
||||
ScatterSplitReplacementPass(vllm_config),
|
||||
generic_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
)
|
||||
# Reset dynamo so the model is recompiled through generic_backend
|
||||
# instead of reusing the cached compilation from above.
|
||||
torch._dynamo.reset()
|
||||
with set_forward_context(None, vllm_config):
|
||||
model_generic = torch.compile(model, backend=generic_backend)
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model_generic.build_attn_metadata(T)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
model_generic(qkv, pos)
|
||||
# op_count reads the post-pass graph, so it also confirms the pass ran
|
||||
# (a no-op pass would raise instead of silently passing on count 0).
|
||||
assert generic_pass.matched_count == 0
|
||||
assert (
|
||||
generic_backend.op_count(
|
||||
torch.ops.vllm.fused_rope_and_unified_kv_cache_update.default
|
||||
)
|
||||
== 0
|
||||
)
|
||||
|
||||
if dtype == torch.float16:
|
||||
ATOL, RTOL = (2e-3, 2e-3)
|
||||
else:
|
||||
ATOL, RTOL = (1e-2, 1e-2)
|
||||
|
||||
# TODO(charlifu): switch back to ATOL, RTOL after aiter fix is merged.
|
||||
torch.testing.assert_close(
|
||||
q_unfused.to(torch.float32),
|
||||
q_fused.to(torch.float32),
|
||||
atol=1e-1,
|
||||
rtol=1e-1,
|
||||
)
|
||||
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
|
||||
torch.testing.assert_close(v_unfused, v_fused, atol=ATOL, rtol=RTOL)
|
||||
# TODO(charlifu): switch back to ATOL, RTOL after aiter fix is merged.
|
||||
torch.testing.assert_close(
|
||||
kv_cache_unfused.to(dtype),
|
||||
kv_cache_fused.to(dtype),
|
||||
atol=1e-1,
|
||||
rtol=1e-1,
|
||||
)
|
||||
@@ -1,110 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm
|
||||
from tests.compile.backend import TestBackend
|
||||
from vllm.compilation.passes.utility.scatter_split_replace import (
|
||||
ScatterSplitReplacementPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
|
||||
from vllm.config import CompilationConfig, CompilationMode, VllmConfig
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
class ScatterSplitReplacementModel(nn.Module):
|
||||
"""Model with a rope+getitem+slice_scatter+split_with_sizes sequence."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
super().__init__()
|
||||
self.q_size = num_heads * head_size
|
||||
self.kv_size = num_kv_heads * head_size
|
||||
|
||||
self.rotary_emb = RotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim=head_size,
|
||||
max_position_embeddings=4096,
|
||||
base=10000,
|
||||
is_neox_style=True,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
def forward(self, qkv: torch.Tensor, positions: torch.Tensor):
|
||||
# Create copy so inplace ops do not modify the original tensors
|
||||
qkv = qkv.clone()
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
q = q + 1
|
||||
k = k + 2
|
||||
v = v + 3
|
||||
return q, k, v
|
||||
|
||||
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
|
||||
return [
|
||||
torch.ops.aten.slice_scatter.default,
|
||||
torch.ops.aten.split_with_sizes.default,
|
||||
torch.ops.aten.getitem.default,
|
||||
]
|
||||
|
||||
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
|
||||
return [torch.ops.aten.getitem.default]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_scatter_split_replace(dtype):
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
num_heads = 8
|
||||
num_kv_heads = 4
|
||||
head_size = 64
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=["+rotary_embedding"],
|
||||
),
|
||||
)
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
# ScatterSplitReplacementPass requires SplitCoalescingPass to be run before it
|
||||
coalesce_pass = SplitCoalescingPass(vllm_config)
|
||||
replace_pass = ScatterSplitReplacementPass(vllm_config)
|
||||
passes = [coalesce_pass, replace_pass]
|
||||
backend = TestBackend(*passes)
|
||||
|
||||
model = ScatterSplitReplacementModel(num_heads, num_kv_heads, head_size, dtype)
|
||||
|
||||
T = 5
|
||||
qkv = torch.randn(
|
||||
T, num_heads * head_size + 2 * num_kv_heads * head_size, dtype=dtype
|
||||
)
|
||||
pos = torch.arange(T, dtype=torch.long)
|
||||
|
||||
qkv_eager = qkv.clone()
|
||||
pos_eager = pos.clone()
|
||||
result_eager = model(qkv_eager, pos_eager)
|
||||
|
||||
torch._dynamo.mark_dynamic(qkv, 0)
|
||||
torch._dynamo.mark_dynamic(pos, 0)
|
||||
|
||||
model_compiled = torch.compile(model, backend=backend)
|
||||
result_compiled = model_compiled(qkv, pos)
|
||||
|
||||
for eager, compiled in zip(result_eager, result_compiled):
|
||||
torch.testing.assert_close(eager, compiled)
|
||||
|
||||
assert backend.op_count(torch.ops.aten.slice_scatter.default) == 0
|
||||
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
|
||||
@@ -1,373 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import itertools
|
||||
from functools import partial
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.kernels.quantization.nvfp4_utils import quant_nvfp4_tensor
|
||||
from tests.utils import TestFP8Layer
|
||||
from vllm._aiter_ops import IS_AITER_FOUND, rocm_aiter_ops
|
||||
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
from vllm.compilation.passes.fusion.act_quant_fusion import (
|
||||
FUSED_OPS,
|
||||
SILU_MUL_OP,
|
||||
ActivationQuantFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.fusion.rms_quant_fusion import QUANT_OPS
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
FlashInferFP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ROCmFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
create_fp8_quant_key,
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import is_deep_gemm_supported
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
FP4_DTYPE = torch.uint8
|
||||
|
||||
|
||||
def is_nvfp4_supported():
|
||||
return current_platform.has_device_capability(100)
|
||||
|
||||
|
||||
class TestSiluMulFp8QuantModel(torch.nn.Module):
|
||||
quant_key = kFp8StaticTensorSym
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
force_kernel: type[FP8ScaledMMLinearKernel],
|
||||
dtype: torch.dtype,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
self.silu_and_mul = SiluAndMul()
|
||||
|
||||
self.fp8_linear = TestFP8Layer(
|
||||
weight_shape=(hidden_size, hidden_size),
|
||||
activation_quant_key=self.quant_key,
|
||||
weight_quant_key=self.quant_key,
|
||||
force_kernel=force_kernel,
|
||||
input_dtype=dtype,
|
||||
)
|
||||
|
||||
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
|
||||
self.enable_quant_fp8_custom_op = self.fp8_linear.is_quant_fp8_enabled()
|
||||
|
||||
def forward(self, x):
|
||||
y = self.silu_and_mul(x)
|
||||
x2 = self.fp8_linear(y)
|
||||
return x2
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul,
|
||||
(
|
||||
QUANT_OPS[kFp8StaticTensorSym]
|
||||
if self.enable_quant_fp8_custom_op
|
||||
else torch.ops.aten.reciprocal
|
||||
),
|
||||
]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [FUSED_OPS[kFp8StaticTensorSym]]
|
||||
|
||||
|
||||
class TestSiluMulNvfp4QuantModel(torch.nn.Module):
|
||||
def __init__(self, hidden_size: int, x: torch.Tensor, **kwargs):
|
||||
super().__init__()
|
||||
from vllm.compilation.passes.fusion.act_quant_fusion import (
|
||||
silu_and_mul_nvfp4_quant_supported,
|
||||
)
|
||||
|
||||
assert silu_and_mul_nvfp4_quant_supported
|
||||
|
||||
self.silu_and_mul = SiluAndMul()
|
||||
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
|
||||
|
||||
# create nvfp4 weight
|
||||
w = torch.rand((hidden_size, hidden_size))
|
||||
self.w, self.w_block_scale, self.w_global_scale = quant_nvfp4_tensor(w)
|
||||
|
||||
# get global scale offline
|
||||
_, _, self.y_global_scale = quant_nvfp4_tensor(self.silu_and_mul(x))
|
||||
|
||||
self.alpha = 1.0 / (self.w_global_scale * self.y_global_scale)
|
||||
|
||||
def forward(self, x):
|
||||
y = self.silu_and_mul(x)
|
||||
y_quant, y_block_scale = scaled_fp4_quant(y, self.y_global_scale)
|
||||
out = cutlass_scaled_fp4_mm(
|
||||
a=y_quant,
|
||||
b=self.w,
|
||||
block_scale_a=y_block_scale,
|
||||
block_scale_b=self.w_block_scale,
|
||||
alpha=self.alpha,
|
||||
out_dtype=y.dtype,
|
||||
)
|
||||
return out
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul,
|
||||
QUANT_OPS[kNvfp4Dynamic],
|
||||
]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [FUSED_OPS[kNvfp4Dynamic]]
|
||||
|
||||
|
||||
class TestSiluMulGroupFp8QuantModel(torch.nn.Module):
|
||||
act_quant_key = kFp8Dynamic128Sym
|
||||
|
||||
def __init__(self, hidden_size: int, dtype: torch.dtype, **kwargs):
|
||||
super().__init__()
|
||||
self.silu_and_mul = SiluAndMul()
|
||||
self.weight_quant_key = create_fp8_quant_key(
|
||||
static=True, group_shape=GroupShape(hidden_size, hidden_size)
|
||||
)
|
||||
|
||||
self.w8a8_block_fp8_linear = TestFP8Layer(
|
||||
weight_shape=(hidden_size, hidden_size),
|
||||
weight_quant_key=self.weight_quant_key,
|
||||
activation_quant_key=self.act_quant_key,
|
||||
input_dtype=dtype,
|
||||
)
|
||||
|
||||
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
|
||||
|
||||
def forward(self, x):
|
||||
y = self.silu_and_mul(x)
|
||||
x2 = self.w8a8_block_fp8_linear(y)
|
||||
return x2
|
||||
|
||||
def ops_in_model_before(self):
|
||||
return [
|
||||
SILU_MUL_OP if self.enable_silu_mul_custom_op else torch.ops.aten.mul,
|
||||
rocm_aiter_ops.get_group_quant_op(),
|
||||
]
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [torch.ops.vllm.rocm_aiter_act_mul_and_fp8_group_quant]
|
||||
|
||||
|
||||
class TestSiluMulBlockQuantModel(torch.nn.Module):
|
||||
quant_key = kFp8Dynamic128Sym
|
||||
|
||||
def __init__(self, hidden_size: int, is_scale_transposed: bool = False, **kwargs):
|
||||
super().__init__()
|
||||
self.silu_and_mul = SiluAndMul()
|
||||
self.is_scale_transposed = is_scale_transposed
|
||||
self.quant_fp8 = QuantFP8(
|
||||
static=False,
|
||||
group_shape=GroupShape(1, 128),
|
||||
column_major_scales=is_scale_transposed,
|
||||
compile_native=False,
|
||||
)
|
||||
|
||||
self.enable_silu_mul_custom_op = self.silu_and_mul.enabled()
|
||||
self.enable_quant_fp8_custom_op = self.quant_fp8.enabled()
|
||||
|
||||
def forward(self, x):
|
||||
y = self.silu_and_mul(x)
|
||||
out, scale = self.quant_fp8(y)
|
||||
group_size = self.quant_key.scale.group_shape[1]
|
||||
scale_expanded = scale.repeat_interleave(group_size, dim=1)
|
||||
dequant = out.to(dtype=torch.float32) * scale_expanded
|
||||
return (dequant,)
|
||||
|
||||
def ops_in_model_before(self):
|
||||
ops = []
|
||||
if self.enable_silu_mul_custom_op:
|
||||
ops.append(SILU_MUL_OP)
|
||||
# When silu custom op is disabled, aten.mul.Tensor also appears
|
||||
# in dequant code, so we skip checking it to avoid false positives.
|
||||
ops.append(
|
||||
QUANT_OPS[self.quant_key]
|
||||
if self.enable_quant_fp8_custom_op
|
||||
else torch.ops.aten.reciprocal.default
|
||||
)
|
||||
return ops
|
||||
|
||||
def ops_in_model_after(self):
|
||||
return [FUSED_OPS[self.quant_key]]
|
||||
|
||||
|
||||
ROCM_KERNELS = [ROCmFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel]
|
||||
CUDA_KERNELS = [
|
||||
FlashInferFP8ScaledMMLinearKernel,
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
]
|
||||
TEST_KERNELS = ROCM_KERNELS if current_platform.is_rocm() else CUDA_KERNELS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [32, 64])
|
||||
@pytest.mark.parametrize("hidden_size", [128, 256])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("enable_silu_mul_custom_op", [True, False])
|
||||
@pytest.mark.parametrize(
|
||||
"model_class, enable_quant_fp8_custom_op, force_kernel",
|
||||
list(itertools.product([TestSiluMulFp8QuantModel], [True, False], TEST_KERNELS))
|
||||
+ [
|
||||
pytest.param(
|
||||
TestSiluMulNvfp4QuantModel,
|
||||
False,
|
||||
None,
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUDA only"
|
||||
),
|
||||
),
|
||||
# GroupFP8Quant fusion only works with AITER on ROCm.
|
||||
# and the enable_quant_fp8_custom_op must be True.
|
||||
pytest.param(
|
||||
TestSiluMulGroupFp8QuantModel,
|
||||
True,
|
||||
None,
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_rocm(), reason="ROCm only"
|
||||
),
|
||||
),
|
||||
# Block quant fusion for per-group FP8 (CUDA only).
|
||||
*[
|
||||
pytest.param(
|
||||
partial(TestSiluMulBlockQuantModel, is_scale_transposed=transposed),
|
||||
True,
|
||||
None,
|
||||
marks=pytest.mark.skipif(
|
||||
not current_platform.is_cuda(), reason="CUDA only"
|
||||
),
|
||||
id=f"TestSiluMulBlockQuant-transposed={transposed}",
|
||||
)
|
||||
for transposed in [False, True]
|
||||
],
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
|
||||
)
|
||||
def test_fusion_silu_and_mul_quant(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
model_class: type[
|
||||
TestSiluMulFp8QuantModel
|
||||
| TestSiluMulNvfp4QuantModel
|
||||
| TestSiluMulGroupFp8QuantModel
|
||||
| TestSiluMulBlockQuantModel
|
||||
],
|
||||
enable_silu_mul_custom_op: bool,
|
||||
enable_quant_fp8_custom_op: bool,
|
||||
force_kernel: type[FP8ScaledMMLinearKernel] | None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
if model_class is TestSiluMulNvfp4QuantModel and not is_nvfp4_supported():
|
||||
pytest.skip("NVFP4 is not supported on this GPU.")
|
||||
if model_class is TestSiluMulGroupFp8QuantModel and not IS_AITER_FOUND:
|
||||
pytest.skip("AITER is not supported on this GPU.")
|
||||
if (
|
||||
isinstance(model_class, partial)
|
||||
and model_class.func is TestSiluMulBlockQuantModel
|
||||
and is_deep_gemm_supported()
|
||||
):
|
||||
pytest.skip("SiluMul+BlockQuant fusion not applicable with DeepGemm")
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
x = torch.rand(num_tokens, hidden_size * 2)
|
||||
|
||||
# Reshape pass is needed for the fusion pass to work
|
||||
custom_ops = ["none"]
|
||||
if enable_silu_mul_custom_op:
|
||||
custom_ops.append("+silu_and_mul")
|
||||
if enable_quant_fp8_custom_op:
|
||||
custom_ops.append("+quant_fp8")
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=custom_ops,
|
||||
backend="eager", # avoid compilation for SiluAndMul and QuantFP8
|
||||
pass_config=PassConfig(fuse_act_quant=True, eliminate_noops=True),
|
||||
),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(config), monkeypatch.context() as m:
|
||||
fusion_passes = [ActivationQuantFusionPass(config)]
|
||||
if IS_AITER_FOUND and model_class is TestSiluMulGroupFp8QuantModel:
|
||||
from vllm.compilation.passes.fusion.rocm_aiter_fusion import (
|
||||
RocmAiterSiluMulFp8GroupQuantFusionPass,
|
||||
)
|
||||
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
fusion_passes += [RocmAiterSiluMulFp8GroupQuantFusionPass(config)]
|
||||
|
||||
passes = [NoOpEliminationPass(config), *fusion_passes, PostCleanupPass(config)]
|
||||
backend = TestBackend(*passes)
|
||||
model = model_class(
|
||||
hidden_size=hidden_size,
|
||||
force_kernel=force_kernel,
|
||||
x=x,
|
||||
dtype=dtype,
|
||||
)
|
||||
|
||||
# First dimension dynamic
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
|
||||
result = model(x)
|
||||
|
||||
model2 = torch.compile(model, backend=backend)
|
||||
result2 = model2(x)
|
||||
|
||||
# Check that it gives the same answer
|
||||
if isinstance(model, TestSiluMulFp8QuantModel):
|
||||
atol, rtol = 1e-3, 1e-3
|
||||
elif isinstance(model, TestSiluMulNvfp4QuantModel):
|
||||
atol, rtol = 1e-1, 1e-1
|
||||
elif isinstance(model, TestSiluMulGroupFp8QuantModel):
|
||||
atol, rtol = 5e-2, 5e-2
|
||||
elif isinstance(model, TestSiluMulBlockQuantModel):
|
||||
if current_platform.is_rocm():
|
||||
atol, rtol = 1e-3, 1e-3
|
||||
else:
|
||||
# CUDA fused kernel computes silu*mul in fp32 while the reference
|
||||
# goes through bf16/fp16 storage, so group maxima (and thus scales)
|
||||
# can shift by one FP8-e4m3 code (~1/8 relative step).
|
||||
atol, rtol = 5e-2, 5e-2
|
||||
|
||||
torch.testing.assert_close(
|
||||
result[0].to(dtype=dtype), result2[0].to(dtype=dtype), atol=atol, rtol=rtol
|
||||
)
|
||||
|
||||
assert sum([p.matched_count for p in fusion_passes]) == 1
|
||||
|
||||
# In pre-nodes, quant op should be present and fused kernels should not
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
|
||||
# In post-nodes, fused kernels should be present and quant op should not
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
@@ -1,379 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for manual fusion via maybe_fused_act_quant.
|
||||
|
||||
Tests all fusion paths in _FUSED_ACT_QUANT:
|
||||
- kFp8StaticTensorSym: all platforms
|
||||
- kFp8Dynamic128Sym: CUDA only
|
||||
- kNvfp4Dynamic: CUDA SM100+ only
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from tests.utils import TestFP8Layer
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
FlashInferFP8ScaledMMLinearKernel,
|
||||
FP8ScaledMMLinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
ROCmFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.fusion.fused_act_quant import (
|
||||
_FUSED_ACT_QUANT,
|
||||
maybe_fused_act_quant,
|
||||
)
|
||||
from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
QuantizedActivation,
|
||||
expose_input_quant_key,
|
||||
get_input_quant_key,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
kFp8Dynamic128Sym,
|
||||
kFp8StaticTensorSym,
|
||||
kNvfp4Dynamic,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
# Mock linear layer for testing fusion paths that don't have real kernel support
|
||||
class MockLinearForFusion(torch.nn.Module):
|
||||
"""Mock linear layer that exposes an input key for fusion testing."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
quant_key,
|
||||
input_scale=None,
|
||||
input_global_scale=None,
|
||||
input_global_scale_inv=None,
|
||||
):
|
||||
super().__init__()
|
||||
self._input_quant_key = quant_key
|
||||
if input_scale is not None:
|
||||
self.input_scale = input_scale
|
||||
if input_global_scale is not None:
|
||||
self.input_global_scale = input_global_scale
|
||||
if input_global_scale_inv is not None:
|
||||
self.input_global_scale_inv = input_global_scale_inv
|
||||
|
||||
|
||||
ROCM_KERNELS = [ROCmFP8ScaledMMLinearKernel, PerTensorTorchFP8ScaledMMLinearKernel]
|
||||
CUDA_KERNELS = [
|
||||
FlashInferFP8ScaledMMLinearKernel,
|
||||
CutlassFP8ScaledMMLinearKernel,
|
||||
PerTensorTorchFP8ScaledMMLinearKernel,
|
||||
]
|
||||
TEST_KERNELS = ROCM_KERNELS if current_platform.is_rocm() else CUDA_KERNELS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [32, 64])
|
||||
@pytest.mark.parametrize("hidden_size", [128, 256])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.parametrize("force_kernel", TEST_KERNELS)
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
|
||||
)
|
||||
def test_manual_fusion_fp8_static_with_linear(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
force_kernel: type[FP8ScaledMMLinearKernel],
|
||||
):
|
||||
"""Test manual fusion with real FP8 linear layer (kFp8StaticTensorSym).
|
||||
|
||||
This is an end-to-end test that verifies the full flow:
|
||||
unfused (silu_and_mul -> in-kernel quant) vs fused (silu_and_mul_quant).
|
||||
"""
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
x = torch.rand(num_tokens, hidden_size * 2)
|
||||
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(custom_ops=["none"]),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
silu_and_mul = SiluAndMul()
|
||||
fp8_linear = TestFP8Layer(
|
||||
weight_shape=(hidden_size, hidden_size),
|
||||
activation_quant_key=kFp8StaticTensorSym,
|
||||
weight_quant_key=kFp8StaticTensorSym,
|
||||
force_kernel=force_kernel,
|
||||
input_dtype=dtype,
|
||||
)
|
||||
|
||||
# Run without fusion: silu_and_mul returns plain tensor
|
||||
y_unfused = maybe_fused_act_quant(silu_and_mul, x, fp8_linear)
|
||||
assert isinstance(y_unfused, torch.Tensor)
|
||||
result_unfused = fp8_linear(y_unfused)
|
||||
|
||||
# Enable fusion
|
||||
expose_input_quant_key(fp8_linear, fp8_linear.kernel)
|
||||
|
||||
if get_input_quant_key(fp8_linear) is None:
|
||||
pytest.skip(
|
||||
f"Kernel {force_kernel.__name__} doesn't support input_quant_key"
|
||||
)
|
||||
|
||||
# Run with fusion: silu_and_mul returns QuantizedActivation
|
||||
y_fused = maybe_fused_act_quant(silu_and_mul, x, fp8_linear)
|
||||
assert isinstance(y_fused, QuantizedActivation)
|
||||
assert y_fused.quant_key == kFp8StaticTensorSym
|
||||
assert y_fused.data.dtype == current_platform.fp8_dtype()
|
||||
assert y_fused.data.shape == (num_tokens, hidden_size)
|
||||
assert y_fused.orig_dtype == dtype
|
||||
assert y_fused.orig_shape == (num_tokens, hidden_size)
|
||||
|
||||
result_fused = fp8_linear(y_fused)
|
||||
|
||||
torch.testing.assert_close(
|
||||
result_fused.to(dtype=dtype),
|
||||
result_unfused.to(dtype=dtype),
|
||||
atol=5e-2,
|
||||
rtol=5e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="Dynamic block quant CUDA only"
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
|
||||
)
|
||||
def test_manual_fusion_fp8_dynamic_128(dtype: torch.dtype):
|
||||
"""Test kFp8Dynamic128Sym fusion path (group_size=128).
|
||||
|
||||
Compares fused (silu_and_mul_per_block_quant) vs unfused (silu_and_mul)
|
||||
by dequantizing the fused result and comparing with unfused.
|
||||
"""
|
||||
if (SiluAndMul, kFp8Dynamic128Sym) not in _FUSED_ACT_QUANT:
|
||||
pytest.skip("kFp8Dynamic128Sym fusion not available")
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
# hidden_size must be divisible by group_size (128)
|
||||
num_tokens, hidden_size = 32, 256
|
||||
group_size = 128
|
||||
x = torch.rand(num_tokens, hidden_size * 2)
|
||||
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(custom_ops=["none"]),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
silu_and_mul = SiluAndMul()
|
||||
|
||||
# Unfused path: just apply silu_and_mul
|
||||
mock_linear_no_key = torch.nn.Linear(hidden_size, hidden_size)
|
||||
result_unfused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_no_key)
|
||||
assert isinstance(result_unfused, torch.Tensor)
|
||||
|
||||
# Fused path: apply silu_and_mul + per-block quantization
|
||||
mock_linear_with_key = MockLinearForFusion(kFp8Dynamic128Sym)
|
||||
result_fused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_with_key)
|
||||
|
||||
# Verify fused result structure
|
||||
assert isinstance(result_fused, QuantizedActivation)
|
||||
assert result_fused.quant_key == kFp8Dynamic128Sym
|
||||
assert result_fused.data.dtype == current_platform.fp8_dtype()
|
||||
assert result_fused.data.shape == (num_tokens, hidden_size)
|
||||
assert result_fused.orig_dtype == dtype
|
||||
|
||||
# Check scale shape
|
||||
expected_num_groups = hidden_size // group_size
|
||||
assert result_fused.scale.shape == (num_tokens, expected_num_groups)
|
||||
|
||||
# Dequantize fused result and compare with unfused
|
||||
# Per-block dequant: data * scale (broadcast scale across group)
|
||||
dequant_data = result_fused.data.to(dtype).view(
|
||||
num_tokens, expected_num_groups, group_size
|
||||
)
|
||||
scales_expanded = result_fused.scale.unsqueeze(
|
||||
-1
|
||||
) # (num_tokens, num_groups, 1)
|
||||
dequant_result = (
|
||||
(dequant_data * scales_expanded).view(num_tokens, hidden_size).to(dtype)
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
dequant_result,
|
||||
result_unfused,
|
||||
atol=5e-2,
|
||||
rtol=5e-2,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
# Cover M < 128 (padded swizzled SF tiles) and hidden_size that makes the
|
||||
# kernel launch grid.y > 1 (num_packed_cols = hidden_size / 16 > 512).
|
||||
@pytest.mark.parametrize("num_tokens", [1, 127, 128])
|
||||
@pytest.mark.parametrize("hidden_size", [256, 14336])
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="NVFP4 CUDA only")
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.has_device_capability(100), reason="NVFP4 requires SM100+"
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
|
||||
)
|
||||
def test_manual_fusion_nvfp4_dynamic(
|
||||
dtype: torch.dtype, num_tokens: int, hidden_size: int
|
||||
):
|
||||
"""Test kNvfp4Dynamic fusion path.
|
||||
|
||||
Compares fused (silu_and_mul_nvfp4_quant) vs unfused (silu_and_mul)
|
||||
by dequantizing the fused result and comparing with unfused.
|
||||
"""
|
||||
if (SiluAndMul, kNvfp4Dynamic) not in _FUSED_ACT_QUANT:
|
||||
pytest.skip("kNvfp4Dynamic fusion not available")
|
||||
|
||||
from tests.kernels.quantization.nvfp4_utils import dequantize_nvfp4_to_dtype
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
# NVFP4 requires hidden_size divisible by 16 (block size) and by 2 (packing).
|
||||
x = torch.rand(num_tokens, hidden_size * 2)
|
||||
# Non-1.0 global scale so the test is sensitive to the scale direction:
|
||||
# the fused producer must quantize with input_global_scale_inv (the GEMM's
|
||||
# alpha divides by input_global_scale), matching the unfused path.
|
||||
input_global_scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
|
||||
input_global_scale_inv = 1.0 / input_global_scale
|
||||
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(custom_ops=["none"]),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
silu_and_mul = SiluAndMul()
|
||||
|
||||
# Unfused path: just apply silu_and_mul
|
||||
mock_linear_no_key = torch.nn.Linear(hidden_size, hidden_size)
|
||||
result_unfused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_no_key)
|
||||
assert isinstance(result_unfused, torch.Tensor)
|
||||
|
||||
# Fused path: apply silu_and_mul + NVFP4 quantization
|
||||
mock_linear_with_key = MockLinearForFusion(
|
||||
kNvfp4Dynamic,
|
||||
input_global_scale=input_global_scale,
|
||||
input_global_scale_inv=input_global_scale_inv,
|
||||
)
|
||||
result_fused = maybe_fused_act_quant(silu_and_mul, x, mock_linear_with_key)
|
||||
|
||||
# Verify fused result structure
|
||||
assert isinstance(result_fused, QuantizedActivation)
|
||||
assert result_fused.quant_key == kNvfp4Dynamic
|
||||
# NVFP4 packs 2 values into 1 byte
|
||||
assert result_fused.data.dtype == torch.uint8
|
||||
assert result_fused.data.shape == (num_tokens, hidden_size // 2)
|
||||
assert result_fused.orig_dtype == dtype
|
||||
assert result_fused.orig_shape == (num_tokens, hidden_size)
|
||||
|
||||
# The scale tensor must use the padded 128x4-tile swizzled layout,
|
||||
# same as scaled_fp4_quant's allocator; the kernel writes a full
|
||||
# 128-row tile even when M is smaller.
|
||||
from vllm.utils.math_utils import round_up
|
||||
|
||||
assert result_fused.scale.dtype == current_platform.fp8_dtype()
|
||||
assert result_fused.scale.shape == (
|
||||
round_up(num_tokens, 128),
|
||||
round_up(hidden_size // 16, 4),
|
||||
)
|
||||
|
||||
# Dequantize fused result and compare with unfused. The block scales
|
||||
# must have been folded with input_global_scale_inv, so dequantize
|
||||
# with the same inverse scale to recover the activation.
|
||||
dequant_result = dequantize_nvfp4_to_dtype(
|
||||
tensor_fp4=result_fused.data,
|
||||
tensor_sf=result_fused.scale,
|
||||
global_scale=input_global_scale_inv,
|
||||
dtype=dtype,
|
||||
device="cuda",
|
||||
block_size=16,
|
||||
is_sf_128x4_layout=True,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
dequant_result,
|
||||
result_unfused,
|
||||
atol=3e-1,
|
||||
rtol=3e-1,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
|
||||
)
|
||||
def test_manual_fusion_fallback_no_key(dtype: torch.dtype):
|
||||
"""Test that maybe_fused_act_quant falls back without an input key."""
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
x = torch.rand(32, 256)
|
||||
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(custom_ops=["none"]),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
silu_and_mul = SiluAndMul()
|
||||
# Linear without an input quantization key
|
||||
mock_linear = torch.nn.Linear(128, 128)
|
||||
|
||||
result = maybe_fused_act_quant(silu_and_mul, x, mock_linear)
|
||||
|
||||
# Should fall back to plain silu_and_mul
|
||||
assert isinstance(result, torch.Tensor)
|
||||
assert result.shape == (32, 128)
|
||||
assert result.dtype == dtype
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
@pytest.mark.skipif(
|
||||
envs.VLLM_TARGET_DEVICE not in ["cuda", "rocm"], reason="Only test on CUDA and ROCm"
|
||||
)
|
||||
def test_manual_fusion_fallback_unsupported_key(dtype: torch.dtype):
|
||||
"""Test that maybe_fused_act_quant falls back for unsupported quant keys."""
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
QuantKey,
|
||||
ScaleDesc,
|
||||
)
|
||||
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(dtype)
|
||||
|
||||
x = torch.rand(32, 256)
|
||||
|
||||
config = VllmConfig(
|
||||
compilation_config=CompilationConfig(custom_ops=["none"]),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(config):
|
||||
# Create an unsupported quant key
|
||||
unsupported_key = QuantKey(
|
||||
dtype=torch.int8,
|
||||
scale=ScaleDesc(
|
||||
dtype=torch.float32, static=False, group_shape=GroupShape(1, 1)
|
||||
),
|
||||
)
|
||||
|
||||
silu_and_mul = SiluAndMul()
|
||||
mock_linear = MockLinearForFusion(unsupported_key)
|
||||
|
||||
result = maybe_fused_act_quant(silu_and_mul, x, mock_linear)
|
||||
|
||||
# Should fall back to plain silu_and_mul since key not in _FUSED_ACT_QUANT
|
||||
assert isinstance(result, torch.Tensor)
|
||||
assert result.shape == (32, 128)
|
||||
assert result.dtype == dtype
|
||||
@@ -1,273 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import operator
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm
|
||||
from tests.compile.backend import TestBackend
|
||||
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
|
||||
from vllm.config import CompilationConfig, CompilationMode, PassConfig, VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
class SplitCoalescingModel(torch.nn.Module):
|
||||
"""Model with 3 separate split_with_sizes calls on the same input,
|
||||
simulating the B200+FP8 graph where CSE fails to merge them."""
|
||||
|
||||
def __init__(self, q_size: int, kv_size: int) -> None:
|
||||
super().__init__()
|
||||
self.q_size = q_size
|
||||
self.kv_size = kv_size
|
||||
|
||||
def forward(self, qkv: torch.Tensor):
|
||||
q, _, _ = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
_, k, _ = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
_, _, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
return q + 1, k + 2, v + 3
|
||||
|
||||
|
||||
class SliceCoalescingModel(torch.nn.Module):
|
||||
"""Model with a complete QKV partition represented as contiguous slices."""
|
||||
|
||||
def __init__(self, q_size: int, kv_size: int) -> None:
|
||||
super().__init__()
|
||||
self.q_size = q_size
|
||||
self.kv_size = kv_size
|
||||
|
||||
def forward(self, qkv: torch.Tensor):
|
||||
q_end = self.q_size
|
||||
k_end = q_end + self.kv_size
|
||||
v_end = k_end + self.kv_size
|
||||
q = torch.ops.aten.slice.Tensor(qkv, -1, 0, q_end)
|
||||
k = torch.ops.aten.slice.Tensor(qkv, -1, q_end, k_end)
|
||||
v = torch.ops.aten.slice.Tensor(qkv, -1, k_end, v_end)
|
||||
return q + 1, k + 2, v + 3
|
||||
|
||||
|
||||
def make_vllm_config() -> VllmConfig:
|
||||
return VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
pass_config=PassConfig(),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_split_coalescing(dtype):
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
q_size, kv_size = 2048, 512
|
||||
|
||||
vllm_config = make_vllm_config()
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
coalesce_pass = SplitCoalescingPass(vllm_config)
|
||||
backend = TestBackend(coalesce_pass)
|
||||
|
||||
model = SplitCoalescingModel(q_size, kv_size)
|
||||
|
||||
T = 5
|
||||
qkv = torch.randn(T, q_size + 2 * kv_size)
|
||||
torch._dynamo.mark_dynamic(qkv, 0)
|
||||
|
||||
result_eager = model(qkv)
|
||||
|
||||
model_compiled = torch.compile(model, backend=backend)
|
||||
result_compiled = model_compiled(qkv)
|
||||
|
||||
ATOL, RTOL = (2e-3, 2e-3)
|
||||
for eager, compiled in zip(result_eager, result_compiled):
|
||||
torch.testing.assert_close(eager, compiled, atol=ATOL, rtol=RTOL)
|
||||
|
||||
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16])
|
||||
def test_slice_partition_canonicalization(dtype):
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
q_size, kv_size = 2048, 512
|
||||
vllm_config = make_vllm_config()
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
coalesce_pass = SplitCoalescingPass(vllm_config)
|
||||
backend = TestBackend(coalesce_pass)
|
||||
model = SliceCoalescingModel(q_size, kv_size)
|
||||
|
||||
qkv = torch.randn(5, q_size + 2 * kv_size)
|
||||
torch._dynamo.mark_dynamic(qkv, 0)
|
||||
result_eager = model(qkv)
|
||||
result_compiled = torch.compile(model, backend=backend, fullgraph=True)(qkv)
|
||||
|
||||
for eager, compiled in zip(result_eager, result_compiled):
|
||||
torch.testing.assert_close(eager, compiled, atol=2e-3, rtol=2e-3)
|
||||
|
||||
assert backend.op_count(torch.ops.aten.slice.Tensor, before=True) == 3
|
||||
assert backend.op_count(torch.ops.aten.slice.Tensor) == 0
|
||||
assert backend.op_count(torch.ops.aten.split_with_sizes.default) == 1
|
||||
|
||||
split = next(
|
||||
node
|
||||
for node in backend.final_graph.nodes
|
||||
if node.target == torch.ops.aten.split_with_sizes.default
|
||||
)
|
||||
assert list(split.args[1]) == [q_size, kv_size, kv_size]
|
||||
assert split.args[2] == -1
|
||||
|
||||
getitems = [
|
||||
node
|
||||
for node in split.users
|
||||
if node.op == "call_function" and node.target == operator.getitem
|
||||
]
|
||||
assert sorted(node.args[1] for node in getitems) == [0, 1, 2]
|
||||
assert isinstance(split.meta["val"], list)
|
||||
for getitem in getitems:
|
||||
index = getitem.args[1]
|
||||
assert getitem.meta["val"] is split.meta["val"][index]
|
||||
|
||||
|
||||
def make_slice_graph(
|
||||
ranges: list[tuple[int, int, int]],
|
||||
) -> torch.fx.Graph:
|
||||
graph = torch.fx.Graph()
|
||||
source = graph.placeholder("qkv")
|
||||
source.meta["val"] = torch.empty((2, 8), device="meta")
|
||||
|
||||
slices = []
|
||||
for start, end, step in ranges:
|
||||
node = graph.call_function(
|
||||
torch.ops.aten.slice.Tensor,
|
||||
args=(source, -1, start, end, step),
|
||||
)
|
||||
node.meta["val"] = torch.empty(
|
||||
(2, len(range(start, min(end, 8), step))), device="meta"
|
||||
)
|
||||
slices.append(node)
|
||||
graph.output(tuple(slices))
|
||||
return graph
|
||||
|
||||
|
||||
def test_slice_partition_accepts_safe_group():
|
||||
graph = make_slice_graph([(0, 4, 1), (4, 8, 1)])
|
||||
vllm_config = make_vllm_config()
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
SplitCoalescingPass(vllm_config)(graph)
|
||||
|
||||
graph.lint()
|
||||
assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0
|
||||
assert (
|
||||
sum(
|
||||
node.target == torch.ops.aten.split_with_sizes.default
|
||||
for node in graph.nodes
|
||||
)
|
||||
== 1
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"ranges",
|
||||
[
|
||||
[(0, 3, 1), (4, 8, 1)], # gap
|
||||
[(0, 5, 1), (4, 8, 1)], # overlap
|
||||
[(1, 4, 1), (4, 8, 1)], # missing prefix
|
||||
[(0, 4, 1), (4, 7, 1)], # missing suffix
|
||||
[(0, 8, 1)], # single full-range slice
|
||||
[(0, 8, 2)], # non-unit step
|
||||
],
|
||||
)
|
||||
def test_slice_partition_rejects_unsafe_groups(ranges):
|
||||
graph = make_slice_graph(ranges)
|
||||
vllm_config = make_vllm_config()
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
SplitCoalescingPass(vllm_config)(graph)
|
||||
|
||||
slice_count = sum(
|
||||
node.target == torch.ops.aten.slice.Tensor for node in graph.nodes
|
||||
)
|
||||
split_count = sum(
|
||||
node.target == torch.ops.aten.split_with_sizes.default for node in graph.nodes
|
||||
)
|
||||
assert slice_count == len(ranges)
|
||||
assert split_count == 0
|
||||
|
||||
|
||||
def test_nested_slice_partitions_use_live_source():
|
||||
"""Use the rewired source when a slice partition consumes another slice."""
|
||||
graph = torch.fx.Graph()
|
||||
source = graph.placeholder("input")
|
||||
source.meta["val"] = torch.empty((2, 8), device="meta")
|
||||
|
||||
outer_left = graph.call_function(
|
||||
torch.ops.aten.slice.Tensor, args=(source, -1, 0, 4)
|
||||
)
|
||||
outer_left.meta["val"] = torch.empty((2, 4), device="meta")
|
||||
outer_right = graph.call_function(
|
||||
torch.ops.aten.slice.Tensor, args=(source, -1, 4, 8)
|
||||
)
|
||||
outer_right.meta["val"] = torch.empty((2, 4), device="meta")
|
||||
inner_left = graph.call_function(
|
||||
torch.ops.aten.slice.Tensor, args=(outer_left, -1, 0, 2)
|
||||
)
|
||||
inner_left.meta["val"] = torch.empty((2, 2), device="meta")
|
||||
inner_right = graph.call_function(
|
||||
torch.ops.aten.slice.Tensor, args=(outer_left, -1, 2, 4)
|
||||
)
|
||||
inner_right.meta["val"] = torch.empty((2, 2), device="meta")
|
||||
graph.output((inner_left, inner_right, outer_right))
|
||||
|
||||
vllm_config = make_vllm_config()
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
SplitCoalescingPass(vllm_config)(graph)
|
||||
|
||||
graph.lint()
|
||||
assert sum(node.target == torch.ops.aten.slice.Tensor for node in graph.nodes) == 0
|
||||
splits = [
|
||||
node
|
||||
for node in graph.nodes
|
||||
if node.target == torch.ops.aten.split_with_sizes.default
|
||||
]
|
||||
assert len(splits) == 2
|
||||
|
||||
outer_split = next(node for node in splits if list(node.args[1]) == [4, 4])
|
||||
inner_split = next(node for node in splits if list(node.args[1]) == [2, 2])
|
||||
assert outer_split.args[0] is source
|
||||
assert inner_split.args[0].target == operator.getitem
|
||||
assert inner_split.args[0].args == (outer_split, 0)
|
||||
|
||||
|
||||
def test_split_coalescing_preserves_different_dimensions():
|
||||
"""Keep equal-sized splits on different dimensions as separate nodes."""
|
||||
graph = torch.fx.Graph()
|
||||
source = graph.placeholder("input")
|
||||
source.meta["val"] = torch.empty((8, 8), device="meta")
|
||||
|
||||
outputs = []
|
||||
for dim in (0, 1):
|
||||
split = graph.call_function(
|
||||
torch.ops.aten.split_with_sizes.default,
|
||||
args=(source, [4, 4], dim),
|
||||
)
|
||||
for index in range(2):
|
||||
outputs.append(graph.call_function(operator.getitem, args=(split, index)))
|
||||
graph.output(tuple(outputs))
|
||||
|
||||
vllm_config = make_vllm_config()
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
SplitCoalescingPass(vllm_config)(graph)
|
||||
|
||||
splits = [
|
||||
node
|
||||
for node in graph.nodes
|
||||
if node.target == torch.ops.aten.split_with_sizes.default
|
||||
]
|
||||
assert len(splits) == 2
|
||||
assert all(split.args[0] is source for split in splits)
|
||||
assert {split.args[2] for split in splits} == {0, 1}
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.config
|
||||
from tests.compile.backend import TestBackend
|
||||
from vllm.compilation.passes.vllm_inductor_pass import (
|
||||
VllmFusionPatternMatcherPass,
|
||||
VllmPatternMatcherPass,
|
||||
VllmPatternReplacement,
|
||||
)
|
||||
from vllm.config import CompilationConfig, CompilationMode, VllmConfig
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
|
||||
class ReluToAbsPattern(VllmPatternReplacement):
|
||||
"""Replaces relu(x) with abs(x) — a minimal test fixture."""
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
def _pattern(x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.ops.aten.relu.default(x)
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
def _replacement(x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.ops.aten.abs.default(x)
|
||||
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
return [self.empty_fp32(4)]
|
||||
|
||||
|
||||
class ExpToSqrtPattern(VllmPatternReplacement):
|
||||
"""A second distinct pattern type — used to test uuid differentiation."""
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
def _pattern(x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.ops.aten.exp.default(x)
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self):
|
||||
def _replacement(x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.ops.aten.sqrt.default(x)
|
||||
|
||||
return _replacement
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
return [self.empty_fp32(4)]
|
||||
|
||||
|
||||
class ReluFusionPass(VllmFusionPatternMatcherPass):
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "test_relu_fusion")
|
||||
self.register(ReluToAbsPattern())
|
||||
|
||||
|
||||
class TwoPatternFusionPass(VllmFusionPatternMatcherPass):
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config, "test_two_pattern_fusion")
|
||||
self.register(ReluToAbsPattern())
|
||||
self.register(ExpToSqrtPattern())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def vllm_config():
|
||||
return VllmConfig(
|
||||
compilation_config=CompilationConfig(mode=CompilationMode.VLLM_COMPILE),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Requires CUDA")
|
||||
def test_register_tracks_patterns(vllm_config):
|
||||
"""register() appends each VllmPatternReplacement to _pattern_replacements."""
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
single = ReluFusionPass(vllm_config)
|
||||
two = TwoPatternFusionPass(vllm_config)
|
||||
|
||||
assert len(single._pattern_replacements) == 1
|
||||
assert len(two._pattern_replacements) == 2
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Requires CUDA")
|
||||
def test_uuid_stable(vllm_config):
|
||||
"""Two instances of the same pass class produce identical uuids."""
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
p1 = ReluFusionPass(vllm_config)
|
||||
p2 = ReluFusionPass(vllm_config)
|
||||
p3 = TwoPatternFusionPass(vllm_config)
|
||||
|
||||
assert p1.uuid() == p2.uuid()
|
||||
assert p1.uuid() != p3.uuid()
|
||||
assert p2.uuid() != p3.uuid()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda_alike(), reason="Requires CUDA")
|
||||
@pytest.mark.parametrize("N", [1, 2, 4])
|
||||
def test_matched_count_and_match_table(vllm_config, N):
|
||||
"""matched_count and match_table reflect the number of matched patterns."""
|
||||
|
||||
class Model(torch.nn.Module):
|
||||
def forward(self, *inputs):
|
||||
# N independent relus
|
||||
return sum(torch.relu(x) for x in inputs)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config):
|
||||
torch.set_default_device("cuda")
|
||||
torch.set_default_dtype(torch.float32)
|
||||
|
||||
fusion_pass = ReluFusionPass(vllm_config)
|
||||
backend = TestBackend(fusion_pass)
|
||||
model = torch.compile(Model(), backend=backend)
|
||||
|
||||
inputs = [torch.rand(8) for _ in range(N)]
|
||||
model(*inputs)
|
||||
|
||||
assert fusion_pass.matched_count == N
|
||||
assert VllmPatternMatcherPass.match_table["test_relu_fusion"] >= N
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1,752 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import operator
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch._dynamo
|
||||
import torch.fx as fx
|
||||
from torch.fx.experimental.proxy_tensor import make_fx
|
||||
|
||||
from vllm.compilation.backends import (
|
||||
_decompose_size_nodes,
|
||||
_is_empty_allocation_node,
|
||||
split_graph,
|
||||
)
|
||||
from vllm.compilation.passes.fx_utils import find_op_nodes
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# This import automatically registers `torch.ops.silly.attention`
|
||||
from . import silly_attention # noqa: F401
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def test_getitem_moved_to_producer_subgraph():
|
||||
"""Test that getitem operations are moved to the same subgraph as their input,
|
||||
preventing tuple inputs to submodules.
|
||||
"""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
# torch.split returns a tuple, creating real getitem operations
|
||||
# Should become first submodule that produces tuple
|
||||
chunks = torch.split(x, x.shape[0] // 2, dim=0)
|
||||
|
||||
# Following ops should become second submodule that consumes tuple
|
||||
result_0 = torch.relu(chunks[0])
|
||||
result_1 = torch.relu(chunks[1])
|
||||
return torch.cat([result_0, result_1], dim=0)
|
||||
|
||||
x = torch.randn(4, 3)
|
||||
gm = make_fx(model_fn)(x)
|
||||
|
||||
has_getitem = any(
|
||||
node.op == "call_function" and node.target == operator.getitem
|
||||
for node in gm.graph.nodes
|
||||
)
|
||||
assert has_getitem, "Test setup failed: graph should contain getitem operations"
|
||||
|
||||
# Split on tuple producer aten::split
|
||||
split_ops = ["aten::split.Tensor"]
|
||||
split_gm, split_items = split_graph(gm, split_ops)
|
||||
assert len(split_items) == 2, "Graph should be split into 2 submodules"
|
||||
|
||||
for split_item in split_items:
|
||||
submodule = split_item.graph
|
||||
|
||||
getitem_on_placeholder = []
|
||||
for node in submodule.graph.nodes:
|
||||
if (
|
||||
node.op == "call_function"
|
||||
and node.target == operator.getitem
|
||||
and node.args[0].op == "placeholder"
|
||||
):
|
||||
getitem_on_placeholder.append(node)
|
||||
|
||||
assert len(getitem_on_placeholder) == 0, (
|
||||
f"Submodule {split_item.submod_name} has getitem operations on "
|
||||
f"placeholder nodes: {[n.name for n in getitem_on_placeholder]}. "
|
||||
"This means tuple inputs were not properly eliminated."
|
||||
)
|
||||
|
||||
new_x = torch.randn(4, 3)
|
||||
output_original = gm(new_x)
|
||||
output_split = split_gm(new_x)
|
||||
|
||||
assert torch.allclose(output_original, output_split), "Output mismatch"
|
||||
|
||||
|
||||
def test_no_tuple_inputs_with_multiple_consumers():
|
||||
"""Test that when a tuple is consumed by multiple split operations,
|
||||
getitem operations are properly moved to avoid tuple inputs.
|
||||
"""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
# torch.split returns a tuple, creating real getitem operations
|
||||
# Should become first submodule that produces tuple
|
||||
chunks = torch.split(x, x.shape[0] // 2, dim=0)
|
||||
|
||||
# These should become second submodule consuming tuple
|
||||
result_1 = torch.relu(chunks[0])
|
||||
result_2 = torch.relu(chunks[1])
|
||||
|
||||
# Artificial graph splitting point to create another
|
||||
# independent submodule that consumes tuple later
|
||||
# This would become the third submodule
|
||||
result_1 = torch.sigmoid(result_1)
|
||||
|
||||
# Fourth submodule that consumes tuple
|
||||
result = torch.cat([chunks[0], chunks[1], result_1, result_2])
|
||||
return result
|
||||
|
||||
x = torch.randn(4, 3)
|
||||
gm = make_fx(model_fn)(x)
|
||||
|
||||
has_getitem = any(
|
||||
node.op == "call_function" and node.target == operator.getitem
|
||||
for node in gm.graph.nodes
|
||||
)
|
||||
assert has_getitem, "Test setup failed: graph should contain getitem operations"
|
||||
|
||||
split_ops = ["aten::split.Tensor", "aten::sigmoid"]
|
||||
split_gm, split_items = split_graph(gm, split_ops)
|
||||
assert len(split_items) == 4, "Graph should be split into 4 submodules"
|
||||
|
||||
for split_item in split_items:
|
||||
submodule = split_item.graph
|
||||
|
||||
for node in submodule.graph.nodes:
|
||||
if (
|
||||
node.op == "call_function"
|
||||
and node.target == operator.getitem
|
||||
and node.args[0].op == "placeholder"
|
||||
):
|
||||
pytest.fail(
|
||||
f"Submodule {split_item.submod_name} has getitem on "
|
||||
f"placeholder {node.args[0].name}, indicating it receives "
|
||||
"a tuple input"
|
||||
)
|
||||
|
||||
new_x = torch.randn(4, 3)
|
||||
output_original = gm(new_x)
|
||||
output_split = split_gm(new_x)
|
||||
|
||||
assert torch.allclose(output_original, output_split), "Output mismatch after split"
|
||||
|
||||
|
||||
def test_consecutive_ops_in_split():
|
||||
"""Test that consecutive splitting operations are grouped into the same subgraph."""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Define a simple model where consecutive operations create opportunities
|
||||
for splitting subgraphs.
|
||||
"""
|
||||
# Apply silly attention followed by consecutive operations
|
||||
intermediate = torch.relu(x)
|
||||
attn_inout = torch.sqrt(intermediate)
|
||||
torch.ops.silly.attention(intermediate, intermediate, attn_inout, attn_inout)
|
||||
final_result = torch.sigmoid(attn_inout)
|
||||
return final_result
|
||||
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
|
||||
# Create the traced FX graph for the model
|
||||
x = torch.randn(8, 4)
|
||||
|
||||
gm = make_fx(model_fn)(x)
|
||||
|
||||
# Assert presence of the expected operations in the setup
|
||||
assert (
|
||||
len(list(find_op_nodes(torch.ops.aten.relu, gm.graph))) == 1
|
||||
and len(list(find_op_nodes(torch.ops.aten.sqrt, gm.graph))) == 1
|
||||
), "Test setup failed: Expected sqrt and relu operations in the graph."
|
||||
|
||||
# Configure split operations to test
|
||||
splitting_ops = ["silly::attention", "aten::sqrt"]
|
||||
split_gm, split_items = split_graph(gm, splitting_ops)
|
||||
|
||||
# Validate the number of partitions
|
||||
assert len(split_items) == 3, (
|
||||
"Consecutive splitting operations were not grouped correctly."
|
||||
)
|
||||
|
||||
# Validate that correctness is preserved
|
||||
new_x = torch.randn(8, 4)
|
||||
output_original = gm(new_x)
|
||||
output_split = split_gm(new_x)
|
||||
assert torch.allclose(output_original, output_split), (
|
||||
"Output mismatch after splitting."
|
||||
)
|
||||
|
||||
# Check the splitting item has 2 nodes exactly (relu and attn)
|
||||
splitting_items = list(s for s in split_items if s.is_splitting_graph)
|
||||
assert len(splitting_items) == 1, "Expecting a single splitting graph"
|
||||
print(splitting_items[0].graph.graph)
|
||||
splitting_gm = splitting_items[0].graph
|
||||
assert len(splitting_gm.graph.nodes) == 4, "Expecting 4 nodes in splitting graph"
|
||||
assert [node.op for node in splitting_gm.graph.nodes] == ["placeholder"] + 2 * [
|
||||
"call_function"
|
||||
] + ["output"]
|
||||
|
||||
|
||||
def _get_empty_nodes(split_item):
|
||||
return [
|
||||
node for node in split_item.graph.graph.nodes if _is_empty_allocation_node(node)
|
||||
]
|
||||
|
||||
|
||||
def _subgraphs_with_empty_nodes(split_items, *, is_splitting_graph):
|
||||
return [
|
||||
split_item
|
||||
for split_item in split_items
|
||||
if split_item.is_splitting_graph == is_splitting_graph
|
||||
and _get_empty_nodes(split_item)
|
||||
]
|
||||
|
||||
|
||||
def test_empty_only_partition_stays_separate_after_splitting_predecessor():
|
||||
"""Empty-only subgraphs should not be merged when the only predecessor is
|
||||
a splitting-op subgraph.
|
||||
"""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
y = torch.sin(x)
|
||||
out = torch.empty_like(y)
|
||||
torch.ops.aten.cos.out(y, out=out)
|
||||
return out
|
||||
|
||||
x = torch.randn(4, 3)
|
||||
gm = make_fx(model_fn)(x)
|
||||
|
||||
split_ops = ["aten::sin", "aten::cos.out"]
|
||||
split_gm, split_items = split_graph(gm, split_ops)
|
||||
|
||||
# Graph partitioning for this pattern is:
|
||||
# [sin], [empty_like], [cos.out].
|
||||
assert len(split_items) == 3, (
|
||||
"Empty-only partition should not merge into splitting-op subgraph"
|
||||
)
|
||||
|
||||
splitting_with_empty = _subgraphs_with_empty_nodes(
|
||||
split_items, is_splitting_graph=True
|
||||
)
|
||||
assert len(splitting_with_empty) == 0, (
|
||||
"Splitting-op subgraphs should not contain empty allocation nodes: "
|
||||
f"{[item.submod_name for item in splitting_with_empty]}"
|
||||
)
|
||||
|
||||
output_original = gm(x)
|
||||
output_split = split_gm(x)
|
||||
assert torch.allclose(output_original, output_split), "Output mismatch after split"
|
||||
|
||||
|
||||
def test_empty_only_partition_is_merged():
|
||||
"""Empty-only subgraphs should still be merged when a non-splitting predecessor
|
||||
exists. The merged empty node must remain outside splitting-op subgraphs.
|
||||
"""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
base = x + 1
|
||||
y = torch.sin(base)
|
||||
out = torch.empty_like(base)
|
||||
torch.ops.aten.cos.out(base, out=out)
|
||||
return out + y
|
||||
|
||||
x = torch.randn(4, 3)
|
||||
gm = make_fx(model_fn)(x)
|
||||
split_gm, split_items = split_graph(gm, ["aten::sin", "aten::cos.out"])
|
||||
|
||||
# Partitioning should be:
|
||||
# [add, empty_like], [sin], [cos.out], [add].
|
||||
assert len(split_items) == 4, (
|
||||
"Empty-only partition should be merged into non-splitting predecessor"
|
||||
)
|
||||
|
||||
splitting_with_empty = _subgraphs_with_empty_nodes(
|
||||
split_items, is_splitting_graph=True
|
||||
)
|
||||
assert len(splitting_with_empty) == 0, (
|
||||
"Splitting-op subgraphs should not contain empty allocation nodes: "
|
||||
f"{[item.submod_name for item in splitting_with_empty]}"
|
||||
)
|
||||
|
||||
non_splitting_with_empty = _subgraphs_with_empty_nodes(
|
||||
split_items, is_splitting_graph=False
|
||||
)
|
||||
assert len(non_splitting_with_empty) == 1, (
|
||||
"Exactly one non-splitting subgraph should contain the merged empty node"
|
||||
)
|
||||
assert len(_get_empty_nodes(non_splitting_with_empty[0])) == 1, (
|
||||
"Expected exactly one empty allocation node in merged subgraph"
|
||||
)
|
||||
|
||||
output_original = gm(x)
|
||||
output_split = split_gm(x)
|
||||
assert torch.allclose(output_original, output_split), "Output mismatch after split"
|
||||
|
||||
|
||||
def test_builtin_empty_only_partition_is_merged():
|
||||
"""In Dynamo graphs, torch.empty/empty_like may appear as builtin call targets
|
||||
(not aten OpOverload). Ensure empty-only partitions are still merged.
|
||||
"""
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
hidden = x + 1
|
||||
out1 = torch.empty_like(hidden)
|
||||
torch.ops.silly.attention(hidden, hidden, hidden, out1)
|
||||
out2 = torch.empty_like(hidden)
|
||||
torch.ops.silly.attention(out1, out1, hidden, out2)
|
||||
return out2 + hidden
|
||||
|
||||
gm = torch.fx.symbolic_trace(model_fn)
|
||||
split_gm, split_items = split_graph(gm, ["silly::attention"])
|
||||
|
||||
# Without empty-only merge, this graph would split into:
|
||||
# [add, empty_like], [attention], [empty_like], [attention], [add].
|
||||
assert len(split_items) == 4, "Builtin empty-only partition should be merged"
|
||||
|
||||
splitting_with_empty = _subgraphs_with_empty_nodes(
|
||||
split_items, is_splitting_graph=True
|
||||
)
|
||||
assert len(splitting_with_empty) == 0, (
|
||||
"Splitting-op subgraphs should not contain empty allocation nodes: "
|
||||
f"{[item.submod_name for item in splitting_with_empty]}"
|
||||
)
|
||||
|
||||
non_splitting_with_empty = _subgraphs_with_empty_nodes(
|
||||
split_items, is_splitting_graph=False
|
||||
)
|
||||
assert len(non_splitting_with_empty) == 1, (
|
||||
"Exactly one non-splitting subgraph should contain merged empty nodes"
|
||||
)
|
||||
assert len(_get_empty_nodes(non_splitting_with_empty[0])) == 2, (
|
||||
"Expected two builtin empty_like nodes in merged non-splitting subgraph"
|
||||
)
|
||||
|
||||
x = torch.randn(2, 3, device=DEVICE_TYPE)
|
||||
output_original = gm(x)
|
||||
output_split = split_gm(x)
|
||||
assert torch.allclose(output_original, output_split), "Output mismatch after split"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_sym_size_whole_shape_boundary():
|
||||
"""Test that using x.size() (whole shape) across a split boundary can be
|
||||
compiled by standalone_compile.
|
||||
|
||||
The dynamo graph looks like:
|
||||
shape = x.size()
|
||||
y = sigmoid(x) # split point
|
||||
z = y.clone().view(shape)
|
||||
|
||||
Which splits into:
|
||||
subgraph0(x) -> shape # returns torch.Size — problematic
|
||||
subgraph1(x) -> y # sigmoid
|
||||
subgraph2(y, shape) -> z # view
|
||||
|
||||
Two approaches to fix the torch.Size crossing:
|
||||
|
||||
Approach 1 — move sym_size to consumer (memory implication: x passed to
|
||||
subgraph2 just for .size()):
|
||||
subgraph0(x) -> # empty
|
||||
subgraph1(x) -> y
|
||||
subgraph2(y, x) -> z # computes shape locally from x
|
||||
|
||||
Approach 2 — decompose shape into individual int/SymInt values:
|
||||
subgraph0(x) -> s0, val # returns individual scalars, not Size
|
||||
subgraph1(x) -> y
|
||||
subgraph2(y, s0, val) -> z # reconstructs view args from scalars
|
||||
"""
|
||||
from torch._inductor import standalone_compile
|
||||
|
||||
captured_graph = None
|
||||
|
||||
def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
|
||||
nonlocal captured_graph
|
||||
captured_graph = gm
|
||||
return gm
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
shape = x.size()
|
||||
x = torch.ops.aten.sigmoid.default(x)
|
||||
x = x.clone().view(shape)
|
||||
return x
|
||||
|
||||
x = torch.randn(4, 8)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
compiled_fn = torch.compile(model_fn, backend=capturing_backend)
|
||||
compiled_fn(x)
|
||||
|
||||
split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
|
||||
assert len(split_items) == 3
|
||||
|
||||
submod_0 = split_gm.submod_0
|
||||
example_input = torch.randn(4, 8)
|
||||
compiled = standalone_compile(
|
||||
submod_0, [example_input, 4], dynamic_shapes="from_example_inputs"
|
||||
)
|
||||
assert compiled is not None
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_symint_crosses_split_boundary():
|
||||
"""Test that SymInt placeholders from torch.compile + mark_dynamic
|
||||
cross split boundaries safely via split_module's natural threading.
|
||||
|
||||
SymInt values are threaded through subgraphs by split_module and
|
||||
handled correctly by inductor — no special replacement is needed.
|
||||
"""
|
||||
captured_graph = None
|
||||
|
||||
def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
|
||||
nonlocal captured_graph
|
||||
captured_graph = gm
|
||||
return gm
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
batch_size = x.shape[0]
|
||||
hidden_size = x.shape[1]
|
||||
x = torch.ops.aten.sigmoid.default(x)
|
||||
x = x.clone().view(batch_size, hidden_size)
|
||||
x = torch.ops.aten.sigmoid.default(x)
|
||||
x = x.clone().view(batch_size, hidden_size)
|
||||
x = torch.ops.aten.sigmoid.default(x)
|
||||
x = x.clone().view(batch_size, hidden_size)
|
||||
return x
|
||||
|
||||
x = torch.randn(4, 8)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
|
||||
compiled_fn = torch.compile(model_fn, backend=capturing_backend)
|
||||
compiled_fn(x)
|
||||
|
||||
assert captured_graph is not None, "Graph should be captured by backend"
|
||||
|
||||
# SymInt placeholders should exist in the captured graph
|
||||
symint_placeholders = [
|
||||
node
|
||||
for node in captured_graph.graph.nodes
|
||||
if node.op == "placeholder"
|
||||
and isinstance(node.meta.get("example_value"), torch.SymInt)
|
||||
]
|
||||
assert len(symint_placeholders) > 0, (
|
||||
"Captured graph should have SymInt placeholders from mark_dynamic."
|
||||
)
|
||||
|
||||
# split_graph should handle SymInt placeholders without error
|
||||
split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
|
||||
|
||||
# Should have 3 splitting subgraphs (3 sigmoids)
|
||||
splitting_subgraphs = [item for item in split_items if item.is_splitting_graph]
|
||||
assert len(splitting_subgraphs) == 3, (
|
||||
f"Expected 3 splitting subgraphs (3 sigmoids), got {len(splitting_subgraphs)}"
|
||||
)
|
||||
assert len(split_items) >= 6, (
|
||||
f"Expected at least 6 total subgraphs, got {len(split_items)}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_shape_boundary_standalone_compile():
|
||||
"""Repro for the original production bug:
|
||||
|
||||
AssertionError: out_spec mismatch
|
||||
TreeSpec(tuple, None, [*, *, TreeSpec(Size, None, [*, *]), *])
|
||||
vs
|
||||
TreeSpec(tuple, None, [*, *, *, *])
|
||||
|
||||
A subgraph outputs torch.Size (e.g. torch.Size([s72, 2048])) as one of
|
||||
its values when shape info crosses a split boundary. aot_autograd / inductor
|
||||
expect all submodule outputs to be flat tensors or scalars, not torch.Size.
|
||||
|
||||
With the fix, x.size() is decomposed into individual sym_size.int calls
|
||||
so only scalar SymInts cross the boundary — not the torch.Size.
|
||||
"""
|
||||
from torch._inductor import standalone_compile
|
||||
|
||||
captured_graph = None
|
||||
|
||||
def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
|
||||
nonlocal captured_graph
|
||||
captured_graph = gm
|
||||
return gm
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
shape = x.size()
|
||||
x = torch.ops.aten.sigmoid.default(x)
|
||||
x = x.clone().view(shape)
|
||||
return x
|
||||
|
||||
x = torch.randn(4, 8)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
torch.compile(model_fn, backend=capturing_backend)(x)
|
||||
|
||||
split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
|
||||
assert len(split_items) == 3
|
||||
|
||||
# Verify that the consumer subgraph only has a placeholder for the dynamic
|
||||
# dim (SymInt) — the static dim (8) should be inlined as a literal, not
|
||||
# threaded as a placeholder.
|
||||
consumer = split_items[-1] # valid since len == 3: [producer, sigmoid, consumer]
|
||||
symint_placeholders = [
|
||||
n
|
||||
for n in consumer.graph.graph.nodes
|
||||
if n.op == "placeholder"
|
||||
and isinstance(n.meta.get("example_value"), torch.SymInt)
|
||||
]
|
||||
static_int_placeholders = [
|
||||
n
|
||||
for n in consumer.graph.graph.nodes
|
||||
if n.op == "placeholder"
|
||||
and isinstance(n.meta.get("example_value"), int)
|
||||
and not isinstance(n.meta.get("example_value"), torch.SymInt)
|
||||
]
|
||||
assert len(symint_placeholders) >= 1, (
|
||||
"Consumer should have a SymInt placeholder for the dynamic dim."
|
||||
)
|
||||
assert len(static_int_placeholders) == 0, (
|
||||
"Static dims should be inlined as literals, not threaded as placeholders."
|
||||
)
|
||||
|
||||
submod_0 = split_gm.submod_0
|
||||
|
||||
standalone_compile(
|
||||
submod_0, [torch.randn(4, 8), 4], dynamic_shapes="from_example_inputs"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_size_used_in_multiple_consumer_subgraphs():
|
||||
"""Validates that x.size() (whole shape) used by multiple downstream subgraphs
|
||||
does not cause torch.Size to cross split boundaries.
|
||||
|
||||
Model:
|
||||
shape = x.size() # whole shape — must not cross as torch.Size
|
||||
z1 = sigmoid(x) # split point 1
|
||||
y1 = y.view(shape) # consumer 1 uses shape
|
||||
z2 = sigmoid(z1) # split point 2
|
||||
y2 = y.view(shape) # consumer 2 uses shape again
|
||||
|
||||
Without the fix, torch.Size crosses the boundary as a submodule output,
|
||||
which aot_autograd / standalone_compile rejects.
|
||||
"""
|
||||
captured_graph = None
|
||||
captured_inputs = None
|
||||
|
||||
def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
|
||||
nonlocal captured_graph, captured_inputs
|
||||
captured_graph = gm
|
||||
captured_inputs = example_inputs
|
||||
return gm
|
||||
|
||||
def model_fn(x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
|
||||
shape = x.size()
|
||||
z1 = torch.ops.aten.sigmoid.default(x)
|
||||
y1 = y.view(shape)
|
||||
z2 = torch.ops.aten.sigmoid.default(z1)
|
||||
y2 = y.view(shape)
|
||||
return z2 + y1 + y2
|
||||
|
||||
x = torch.randn(4, 8)
|
||||
y = torch.randn(4, 8) # same shape as x so view(shape) doesn't specialize dim 0
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
torch._dynamo.mark_dynamic(y, 0)
|
||||
torch.compile(model_fn, backend=capturing_backend)(x, y)
|
||||
assert captured_graph is not None, "Graph should be captured by backend"
|
||||
assert captured_inputs is not None, "Example inputs should be captured by backend"
|
||||
|
||||
split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
|
||||
|
||||
splitting_items = [item for item in split_items if item.is_splitting_graph]
|
||||
assert len(splitting_items) == 2
|
||||
|
||||
# Verify functional correctness — fails without the fix because torch.Size
|
||||
# would cross a split boundary as a submodule output
|
||||
output_original = model_fn(x, y)
|
||||
output_split = split_gm(*captured_inputs)
|
||||
if isinstance(output_split, tuple):
|
||||
output_split = next(o for o in output_split if isinstance(o, torch.Tensor))
|
||||
assert torch.allclose(output_original, output_split), "Output mismatch after split"
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
def test_sym_size_metadata_propagated():
|
||||
"""Validates that new sym_size.int nodes created by the pre-pass have
|
||||
example_value metadata set. Without it, placeholder metadata in consumer
|
||||
subgraphs would be None, breaking any code that dynamically builds
|
||||
example inputs from metadata (e.g. standalone_compile per-submodule).
|
||||
"""
|
||||
from torch._inductor import standalone_compile
|
||||
|
||||
captured_graph = None
|
||||
|
||||
def capturing_backend(gm: fx.GraphModule, example_inputs: list) -> fx.GraphModule:
|
||||
nonlocal captured_graph
|
||||
captured_graph = gm
|
||||
return gm
|
||||
|
||||
def model_fn(x: torch.Tensor) -> torch.Tensor:
|
||||
shape = x.size()
|
||||
x = torch.ops.aten.sigmoid.default(x)
|
||||
x = x.clone().view(shape)
|
||||
return x
|
||||
|
||||
x = torch.randn(4, 8)
|
||||
torch._dynamo.mark_dynamic(x, 0)
|
||||
torch.compile(model_fn, backend=capturing_backend)(x)
|
||||
|
||||
split_gm, split_items = split_graph(captured_graph, ["aten::sigmoid"])
|
||||
|
||||
# For each submodule, build example inputs purely from placeholder metadata.
|
||||
# This fails if example_value is None on any placeholder (i.e. metadata
|
||||
# was not propagated to the sym_size.int nodes we created).
|
||||
for item in split_items:
|
||||
submod = item.graph
|
||||
example_inputs = []
|
||||
for n in submod.graph.nodes:
|
||||
if n.op != "placeholder":
|
||||
continue
|
||||
ev = n.meta.get("example_value")
|
||||
assert ev is not None, (
|
||||
f"Placeholder '{n.name}' in {item.submod_name} has no "
|
||||
"example_value metadata. sym_size.int nodes must propagate "
|
||||
"metadata so consumer subgraphs can be introspected."
|
||||
)
|
||||
if isinstance(ev, torch.Tensor):
|
||||
example_inputs.append(torch.randn(*(int(d) for d in ev.shape)))
|
||||
else:
|
||||
example_inputs.append(int(ev))
|
||||
standalone_compile(submod, example_inputs, dynamic_shapes="from_example_inputs")
|
||||
|
||||
|
||||
def test_decompose_size_with_getitem_user():
|
||||
"""Regression test: _decompose_size_nodes must handle getitem users of size()
|
||||
correctly.
|
||||
|
||||
When a graph contains x.shape[i], it can appear as:
|
||||
|
||||
%size = call_method[target="size"](args = (%x,))
|
||||
%getitem = call_function[target=operator.getitem](args = (%size, 1))
|
||||
|
||||
The old code spliced *all* per-dim values into every user's args
|
||||
unconditionally, turning the 2-arg getitem into a malformed 3-arg node:
|
||||
|
||||
%getitem(args = (%sym_size_int, 5120, 1)) # TypeError at runtime
|
||||
|
||||
The fix detects getitem users and replaces them with dims[idx] directly.
|
||||
"""
|
||||
# Build a graph manually to guarantee the size() + getitem pattern.
|
||||
#
|
||||
# Graph:
|
||||
# %x = placeholder
|
||||
# %size = x.size()
|
||||
# %dim1 = getitem(%size, 1) <-- the getitem branch we're testing
|
||||
# %relu = relu(%x)
|
||||
# %view = view(%relu, -1, %dim1)
|
||||
# return %view
|
||||
graph = fx.Graph()
|
||||
x = graph.placeholder("x")
|
||||
size_node = graph.call_method("size", args=(x,))
|
||||
getitem_node = graph.call_function(operator.getitem, args=(size_node, 1))
|
||||
relu_node = graph.call_function(torch.ops.aten.relu.default, args=(x,))
|
||||
view_node = graph.call_function(
|
||||
torch.ops.aten.view.default, args=(relu_node, [-1, getitem_node])
|
||||
)
|
||||
graph.output(view_node)
|
||||
|
||||
# Attach example_value metadata so _decompose_size_nodes can inspect dims.
|
||||
# dim 0 is dynamic (SymInt), dim 1 is static (8).
|
||||
from torch._dynamo.source import LocalSource
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
from torch.fx.experimental.symbolic_shapes import ShapeEnv
|
||||
|
||||
shape_env = ShapeEnv()
|
||||
src = LocalSource("batch_size")
|
||||
sym_batch = shape_env.create_symintnode(shape_env.create_symbol(4, src), hint=4)
|
||||
fake_mode = FakeTensorMode(shape_env=shape_env)
|
||||
with fake_mode:
|
||||
fake_x = torch.empty_strided((sym_batch, 8), (8, 1))
|
||||
x.meta["example_value"] = fake_x
|
||||
|
||||
gm = fx.GraphModule(torch.nn.Module(), graph)
|
||||
|
||||
# Run decomposition — this would produce a 3-arg getitem without the fix
|
||||
_decompose_size_nodes(gm)
|
||||
|
||||
# Verify no size() nodes remain
|
||||
remaining_size_nodes = list(gm.graph.find_nodes(op="call_method", target="size"))
|
||||
assert len(remaining_size_nodes) == 0, (
|
||||
f"size() nodes should be fully decomposed, found {len(remaining_size_nodes)}"
|
||||
)
|
||||
|
||||
# Verify no malformed getitem nodes (3+ args)
|
||||
for node in gm.graph.nodes:
|
||||
if node.op == "call_function" and node.target is operator.getitem:
|
||||
assert len(node.args) == 2, (
|
||||
f"getitem node '{node.name}' has {len(node.args)} args "
|
||||
f"(expected 2): {node.args}"
|
||||
)
|
||||
|
||||
|
||||
def test_decompose_size_leaves_scalar_size_with_dim():
|
||||
"""Regression test: _decompose_size_nodes must leave x.size(dim) alone.
|
||||
|
||||
x.size() returns a torch.Size tuple that can't cross split boundaries and
|
||||
must be decomposed. x.size(dim), however, already returns a scalar
|
||||
SymInt/int that crosses fine, so the pass must not touch it.
|
||||
|
||||
The punica LoRA path traces token_lora_mapping[:x.size(0)] under a dynamic
|
||||
batch dim, so the size(0) node ends up nested inside a slice object:
|
||||
|
||||
%size = call_method[target="size"](args = (%x, 0))
|
||||
%slice = call_function[target=getitem](
|
||||
args = (%mapping, slice(None, %size, None)))
|
||||
|
||||
The old pass tried to decompose this scalar node too and then erase it, but
|
||||
the slice still referenced it, raising "Tried to erase Node size but it
|
||||
still had N users". The fix skips size calls that carry a dim argument.
|
||||
"""
|
||||
from torch._dynamo.source import LocalSource
|
||||
from torch._subclasses.fake_tensor import FakeTensorMode
|
||||
from torch.fx.experimental.symbolic_shapes import ShapeEnv
|
||||
|
||||
# Build graph:
|
||||
# %x = placeholder
|
||||
# %mapping = placeholder
|
||||
# %size = x.size(0) # scalar, with a dim arg
|
||||
# %sliced = mapping[slice(None, %size, None)] # size node inside a slice
|
||||
graph = fx.Graph()
|
||||
x = graph.placeholder("x")
|
||||
mapping = graph.placeholder("token_lora_mapping")
|
||||
size_node = graph.call_method("size", args=(x, 0))
|
||||
sliced_node = graph.call_function(
|
||||
operator.getitem,
|
||||
args=(mapping, slice(None, size_node, None)),
|
||||
)
|
||||
graph.output((sliced_node,))
|
||||
|
||||
# dim 0 dynamic (SymInt) — the realistic Unsloth + LoRA case. Without the
|
||||
# skip, the pass would build per-dim replacements and then crash trying to
|
||||
# erase the still-referenced size node.
|
||||
shape_env = ShapeEnv()
|
||||
src = LocalSource("tokens")
|
||||
sym_tokens = shape_env.create_symintnode(shape_env.create_symbol(4, src), hint=4)
|
||||
fake_mode = FakeTensorMode(shape_env=shape_env)
|
||||
with fake_mode:
|
||||
fake_x = torch.empty_strided((sym_tokens, 8), (8, 1))
|
||||
x.meta["example_value"] = fake_x
|
||||
|
||||
gm = fx.GraphModule(torch.nn.Module(), graph)
|
||||
|
||||
# Must not raise "Tried to erase Node ... still had N users".
|
||||
_decompose_size_nodes(gm)
|
||||
|
||||
# The scalar x.size(0) node is left in place, untouched.
|
||||
remaining = list(gm.graph.find_nodes(op="call_method", target="size"))
|
||||
assert len(remaining) == 1, (
|
||||
f"x.size(0) should be left untouched, found {len(remaining)} size nodes"
|
||||
)
|
||||
assert remaining[0].args == (x, 0), (
|
||||
f"size node args changed: {remaining[0].args} (expected (x, 0))"
|
||||
)
|
||||
@@ -1,70 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import (
|
||||
CompilationConfig,
|
||||
ModelConfig,
|
||||
VllmConfig,
|
||||
set_current_vllm_config,
|
||||
)
|
||||
from vllm.config.compilation import CompilationMode, CUDAGraphMode
|
||||
from vllm.model_executor.layers.rotary_embedding import get_rope
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class RotaryEmbeddingCompileModule(torch.nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
super().__init__()
|
||||
self.rotary_emb = get_rope(
|
||||
head_size=32,
|
||||
max_position=128,
|
||||
dtype=torch.float32,
|
||||
rope_parameters={"rope_type": "default", "rope_theta": 10000},
|
||||
is_neox_style=True,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self, positions: torch.Tensor, query: torch.Tensor, key: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
q_rot, k_rot = self.rotary_emb(positions, query, key)
|
||||
return q_rot + k_rot
|
||||
|
||||
|
||||
@pytest.mark.skipif(current_platform.is_cpu(), reason="Requires GPU for torch.compile")
|
||||
def test_rotary_embedding_torch_compile_with_custom_op(monkeypatch):
|
||||
# Ensure env toggles take effect for this test only.
|
||||
# The bytecode hook is required to detect buffer mutation in compiled code,
|
||||
# and AOT compile bypasses that hook entirely.
|
||||
envs.disable_envs_cache()
|
||||
monkeypatch.setenv("VLLM_USE_BYTECODE_HOOK", "1")
|
||||
monkeypatch.setenv("VLLM_USE_AOT_COMPILE", "0")
|
||||
|
||||
device = DEVICE_TYPE
|
||||
positions = torch.arange(16, device=device)
|
||||
query = torch.randn(16, 32, device=device, dtype=torch.bfloat16)
|
||||
key = torch.randn(16, 32, device=device, dtype=torch.bfloat16)
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=torch.bfloat16),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
backend="inductor",
|
||||
custom_ops=["+rotary_embedding"],
|
||||
cudagraph_mode=CUDAGraphMode.NONE,
|
||||
cudagraph_num_of_warmups=0,
|
||||
),
|
||||
)
|
||||
|
||||
with set_current_vllm_config(vllm_config):
|
||||
model = RotaryEmbeddingCompileModule(vllm_config=vllm_config)
|
||||
model(positions, query, key)
|
||||
assert model._compiled_bytecode is not None
|
||||
assert "update" not in model._compiled_bytecode.co_names
|
||||
@@ -1,124 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
import tests.compile.silly_attention # noqa
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.compilation import (
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
CUDAGraphMode,
|
||||
)
|
||||
from vllm.config.scheduler import SchedulerConfig
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
MLP_SIZE = 64
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
@support_torch_compile
|
||||
class SimpleModel(nn.Module):
|
||||
"""A simple model with a splitting op for piecewise compilation."""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "", **kwargs):
|
||||
super().__init__()
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = x + x
|
||||
attn_output = torch.empty_like(x)
|
||||
torch.ops.silly.attention(x, x, x, attn_output)
|
||||
x = attn_output * 2
|
||||
return x
|
||||
|
||||
|
||||
class TraceStructuredCapture:
|
||||
"""Captures trace_structured calls for testing."""
|
||||
|
||||
def __init__(self):
|
||||
self.calls: list[dict] = []
|
||||
|
||||
def __call__(self, event_type: str, metadata_fn=None, payload_fn=None, **kwargs):
|
||||
"""Capture a trace_structured call."""
|
||||
metadata = metadata_fn() if metadata_fn else {}
|
||||
self.calls.append(
|
||||
{
|
||||
"event_type": event_type,
|
||||
"metadata": metadata,
|
||||
}
|
||||
)
|
||||
|
||||
def get(self, event_type: str, name_pattern: str) -> list[dict]:
|
||||
"""Get all calls with the given event type and name matching pattern.
|
||||
|
||||
Args:
|
||||
event_type: The event type to filter by (e.g., "artifact", "graph_dump")
|
||||
name_pattern: Regex pattern to match against the artifact name
|
||||
|
||||
"""
|
||||
regex = re.compile(name_pattern)
|
||||
return [
|
||||
c
|
||||
for c in self.calls
|
||||
if c["event_type"] == event_type
|
||||
and regex.fullmatch(c.get("metadata", {}).get("name", ""))
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_vllm_structured_logging_artifacts(use_fresh_inductor_cache):
|
||||
"""Test that all expected vLLM artifacts are logged during compilation."""
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
|
||||
capture = TraceStructuredCapture()
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
cudagraph_mode=CUDAGraphMode.PIECEWISE,
|
||||
compile_sizes=[8],
|
||||
splitting_ops=["silly::attention"],
|
||||
),
|
||||
scheduler_config=SchedulerConfig(
|
||||
max_num_seqs=8,
|
||||
max_model_len=8192,
|
||||
is_encoder_decoder=False,
|
||||
),
|
||||
)
|
||||
|
||||
# Patch trace_structured to capture calls
|
||||
with (
|
||||
patch("vllm.compilation.backends.trace_structured", capture),
|
||||
patch("vllm.compilation.piecewise_backend.trace_structured", capture),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
model = SimpleModel(vllm_config=vllm_config, prefix="test")
|
||||
with set_forward_context({}, vllm_config=vllm_config):
|
||||
model(torch.randn(8, MLP_SIZE))
|
||||
|
||||
config_artifacts = capture.get("artifact", "vllm_compilation_config")
|
||||
assert len(config_artifacts) == 1, (
|
||||
f"Expected 1 vllm_compilation_config, got {len(config_artifacts)}"
|
||||
)
|
||||
vllm_piecewise_split_graph = capture.get("graph_dump", "vllm_piecewise_split_graph")
|
||||
assert len(vllm_piecewise_split_graph) == 1, (
|
||||
"Expected 1 toplevel piecewise split graph, "
|
||||
f"got {len(vllm_piecewise_split_graph)}"
|
||||
)
|
||||
compile_start_artifacts = capture.get("artifact", "vllm_piecewise_compile_start")
|
||||
assert len(compile_start_artifacts) == 4, (
|
||||
"Expected 4 vllm_piecewise_compile_start "
|
||||
"(2 subgraphs x 2 ranges each: dynamic + compile size), "
|
||||
f"got {len(compile_start_artifacts)}"
|
||||
)
|
||||
submod_dumps = capture.get("graph_dump", r"vllm_submod_.*")
|
||||
assert len(submod_dumps) == 2, (
|
||||
"Expected 2 submods (one before attention, one after attention), "
|
||||
f"got {len(submod_dumps)}"
|
||||
)
|
||||
Reference in New Issue
Block a user