[ROCm][Compile] Fuse AITER static FP8 attention output (#58099)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Andreas Karatzas
2026-09-23 03:16:52 -05:00
committed by GitHub
co-authored by OpenAI Codex
parent 973a3be780
commit 26879f3260
5 changed files with 251 additions and 12 deletions
+23
View File
@@ -3452,6 +3452,29 @@ steps:
commands:
- pytest -v -s compile/h100/test_startup.py
- label: ":amd: (MI355) Fusion and Compile FP8"
timeout_in_minutes: 45
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
dind: false
agent_pool: mi355_2
num_gpus: 2
optional: true
working_dir: "/vllm-workspace/"
source_file_dependencies:
- vllm/compilation/
- vllm/model_executor/layers/
- vllm/v1/attention/
- vllm/distributed/
- vllm/_aiter_ops.py
- tests/compile/passes/
- tests/compile/correctness_e2e/test_attn_quant.py
- vllm/model_executor/kernels/linear/
- vllm/model_executor/models/llama.py
commands:
- amd-smi
- pytest -v -s tests/compile/passes/test_fusion_attn.py tests/compile/passes/test_silu_mul_quant_fusion.py tests/compile/passes/test_silu_mul_quant_manual_fusion.py
- pytest -v -s tests/compile/correctness_e2e/test_attn_quant.py
#-------------------------------------------------------- mi355 · distributed --------------------------------------------------------#
- label: ":amd: (MI355) Sharded RDT Weight Transfer"
+25
View File
@@ -67,6 +67,31 @@ steps:
- pytest -v -s tests/compile/passes/test_silu_mul_quant_manual_fusion.py
# this runner has 2 GPUs available even though num_devices=2 is not set
- pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py
mirror:
amd:
label: ":amd: (MI355) Fusion and Compile FP8"
dind: false
device: mi355_2
num_devices: 2
working_dir: "/vllm-workspace/"
timeout_in_minutes: 45
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/compilation/
- vllm/model_executor/layers/
- vllm/v1/attention/
- vllm/distributed/
- vllm/_aiter_ops.py
- tests/compile/passes/
- tests/compile/correctness_e2e/test_attn_quant.py
- vllm/model_executor/kernels/linear/
- vllm/model_executor/models/llama.py
commands:
- amd-smi
# MLA static FP8 and distributed AR/RMS are owned by the existing ROCm jobs.
- pytest -v -s tests/compile/passes/test_fusion_attn.py tests/compile/passes/test_silu_mul_quant_fusion.py tests/compile/passes/test_silu_mul_quant_manual_fusion.py
- pytest -v -s tests/compile/correctness_e2e/test_attn_quant.py
- label: ":nvidia: (H100) Fusion E2E Quick"
key: fusion-e2e-quick-h100
@@ -0,0 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import logging
import pytest
import regex as re
from tests.utils import compare_two_settings
from vllm._aiter_ops import is_aiter_found_and_supported
from vllm.config import CompilationMode
from vllm.platforms import current_platform
from vllm.transformers_utils.repo_utils import hf_api
@pytest.mark.skipif(
not current_platform.is_rocm() or not is_aiter_found_and_supported(),
reason="Requires ROCm AITER attention and quantization",
)
def test_rocm_aiter_static_attention_output_correctness(caplog_mp_spawn):
"""Trained FP8 outputs agree when actual AITER output quantization is fused."""
model = hf_api().snapshot_download(
"RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
revision="870a1177ed082c0ba5dbefd62573ab8f91803f6a",
allow_patterns=["*.json", "*.safetensors", "*.model", "*.txt", "*.jinja"],
)
common_args = [
"--dtype",
"bfloat16",
"--load-format",
"auto",
"--max-model-len",
"512",
"--max-num-batched-tokens",
"512",
"--max-num-seqs",
"8",
"--kv-cache-memory-bytes",
str(256 * 1024**2),
"--gpu-memory-utilization",
"0.01",
"--attention-backend",
"ROCM_AITER_UNIFIED_ATTN",
]
def settings(enabled):
return [
*common_args,
"--compilation-config",
json.dumps(
{
"mode": CompilationMode.VLLM_COMPILE,
"cudagraph_mode": "NONE",
"splitting_ops": [],
"inductor_compile_config": {"force_disable_caches": True},
"custom_ops": ["+quant_fp8", "-rms_norm"],
"pass_config": {
"fuse_attn_quant": enabled,
"fuse_norm_quant": False,
"fuse_act_quant": False,
"fuse_allreduce_rms": False,
},
}
),
]
env = {
"VLLM_ROCM_USE_AITER": "1",
"VLLM_ROCM_USE_AITER_LINEAR": "1",
"VLLM_DISABLE_COMPILE_CACHE": "1",
}
with caplog_mp_spawn(logging.DEBUG) as logs:
compare_two_settings(
model,
settings(False),
settings(True),
env1=env,
env2=env,
method="generate",
force_v1_runner=True,
)
matches = [
int(count) for count in re.findall(r"'attn_quant_fusion': (\d+)", logs.text)
]
print(f"Attention output quantization rewrite counts: {matches}")
assert matches and all(count == 32 for count in matches), (
f"Expected output quantization fused in all 32 Llama layers, found {matches}"
)
+61 -5
View File
@@ -12,6 +12,7 @@ from tests.v1.attention.utils import (
create_common_attn_metadata,
dense_kv_cache_views,
)
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
from vllm.compilation.passes.fusion.attn_quant_fusion import (
ATTN_OP,
@@ -189,6 +190,17 @@ class TestAttentionFp8StaticQuantPatternModel(AttentionQuantPatternModel):
return self.fp8_linear(attn_output)
class TestAttentionAiterFp8StaticQuantPatternModel(
TestAttentionFp8StaticQuantPatternModel
):
aiter_quant = True
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.fp8_linear.kernel.quant_fp8.use_aiter = True
self.fp8_linear.input_scale.fill_(0.125)
class TestAttentionNvfp4QuantPatternModel(AttentionQuantPatternModel):
"""Test model for AttentionNvfp4QuantPattern fusion."""
@@ -455,11 +467,12 @@ def test_attention_quant_pattern(
)
# Check quantization ops in the graph before and after fusion
quant_op = (
torch.ops.aten.reciprocal
if "-quant_fp8" in custom_ops_list
else QUANT_OPS[quant_key]
)
if "-quant_fp8" in custom_ops_list:
quant_op = torch.ops.aten.reciprocal
elif getattr(model_class, "aiter_quant", False):
quant_op = torch.ops.vllm.rocm_aiter_per_tensor_quant.default
else:
quant_op = QUANT_OPS[quant_key]
if expect_fusion:
# Note: for fp8, fully_replaced=False because query quant ops remain in
@@ -523,3 +536,46 @@ def test_attention_quant_pattern(
# Check that results are close
torch.testing.assert_close(result_unfused, result_fused, atol=1e-2, rtol=1e-2)
if getattr(model_class, "aiter_quant", False):
# Query quantization remains; exactly the output quant op is removed.
assert test_backend.op_count(quant_op, before=True) == (
test_backend.op_count(quant_op) + 1
)
for model in (model_unfused, model_fused):
torch.testing.assert_close(
model.fp8_linear.input_scale,
torch.full_like(model.fp8_linear.input_scale, 0.125),
atol=0,
rtol=0,
)
@pytest.mark.skipif(
not current_platform.is_rocm() or not is_aiter_found_and_supported(),
reason="Requires ROCm AITER attention and quantization",
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
def test_rocm_aiter_static_quant_attention(
dtype, dist_init, disable_vllm_compile_cache, monkeypatch
):
"""Fuse the actual AITER op without changing its static scale or outputs."""
try:
with monkeypatch.context() as patch:
patch.setenv("VLLM_ROCM_USE_AITER", "1")
rocm_aiter_ops.refresh_env_variables()
test_attention_quant_pattern(
num_qo_heads=32,
num_kv_heads=8,
head_size=128,
batch_size=8,
dtype=dtype,
kv_cache_dtype="fp8",
custom_ops="+quant_fp8",
model_name="amd/Llama-3.1-8B-Instruct-FP8-KV",
model_class=TestAttentionAiterFp8StaticQuantPatternModel,
backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
dist_init=dist_init,
disable_vllm_compile_cache=disable_vllm_compile_cache,
)
finally:
rocm_aiter_ops.refresh_env_variables()
@@ -7,6 +7,7 @@ from collections.abc import Callable
import torch
from torch._higher_order_ops.auto_functionalize import auto_functionalized
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention
@@ -35,7 +36,9 @@ RESHAPE_OP = torch.ops.aten.reshape.default
_FP8_QUANT_KEY = QuantKey(dtype=FP8_DTYPE, scale=kStaticTensorScale, symmetric=True)
class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
class AttnFp8StaticQuantPattern(
VllmPatternReplacement[..., torch.Tensor | tuple[torch.Tensor, torch.Tensor]]
):
"""Fusion for Attention+Fp8StaticQuant.
Only triggers when the attention implementation returns True in
@@ -51,8 +54,20 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
self._dtype = dtype
self._quant_matcher = MatcherQuantFP8(_FP8_QUANT_KEY)
def _quantize(
self, value: torch.Tensor, scale: torch.Tensor
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
return self._quant_matcher(value, scale)[0]
def _output(
self, value: torch.Tensor, scale: torch.Tensor
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
return value
@property
def pattern(self) -> Callable[..., torch.Tensor]:
def pattern(
self,
) -> Callable[..., torch.Tensor | tuple[torch.Tensor, torch.Tensor]]:
# When _USE_LAYERNAME is enabled (torch >= 2.11), layer_name is
# passed as an explicit pattern input so the pattern matcher
# treats it as a wildcard matching hoisted LayerName placeholders.
@@ -78,7 +93,7 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
attn_out_view = RESHAPE_OP(
at1[1], [q.shape[0], self._num_heads * self._head_size]
)
return self._quant_matcher(attn_out_view, scale)[0]
return self._quantize(attn_out_view, scale)
return _pattern_with_ln
@@ -97,12 +112,14 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
attn_out_view = RESHAPE_OP(
at1[1], [q.shape[0], self._num_heads * self._head_size]
)
return self._quant_matcher(attn_out_view, scale)[0]
return self._quantize(attn_out_view, scale)
return _pattern
@property
def replacement(self) -> Callable[..., torch.Tensor]:
def replacement(
self,
) -> Callable[..., torch.Tensor | tuple[torch.Tensor, torch.Tensor]]:
_ln = _encode_layer_name(self._layer_name)
if _USE_LAYERNAME:
@@ -126,7 +143,9 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
return RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size])
return self._output(
RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size]), scale
)
return _replacement_with_ln
@@ -147,7 +166,9 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
output_block_scale=None,
kv_cache_dummy_dep=kv_cache_dummy_dep,
)
return RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size])
return self._output(
RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size]), scale
)
return _replacement
@@ -168,6 +189,30 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
return inputs
class RocmAttnFp8StaticQuantPattern(AttnFp8StaticQuantPattern):
"""Fuse AITER static quantization while preserving its scale output."""
def _quantize(
self, value: torch.Tensor, scale: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
out = torch.empty_like(value, dtype=FP8_DTYPE)
_, out, scale = auto_functionalized(
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
out=out,
x=value,
scale=scale,
is_dynamic=False,
)
# The shared AITER op declares scale mutable for its dynamic variant.
# Static quant preserves it; keep this graph output when replacing it.
return out, scale
def _output(
self, value: torch.Tensor, scale: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
return value, scale
class AttnNvfp4QuantPattern(
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
):
@@ -387,6 +432,8 @@ class AttnQuantFusionPass(VllmFusionPatternMatcherPass):
for layer in layers:
if layer.impl.fused_output_quant_supported(_FP8_QUANT_KEY):
self.register(AttnFp8StaticQuantPattern(layer, dtype))
if current_platform.is_rocm() and rocm_aiter_ops.is_enabled():
self.register(RocmAttnFp8StaticQuantPattern(layer, dtype))
if _USE_LAYERNAME:
break