[ROCm][Compile] Fuse AITER static FP8 attention output (#58099)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
973a3be780
commit
26879f3260
@@ -3452,6 +3452,29 @@ steps:
|
||||
commands:
|
||||
- pytest -v -s compile/h100/test_startup.py
|
||||
|
||||
- label: ":amd: (MI355) Fusion and Compile FP8"
|
||||
timeout_in_minutes: 45
|
||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355]
|
||||
dind: false
|
||||
agent_pool: mi355_2
|
||||
num_gpus: 2
|
||||
optional: true
|
||||
working_dir: "/vllm-workspace/"
|
||||
source_file_dependencies:
|
||||
- vllm/compilation/
|
||||
- vllm/model_executor/layers/
|
||||
- vllm/v1/attention/
|
||||
- vllm/distributed/
|
||||
- vllm/_aiter_ops.py
|
||||
- tests/compile/passes/
|
||||
- tests/compile/correctness_e2e/test_attn_quant.py
|
||||
- vllm/model_executor/kernels/linear/
|
||||
- vllm/model_executor/models/llama.py
|
||||
commands:
|
||||
- amd-smi
|
||||
- pytest -v -s tests/compile/passes/test_fusion_attn.py tests/compile/passes/test_silu_mul_quant_fusion.py tests/compile/passes/test_silu_mul_quant_manual_fusion.py
|
||||
- pytest -v -s tests/compile/correctness_e2e/test_attn_quant.py
|
||||
|
||||
#-------------------------------------------------------- mi355 · distributed --------------------------------------------------------#
|
||||
|
||||
- label: ":amd: (MI355) Sharded RDT Weight Transfer"
|
||||
|
||||
@@ -67,6 +67,31 @@ steps:
|
||||
- pytest -v -s tests/compile/passes/test_silu_mul_quant_manual_fusion.py
|
||||
# this runner has 2 GPUs available even though num_devices=2 is not set
|
||||
- pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI355) Fusion and Compile FP8"
|
||||
dind: false
|
||||
device: mi355_2
|
||||
num_devices: 2
|
||||
working_dir: "/vllm-workspace/"
|
||||
timeout_in_minutes: 45
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
- vllm/compilation/
|
||||
- vllm/model_executor/layers/
|
||||
- vllm/v1/attention/
|
||||
- vllm/distributed/
|
||||
- vllm/_aiter_ops.py
|
||||
- tests/compile/passes/
|
||||
- tests/compile/correctness_e2e/test_attn_quant.py
|
||||
- vllm/model_executor/kernels/linear/
|
||||
- vllm/model_executor/models/llama.py
|
||||
commands:
|
||||
- amd-smi
|
||||
# MLA static FP8 and distributed AR/RMS are owned by the existing ROCm jobs.
|
||||
- pytest -v -s tests/compile/passes/test_fusion_attn.py tests/compile/passes/test_silu_mul_quant_fusion.py tests/compile/passes/test_silu_mul_quant_manual_fusion.py
|
||||
- pytest -v -s tests/compile/correctness_e2e/test_attn_quant.py
|
||||
|
||||
- label: ":nvidia: (H100) Fusion E2E Quick"
|
||||
key: fusion-e2e-quick-h100
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import logging
|
||||
|
||||
import pytest
|
||||
import regex as re
|
||||
|
||||
from tests.utils import compare_two_settings
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.config import CompilationMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.repo_utils import hf_api
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm() or not is_aiter_found_and_supported(),
|
||||
reason="Requires ROCm AITER attention and quantization",
|
||||
)
|
||||
def test_rocm_aiter_static_attention_output_correctness(caplog_mp_spawn):
|
||||
"""Trained FP8 outputs agree when actual AITER output quantization is fused."""
|
||||
model = hf_api().snapshot_download(
|
||||
"RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8",
|
||||
revision="870a1177ed082c0ba5dbefd62573ab8f91803f6a",
|
||||
allow_patterns=["*.json", "*.safetensors", "*.model", "*.txt", "*.jinja"],
|
||||
)
|
||||
common_args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--load-format",
|
||||
"auto",
|
||||
"--max-model-len",
|
||||
"512",
|
||||
"--max-num-batched-tokens",
|
||||
"512",
|
||||
"--max-num-seqs",
|
||||
"8",
|
||||
"--kv-cache-memory-bytes",
|
||||
str(256 * 1024**2),
|
||||
"--gpu-memory-utilization",
|
||||
"0.01",
|
||||
"--attention-backend",
|
||||
"ROCM_AITER_UNIFIED_ATTN",
|
||||
]
|
||||
|
||||
def settings(enabled):
|
||||
return [
|
||||
*common_args,
|
||||
"--compilation-config",
|
||||
json.dumps(
|
||||
{
|
||||
"mode": CompilationMode.VLLM_COMPILE,
|
||||
"cudagraph_mode": "NONE",
|
||||
"splitting_ops": [],
|
||||
"inductor_compile_config": {"force_disable_caches": True},
|
||||
"custom_ops": ["+quant_fp8", "-rms_norm"],
|
||||
"pass_config": {
|
||||
"fuse_attn_quant": enabled,
|
||||
"fuse_norm_quant": False,
|
||||
"fuse_act_quant": False,
|
||||
"fuse_allreduce_rms": False,
|
||||
},
|
||||
}
|
||||
),
|
||||
]
|
||||
|
||||
env = {
|
||||
"VLLM_ROCM_USE_AITER": "1",
|
||||
"VLLM_ROCM_USE_AITER_LINEAR": "1",
|
||||
"VLLM_DISABLE_COMPILE_CACHE": "1",
|
||||
}
|
||||
with caplog_mp_spawn(logging.DEBUG) as logs:
|
||||
compare_two_settings(
|
||||
model,
|
||||
settings(False),
|
||||
settings(True),
|
||||
env1=env,
|
||||
env2=env,
|
||||
method="generate",
|
||||
force_v1_runner=True,
|
||||
)
|
||||
matches = [
|
||||
int(count) for count in re.findall(r"'attn_quant_fusion': (\d+)", logs.text)
|
||||
]
|
||||
print(f"Attention output quantization rewrite counts: {matches}")
|
||||
assert matches and all(count == 32 for count in matches), (
|
||||
f"Expected output quantization fused in all 32 Llama layers, found {matches}"
|
||||
)
|
||||
@@ -12,6 +12,7 @@ from tests.v1.attention.utils import (
|
||||
create_common_attn_metadata,
|
||||
dense_kv_cache_views,
|
||||
)
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant
|
||||
from vllm.compilation.passes.fusion.attn_quant_fusion import (
|
||||
ATTN_OP,
|
||||
@@ -189,6 +190,17 @@ class TestAttentionFp8StaticQuantPatternModel(AttentionQuantPatternModel):
|
||||
return self.fp8_linear(attn_output)
|
||||
|
||||
|
||||
class TestAttentionAiterFp8StaticQuantPatternModel(
|
||||
TestAttentionFp8StaticQuantPatternModel
|
||||
):
|
||||
aiter_quant = True
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fp8_linear.kernel.quant_fp8.use_aiter = True
|
||||
self.fp8_linear.input_scale.fill_(0.125)
|
||||
|
||||
|
||||
class TestAttentionNvfp4QuantPatternModel(AttentionQuantPatternModel):
|
||||
"""Test model for AttentionNvfp4QuantPattern fusion."""
|
||||
|
||||
@@ -455,11 +467,12 @@ def test_attention_quant_pattern(
|
||||
)
|
||||
|
||||
# Check quantization ops in the graph before and after fusion
|
||||
quant_op = (
|
||||
torch.ops.aten.reciprocal
|
||||
if "-quant_fp8" in custom_ops_list
|
||||
else QUANT_OPS[quant_key]
|
||||
)
|
||||
if "-quant_fp8" in custom_ops_list:
|
||||
quant_op = torch.ops.aten.reciprocal
|
||||
elif getattr(model_class, "aiter_quant", False):
|
||||
quant_op = torch.ops.vllm.rocm_aiter_per_tensor_quant.default
|
||||
else:
|
||||
quant_op = QUANT_OPS[quant_key]
|
||||
|
||||
if expect_fusion:
|
||||
# Note: for fp8, fully_replaced=False because query quant ops remain in
|
||||
@@ -523,3 +536,46 @@ def test_attention_quant_pattern(
|
||||
|
||||
# Check that results are close
|
||||
torch.testing.assert_close(result_unfused, result_fused, atol=1e-2, rtol=1e-2)
|
||||
if getattr(model_class, "aiter_quant", False):
|
||||
# Query quantization remains; exactly the output quant op is removed.
|
||||
assert test_backend.op_count(quant_op, before=True) == (
|
||||
test_backend.op_count(quant_op) + 1
|
||||
)
|
||||
for model in (model_unfused, model_fused):
|
||||
torch.testing.assert_close(
|
||||
model.fp8_linear.input_scale,
|
||||
torch.full_like(model.fp8_linear.input_scale, 0.125),
|
||||
atol=0,
|
||||
rtol=0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm() or not is_aiter_found_and_supported(),
|
||||
reason="Requires ROCm AITER attention and quantization",
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
|
||||
def test_rocm_aiter_static_quant_attention(
|
||||
dtype, dist_init, disable_vllm_compile_cache, monkeypatch
|
||||
):
|
||||
"""Fuse the actual AITER op without changing its static scale or outputs."""
|
||||
try:
|
||||
with monkeypatch.context() as patch:
|
||||
patch.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
test_attention_quant_pattern(
|
||||
num_qo_heads=32,
|
||||
num_kv_heads=8,
|
||||
head_size=128,
|
||||
batch_size=8,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype="fp8",
|
||||
custom_ops="+quant_fp8",
|
||||
model_name="amd/Llama-3.1-8B-Instruct-FP8-KV",
|
||||
model_class=TestAttentionAiterFp8StaticQuantPatternModel,
|
||||
backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
dist_init=dist_init,
|
||||
disable_vllm_compile_cache=disable_vllm_compile_cache,
|
||||
)
|
||||
finally:
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
@@ -7,6 +7,7 @@ from collections.abc import Callable
|
||||
import torch
|
||||
from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
@@ -35,7 +36,9 @@ RESHAPE_OP = torch.ops.aten.reshape.default
|
||||
_FP8_QUANT_KEY = QuantKey(dtype=FP8_DTYPE, scale=kStaticTensorScale, symmetric=True)
|
||||
|
||||
|
||||
class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
class AttnFp8StaticQuantPattern(
|
||||
VllmPatternReplacement[..., torch.Tensor | tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
"""Fusion for Attention+Fp8StaticQuant.
|
||||
|
||||
Only triggers when the attention implementation returns True in
|
||||
@@ -51,8 +54,20 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
self._dtype = dtype
|
||||
self._quant_matcher = MatcherQuantFP8(_FP8_QUANT_KEY)
|
||||
|
||||
def _quantize(
|
||||
self, value: torch.Tensor, scale: torch.Tensor
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
return self._quant_matcher(value, scale)[0]
|
||||
|
||||
def _output(
|
||||
self, value: torch.Tensor, scale: torch.Tensor
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
return value
|
||||
|
||||
@property
|
||||
def pattern(self) -> Callable[..., torch.Tensor]:
|
||||
def pattern(
|
||||
self,
|
||||
) -> Callable[..., torch.Tensor | tuple[torch.Tensor, torch.Tensor]]:
|
||||
# When _USE_LAYERNAME is enabled (torch >= 2.11), layer_name is
|
||||
# passed as an explicit pattern input so the pattern matcher
|
||||
# treats it as a wildcard matching hoisted LayerName placeholders.
|
||||
@@ -78,7 +93,7 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
attn_out_view = RESHAPE_OP(
|
||||
at1[1], [q.shape[0], self._num_heads * self._head_size]
|
||||
)
|
||||
return self._quant_matcher(attn_out_view, scale)[0]
|
||||
return self._quantize(attn_out_view, scale)
|
||||
|
||||
return _pattern_with_ln
|
||||
|
||||
@@ -97,12 +112,14 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
attn_out_view = RESHAPE_OP(
|
||||
at1[1], [q.shape[0], self._num_heads * self._head_size]
|
||||
)
|
||||
return self._quant_matcher(attn_out_view, scale)[0]
|
||||
return self._quantize(attn_out_view, scale)
|
||||
|
||||
return _pattern
|
||||
|
||||
@property
|
||||
def replacement(self) -> Callable[..., torch.Tensor]:
|
||||
def replacement(
|
||||
self,
|
||||
) -> Callable[..., torch.Tensor | tuple[torch.Tensor, torch.Tensor]]:
|
||||
_ln = _encode_layer_name(self._layer_name)
|
||||
|
||||
if _USE_LAYERNAME:
|
||||
@@ -126,7 +143,9 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size])
|
||||
return self._output(
|
||||
RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size]), scale
|
||||
)
|
||||
|
||||
return _replacement_with_ln
|
||||
|
||||
@@ -147,7 +166,9 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
output_block_scale=None,
|
||||
kv_cache_dummy_dep=kv_cache_dummy_dep,
|
||||
)
|
||||
return RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size])
|
||||
return self._output(
|
||||
RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size]), scale
|
||||
)
|
||||
|
||||
return _replacement
|
||||
|
||||
@@ -168,6 +189,30 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]):
|
||||
return inputs
|
||||
|
||||
|
||||
class RocmAttnFp8StaticQuantPattern(AttnFp8StaticQuantPattern):
|
||||
"""Fuse AITER static quantization while preserving its scale output."""
|
||||
|
||||
def _quantize(
|
||||
self, value: torch.Tensor, scale: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
out = torch.empty_like(value, dtype=FP8_DTYPE)
|
||||
_, out, scale = auto_functionalized(
|
||||
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
|
||||
out=out,
|
||||
x=value,
|
||||
scale=scale,
|
||||
is_dynamic=False,
|
||||
)
|
||||
# The shared AITER op declares scale mutable for its dynamic variant.
|
||||
# Static quant preserves it; keep this graph output when replacing it.
|
||||
return out, scale
|
||||
|
||||
def _output(
|
||||
self, value: torch.Tensor, scale: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return value, scale
|
||||
|
||||
|
||||
class AttnNvfp4QuantPattern(
|
||||
VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]]
|
||||
):
|
||||
@@ -387,6 +432,8 @@ class AttnQuantFusionPass(VllmFusionPatternMatcherPass):
|
||||
for layer in layers:
|
||||
if layer.impl.fused_output_quant_supported(_FP8_QUANT_KEY):
|
||||
self.register(AttnFp8StaticQuantPattern(layer, dtype))
|
||||
if current_platform.is_rocm() and rocm_aiter_ops.is_enabled():
|
||||
self.register(RocmAttnFp8StaticQuantPattern(layer, dtype))
|
||||
if _USE_LAYERNAME:
|
||||
break
|
||||
|
||||
|
||||
Reference in New Issue
Block a user