diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 55dde1ec1c..61b74d07ae 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -3452,6 +3452,29 @@ steps: commands: - pytest -v -s compile/h100/test_startup.py +- label: ":amd: (MI355) Fusion and Compile FP8" + timeout_in_minutes: 45 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] + dind: false + agent_pool: mi355_2 + num_gpus: 2 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/layers/ + - vllm/v1/attention/ + - vllm/distributed/ + - vllm/_aiter_ops.py + - tests/compile/passes/ + - tests/compile/correctness_e2e/test_attn_quant.py + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/models/llama.py + commands: + - amd-smi + - pytest -v -s tests/compile/passes/test_fusion_attn.py tests/compile/passes/test_silu_mul_quant_fusion.py tests/compile/passes/test_silu_mul_quant_manual_fusion.py + - pytest -v -s tests/compile/correctness_e2e/test_attn_quant.py + #-------------------------------------------------------- mi355 ยท distributed --------------------------------------------------------# - label: ":amd: (MI355) Sharded RDT Weight Transfer" diff --git a/.buildkite/test_areas/compile.yaml b/.buildkite/test_areas/compile.yaml index e7be6e3ac6..6a7a6f909f 100644 --- a/.buildkite/test_areas/compile.yaml +++ b/.buildkite/test_areas/compile.yaml @@ -67,6 +67,31 @@ steps: - pytest -v -s tests/compile/passes/test_silu_mul_quant_manual_fusion.py # this runner has 2 GPUs available even though num_devices=2 is not set - pytest -v -s tests/compile/passes/distributed/test_fusion_all_reduce.py + mirror: + amd: + label: ":amd: (MI355) Fusion and Compile FP8" + dind: false + device: mi355_2 + num_devices: 2 + working_dir: "/vllm-workspace/" + timeout_in_minutes: 45 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/compilation/ + - vllm/model_executor/layers/ + - vllm/v1/attention/ + - vllm/distributed/ + - vllm/_aiter_ops.py + - tests/compile/passes/ + - tests/compile/correctness_e2e/test_attn_quant.py + - vllm/model_executor/kernels/linear/ + - vllm/model_executor/models/llama.py + commands: + - amd-smi + # MLA static FP8 and distributed AR/RMS are owned by the existing ROCm jobs. + - pytest -v -s tests/compile/passes/test_fusion_attn.py tests/compile/passes/test_silu_mul_quant_fusion.py tests/compile/passes/test_silu_mul_quant_manual_fusion.py + - pytest -v -s tests/compile/correctness_e2e/test_attn_quant.py - label: ":nvidia: (H100) Fusion E2E Quick" key: fusion-e2e-quick-h100 diff --git a/tests/compile/correctness_e2e/test_attn_quant.py b/tests/compile/correctness_e2e/test_attn_quant.py new file mode 100644 index 0000000000..c7fc849ae5 --- /dev/null +++ b/tests/compile/correctness_e2e/test_attn_quant.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json +import logging + +import pytest +import regex as re + +from tests.utils import compare_two_settings +from vllm._aiter_ops import is_aiter_found_and_supported +from vllm.config import CompilationMode +from vllm.platforms import current_platform +from vllm.transformers_utils.repo_utils import hf_api + + +@pytest.mark.skipif( + not current_platform.is_rocm() or not is_aiter_found_and_supported(), + reason="Requires ROCm AITER attention and quantization", +) +def test_rocm_aiter_static_attention_output_correctness(caplog_mp_spawn): + """Trained FP8 outputs agree when actual AITER output quantization is fused.""" + model = hf_api().snapshot_download( + "RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8", + revision="870a1177ed082c0ba5dbefd62573ab8f91803f6a", + allow_patterns=["*.json", "*.safetensors", "*.model", "*.txt", "*.jinja"], + ) + common_args = [ + "--dtype", + "bfloat16", + "--load-format", + "auto", + "--max-model-len", + "512", + "--max-num-batched-tokens", + "512", + "--max-num-seqs", + "8", + "--kv-cache-memory-bytes", + str(256 * 1024**2), + "--gpu-memory-utilization", + "0.01", + "--attention-backend", + "ROCM_AITER_UNIFIED_ATTN", + ] + + def settings(enabled): + return [ + *common_args, + "--compilation-config", + json.dumps( + { + "mode": CompilationMode.VLLM_COMPILE, + "cudagraph_mode": "NONE", + "splitting_ops": [], + "inductor_compile_config": {"force_disable_caches": True}, + "custom_ops": ["+quant_fp8", "-rms_norm"], + "pass_config": { + "fuse_attn_quant": enabled, + "fuse_norm_quant": False, + "fuse_act_quant": False, + "fuse_allreduce_rms": False, + }, + } + ), + ] + + env = { + "VLLM_ROCM_USE_AITER": "1", + "VLLM_ROCM_USE_AITER_LINEAR": "1", + "VLLM_DISABLE_COMPILE_CACHE": "1", + } + with caplog_mp_spawn(logging.DEBUG) as logs: + compare_two_settings( + model, + settings(False), + settings(True), + env1=env, + env2=env, + method="generate", + force_v1_runner=True, + ) + matches = [ + int(count) for count in re.findall(r"'attn_quant_fusion': (\d+)", logs.text) + ] + print(f"Attention output quantization rewrite counts: {matches}") + assert matches and all(count == 32 for count in matches), ( + f"Expected output quantization fused in all 32 Llama layers, found {matches}" + ) diff --git a/tests/compile/passes/test_fusion_attn.py b/tests/compile/passes/test_fusion_attn.py index 37f0d4126c..ccc7e624db 100644 --- a/tests/compile/passes/test_fusion_attn.py +++ b/tests/compile/passes/test_fusion_attn.py @@ -12,6 +12,7 @@ from tests.v1.attention.utils import ( create_common_attn_metadata, dense_kv_cache_views, ) +from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops from vllm._custom_ops import cutlass_scaled_fp4_mm, scaled_fp4_quant from vllm.compilation.passes.fusion.attn_quant_fusion import ( ATTN_OP, @@ -189,6 +190,17 @@ class TestAttentionFp8StaticQuantPatternModel(AttentionQuantPatternModel): return self.fp8_linear(attn_output) +class TestAttentionAiterFp8StaticQuantPatternModel( + TestAttentionFp8StaticQuantPatternModel +): + aiter_quant = True + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fp8_linear.kernel.quant_fp8.use_aiter = True + self.fp8_linear.input_scale.fill_(0.125) + + class TestAttentionNvfp4QuantPatternModel(AttentionQuantPatternModel): """Test model for AttentionNvfp4QuantPattern fusion.""" @@ -455,11 +467,12 @@ def test_attention_quant_pattern( ) # Check quantization ops in the graph before and after fusion - quant_op = ( - torch.ops.aten.reciprocal - if "-quant_fp8" in custom_ops_list - else QUANT_OPS[quant_key] - ) + if "-quant_fp8" in custom_ops_list: + quant_op = torch.ops.aten.reciprocal + elif getattr(model_class, "aiter_quant", False): + quant_op = torch.ops.vllm.rocm_aiter_per_tensor_quant.default + else: + quant_op = QUANT_OPS[quant_key] if expect_fusion: # Note: for fp8, fully_replaced=False because query quant ops remain in @@ -523,3 +536,46 @@ def test_attention_quant_pattern( # Check that results are close torch.testing.assert_close(result_unfused, result_fused, atol=1e-2, rtol=1e-2) + if getattr(model_class, "aiter_quant", False): + # Query quantization remains; exactly the output quant op is removed. + assert test_backend.op_count(quant_op, before=True) == ( + test_backend.op_count(quant_op) + 1 + ) + for model in (model_unfused, model_fused): + torch.testing.assert_close( + model.fp8_linear.input_scale, + torch.full_like(model.fp8_linear.input_scale, 0.125), + atol=0, + rtol=0, + ) + + +@pytest.mark.skipif( + not current_platform.is_rocm() or not is_aiter_found_and_supported(), + reason="Requires ROCm AITER attention and quantization", +) +@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16]) +def test_rocm_aiter_static_quant_attention( + dtype, dist_init, disable_vllm_compile_cache, monkeypatch +): + """Fuse the actual AITER op without changing its static scale or outputs.""" + try: + with monkeypatch.context() as patch: + patch.setenv("VLLM_ROCM_USE_AITER", "1") + rocm_aiter_ops.refresh_env_variables() + test_attention_quant_pattern( + num_qo_heads=32, + num_kv_heads=8, + head_size=128, + batch_size=8, + dtype=dtype, + kv_cache_dtype="fp8", + custom_ops="+quant_fp8", + model_name="amd/Llama-3.1-8B-Instruct-FP8-KV", + model_class=TestAttentionAiterFp8StaticQuantPatternModel, + backend=AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + dist_init=dist_init, + disable_vllm_compile_cache=disable_vllm_compile_cache, + ) + finally: + rocm_aiter_ops.refresh_env_variables() diff --git a/vllm/compilation/passes/fusion/attn_quant_fusion.py b/vllm/compilation/passes/fusion/attn_quant_fusion.py index 432d59e5ab..bdbca79e5d 100644 --- a/vllm/compilation/passes/fusion/attn_quant_fusion.py +++ b/vllm/compilation/passes/fusion/attn_quant_fusion.py @@ -7,6 +7,7 @@ from collections.abc import Callable import torch from torch._higher_order_ops.auto_functionalize import auto_functionalized +from vllm._aiter_ops import rocm_aiter_ops from vllm.config import VllmConfig, get_layers_from_vllm_config from vllm.logger import init_logger from vllm.model_executor.layers.attention import Attention @@ -35,7 +36,9 @@ RESHAPE_OP = torch.ops.aten.reshape.default _FP8_QUANT_KEY = QuantKey(dtype=FP8_DTYPE, scale=kStaticTensorScale, symmetric=True) -class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): +class AttnFp8StaticQuantPattern( + VllmPatternReplacement[..., torch.Tensor | tuple[torch.Tensor, torch.Tensor]] +): """Fusion for Attention+Fp8StaticQuant. Only triggers when the attention implementation returns True in @@ -51,8 +54,20 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): self._dtype = dtype self._quant_matcher = MatcherQuantFP8(_FP8_QUANT_KEY) + def _quantize( + self, value: torch.Tensor, scale: torch.Tensor + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + return self._quant_matcher(value, scale)[0] + + def _output( + self, value: torch.Tensor, scale: torch.Tensor + ) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]: + return value + @property - def pattern(self) -> Callable[..., torch.Tensor]: + def pattern( + self, + ) -> Callable[..., torch.Tensor | tuple[torch.Tensor, torch.Tensor]]: # When _USE_LAYERNAME is enabled (torch >= 2.11), layer_name is # passed as an explicit pattern input so the pattern matcher # treats it as a wildcard matching hoisted LayerName placeholders. @@ -78,7 +93,7 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): attn_out_view = RESHAPE_OP( at1[1], [q.shape[0], self._num_heads * self._head_size] ) - return self._quant_matcher(attn_out_view, scale)[0] + return self._quantize(attn_out_view, scale) return _pattern_with_ln @@ -97,12 +112,14 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): attn_out_view = RESHAPE_OP( at1[1], [q.shape[0], self._num_heads * self._head_size] ) - return self._quant_matcher(attn_out_view, scale)[0] + return self._quantize(attn_out_view, scale) return _pattern @property - def replacement(self) -> Callable[..., torch.Tensor]: + def replacement( + self, + ) -> Callable[..., torch.Tensor | tuple[torch.Tensor, torch.Tensor]]: _ln = _encode_layer_name(self._layer_name) if _USE_LAYERNAME: @@ -126,7 +143,9 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): output_block_scale=None, kv_cache_dummy_dep=kv_cache_dummy_dep, ) - return RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size]) + return self._output( + RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size]), scale + ) return _replacement_with_ln @@ -147,7 +166,9 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): output_block_scale=None, kv_cache_dummy_dep=kv_cache_dummy_dep, ) - return RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size]) + return self._output( + RESHAPE_OP(at1[1], [-1, self._num_heads * self._head_size]), scale + ) return _replacement @@ -168,6 +189,30 @@ class AttnFp8StaticQuantPattern(VllmPatternReplacement[..., torch.Tensor]): return inputs +class RocmAttnFp8StaticQuantPattern(AttnFp8StaticQuantPattern): + """Fuse AITER static quantization while preserving its scale output.""" + + def _quantize( + self, value: torch.Tensor, scale: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + out = torch.empty_like(value, dtype=FP8_DTYPE) + _, out, scale = auto_functionalized( + torch.ops.vllm.rocm_aiter_per_tensor_quant.default, + out=out, + x=value, + scale=scale, + is_dynamic=False, + ) + # The shared AITER op declares scale mutable for its dynamic variant. + # Static quant preserves it; keep this graph output when replacing it. + return out, scale + + def _output( + self, value: torch.Tensor, scale: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor]: + return value, scale + + class AttnNvfp4QuantPattern( VllmPatternReplacement[..., tuple[torch.Tensor, torch.Tensor]] ): @@ -387,6 +432,8 @@ class AttnQuantFusionPass(VllmFusionPatternMatcherPass): for layer in layers: if layer.impl.fused_output_quant_supported(_FP8_QUANT_KEY): self.register(AttnFp8StaticQuantPattern(layer, dtype)) + if current_platform.is_rocm() and rocm_aiter_ops.is_enabled(): + self.register(RocmAttnFp8StaticQuantPattern(layer, dtype)) if _USE_LAYERNAME: break