From 2cab53ddeea3851a083beff8c192f068c4518b67 Mon Sep 17 00:00:00 2001 From: Jack Hu Date: Thu, 16 Jul 2026 18:39:04 -0400 Subject: [PATCH] [Model][Hardware][AMD]: Part 1/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_AITER_FA, and ROCM_AITER_UNIFIED_ATTN (#42749) Signed-off-by: Jack Hu Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> --- ..._rocm_aiter_qk_norm_rope_kvcache_fusion.py | 478 +++++++++++++++++ tests/quantization/test_fp8.py | 41 ++ vllm/_aiter_ops.py | 122 +++++ .../passes/fusion/qk_norm_rope_fusion.py | 23 +- .../fusion/qk_norm_rope_kvcache_fusion.py | 492 ++++++++++++++++++ vllm/compilation/passes/pass_manager.py | 6 + vllm/config/compilation.py | 38 +- vllm/config/vllm.py | 32 ++ .../layers/attention/attention.py | 6 +- .../layers/attention/mla_attention.py | 2 + .../compressed_tensors/compressed_tensors.py | 4 + .../layers/quantization/kv_cache.py | 3 + vllm/v1/attention/backend.py | 33 ++ vllm/v1/attention/backends/rocm_aiter_fa.py | 41 ++ .../backends/rocm_aiter_unified_attn.py | 44 ++ 15 files changed, 1361 insertions(+), 4 deletions(-) create mode 100644 tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py create mode 100644 vllm/compilation/passes/fusion/qk_norm_rope_kvcache_fusion.py diff --git a/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py b/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py new file mode 100644 index 0000000000..b5e354ddfd --- /dev/null +++ b/tests/compile/passes/test_rocm_aiter_qk_norm_rope_kvcache_fusion.py @@ -0,0 +1,478 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import os + +import pytest +import torch + +import vllm.config +from tests.compile.backend import TestBackend +from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata +from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops +from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP +from vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion import ( + QkNormRopeKvCacheFusionPass, +) +from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass +from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass +from vllm.compilation.passes.utility.scatter_split_replace import ( + ScatterSplitReplacementPass, +) +from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass +from vllm.config import ( + CacheConfig, + CompilationConfig, + CompilationMode, + ModelConfig, + PassConfig, + VllmConfig, +) +from vllm.forward_context import get_forward_context, set_forward_context +from vllm.model_executor.layers.attention import Attention +from vllm.model_executor.layers.layernorm import RMSNorm +from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding +from vllm.platforms import current_platform +from vllm.v1.attention.backend import ( + AttentionBackend, + CommonAttentionMetadata, +) +from vllm.v1.attention.backends.registry import AttentionBackendEnum +from vllm.v1.kv_cache_interface import AttentionSpec + +INDEX_SELECT_OP = torch.ops.aten.index.Tensor +FP8_DTYPE = current_platform.fp8_dtype() + + +class QKNormRoPEKVCacheTestModel(torch.nn.Module): + """Minimal model that reproduces the QK-norm + RoPE + KV cache update + pattern matched by QkNormRopeKvCacheFusionPass: + + q, k, v = split(qkv) + q = rms_norm(q.view(heads, dim), q_weight).view(flat) + k = rms_norm(k.view(heads, dim), k_weight).view(flat) + q, k = rotary_emb(positions, q, k) + q = q.view(num_heads, head_dim) + k = k.view(num_kv_heads, head_dim) + v = v.view(num_kv_heads, head_dim) + dummy = unified_kv_cache_update(k, v, layer_name) + """ + + def __init__( + self, + vllm_config: VllmConfig, + attn_backend: AttentionBackendEnum, + num_heads: int, + num_kv_heads: int, + head_size: int, + is_neox: bool, + rms_norm_eps: float, + dtype: torch.dtype, + device: torch.device, + rotary_dim: int | None = None, + prefix: str = "model.layers.0.self_attn.attn", + ): + super().__init__() + self.num_heads = num_heads + self.num_kv_heads = num_kv_heads + self.head_size = head_size + self.rotary_dim = rotary_dim if rotary_dim is not None else head_size + self.block_size = vllm_config.cache_config.block_size + self.q_size = num_heads * head_size + self.kv_size = num_kv_heads * head_size + self.is_neox = is_neox + self.dtype = dtype + self.device = device + self.layer_name = prefix + + self.q_norm = RMSNorm(head_size, eps=rms_norm_eps) + self.k_norm = RMSNorm(head_size, eps=rms_norm_eps) + + self.rotary_emb = RotaryEmbedding( + head_size, + rotary_dim=self.rotary_dim, + max_position_embeddings=4096, + base=10000, + is_neox_style=is_neox, + dtype=self.dtype, + ) + + self.enable_rope_custom_op = self.rotary_emb.enabled() + + self.attn = Attention( + num_heads=num_heads, + head_size=head_size, + scale=1.0 / head_size**0.5, + num_kv_heads=num_kv_heads, + cache_config=vllm_config.cache_config, + quant_config=vllm_config.quant_config, + prefix=prefix, + attn_backend=attn_backend.get_class(), + ) + self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend() + assert not self.attn_backend.forward_includes_kv_cache_update, ( + f"Attention backend {self.attn_backend} does not support " + "fuse_qk_norm_rope_kvcache." + ) + kv_cache_dtype_str = vllm_config.cache_config.cache_dtype + self.kv_cache_dtype = ( + FP8_DTYPE if kv_cache_dtype_str.startswith("fp8") else self.dtype + ) + + if self.kv_cache_dtype != self.dtype: + self.attn._k_scale = torch.tensor(1.0, dtype=torch.float32, device=device) + self.attn._v_scale = torch.tensor(1.0, dtype=torch.float32, device=device) + self.attn._k_scale_float = 1.0 + self.attn._v_scale_float = 1.0 + else: + self.attn._k_scale = self.attn._k_scale.to(device) + self.attn._v_scale = self.attn._v_scale.to(device) + + self.builder = self.attn.attn_backend.get_builder_cls()( + kv_cache_spec=AttentionSpec( + block_size=self.block_size, + num_kv_heads=self.num_kv_heads, + head_size=head_size, + dtype=self.kv_cache_dtype, + ), + layer_names=[self.attn.layer_name], + vllm_config=vllm_config, + device=device, + ) + + def build_attn_metadata( + self, batch_size: int, kv_stride_order: tuple[int, ...] | None = None + ) -> CommonAttentionMetadata: + batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size) + common_attn_metadata = create_common_attn_metadata( + batch_spec, self.block_size, self.device, arange_block_indices=True + ) + + max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size + num_blocks = batch_size * max_blocks + + attn_backend = self.attn.attn_backend + kv_cache_shape = attn_backend.get_kv_cache_shape( + num_blocks, self.block_size, self.num_kv_heads, self.head_size + ) + # Caller can force a physical layout; else use the backend's. + if kv_stride_order is None: + try: + kv_stride_order = attn_backend.get_kv_cache_stride_order() + except (AttributeError, NotImplementedError): + kv_stride_order = tuple(range(len(kv_cache_shape))) + + kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_stride_order) + inv_order = [kv_stride_order.index(i) for i in range(len(kv_stride_order))] + + raw_tensor = torch.zeros( + 2 * num_blocks * self.block_size * self.num_kv_heads * self.head_size, + dtype=self.kv_cache_dtype, + device=self.device, + ) + raw_tensor = raw_tensor.view(kv_cache_shape) + kv_cache = raw_tensor.permute(*inv_order) + + # Store as a bare tensor (not wrapped in a list) to match production + # `bind_kv_cache` behavior. `get_attention_context` returns this + # attribute directly to the fused/unfused `do_kv_cache_update` impls, + # which call `kv_cache.unbind(0)` and therefore require a tensor. + self.attn.kv_cache = kv_cache + + attn_metadata = self.builder.build( + common_prefix_len=0, common_attn_metadata=common_attn_metadata + ) + + return attn_metadata + + def forward( + self, qkv: torch.Tensor, positions: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + qkv = qkv.clone() + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) + + # QK-norm: RMSNorm on per-head Q and K + q = q.view(-1, self.num_heads, self.head_size) + q = self.q_norm(q) + q = q.view(-1, self.q_size) + + k = k.view(-1, self.num_kv_heads, self.head_size) + k = self.k_norm(k) + k = k.view(-1, self.kv_size) + + # RoPE + q, k = self.rotary_emb(positions, q, k) + + # Mirror Attention.forward: quant-query impls consume an fp8 q. + if ( + self.kv_cache_dtype != self.dtype + and self.attn.impl.supports_quant_query_input + ): + q_fp8 = torch.empty_like(q, dtype=FP8_DTYPE) + torch.ops.vllm.rocm_aiter_per_tensor_quant( + q_fp8, q, self.attn._q_scale, False + ) + q = q_fp8 + + # Final views + KV cache update + q = q.view(-1, self.num_heads, self.head_size) + k = k.view(-1, self.num_kv_heads, self.head_size) + v = v.view(-1, self.num_kv_heads, self.head_size) + kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update( + k, v, self.layer_name + ) + return q, k, v, kv_cache_dummy_dep + + def ops_in_model_before(self) -> list[torch._ops.OpOverload]: + ops: list[torch._ops.OpOverload] = [] + # RoPE is not yet IR-migrated, so its custom op still surfaces + # directly in the graph based on `enable_rope_custom_op`. + if self.enable_rope_custom_op: + if rocm_aiter_ops.is_triton_rotary_embed_enabled(): + ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default) + else: + ops.append(ROTARY_OP) + else: + ops.append(INDEX_SELECT_OP) + ops.append(torch.ops.vllm.unified_kv_cache_update.default) + return ops + + def ops_in_model_after(self) -> list[torch._ops.OpOverload]: + return [torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default] + + +def _run_qk_norm_rope_kvcache_fusion_test( + *, + attn_backend: AttentionBackendEnum, + enable_aiter_triton_rope: bool, + num_tokens: int, + num_heads: int, + num_kv_heads: int, + head_size: int, + rotary_dim: int, + block_size: int, + is_neox: bool, + use_shuffle_kv_layout: str, + kv_stride_order: tuple[int, ...], + dtype: torch.dtype, + kv_cache_dtype: str, + rms_norm_eps: float, + custom_op: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda") + torch.set_default_device(device) + torch.set_default_dtype(dtype) + torch.manual_seed(0) + + vllm_config = VllmConfig( + model_config=ModelConfig(dtype=dtype), + cache_config=CacheConfig( + block_size=block_size, + cache_dtype=kv_cache_dtype, + ), + compilation_config=CompilationConfig( + mode=CompilationMode.VLLM_COMPILE, + custom_ops=[custom_op], + pass_config=PassConfig( + fuse_qk_norm_rope_kvcache=True, + eliminate_noops=True, + ), + ), + ) + + with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m: + m.setenv("VLLM_ROCM_USE_AITER", "1") + m.setenv( + "VLLM_ROCM_USE_AITER_TRITON_ROPE", + "1" if enable_aiter_triton_rope else "0", + ) + m.setenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", use_shuffle_kv_layout) + rocm_aiter_ops.refresh_env_variables() + + model = QKNormRoPEKVCacheTestModel( + vllm_config=vllm_config, + attn_backend=attn_backend, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_size=head_size, + rotary_dim=rotary_dim, + is_neox=is_neox, + rms_norm_eps=rms_norm_eps, + dtype=dtype, + device=torch.get_default_device(), + ) + + fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config) + passes = [ + NoOpEliminationPass(vllm_config), + SplitCoalescingPass(vllm_config), + ScatterSplitReplacementPass(vllm_config), + fusion_pass, + PostCleanupPass(vllm_config), + ] + backend = TestBackend(*passes) + + qkv = torch.randn( + num_tokens, + num_heads * head_size + 2 * num_kv_heads * head_size, + dtype=dtype, + ) + pos = torch.arange(num_tokens, dtype=torch.long) + + qkv_unfused = qkv.clone() + pos_unfused = pos.clone() + + # Run unfused (eager) forward + with set_forward_context(None, vllm_config): + forward_context = get_forward_context() + attn_metadata = model.build_attn_metadata(num_tokens, kv_stride_order) + forward_context.slot_mapping = { + model.layer_name: attn_metadata.slot_mapping + } + q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused) + attn_layer = forward_context.no_compile_layers[model.layer_name] + kv_cache_unfused = attn_layer.kv_cache + del dummy + + # Run fused (compiled) forward + torch._dynamo.mark_dynamic(qkv, 0) + torch._dynamo.mark_dynamic(pos, 0) + with set_forward_context(None, vllm_config): + model_fused = torch.compile(model, backend=backend) + forward_context = get_forward_context() + attn_metadata = model_fused.build_attn_metadata(num_tokens, kv_stride_order) + forward_context.slot_mapping = { + model.layer_name: attn_metadata.slot_mapping + } + q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos) + attn_layer = forward_context.no_compile_layers[model.layer_name] + kv_cache_fused = attn_layer.kv_cache + del dummy + + assert fusion_pass.matched_count == 1 + + backend.check_before_ops(model.ops_in_model_before()) + backend.check_after_ops(model.ops_in_model_after()) + + # Sweep-backed (18.2k pts, PR #42749): native-rope ref worst 7.7e-3 -> 1e-2; + # AITER-triton-rope ref is itself approximate (plateau 1.28e-2) -> 2e-2. + ATOL, RTOL = (2e-2, 2e-2) if enable_aiter_triton_rope else (1e-2, 1e-2) + is_fp8_cache = model.kv_cache_dtype != dtype + + if q_fused.dtype == FP8_DTYPE: + # Quant-query path: both q are fp8; compare dequant within 1 fp8 ULP. + torch.testing.assert_close( + q_unfused.float(), q_fused.float(), atol=1.25e-1, rtol=1.25e-1 + ) + else: + torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL) + + if not is_fp8_cache: + # The AITER PTS kernel populates k_out only for non-FP8 caches. + # With FP8, the kernel writes quantized K directly to the cache + # and may leave k_out uninitialised. In production this is fine + # because downstream attention reads K from the cache. + torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL) + + # Should be bit exact since no processing had been done on v for both paths + torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0) + + # fp8 vs triton-rope ref requires loosening tolerance to 1.25e-1. + if is_fp8_cache and enable_aiter_triton_rope: + cache_atol = cache_rtol = 1.25e-1 + else: + cache_atol, cache_rtol = ATOL, RTOL + + torch.testing.assert_close( + kv_cache_unfused[0].float(), + kv_cache_fused[0].float(), + atol=cache_atol, + rtol=cache_rtol, + ) + + +_FUSION_CONFIGS = [ + # Full rotary, both neox styles (the original coverage). + pytest.param(64, 8, 64, 64, True, id="full-neox"), + pytest.param(64, 8, 64, 64, False, id="full-non_neox"), + # GLM-4.5/4.6/4.7 (glm4_moe.py:275 partial_rotary_factor=0.5, neox-style) + pytest.param(32, 8, 128, 64, True, id="glm4_moe"), + # GLM-4 dense (glm4.py:97,124 partial_rotary_factor=0.5, non-neox) + pytest.param(32, 2, 128, 64, False, id="glm4_dense"), + # Moondream3-style small head (head_size=64, rotary_dim=32) + pytest.param(16, 2, 64, 32, True, id="partial_small_head"), +] + + +@pytest.mark.parametrize( + "num_heads, num_kv_heads, head_size, rotary_dim, is_neox", + _FUSION_CONFIGS, +) +@pytest.mark.parametrize( + "attn_backend", + [ + AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN, + AttentionBackendEnum.ROCM_AITER_FA, + ], +) +@pytest.mark.parametrize("num_tokens", [5, 16, 2048]) +@pytest.mark.parametrize("use_shuffle_kv_layout", ["1", "0"]) +@pytest.mark.parametrize( + "kv_stride_order", + [ + pytest.param((0, 1, 2, 3, 4), id="block_first"), + pytest.param((1, 0, 2, 3, 4), id="kv_first"), + ], +) +@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False]) +@pytest.mark.parametrize("block_size", [16, 32, 64]) +@pytest.mark.parametrize("dtype", [torch.bfloat16]) +@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"]) +@pytest.mark.parametrize("rms_norm_eps", [1e-5, 1e-6]) +@pytest.mark.parametrize("custom_op", ["+rotary_embedding", "+rms_norm"]) +@pytest.mark.skipif( + not is_aiter_found_and_supported(), + reason="Only test on ROCm with AITER installed and supported", +) +def test_qk_norm_rope_kvcache_fusion( + num_tokens: int, + num_heads: int, + num_kv_heads: int, + head_size: int, + rotary_dim: int, + is_neox: bool, + attn_backend: AttentionBackendEnum, + enable_aiter_triton_rope: bool, + use_shuffle_kv_layout: str, + kv_stride_order: tuple[int, ...], + block_size: int, + dtype: torch.dtype, + kv_cache_dtype: str, + rms_norm_eps: float, + custom_op: str, + monkeypatch: pytest.MonkeyPatch, +): + if ( + attn_backend == AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN + and use_shuffle_kv_layout == "1" + ): + pytest.skip("ROCM_AITER_UNIFIED_ATTN is NHD-only; shuffle env is ignored") + _run_qk_norm_rope_kvcache_fusion_test( + attn_backend=attn_backend, + enable_aiter_triton_rope=enable_aiter_triton_rope, + num_tokens=num_tokens, + num_heads=num_heads, + num_kv_heads=num_kv_heads, + head_size=head_size, + rotary_dim=rotary_dim, + block_size=block_size, + is_neox=is_neox, + use_shuffle_kv_layout=use_shuffle_kv_layout, + kv_stride_order=kv_stride_order, + dtype=dtype, + kv_cache_dtype=kv_cache_dtype, + rms_norm_eps=rms_norm_eps, + custom_op=custom_op, + monkeypatch=monkeypatch, + ) diff --git a/tests/quantization/test_fp8.py b/tests/quantization/test_fp8.py index 571e180d6c..abb995257e 100644 --- a/tests/quantization/test_fp8.py +++ b/tests/quantization/test_fp8.py @@ -17,6 +17,10 @@ from vllm.config.model import ModelConfig from vllm.model_executor.kernels.linear.scaled_mm import ( MarlinFP8ScaledMMLinearKernel, ) +from vllm.model_executor.layers.attention.attention import ( + Attention, + set_default_quant_scales, +) from vllm.model_executor.layers.fused_moe import FusedMoE from vllm.model_executor.layers.quantization.fp8 import ( Fp8Config, @@ -24,6 +28,7 @@ from vllm.model_executor.layers.quantization.fp8 import ( Fp8LinearMethod, Fp8MoEMethod, ) +from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod from vllm.model_executor.layers.quantization.online.fp8 import ( Fp8PerTensorOnlineLinearMethod, ) @@ -422,6 +427,42 @@ def test_fp8_reloading( method.process_weights_after_loading(layer) +@pytest.mark.parametrize("source", ["checkpoint", "runtime_calc"]) +def test_kv_cache_scale_sync_to_host_copies(source): + """Test device-to-host sync of the k/v quantization scales, for both the + checkpoint-load and runtime-calc paths that produce them. + """ + layer = torch.nn.Module() + set_default_quant_scales(layer, register_buffer=True) + layer.kv_cache_dtype = "fp8" + + if source == "checkpoint": + # Scales come from the checkpoint, so runtime calc is disabled. + layer.calculate_kv_scales = False + method = BaseKVCacheMethod(quant_config=None) + method.create_weights(layer) + # 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale. + checkpoint_scale = torch.tensor(0.3, dtype=torch.float32) + layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale) + layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale) + method.process_weights_after_loading(layer) + else: + # First forward computes distinct, non-unity scales from live k/v. + layer.calculate_kv_scales = True + query = torch.full((4, 8), 10.0) + key = torch.full((4, 8), 60.0) + value = torch.full((4, 8), 50.0) + Attention.calc_kv_scales(layer, query, key, value) + + assert layer._k_scale_float != 1.0 + assert layer._v_scale_float != 1.0 + # Host copy must mirror both the float and the device scale tensor. + assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale_float) + assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale_float) + assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale.item()) + assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale.item()) + + @pytest.mark.skipif( not is_quant_method_supported("fp8"), reason="FP8 is not supported on this GPU type.", diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 3c018313d0..8ab06d446b 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -2458,6 +2458,128 @@ class rocm_aiter_ops: gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y) return y + @staticmethod + def fused_qk_norm_rope_and_cache( + qkv: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + positions: torch.Tensor, + num_heads_q: int, + num_heads_k: int, + num_heads_v: int, + head_dim: int, + is_neox: bool, + rms_norm_eps: float, + q_out: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + slot_mapping: torch.Tensor, + k_scale: torch.Tensor, + v_scale: torch.Tensor, + k_out: torch.Tensor | None, + v_out: torch.Tensor | None, + return_kv: bool, + use_shuffle_layout: bool, + block_size: int, + x: int, + rotary_dim: int = 0, + ): + from aiter.ops.fused_qk_norm_rope_cache_quant import ( + fused_qk_norm_rope_cache_pts_quant_shuffle, + ) + + fused_qk_norm_rope_cache_pts_quant_shuffle( + qkv, + q_weight, + k_weight, + cos_sin_cache, + positions, + qkv.size(0), + num_heads_q, + num_heads_k, + num_heads_v, + head_dim, + is_neox, + rms_norm_eps, + q_out, + k_cache, + v_cache, + slot_mapping, + k_scale, + v_scale, + k_out, + v_out, + return_kv, + use_shuffle_layout, + block_size, + x, + rotary_dim, + ) + + @staticmethod + def do_qk_norm_rope_kvcache_update( + qkv: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + positions: torch.Tensor, + num_heads_q: int, + num_heads_k: int, + head_dim: int, + is_neox: bool, + rms_norm_eps: float, + q_out: torch.Tensor, + k_out: torch.Tensor, + key_cache: torch.Tensor, + value_cache: torch.Tensor, + slot_mapping: torch.Tensor, + k_scale: torch.Tensor, + v_scale: torch.Tensor, + kv_cache_dtype: str, + use_shuffle_layout: bool, + ) -> None: + """Run the fused QK-norm+RoPE+KV-cache op on already-split k/v caches. + + Shared by the AITER FA and unified-attention impls. The caller splits + kv_cache, since the unbind dim depends on the layout (e.g. the unified + encoder-decoder path is K/V-first), and passes use_shuffle_layout + (unified reads NHD and must pass False). + """ + if kv_cache_dtype.startswith("fp8"): + key_cache = key_cache.view(current_platform.fp8_dtype()) + value_cache = value_cache.view(current_platform.fp8_dtype()) + # Partial-rotary support (e.g. GLM-4.7 applies rotary to only a prefix + # of each head's channel dim). + rotary_dim = cos_sin_cache.shape[-1] + kernel_rotary_dim = 0 if rotary_dim == head_dim else rotary_dim + rocm_aiter_ops.fused_qk_norm_rope_and_cache( + qkv=qkv, + q_weight=q_weight, + k_weight=k_weight, + cos_sin_cache=cos_sin_cache, + positions=positions, + num_heads_q=num_heads_q, + num_heads_k=num_heads_k, + num_heads_v=num_heads_k, + head_dim=head_dim, + is_neox=is_neox, + rms_norm_eps=rms_norm_eps, + q_out=q_out, + k_cache=key_cache, + v_cache=value_cache, + slot_mapping=slot_mapping, + k_scale=k_scale, + v_scale=v_scale, + k_out=k_out, + v_out=None, + return_kv=True, + use_shuffle_layout=use_shuffle_layout, + block_size=key_cache.shape[1], + x=16 // key_cache.element_size(), + rotary_dim=kernel_rotary_dim, + ) + @staticmethod def triton_rope_and_cache( query: torch.Tensor, diff --git a/vllm/compilation/passes/fusion/qk_norm_rope_fusion.py b/vllm/compilation/passes/fusion/qk_norm_rope_fusion.py index e322e109c4..2c8b9c43bd 100644 --- a/vllm/compilation/passes/fusion/qk_norm_rope_fusion.py +++ b/vllm/compilation/passes/fusion/qk_norm_rope_fusion.py @@ -25,6 +25,11 @@ logger = init_logger(__name__) FUSED_QK_ROPE_OP = torch.ops._C.fused_qk_norm_rope.default +# Head dimensions supported by csrc/fused_qknorm_rope_kernel.cu's +# launchFusedQKNormRope and launchFusedQKNormRopeNTokenHeads dispatchers. +# Keep in sync with the switch statements in that file. +SUPPORTED_FUSED_QK_NORM_ROPE_HEAD_DIMS: tuple[int, ...] = (64, 128, 256) + P = ParamSpec("P") @@ -190,7 +195,12 @@ class QkNormRopePattern: class QKNormRoPEFusionPass(VllmPatternMatcherPass): - """Fuse Q/K RMSNorm + RoPE into fused_qk_norm_rope when the custom op exists.""" + """Fuse Q/K RMSNorm + RoPE into fused_qk_norm_rope when the custom op exists. + + Registers patterns for both standard vLLM ops and ROCm AITER ops + (when AITER is enabled), so the fusion fires regardless of which + RMSNorm/RoPE implementation the graph uses. + """ @enable_fake_mode def __init__(self, config: VllmConfig) -> None: @@ -216,6 +226,17 @@ class QKNormRoPEFusionPass(VllmPatternMatcherPass): ) return + for layer in attn_layers.values(): + if layer.head_size not in SUPPORTED_FUSED_QK_NORM_ROPE_HEAD_DIMS: + logger.warning_once( + "QK Norm+RoPE fusion not enabled: layer head_size=%d is not " + "supported by fused_qk_norm_rope kernel (supported: %s). " + "Falling back to unfused QK norm + RoPE path.", + layer.head_size, + SUPPORTED_FUSED_QK_NORM_ROPE_HEAD_DIMS, + ) + return + self._attention_geometries = tuple( sorted( { diff --git a/vllm/compilation/passes/fusion/qk_norm_rope_kvcache_fusion.py b/vllm/compilation/passes/fusion/qk_norm_rope_kvcache_fusion.py new file mode 100644 index 0000000000..85d82c191b --- /dev/null +++ b/vllm/compilation/passes/fusion/qk_norm_rope_kvcache_fusion.py @@ -0,0 +1,492 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import inspect +from collections.abc import Callable +from typing import ParamSpec + +import torch +import torch._inductor.pattern_matcher as pm +from torch import fx +from torch._higher_order_ops.auto_functionalize import auto_functionalized +from torch._inductor.pattern_matcher import PatternMatcherPass + +import vllm.ir.ops +from vllm.config import VllmConfig, get_layers_from_vllm_config +from vllm.config.utils import Range +from vllm.logger import init_logger +from vllm.model_executor.layers.attention.attention import ( + Attention, + get_attention_context, +) +from vllm.platforms import current_platform +from vllm.utils.torch_utils import direct_register_custom_op + +from ..inductor_pass import enable_fake_mode +from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass +from .matcher_utils import MatcherRotaryEmbedding +from .rms_quant_fusion import empty_bf16, empty_fp32, empty_i64 + +logger = init_logger(__name__) + +P = ParamSpec("P") + +# Head sizes the fused kernel fused_qk_norm_rope_cache_pts_quant_shuffle() supports +# Other sizes hard-abort, so skip those layers. +SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS: tuple[int, ...] = (64, 128, 256) + + +# --------------------------------------------------------------------------- +# Custom op: fused QK-norm + RoPE + KV cache update +# --------------------------------------------------------------------------- + + +def fused_qk_norm_rope_and_unified_kv_cache_update_impl( + q_out: torch.Tensor, + k_out: torch.Tensor, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + layer_name: str = "", +) -> torch.Tensor: + _, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name) + if layer_slot_mapping is not None: + attn_layer.impl.do_qk_norm_rope_kvcache_update( + attn_layer, + qkv, + q_out, + k_out, + positions, + q_weight, + k_weight, + rms_norm_eps, + cos_sin_cache, + is_neox, + kv_cache, + layer_slot_mapping, + ) + else: + # Profiling/dummy run: define q_out/k_out (consumed by attention). + q_out.zero_() + k_out.zero_() + + return torch.empty(0, device=qkv.device, dtype=qkv.dtype) + + +def fused_qk_norm_rope_and_unified_kv_cache_update_fake( + q_out: torch.Tensor, + k_out: torch.Tensor, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + layer_name: str = "", +) -> torch.Tensor: + return torch.empty(0, device=qkv.device, dtype=qkv.dtype) + + +direct_register_custom_op( + op_name="fused_qk_norm_rope_and_unified_kv_cache_update", + op_func=fused_qk_norm_rope_and_unified_kv_cache_update_impl, + mutates_args=["q_out", "k_out"], + fake_impl=fused_qk_norm_rope_and_unified_kv_cache_update_fake, +) + + +# --------------------------------------------------------------------------- +# Pattern: QK-norm + RoPE + unified_kv_cache_update +# --------------------------------------------------------------------------- + + +class QkNormRopeKvCachePattern: + """ + Match the unfused sequence: + q, k, v = split(qkv, ...) + q = rms_norm(q.view(heads), q_weight).view(flat) + k = rms_norm(k.view(heads), k_weight).view(flat) + q, k = rotary_embedding(positions, q, k, cos_sin_cache, is_neox) + q = q.view(num_heads, head_dim) + k = k.view(num_kv_heads, head_dim) + v = v.view(num_kv_heads, head_dim) + dummy = unified_kv_cache_update(k, v, layer_name) + + Replace with: + q_out = empty(...) + k_out = empty(...) + dummy = fused_qk_norm_rope_and_unified_kv_cache_update( + q_out, k_out, qkv, positions, q_weight, k_weight, + eps, cos_sin_cache, is_neox, layer_name) + v = split(qkv, ...)[2].view(num_kv_heads, head_dim) + """ + + FUSED_OP = torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default + + def __init__( + self, + layer: Attention, + eps: float, + is_neox: bool, + quant_query: bool, + ) -> None: + self.layer_name = layer.layer_name + self.num_heads = layer.num_heads + self.num_kv_heads = layer.num_kv_heads + self.head_size = layer.head_size + self.head_size_v = layer.head_size_v + self.eps = eps + self.is_neox = is_neox + self.quant_query = quant_query + + self.q_size = self.num_heads * self.head_size + self.k_size = self.num_kv_heads * self.head_size + self.v_size = self.num_kv_heads * self.head_size_v + + self.rope_matcher = MatcherRotaryEmbedding( + is_neox=is_neox, + head_size=self.head_size, + num_heads=self.num_heads, + num_kv_heads=self.num_kv_heads, + ) + + def get_inputs(self) -> list[torch.Tensor]: + T = 5 + L = 4096 + qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size) + positions = empty_i64(T) + q_weight = empty_bf16(1, self.head_size) + k_weight = empty_bf16(1, self.head_size) + cos_sin_cache = empty_bf16(L, self.head_size) + inputs = [qkv, positions, q_weight, k_weight, cos_sin_cache] + if self.quant_query: + q_scale = empty_fp32(1) + inputs += [q_scale] + return inputs + + def pattern_non_fp8_quant_query( + self, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) + q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size) + q_normed = vllm.ir.ops.rms_norm(q_by_head, q_weight, self.eps) + q_flat = q_normed.view(-1, self.q_size) + + k_by_head = k.view(-1, self.k_size // self.head_size, self.head_size) + k_normed = vllm.ir.ops.rms_norm(k_by_head, k_weight, self.eps) + k_flat = k_normed.view(-1, self.k_size) + + q_rope, k_rope = self.rope_matcher(positions, q_flat, k_flat, cos_sin_cache) + + q_rope = q_rope.view(-1, self.num_heads, self.head_size) + k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size) + v = v.view(-1, self.num_kv_heads, self.head_size_v) + dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name) + return dummy, q_rope, k_rope, v + + def replacement_non_fp8_quant_query( + self, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + q_out = torch.empty( + qkv.shape[0], + self.num_heads, + self.head_size, + device=qkv.device, + dtype=qkv.dtype, + ) + k_out = torch.empty( + qkv.shape[0], + self.num_kv_heads, + self.head_size, + device=qkv.device, + dtype=qkv.dtype, + ) + _, _, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) + v = v.view(qkv.shape[0], self.num_kv_heads, self.head_size_v) + results = auto_functionalized( + self.FUSED_OP, + q_out=q_out, + k_out=k_out, + qkv=qkv, + positions=positions, + q_weight=q_weight, + k_weight=k_weight, + rms_norm_eps=self.eps, + cos_sin_cache=cos_sin_cache, + is_neox=self.is_neox, + layer_name=self.layer_name, + ) + return results[0], results[1], results[2], v + + def pattern_fp8_quant_query( + self, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + q_scale: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) + q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size) + q_normed = vllm.ir.ops.rms_norm(q_by_head, q_weight, self.eps) + q_flat = q_normed.view(-1, self.q_size) + + k_by_head = k.view(-1, self.k_size // self.head_size, self.head_size) + k_normed = vllm.ir.ops.rms_norm(k_by_head, k_weight, self.eps) + k_flat = k_normed.view(-1, self.k_size) + + q_rope, k_rope = self.rope_matcher(positions, q_flat, k_flat, cos_sin_cache) + # Match the quant-query op Attention.forward inserts (fp8 KV + UNIFIED). + # Explicit auto_functionalized (out=[1]) keeps the quant node in the pattern. + q_out = torch.empty_like(q_rope, dtype=current_platform.fp8_dtype()) + q_quant = auto_functionalized( + torch.ops.vllm.rocm_aiter_per_tensor_quant.default, + out=q_out, + x=q_rope, + scale=q_scale, + is_dynamic=False, + ) + # `scale` is mutable: its copy_ write-back to _q_scale bumps the mutation + # region, so keep q flat (a reshape lands past the barrier and won't match). + q_rope_fp8 = q_quant[1] + q_scale_out = q_quant[2] + + k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size) + v = v.view(-1, self.num_kv_heads, self.head_size_v) + dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name) + return dummy, q_rope_fp8, k_rope, v, q_scale_out + + def replacement_fp8_quant_query( + self, + qkv: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + cos_sin_cache: torch.Tensor, + q_scale: torch.Tensor, + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + q_out = torch.empty( + qkv.shape[0], + self.num_heads, + self.head_size, + device=qkv.device, + dtype=qkv.dtype, + ) + k_out = torch.empty( + qkv.shape[0], + self.num_kv_heads, + self.head_size, + device=qkv.device, + dtype=qkv.dtype, + ) + _, _, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1) + v = v.view(qkv.shape[0], self.num_kv_heads, self.head_size_v) + results = auto_functionalized( + self.FUSED_OP, + q_out=q_out, + k_out=k_out, + qkv=qkv, + positions=positions, + q_weight=q_weight, + k_weight=k_weight, + rms_norm_eps=self.eps, + cos_sin_cache=cos_sin_cache, + is_neox=self.is_neox, + layer_name=self.layer_name, + ) + # Re-apply the quant on the kernel's bf16 q_out; fused op does not quant q. + # Same explicit auto_functionalized form as the pattern: [1] = quantized + # q, [2] = scale (returned so the buffer-writeback use is preserved). + q_fp8_flat = results[1].view(-1, self.q_size) + q_fp8_out = torch.empty_like(q_fp8_flat, dtype=current_platform.fp8_dtype()) + q_requant = auto_functionalized( + torch.ops.vllm.rocm_aiter_per_tensor_quant.default, + out=q_fp8_out, + x=q_fp8_flat, + scale=q_scale, + is_dynamic=False, + ) + q_fp8 = q_requant[1] # flat to mirror the pattern (see note above) + q_scale_out = q_requant[2] + return results[0], q_fp8, results[2], v, q_scale_out + + @staticmethod + def wrap_trace_fn( + trace_fn: Callable[P, fx.GraphModule], + *process_fx_fns: Callable[[fx.GraphModule], None], + ) -> Callable[P, fx.GraphModule]: + def wrapped(*args: P.args, **kwargs: P.kwargs) -> fx.GraphModule: + gm = trace_fn(*args, **kwargs) + for process_fx in process_fx_fns: + process_fx(gm) + + return gm + + return wrapped + + @staticmethod + def fx_view_to_reshape(gm: torch.fx.GraphModule) -> None: + from torch._inductor.fx_passes.post_grad import view_to_reshape + + view_to_reshape(gm) + + def _register(self, pattern, replacement, pm_pass) -> None: + trace_fn = QkNormRopeKvCachePattern.wrap_trace_fn( + pm.fwd_only, + QkNormRopeKvCachePattern.fx_view_to_reshape, + ) + + # Pre-build the search pattern with `ignore_types=(int, torch.SymInt)` + # and pass it via `search_fn_pattern=` so torch skips both of its + # internal `fx_to_pattern` calls and treats dynamic-shape SymInts as + # wildcards. + inputs = self.get_inputs() + argnames = [*inspect.signature(pattern).parameters.keys()] + search_gm = trace_fn(pattern, inputs) + search_fn_pattern = pm.fx_to_pattern( + search_gm, + ignore_types=(int, torch.SymInt), + argnames=argnames, + ) + + pm.register_replacement( + pattern, + replacement, + inputs, + trace_fn, + pm_pass, + search_fn_pattern=search_fn_pattern, + ) + + def register(self, pm_pass: PatternMatcherPass) -> None: + # make_fx counts `self` in bound-method code params; wrap as plain fns. + # Distinct names per branch so mypy doesn't see one name, two signatures. + if self.quant_query: + + def pattern_q(qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale): + return self.pattern_fp8_quant_query( + qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale + ) + + def replacement_q( + qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale + ): + return self.replacement_fp8_quant_query( + qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale + ) + + self._register(pattern_q, replacement_q, pm_pass) + else: + + def pattern_noq(qkv, positions, q_weight, k_weight, cos_sin_cache): + return self.pattern_non_fp8_quant_query( + qkv, positions, q_weight, k_weight, cos_sin_cache + ) + + def replacement_noq(qkv, positions, q_weight, k_weight, cos_sin_cache): + return self.replacement_non_fp8_quant_query( + qkv, positions, q_weight, k_weight, cos_sin_cache + ) + + self._register(pattern_noq, replacement_noq, pm_pass) + + +# --------------------------------------------------------------------------- +# Pass class +# --------------------------------------------------------------------------- + + +class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass): + """ + Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel. + + Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for + attention layers that support the combined operation, eliminating two + separate kernel launches and the intermediate memory traffic. + """ + + @enable_fake_mode + def __init__(self, config: VllmConfig) -> None: + super().__init__(config) + + self.patterns: PatternMatcherPass = PatternMatcherPass( + pass_name="qk_norm_rope_kvcache_fusion_pass" + ) + + cc = config.compilation_config + self.max_token_num = cc.pass_config.rope_kvcache_fusion_max_token_num + + dtype = config.model_config.dtype + if dtype not in (torch.bfloat16, torch.float16): + logger.warning_once( + "QK Norm+RoPE+KVCache fusion not enabled: unsupported dtype %s", dtype + ) + return + + attn_layers = get_layers_from_vllm_config(config, Attention) + + for _, layer in attn_layers.items(): + if not layer.impl.fused_qk_norm_rope_kvcache_supported(): + continue + if layer.head_size not in SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS: + logger.warning_once( + "QK Norm+RoPE+KVCache fusion not enabled for a layer: " + "head_size=%d is not supported by the " + "fused_qk_norm_rope_cache_pts_quant_shuffle kernel " + "(supported: %s). Falling back to the unfused path.", + layer.head_size, + SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS, + ) + continue + if layer.head_size_v != layer.head_size: + # The fused kernel uses a single head_dim for q/k/v. + logger.warning_once( + "QK Norm+RoPE+KVCache fusion not enabled for a layer: " + "head_size_v=%d differs from head_size=%d, which the fused " + "kernel does not support. Falling back to the unfused path.", + layer.head_size_v, + layer.head_size, + ) + continue + for epsilon in [1e-5, 1e-6]: + for neox in [True, False]: + for quant_q in [False, True]: + QkNormRopeKvCachePattern( + layer=layer, + eps=epsilon, + is_neox=neox, + quant_query=quant_q, + ).register(self.patterns) + + self.dump_patterns(config, self.patterns) + + @VllmInductorPass.time_and_log + def __call__(self, graph: fx.Graph) -> None: + self.matched_count = self.patterns.apply(graph) + logger.info( + "QK-Norm+RoPE+KVCache fusion: replaced %s pattern(s) " + "with AITER fused_qk_norm_rope_cache_pts_quant_shuffle", + self.matched_count, + ) + + def is_applicable_for_range(self, compile_range: Range) -> bool: + return compile_range.end <= self.max_token_num + + def uuid(self) -> str: + return VllmInductorPass.hash_source(self, QkNormRopeKvCachePattern) diff --git a/vllm/compilation/passes/pass_manager.py b/vllm/compilation/passes/pass_manager.py index 4b98ac5774..67a3e4cbae 100644 --- a/vllm/compilation/passes/pass_manager.py +++ b/vllm/compilation/passes/pass_manager.py @@ -38,6 +38,7 @@ if current_platform.is_cuda_alike(): from .fusion.mla_attn_quant_fusion import MLAAttnQuantFusionPass from .fusion.mla_rope_kvcache_cat_fusion import MLARoPEKVCacheCatFusionPass from .fusion.qk_norm_rope_fusion import QKNormRoPEFusionPass + from .fusion.qk_norm_rope_kvcache_fusion import QkNormRopeKvCacheFusionPass from .fusion.rms_quant_fusion import RMSNormQuantFusionPass from .fusion.rope_kvcache_fusion import RopeKVCacheFusionPass from .utility.scatter_split_replace import ScatterSplitReplacementPass @@ -171,6 +172,11 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc] if rocm_aiter_ops.is_enabled(): self.passes += [RocmAiterSiluMulFp8GroupQuantFusionPass(config)] + if self.pass_config.fuse_qk_norm_rope_kvcache: + self.passes += [SplitCoalescingPass(config)] + self.passes += [ScatterSplitReplacementPass(config)] + self.passes += [QkNormRopeKvCacheFusionPass(config)] + if ( self.pass_config.fuse_mla_dual_rms_norm and rocm_aiter_ops.is_enabled() diff --git a/vllm/config/compilation.py b/vllm/config/compilation.py index 810c40131f..810e97c4ca 100644 --- a/vllm/config/compilation.py +++ b/vllm/config/compilation.py @@ -146,10 +146,16 @@ class PassConfig: """Fuse paired q/kv RMS norms in MLA attention.""" fuse_rope_kvcache: bool = None # type: ignore[assignment] """Fuse the QK rope + KV cache ops.""" + fuse_qk_norm_rope_kvcache: bool = Field(default=None) # type: ignore[assignment] + """Fuse QK RMSNorm + RoPE + KV cache update into a single AITER HIP + kernel. Supersedes both enable_qk_norm_rope_fusion and fuse_rope_kvcache + for layers that support it. Auto-enabled at O1+ on ROCm for models + with QK-norm (e.g. Qwen3-MoE).""" rope_kvcache_fusion_max_token_num: int = 256 """The threshold for ROCm AITER RoPE+KVCache fusion e.g. for small batch decode. Larger batch sizes e.g. during prefill will use the unfused kernels. + Also applies to the fused QK-Norm+RoPE+KVCache pass. """ fi_allreduce_fusion_max_size_mb: float | None = None @@ -228,6 +234,8 @@ class PassConfig: "fuse_act_padding", "fuse_mla_dual_rms_norm", "fuse_rope_kvcache", + "fuse_qk_norm_rope_kvcache", + "enable_qk_norm_rope_fusion", "fuse_rope_kvcache_cat_mla", mode="wrap", ) @@ -288,6 +296,12 @@ class PassConfig: "The fusion will be disabled." ) self.fuse_rope_kvcache = False + if self.fuse_qk_norm_rope_kvcache and not current_platform.is_rocm(): + logger.warning_once( + "QK-Norm+RoPE+KVCache fusion requires ROCm with AITER. " + "The fusion will be disabled." + ) + self.fuse_qk_norm_rope_kvcache = False if self.fuse_rope_kvcache_cat_mla and not current_platform.is_cuda_alike(): logger.warning_once( "MLA KV cache update with RoPE fusion enabled but the " @@ -302,10 +316,13 @@ class PassConfig: after all defaults are finalized. TODO also log the compile ranges for which this is enabled. """ + fusion_prefixes = ("fuse_", "enable_") enabled_fusions = [ - f.name[len("fuse_") :] + f.name[len(prefix) :] for f in fields(self) # type: ignore[arg-type] - if getattr(self, f.name) and f.name.startswith("fuse_") + if getattr(self, f.name) + for prefix in fusion_prefixes + if f.name.startswith(prefix) ] if enabled_fusions: @@ -947,6 +964,7 @@ class CompilationConfig: # TODO(zhuhaoran): support rope native forward match and remove this. # Linked issue: https://github.com/vllm-project/vllm/issues/28042 self.custom_ops.append("+rotary_embedding") + if ( self.pass_config.fuse_rope_kvcache and "+rotary_embedding" not in self.custom_ops @@ -955,6 +973,12 @@ class CompilationConfig: # Linked issue: https://github.com/vllm-project/vllm/issues/28042 self.custom_ops.append("+rotary_embedding") + if ( + self.pass_config.fuse_qk_norm_rope_kvcache + and "+rotary_embedding" not in self.custom_ops + ): + self.custom_ops.append("+rotary_embedding") + if ( is_torch_equal_or_newer("2.9.0.dev") and "combo_kernels" not in self.inductor_compile_config @@ -1137,6 +1161,16 @@ class CompilationConfig: "to enable RoPE+KV cache fusion." ) self.pass_config.fuse_rope_kvcache = False + if self.pass_config.fuse_qk_norm_rope_kvcache: + logger.warning_once( + "fuse_qk_norm_rope_kvcache is enabled, but " + "splitting_ops is None and Inductor graph partition " + "is not enabled. Disabling fuse_qk_norm_rope_kvcache. " + "Please either set splitting_ops to an empty list [] " + "or set use_inductor_graph_partition to True " + "to enable QK-Norm+RoPE+KV cache fusion." + ) + self.pass_config.fuse_qk_norm_rope_kvcache = False self.splitting_ops.append("vllm::unified_kv_cache_update") self.splitting_ops.append("vllm::unified_mla_kv_cache_update") diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index 057baed4b2..2e57cb1ec8 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -193,6 +193,15 @@ def enable_mla_dual_rms_norm_fusion(cfg: "VllmConfig") -> bool: return rocm_aiter_ops.is_enabled() and check_aiter_fused_qk_rmsnorm() +def enable_qk_norm_rope_kvcache(cfg: "VllmConfig") -> bool: + """Enable fused QK-norm + RoPE + KV cache update on ROCm with AITER.""" + from vllm._aiter_ops import rocm_aiter_ops + + if not rocm_aiter_ops.is_enabled(): + return False + return cfg.compilation_config.is_custom_op_enabled("rotary_embedding") + + OPTIMIZATION_LEVEL_00 = { "compilation_config": { "pass_config": { @@ -205,6 +214,8 @@ OPTIMIZATION_LEVEL_00 = { "fuse_act_padding": False, "fuse_mla_dual_rms_norm": False, "fuse_rope_kvcache": False, + "fuse_qk_norm_rope_kvcache": False, + "enable_qk_norm_rope_fusion": False, "fuse_rope_kvcache_cat_mla": False, }, "cudagraph_mode": CUDAGraphMode.NONE, @@ -226,6 +237,8 @@ OPTIMIZATION_LEVEL_01 = { "fuse_act_padding": enable_norm_pad_fusion, "fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion, "fuse_rope_kvcache": False, + "fuse_qk_norm_rope_kvcache": False, + "enable_qk_norm_rope_fusion": False, "fuse_rope_kvcache_cat_mla": False, }, "cudagraph_mode": CUDAGraphMode.PIECEWISE, @@ -247,6 +260,8 @@ OPTIMIZATION_LEVEL_02 = { "fuse_act_padding": enable_norm_pad_fusion, "fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion, "fuse_rope_kvcache": enable_rope_kvcache_fusion, + "fuse_qk_norm_rope_kvcache": enable_qk_norm_rope_kvcache, + "enable_qk_norm_rope_fusion": False, "fuse_rope_kvcache_cat_mla": enable_rope_kvcache_mla_fusion, }, "cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE, @@ -268,6 +283,8 @@ OPTIMIZATION_LEVEL_03 = { "fuse_act_padding": enable_norm_pad_fusion, "fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion, "fuse_rope_kvcache": enable_rope_kvcache_fusion, + "fuse_qk_norm_rope_kvcache": enable_qk_norm_rope_kvcache, + "enable_qk_norm_rope_fusion": False, "fuse_rope_kvcache_cat_mla": enable_rope_kvcache_mla_fusion, }, "cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE, @@ -1971,6 +1988,21 @@ class VllmConfig: compile_range_end, ) + if compilation_config.pass_config.fuse_qk_norm_rope_kvcache: + max_token_num = ( + compilation_config.pass_config.rope_kvcache_fusion_max_token_num + ) + if max_token_num is not None: + if compile_range_end is not None and max_token_num < compile_range_end: + computed_compile_ranges_endpoints.append(max_token_num) + else: + logger.debug( + "Max num batched tokens below qk_norm+rope+kvcache " + "fusion threshold, fusion enabled for " + "num_tokens <= %d.", + compile_range_end, + ) + if compilation_config.compile_ranges_endpoints is not None: for x in compilation_config.compile_ranges_endpoints: assert isinstance(x, int) diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index d8c4cf80ba..375049d3b7 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -136,10 +136,12 @@ def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) -> # We also keep q/k/v_scale on host (cpu) memory for attention # backends that require the scales to be on host instead of on device. - # e.g. Flashinfer + # e.g. Flashinfer & AITER layer._q_scale_float = 1.0 layer._k_scale_float = 1.0 layer._v_scale_float = 1.0 + layer._k_scale_cpu = torch.tensor(1.0, dtype=torch.float32) + layer._v_scale_cpu = torch.tensor(1.0, dtype=torch.float32) layer._prob_scale_float = 1.0 # Initialize q/k/v range constants used by calc_kv_scales @@ -586,6 +588,8 @@ class Attention(nn.Module, AttentionLayerBase): self._q_scale_float = self._q_scale.item() self._k_scale_float = self._k_scale.item() self._v_scale_float = self._v_scale.item() + self._k_scale_cpu.fill_(self._k_scale_float) + self._v_scale_cpu.fill_(self._v_scale_float) # We only calculate the scales once self.calculate_kv_scales = False diff --git a/vllm/model_executor/layers/attention/mla_attention.py b/vllm/model_executor/layers/attention/mla_attention.py index 108e6ff152..c1b997dfb5 100644 --- a/vllm/model_executor/layers/attention/mla_attention.py +++ b/vllm/model_executor/layers/attention/mla_attention.py @@ -1024,6 +1024,8 @@ class MLAAttention(nn.Module, AttentionLayerBase): self._q_scale_float = self._q_scale.item() self._k_scale_float = self._k_scale.item() self._v_scale_float = self._v_scale.item() + self._k_scale_cpu.fill_(self._k_scale_float) + self._v_scale_cpu.fill_(self._v_scale_float) self.calculate_kv_scales = False def get_attn_backend(self) -> type[AttentionBackend]: diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py index 6f1d20bd7f..260a8309ff 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py @@ -1200,6 +1200,10 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod): layer._v_scale_float = _to_scalar(layer.v_scale) layer._q_scale_float = _to_scalar(layer.q_scale) + # Sync host (cpu) scale copies read by AITER fused kernels. + layer._k_scale_cpu.fill_(layer._k_scale_float) + layer._v_scale_cpu.fill_(layer._v_scale_float) + # Discard all placeholders. del layer.k_scale del layer.v_scale diff --git a/vllm/model_executor/layers/quantization/kv_cache.py b/vllm/model_executor/layers/quantization/kv_cache.py index 100632686b..380ad33f28 100644 --- a/vllm/model_executor/layers/quantization/kv_cache.py +++ b/vllm/model_executor/layers/quantization/kv_cache.py @@ -144,6 +144,9 @@ class BaseKVCacheMethod(QuantizeMethodBase): layer._v_scale.copy_(v_scale) layer._k_scale_float = k_scale layer._v_scale_float = v_scale + # Sync host (cpu) scale copies read by AITER fused kernels. + layer._k_scale_cpu.fill_(k_scale) + layer._v_scale_cpu.fill_(v_scale) if k_scale == 1.0 and v_scale == 1.0 and "e5m2" not in layer.kv_cache_dtype: logger.warning_once( "Using KV cache scaling factor 1.0 for fp8_e4m3. " diff --git a/vllm/v1/attention/backend.py b/vllm/v1/attention/backend.py index 2708e0ab1f..a82d49dadb 100644 --- a/vllm/v1/attention/backend.py +++ b/vllm/v1/attention/backend.py @@ -757,7 +757,9 @@ class AttentionMetadataBuilder(ABC, Generic[M]): class AttentionLayer(Protocol): _q_scale: torch.Tensor _k_scale: torch.Tensor + _k_scale_cpu: torch.Tensor _v_scale: torch.Tensor + _v_scale_cpu: torch.Tensor _q_scale_float: float _k_scale_float: float _v_scale_float: float @@ -923,6 +925,14 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]): """ return False + def fused_qk_norm_rope_kvcache_supported(self): + """ + Does this attention implementation support fused QKNorm+RoPE+KVCache fusion. + This is used by the QkNormRopeKvCachePattern to only fuse the QKNorm ops + with the RoPE ops and the KV cache update for implementations that support it. + """ + return False + def fused_rope_kvcache_supported(self): """ Does this attention implementation support RoPE+KVCache fusion. @@ -931,6 +941,29 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]): """ return False + def do_qk_norm_rope_kvcache_update( + self, + layer: AttentionLayer, + qkv: torch.Tensor, + q_out: torch.Tensor, + k_out: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + kv_cache: torch.Tensor, + layer_slot_mapping: torch.Tensor, + ): + """ + If `fused_qk_norm_rope_kvcache_supported` returns True, this method + will be called by the fused custom op. Applies QK-norm + RoPE and + writes K/V to the KV cache. Results are written to the pre-allocated + q_out and k_out tensors; V is split from QKV at the graph level. + """ + raise NotImplementedError + def do_rope_and_kv_cache_update( self, layer: AttentionLayer, diff --git a/vllm/v1/attention/backends/rocm_aiter_fa.py b/vllm/v1/attention/backends/rocm_aiter_fa.py index b2edc2d051..dd14b2869a 100644 --- a/vllm/v1/attention/backends/rocm_aiter_fa.py +++ b/vllm/v1/attention/backends/rocm_aiter_fa.py @@ -1449,6 +1449,47 @@ class AiterFlashAttentionImpl(AttentionImpl): and not rocm_aiter_ops.is_shuffle_kv_cache_enabled() ) + def fused_qk_norm_rope_kvcache_supported(self): + return rocm_aiter_ops.is_enabled() + + def do_qk_norm_rope_kvcache_update( + self, + layer: AttentionLayer, + qkv: torch.Tensor, + q_out: torch.Tensor, + k_out: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + kv_cache: torch.Tensor, + layer_slot_mapping: torch.Tensor, + ): + key_cache, value_cache = kv_cache.unbind(1) + rocm_aiter_ops.do_qk_norm_rope_kvcache_update( + qkv=qkv, + q_weight=q_weight, + k_weight=k_weight, + cos_sin_cache=cos_sin_cache, + positions=positions, + num_heads_q=self.num_heads, + num_heads_k=self.num_kv_heads, + head_dim=self.head_size, + is_neox=is_neox, + rms_norm_eps=rms_norm_eps, + q_out=q_out, + k_out=k_out, + key_cache=key_cache, + value_cache=value_cache, + slot_mapping=layer_slot_mapping, + k_scale=layer._k_scale_cpu, + v_scale=layer._v_scale_cpu, + kv_cache_dtype=self.kv_cache_dtype, + use_shuffle_layout=rocm_aiter_ops.is_shuffle_kv_cache_enabled(), + ) + def do_rope_and_kv_cache_update( self, layer: AttentionLayer, diff --git a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py index cc45fade5a..08cd869686 100644 --- a/vllm/v1/attention/backends/rocm_aiter_unified_attn.py +++ b/vllm/v1/attention/backends/rocm_aiter_unified_attn.py @@ -311,6 +311,50 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl): def fused_rope_kvcache_supported(self): return rocm_aiter_ops.is_enabled() + def fused_qk_norm_rope_kvcache_supported(self): + return rocm_aiter_ops.is_enabled() + + def do_qk_norm_rope_kvcache_update( + self, + layer: AttentionLayer, + qkv: torch.Tensor, + q_out: torch.Tensor, + k_out: torch.Tensor, + positions: torch.Tensor, + q_weight: torch.Tensor, + k_weight: torch.Tensor, + rms_norm_eps: float, + cos_sin_cache: torch.Tensor, + is_neox: bool, + kv_cache: torch.Tensor, + layer_slot_mapping: torch.Tensor, + ): + # _split_kv_cache picks the unbind dim per layout (incl. the K/V-first + # encoder-decoder path). unified reads NHD, so never write the shuffle + # layout here. + key_cache, value_cache = self._split_kv_cache(kv_cache) + rocm_aiter_ops.do_qk_norm_rope_kvcache_update( + qkv=qkv, + q_weight=q_weight, + k_weight=k_weight, + cos_sin_cache=cos_sin_cache, + positions=positions, + num_heads_q=self.num_heads, + num_heads_k=self.num_kv_heads, + head_dim=self.head_size, + is_neox=is_neox, + rms_norm_eps=rms_norm_eps, + q_out=q_out, + k_out=k_out, + key_cache=key_cache, + value_cache=value_cache, + slot_mapping=layer_slot_mapping, + k_scale=layer._k_scale_cpu, + v_scale=layer._v_scale_cpu, + kv_cache_dtype=self.kv_cache_dtype, + use_shuffle_layout=False, + ) + def do_rope_and_kv_cache_update( self, layer: AttentionLayer,