[Model][Hardware][AMD]: Part 1/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_AITER_FA, and ROCM_AITER_UNIFIED_ATTN (#42749)

Signed-off-by: Jack Hu <Jack.Hu@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Jack Hu
2026-07-16 17:39:04 -05:00
committed by GitHub
co-authored by Douglas Lehr
parent ab0a20d151
commit 2cab53ddee
15 changed files with 1361 additions and 4 deletions
@@ -0,0 +1,478 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import pytest
import torch
import vllm.config
from tests.compile.backend import TestBackend
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP
from vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion import (
QkNormRopeKvCacheFusionPass,
)
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
from vllm.compilation.passes.utility.scatter_split_replace import (
ScatterSplitReplacementPass,
)
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
from vllm.config import (
CacheConfig,
CompilationConfig,
CompilationMode,
ModelConfig,
PassConfig,
VllmConfig,
)
from vllm.forward_context import get_forward_context, set_forward_context
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
from vllm.platforms import current_platform
from vllm.v1.attention.backend import (
AttentionBackend,
CommonAttentionMetadata,
)
from vllm.v1.attention.backends.registry import AttentionBackendEnum
from vllm.v1.kv_cache_interface import AttentionSpec
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
FP8_DTYPE = current_platform.fp8_dtype()
class QKNormRoPEKVCacheTestModel(torch.nn.Module):
"""Minimal model that reproduces the QK-norm + RoPE + KV cache update
pattern matched by QkNormRopeKvCacheFusionPass:
q, k, v = split(qkv)
q = rms_norm(q.view(heads, dim), q_weight).view(flat)
k = rms_norm(k.view(heads, dim), k_weight).view(flat)
q, k = rotary_emb(positions, q, k)
q = q.view(num_heads, head_dim)
k = k.view(num_kv_heads, head_dim)
v = v.view(num_kv_heads, head_dim)
dummy = unified_kv_cache_update(k, v, layer_name)
"""
def __init__(
self,
vllm_config: VllmConfig,
attn_backend: AttentionBackendEnum,
num_heads: int,
num_kv_heads: int,
head_size: int,
is_neox: bool,
rms_norm_eps: float,
dtype: torch.dtype,
device: torch.device,
rotary_dim: int | None = None,
prefix: str = "model.layers.0.self_attn.attn",
):
super().__init__()
self.num_heads = num_heads
self.num_kv_heads = num_kv_heads
self.head_size = head_size
self.rotary_dim = rotary_dim if rotary_dim is not None else head_size
self.block_size = vllm_config.cache_config.block_size
self.q_size = num_heads * head_size
self.kv_size = num_kv_heads * head_size
self.is_neox = is_neox
self.dtype = dtype
self.device = device
self.layer_name = prefix
self.q_norm = RMSNorm(head_size, eps=rms_norm_eps)
self.k_norm = RMSNorm(head_size, eps=rms_norm_eps)
self.rotary_emb = RotaryEmbedding(
head_size,
rotary_dim=self.rotary_dim,
max_position_embeddings=4096,
base=10000,
is_neox_style=is_neox,
dtype=self.dtype,
)
self.enable_rope_custom_op = self.rotary_emb.enabled()
self.attn = Attention(
num_heads=num_heads,
head_size=head_size,
scale=1.0 / head_size**0.5,
num_kv_heads=num_kv_heads,
cache_config=vllm_config.cache_config,
quant_config=vllm_config.quant_config,
prefix=prefix,
attn_backend=attn_backend.get_class(),
)
self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend()
assert not self.attn_backend.forward_includes_kv_cache_update, (
f"Attention backend {self.attn_backend} does not support "
"fuse_qk_norm_rope_kvcache."
)
kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
self.kv_cache_dtype = (
FP8_DTYPE if kv_cache_dtype_str.startswith("fp8") else self.dtype
)
if self.kv_cache_dtype != self.dtype:
self.attn._k_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
self.attn._v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
self.attn._k_scale_float = 1.0
self.attn._v_scale_float = 1.0
else:
self.attn._k_scale = self.attn._k_scale.to(device)
self.attn._v_scale = self.attn._v_scale.to(device)
self.builder = self.attn.attn_backend.get_builder_cls()(
kv_cache_spec=AttentionSpec(
block_size=self.block_size,
num_kv_heads=self.num_kv_heads,
head_size=head_size,
dtype=self.kv_cache_dtype,
),
layer_names=[self.attn.layer_name],
vllm_config=vllm_config,
device=device,
)
def build_attn_metadata(
self, batch_size: int, kv_stride_order: tuple[int, ...] | None = None
) -> CommonAttentionMetadata:
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
common_attn_metadata = create_common_attn_metadata(
batch_spec, self.block_size, self.device, arange_block_indices=True
)
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
num_blocks = batch_size * max_blocks
attn_backend = self.attn.attn_backend
kv_cache_shape = attn_backend.get_kv_cache_shape(
num_blocks, self.block_size, self.num_kv_heads, self.head_size
)
# Caller can force a physical layout; else use the backend's.
if kv_stride_order is None:
try:
kv_stride_order = attn_backend.get_kv_cache_stride_order()
except (AttributeError, NotImplementedError):
kv_stride_order = tuple(range(len(kv_cache_shape)))
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_stride_order)
inv_order = [kv_stride_order.index(i) for i in range(len(kv_stride_order))]
raw_tensor = torch.zeros(
2 * num_blocks * self.block_size * self.num_kv_heads * self.head_size,
dtype=self.kv_cache_dtype,
device=self.device,
)
raw_tensor = raw_tensor.view(kv_cache_shape)
kv_cache = raw_tensor.permute(*inv_order)
# Store as a bare tensor (not wrapped in a list) to match production
# `bind_kv_cache` behavior. `get_attention_context` returns this
# attribute directly to the fused/unfused `do_kv_cache_update` impls,
# which call `kv_cache.unbind(0)` and therefore require a tensor.
self.attn.kv_cache = kv_cache
attn_metadata = self.builder.build(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
return attn_metadata
def forward(
self, qkv: torch.Tensor, positions: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
qkv = qkv.clone()
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
# QK-norm: RMSNorm on per-head Q and K
q = q.view(-1, self.num_heads, self.head_size)
q = self.q_norm(q)
q = q.view(-1, self.q_size)
k = k.view(-1, self.num_kv_heads, self.head_size)
k = self.k_norm(k)
k = k.view(-1, self.kv_size)
# RoPE
q, k = self.rotary_emb(positions, q, k)
# Mirror Attention.forward: quant-query impls consume an fp8 q.
if (
self.kv_cache_dtype != self.dtype
and self.attn.impl.supports_quant_query_input
):
q_fp8 = torch.empty_like(q, dtype=FP8_DTYPE)
torch.ops.vllm.rocm_aiter_per_tensor_quant(
q_fp8, q, self.attn._q_scale, False
)
q = q_fp8
# Final views + KV cache update
q = q.view(-1, self.num_heads, self.head_size)
k = k.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size)
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
k, v, self.layer_name
)
return q, k, v, kv_cache_dummy_dep
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
ops: list[torch._ops.OpOverload] = []
# RoPE is not yet IR-migrated, so its custom op still surfaces
# directly in the graph based on `enable_rope_custom_op`.
if self.enable_rope_custom_op:
if rocm_aiter_ops.is_triton_rotary_embed_enabled():
ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default)
else:
ops.append(ROTARY_OP)
else:
ops.append(INDEX_SELECT_OP)
ops.append(torch.ops.vllm.unified_kv_cache_update.default)
return ops
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
return [torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default]
def _run_qk_norm_rope_kvcache_fusion_test(
*,
attn_backend: AttentionBackendEnum,
enable_aiter_triton_rope: bool,
num_tokens: int,
num_heads: int,
num_kv_heads: int,
head_size: int,
rotary_dim: int,
block_size: int,
is_neox: bool,
use_shuffle_kv_layout: str,
kv_stride_order: tuple[int, ...],
dtype: torch.dtype,
kv_cache_dtype: str,
rms_norm_eps: float,
custom_op: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda")
torch.set_default_device(device)
torch.set_default_dtype(dtype)
torch.manual_seed(0)
vllm_config = VllmConfig(
model_config=ModelConfig(dtype=dtype),
cache_config=CacheConfig(
block_size=block_size,
cache_dtype=kv_cache_dtype,
),
compilation_config=CompilationConfig(
mode=CompilationMode.VLLM_COMPILE,
custom_ops=[custom_op],
pass_config=PassConfig(
fuse_qk_norm_rope_kvcache=True,
eliminate_noops=True,
),
),
)
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
m.setenv("VLLM_ROCM_USE_AITER", "1")
m.setenv(
"VLLM_ROCM_USE_AITER_TRITON_ROPE",
"1" if enable_aiter_triton_rope else "0",
)
m.setenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", use_shuffle_kv_layout)
rocm_aiter_ops.refresh_env_variables()
model = QKNormRoPEKVCacheTestModel(
vllm_config=vllm_config,
attn_backend=attn_backend,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
rotary_dim=rotary_dim,
is_neox=is_neox,
rms_norm_eps=rms_norm_eps,
dtype=dtype,
device=torch.get_default_device(),
)
fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config)
passes = [
NoOpEliminationPass(vllm_config),
SplitCoalescingPass(vllm_config),
ScatterSplitReplacementPass(vllm_config),
fusion_pass,
PostCleanupPass(vllm_config),
]
backend = TestBackend(*passes)
qkv = torch.randn(
num_tokens,
num_heads * head_size + 2 * num_kv_heads * head_size,
dtype=dtype,
)
pos = torch.arange(num_tokens, dtype=torch.long)
qkv_unfused = qkv.clone()
pos_unfused = pos.clone()
# Run unfused (eager) forward
with set_forward_context(None, vllm_config):
forward_context = get_forward_context()
attn_metadata = model.build_attn_metadata(num_tokens, kv_stride_order)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_unfused = attn_layer.kv_cache
del dummy
# Run fused (compiled) forward
torch._dynamo.mark_dynamic(qkv, 0)
torch._dynamo.mark_dynamic(pos, 0)
with set_forward_context(None, vllm_config):
model_fused = torch.compile(model, backend=backend)
forward_context = get_forward_context()
attn_metadata = model_fused.build_attn_metadata(num_tokens, kv_stride_order)
forward_context.slot_mapping = {
model.layer_name: attn_metadata.slot_mapping
}
q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos)
attn_layer = forward_context.no_compile_layers[model.layer_name]
kv_cache_fused = attn_layer.kv_cache
del dummy
assert fusion_pass.matched_count == 1
backend.check_before_ops(model.ops_in_model_before())
backend.check_after_ops(model.ops_in_model_after())
# Sweep-backed (18.2k pts, PR #42749): native-rope ref worst 7.7e-3 -> 1e-2;
# AITER-triton-rope ref is itself approximate (plateau 1.28e-2) -> 2e-2.
ATOL, RTOL = (2e-2, 2e-2) if enable_aiter_triton_rope else (1e-2, 1e-2)
is_fp8_cache = model.kv_cache_dtype != dtype
if q_fused.dtype == FP8_DTYPE:
# Quant-query path: both q are fp8; compare dequant within 1 fp8 ULP.
torch.testing.assert_close(
q_unfused.float(), q_fused.float(), atol=1.25e-1, rtol=1.25e-1
)
else:
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
if not is_fp8_cache:
# The AITER PTS kernel populates k_out only for non-FP8 caches.
# With FP8, the kernel writes quantized K directly to the cache
# and may leave k_out uninitialised. In production this is fine
# because downstream attention reads K from the cache.
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
# Should be bit exact since no processing had been done on v for both paths
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
# fp8 vs triton-rope ref requires loosening tolerance to 1.25e-1.
if is_fp8_cache and enable_aiter_triton_rope:
cache_atol = cache_rtol = 1.25e-1
else:
cache_atol, cache_rtol = ATOL, RTOL
torch.testing.assert_close(
kv_cache_unfused[0].float(),
kv_cache_fused[0].float(),
atol=cache_atol,
rtol=cache_rtol,
)
_FUSION_CONFIGS = [
# Full rotary, both neox styles (the original coverage).
pytest.param(64, 8, 64, 64, True, id="full-neox"),
pytest.param(64, 8, 64, 64, False, id="full-non_neox"),
# GLM-4.5/4.6/4.7 (glm4_moe.py:275 partial_rotary_factor=0.5, neox-style)
pytest.param(32, 8, 128, 64, True, id="glm4_moe"),
# GLM-4 dense (glm4.py:97,124 partial_rotary_factor=0.5, non-neox)
pytest.param(32, 2, 128, 64, False, id="glm4_dense"),
# Moondream3-style small head (head_size=64, rotary_dim=32)
pytest.param(16, 2, 64, 32, True, id="partial_small_head"),
]
@pytest.mark.parametrize(
"num_heads, num_kv_heads, head_size, rotary_dim, is_neox",
_FUSION_CONFIGS,
)
@pytest.mark.parametrize(
"attn_backend",
[
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
AttentionBackendEnum.ROCM_AITER_FA,
],
)
@pytest.mark.parametrize("num_tokens", [5, 16, 2048])
@pytest.mark.parametrize("use_shuffle_kv_layout", ["1", "0"])
@pytest.mark.parametrize(
"kv_stride_order",
[
pytest.param((0, 1, 2, 3, 4), id="block_first"),
pytest.param((1, 0, 2, 3, 4), id="kv_first"),
],
)
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
@pytest.mark.parametrize("block_size", [16, 32, 64])
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
@pytest.mark.parametrize("rms_norm_eps", [1e-5, 1e-6])
@pytest.mark.parametrize("custom_op", ["+rotary_embedding", "+rms_norm"])
@pytest.mark.skipif(
not is_aiter_found_and_supported(),
reason="Only test on ROCm with AITER installed and supported",
)
def test_qk_norm_rope_kvcache_fusion(
num_tokens: int,
num_heads: int,
num_kv_heads: int,
head_size: int,
rotary_dim: int,
is_neox: bool,
attn_backend: AttentionBackendEnum,
enable_aiter_triton_rope: bool,
use_shuffle_kv_layout: str,
kv_stride_order: tuple[int, ...],
block_size: int,
dtype: torch.dtype,
kv_cache_dtype: str,
rms_norm_eps: float,
custom_op: str,
monkeypatch: pytest.MonkeyPatch,
):
if (
attn_backend == AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN
and use_shuffle_kv_layout == "1"
):
pytest.skip("ROCM_AITER_UNIFIED_ATTN is NHD-only; shuffle env is ignored")
_run_qk_norm_rope_kvcache_fusion_test(
attn_backend=attn_backend,
enable_aiter_triton_rope=enable_aiter_triton_rope,
num_tokens=num_tokens,
num_heads=num_heads,
num_kv_heads=num_kv_heads,
head_size=head_size,
rotary_dim=rotary_dim,
block_size=block_size,
is_neox=is_neox,
use_shuffle_kv_layout=use_shuffle_kv_layout,
kv_stride_order=kv_stride_order,
dtype=dtype,
kv_cache_dtype=kv_cache_dtype,
rms_norm_eps=rms_norm_eps,
custom_op=custom_op,
monkeypatch=monkeypatch,
)
+41
View File
@@ -17,6 +17,10 @@ from vllm.config.model import ModelConfig
from vllm.model_executor.kernels.linear.scaled_mm import (
MarlinFP8ScaledMMLinearKernel,
)
from vllm.model_executor.layers.attention.attention import (
Attention,
set_default_quant_scales,
)
from vllm.model_executor.layers.fused_moe import FusedMoE
from vllm.model_executor.layers.quantization.fp8 import (
Fp8Config,
@@ -24,6 +28,7 @@ from vllm.model_executor.layers.quantization.fp8 import (
Fp8LinearMethod,
Fp8MoEMethod,
)
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
from vllm.model_executor.layers.quantization.online.fp8 import (
Fp8PerTensorOnlineLinearMethod,
)
@@ -422,6 +427,42 @@ def test_fp8_reloading(
method.process_weights_after_loading(layer)
@pytest.mark.parametrize("source", ["checkpoint", "runtime_calc"])
def test_kv_cache_scale_sync_to_host_copies(source):
"""Test device-to-host sync of the k/v quantization scales, for both the
checkpoint-load and runtime-calc paths that produce them.
"""
layer = torch.nn.Module()
set_default_quant_scales(layer, register_buffer=True)
layer.kv_cache_dtype = "fp8"
if source == "checkpoint":
# Scales come from the checkpoint, so runtime calc is disabled.
layer.calculate_kv_scales = False
method = BaseKVCacheMethod(quant_config=None)
method.create_weights(layer)
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale)
layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale)
method.process_weights_after_loading(layer)
else:
# First forward computes distinct, non-unity scales from live k/v.
layer.calculate_kv_scales = True
query = torch.full((4, 8), 10.0)
key = torch.full((4, 8), 60.0)
value = torch.full((4, 8), 50.0)
Attention.calc_kv_scales(layer, query, key, value)
assert layer._k_scale_float != 1.0
assert layer._v_scale_float != 1.0
# Host copy must mirror both the float and the device scale tensor.
assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale_float)
assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale_float)
assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale.item())
assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale.item())
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",
+122
View File
@@ -2458,6 +2458,128 @@ class rocm_aiter_ops:
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y)
return y
@staticmethod
def fused_qk_norm_rope_and_cache(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
num_heads_q: int,
num_heads_k: int,
num_heads_v: int,
head_dim: int,
is_neox: bool,
rms_norm_eps: float,
q_out: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
slot_mapping: torch.Tensor,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
k_out: torch.Tensor | None,
v_out: torch.Tensor | None,
return_kv: bool,
use_shuffle_layout: bool,
block_size: int,
x: int,
rotary_dim: int = 0,
):
from aiter.ops.fused_qk_norm_rope_cache_quant import (
fused_qk_norm_rope_cache_pts_quant_shuffle,
)
fused_qk_norm_rope_cache_pts_quant_shuffle(
qkv,
q_weight,
k_weight,
cos_sin_cache,
positions,
qkv.size(0),
num_heads_q,
num_heads_k,
num_heads_v,
head_dim,
is_neox,
rms_norm_eps,
q_out,
k_cache,
v_cache,
slot_mapping,
k_scale,
v_scale,
k_out,
v_out,
return_kv,
use_shuffle_layout,
block_size,
x,
rotary_dim,
)
@staticmethod
def do_qk_norm_rope_kvcache_update(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
num_heads_q: int,
num_heads_k: int,
head_dim: int,
is_neox: bool,
rms_norm_eps: float,
q_out: torch.Tensor,
k_out: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
slot_mapping: torch.Tensor,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
kv_cache_dtype: str,
use_shuffle_layout: bool,
) -> None:
"""Run the fused QK-norm+RoPE+KV-cache op on already-split k/v caches.
Shared by the AITER FA and unified-attention impls. The caller splits
kv_cache, since the unbind dim depends on the layout (e.g. the unified
encoder-decoder path is K/V-first), and passes use_shuffle_layout
(unified reads NHD and must pass False).
"""
if kv_cache_dtype.startswith("fp8"):
key_cache = key_cache.view(current_platform.fp8_dtype())
value_cache = value_cache.view(current_platform.fp8_dtype())
# Partial-rotary support (e.g. GLM-4.7 applies rotary to only a prefix
# of each head's channel dim).
rotary_dim = cos_sin_cache.shape[-1]
kernel_rotary_dim = 0 if rotary_dim == head_dim else rotary_dim
rocm_aiter_ops.fused_qk_norm_rope_and_cache(
qkv=qkv,
q_weight=q_weight,
k_weight=k_weight,
cos_sin_cache=cos_sin_cache,
positions=positions,
num_heads_q=num_heads_q,
num_heads_k=num_heads_k,
num_heads_v=num_heads_k,
head_dim=head_dim,
is_neox=is_neox,
rms_norm_eps=rms_norm_eps,
q_out=q_out,
k_cache=key_cache,
v_cache=value_cache,
slot_mapping=slot_mapping,
k_scale=k_scale,
v_scale=v_scale,
k_out=k_out,
v_out=None,
return_kv=True,
use_shuffle_layout=use_shuffle_layout,
block_size=key_cache.shape[1],
x=16 // key_cache.element_size(),
rotary_dim=kernel_rotary_dim,
)
@staticmethod
def triton_rope_and_cache(
query: torch.Tensor,
@@ -25,6 +25,11 @@ logger = init_logger(__name__)
FUSED_QK_ROPE_OP = torch.ops._C.fused_qk_norm_rope.default
# Head dimensions supported by csrc/fused_qknorm_rope_kernel.cu's
# launchFusedQKNormRope and launchFusedQKNormRopeNTokenHeads dispatchers.
# Keep in sync with the switch statements in that file.
SUPPORTED_FUSED_QK_NORM_ROPE_HEAD_DIMS: tuple[int, ...] = (64, 128, 256)
P = ParamSpec("P")
@@ -190,7 +195,12 @@ class QkNormRopePattern:
class QKNormRoPEFusionPass(VllmPatternMatcherPass):
"""Fuse Q/K RMSNorm + RoPE into fused_qk_norm_rope when the custom op exists."""
"""Fuse Q/K RMSNorm + RoPE into fused_qk_norm_rope when the custom op exists.
Registers patterns for both standard vLLM ops and ROCm AITER ops
(when AITER is enabled), so the fusion fires regardless of which
RMSNorm/RoPE implementation the graph uses.
"""
@enable_fake_mode
def __init__(self, config: VllmConfig) -> None:
@@ -216,6 +226,17 @@ class QKNormRoPEFusionPass(VllmPatternMatcherPass):
)
return
for layer in attn_layers.values():
if layer.head_size not in SUPPORTED_FUSED_QK_NORM_ROPE_HEAD_DIMS:
logger.warning_once(
"QK Norm+RoPE fusion not enabled: layer head_size=%d is not "
"supported by fused_qk_norm_rope kernel (supported: %s). "
"Falling back to unfused QK norm + RoPE path.",
layer.head_size,
SUPPORTED_FUSED_QK_NORM_ROPE_HEAD_DIMS,
)
return
self._attention_geometries = tuple(
sorted(
{
@@ -0,0 +1,492 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import inspect
from collections.abc import Callable
from typing import ParamSpec
import torch
import torch._inductor.pattern_matcher as pm
from torch import fx
from torch._higher_order_ops.auto_functionalize import auto_functionalized
from torch._inductor.pattern_matcher import PatternMatcherPass
import vllm.ir.ops
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.config.utils import Range
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.attention import (
Attention,
get_attention_context,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
from ..inductor_pass import enable_fake_mode
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
from .matcher_utils import MatcherRotaryEmbedding
from .rms_quant_fusion import empty_bf16, empty_fp32, empty_i64
logger = init_logger(__name__)
P = ParamSpec("P")
# Head sizes the fused kernel fused_qk_norm_rope_cache_pts_quant_shuffle() supports
# Other sizes hard-abort, so skip those layers.
SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS: tuple[int, ...] = (64, 128, 256)
# ---------------------------------------------------------------------------
# Custom op: fused QK-norm + RoPE + KV cache update
# ---------------------------------------------------------------------------
def fused_qk_norm_rope_and_unified_kv_cache_update_impl(
q_out: torch.Tensor,
k_out: torch.Tensor,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
layer_name: str = "",
) -> torch.Tensor:
_, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
if layer_slot_mapping is not None:
attn_layer.impl.do_qk_norm_rope_kvcache_update(
attn_layer,
qkv,
q_out,
k_out,
positions,
q_weight,
k_weight,
rms_norm_eps,
cos_sin_cache,
is_neox,
kv_cache,
layer_slot_mapping,
)
else:
# Profiling/dummy run: define q_out/k_out (consumed by attention).
q_out.zero_()
k_out.zero_()
return torch.empty(0, device=qkv.device, dtype=qkv.dtype)
def fused_qk_norm_rope_and_unified_kv_cache_update_fake(
q_out: torch.Tensor,
k_out: torch.Tensor,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
layer_name: str = "",
) -> torch.Tensor:
return torch.empty(0, device=qkv.device, dtype=qkv.dtype)
direct_register_custom_op(
op_name="fused_qk_norm_rope_and_unified_kv_cache_update",
op_func=fused_qk_norm_rope_and_unified_kv_cache_update_impl,
mutates_args=["q_out", "k_out"],
fake_impl=fused_qk_norm_rope_and_unified_kv_cache_update_fake,
)
# ---------------------------------------------------------------------------
# Pattern: QK-norm + RoPE + unified_kv_cache_update
# ---------------------------------------------------------------------------
class QkNormRopeKvCachePattern:
"""
Match the unfused sequence:
q, k, v = split(qkv, ...)
q = rms_norm(q.view(heads), q_weight).view(flat)
k = rms_norm(k.view(heads), k_weight).view(flat)
q, k = rotary_embedding(positions, q, k, cos_sin_cache, is_neox)
q = q.view(num_heads, head_dim)
k = k.view(num_kv_heads, head_dim)
v = v.view(num_kv_heads, head_dim)
dummy = unified_kv_cache_update(k, v, layer_name)
Replace with:
q_out = empty(...)
k_out = empty(...)
dummy = fused_qk_norm_rope_and_unified_kv_cache_update(
q_out, k_out, qkv, positions, q_weight, k_weight,
eps, cos_sin_cache, is_neox, layer_name)
v = split(qkv, ...)[2].view(num_kv_heads, head_dim)
"""
FUSED_OP = torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default
def __init__(
self,
layer: Attention,
eps: float,
is_neox: bool,
quant_query: bool,
) -> None:
self.layer_name = layer.layer_name
self.num_heads = layer.num_heads
self.num_kv_heads = layer.num_kv_heads
self.head_size = layer.head_size
self.head_size_v = layer.head_size_v
self.eps = eps
self.is_neox = is_neox
self.quant_query = quant_query
self.q_size = self.num_heads * self.head_size
self.k_size = self.num_kv_heads * self.head_size
self.v_size = self.num_kv_heads * self.head_size_v
self.rope_matcher = MatcherRotaryEmbedding(
is_neox=is_neox,
head_size=self.head_size,
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
)
def get_inputs(self) -> list[torch.Tensor]:
T = 5
L = 4096
qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size)
positions = empty_i64(T)
q_weight = empty_bf16(1, self.head_size)
k_weight = empty_bf16(1, self.head_size)
cos_sin_cache = empty_bf16(L, self.head_size)
inputs = [qkv, positions, q_weight, k_weight, cos_sin_cache]
if self.quant_query:
q_scale = empty_fp32(1)
inputs += [q_scale]
return inputs
def pattern_non_fp8_quant_query(
self,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size)
q_normed = vllm.ir.ops.rms_norm(q_by_head, q_weight, self.eps)
q_flat = q_normed.view(-1, self.q_size)
k_by_head = k.view(-1, self.k_size // self.head_size, self.head_size)
k_normed = vllm.ir.ops.rms_norm(k_by_head, k_weight, self.eps)
k_flat = k_normed.view(-1, self.k_size)
q_rope, k_rope = self.rope_matcher(positions, q_flat, k_flat, cos_sin_cache)
q_rope = q_rope.view(-1, self.num_heads, self.head_size)
k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size_v)
dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name)
return dummy, q_rope, k_rope, v
def replacement_non_fp8_quant_query(
self,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q_out = torch.empty(
qkv.shape[0],
self.num_heads,
self.head_size,
device=qkv.device,
dtype=qkv.dtype,
)
k_out = torch.empty(
qkv.shape[0],
self.num_kv_heads,
self.head_size,
device=qkv.device,
dtype=qkv.dtype,
)
_, _, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
v = v.view(qkv.shape[0], self.num_kv_heads, self.head_size_v)
results = auto_functionalized(
self.FUSED_OP,
q_out=q_out,
k_out=k_out,
qkv=qkv,
positions=positions,
q_weight=q_weight,
k_weight=k_weight,
rms_norm_eps=self.eps,
cos_sin_cache=cos_sin_cache,
is_neox=self.is_neox,
layer_name=self.layer_name,
)
return results[0], results[1], results[2], v
def pattern_fp8_quant_query(
self,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
q_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size)
q_normed = vllm.ir.ops.rms_norm(q_by_head, q_weight, self.eps)
q_flat = q_normed.view(-1, self.q_size)
k_by_head = k.view(-1, self.k_size // self.head_size, self.head_size)
k_normed = vllm.ir.ops.rms_norm(k_by_head, k_weight, self.eps)
k_flat = k_normed.view(-1, self.k_size)
q_rope, k_rope = self.rope_matcher(positions, q_flat, k_flat, cos_sin_cache)
# Match the quant-query op Attention.forward inserts (fp8 KV + UNIFIED).
# Explicit auto_functionalized (out=[1]) keeps the quant node in the pattern.
q_out = torch.empty_like(q_rope, dtype=current_platform.fp8_dtype())
q_quant = auto_functionalized(
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
out=q_out,
x=q_rope,
scale=q_scale,
is_dynamic=False,
)
# `scale` is mutable: its copy_ write-back to _q_scale bumps the mutation
# region, so keep q flat (a reshape lands past the barrier and won't match).
q_rope_fp8 = q_quant[1]
q_scale_out = q_quant[2]
k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size_v)
dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name)
return dummy, q_rope_fp8, k_rope, v, q_scale_out
def replacement_fp8_quant_query(
self,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
q_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q_out = torch.empty(
qkv.shape[0],
self.num_heads,
self.head_size,
device=qkv.device,
dtype=qkv.dtype,
)
k_out = torch.empty(
qkv.shape[0],
self.num_kv_heads,
self.head_size,
device=qkv.device,
dtype=qkv.dtype,
)
_, _, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
v = v.view(qkv.shape[0], self.num_kv_heads, self.head_size_v)
results = auto_functionalized(
self.FUSED_OP,
q_out=q_out,
k_out=k_out,
qkv=qkv,
positions=positions,
q_weight=q_weight,
k_weight=k_weight,
rms_norm_eps=self.eps,
cos_sin_cache=cos_sin_cache,
is_neox=self.is_neox,
layer_name=self.layer_name,
)
# Re-apply the quant on the kernel's bf16 q_out; fused op does not quant q.
# Same explicit auto_functionalized form as the pattern: [1] = quantized
# q, [2] = scale (returned so the buffer-writeback use is preserved).
q_fp8_flat = results[1].view(-1, self.q_size)
q_fp8_out = torch.empty_like(q_fp8_flat, dtype=current_platform.fp8_dtype())
q_requant = auto_functionalized(
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
out=q_fp8_out,
x=q_fp8_flat,
scale=q_scale,
is_dynamic=False,
)
q_fp8 = q_requant[1] # flat to mirror the pattern (see note above)
q_scale_out = q_requant[2]
return results[0], q_fp8, results[2], v, q_scale_out
@staticmethod
def wrap_trace_fn(
trace_fn: Callable[P, fx.GraphModule],
*process_fx_fns: Callable[[fx.GraphModule], None],
) -> Callable[P, fx.GraphModule]:
def wrapped(*args: P.args, **kwargs: P.kwargs) -> fx.GraphModule:
gm = trace_fn(*args, **kwargs)
for process_fx in process_fx_fns:
process_fx(gm)
return gm
return wrapped
@staticmethod
def fx_view_to_reshape(gm: torch.fx.GraphModule) -> None:
from torch._inductor.fx_passes.post_grad import view_to_reshape
view_to_reshape(gm)
def _register(self, pattern, replacement, pm_pass) -> None:
trace_fn = QkNormRopeKvCachePattern.wrap_trace_fn(
pm.fwd_only,
QkNormRopeKvCachePattern.fx_view_to_reshape,
)
# Pre-build the search pattern with `ignore_types=(int, torch.SymInt)`
# and pass it via `search_fn_pattern=` so torch skips both of its
# internal `fx_to_pattern` calls and treats dynamic-shape SymInts as
# wildcards.
inputs = self.get_inputs()
argnames = [*inspect.signature(pattern).parameters.keys()]
search_gm = trace_fn(pattern, inputs)
search_fn_pattern = pm.fx_to_pattern(
search_gm,
ignore_types=(int, torch.SymInt),
argnames=argnames,
)
pm.register_replacement(
pattern,
replacement,
inputs,
trace_fn,
pm_pass,
search_fn_pattern=search_fn_pattern,
)
def register(self, pm_pass: PatternMatcherPass) -> None:
# make_fx counts `self` in bound-method code params; wrap as plain fns.
# Distinct names per branch so mypy doesn't see one name, two signatures.
if self.quant_query:
def pattern_q(qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale):
return self.pattern_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
)
def replacement_q(
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
):
return self.replacement_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
)
self._register(pattern_q, replacement_q, pm_pass)
else:
def pattern_noq(qkv, positions, q_weight, k_weight, cos_sin_cache):
return self.pattern_non_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache
)
def replacement_noq(qkv, positions, q_weight, k_weight, cos_sin_cache):
return self.replacement_non_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache
)
self._register(pattern_noq, replacement_noq, pm_pass)
# ---------------------------------------------------------------------------
# Pass class
# ---------------------------------------------------------------------------
class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
"""
Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for
attention layers that support the combined operation, eliminating two
separate kernel launches and the intermediate memory traffic.
"""
@enable_fake_mode
def __init__(self, config: VllmConfig) -> None:
super().__init__(config)
self.patterns: PatternMatcherPass = PatternMatcherPass(
pass_name="qk_norm_rope_kvcache_fusion_pass"
)
cc = config.compilation_config
self.max_token_num = cc.pass_config.rope_kvcache_fusion_max_token_num
dtype = config.model_config.dtype
if dtype not in (torch.bfloat16, torch.float16):
logger.warning_once(
"QK Norm+RoPE+KVCache fusion not enabled: unsupported dtype %s", dtype
)
return
attn_layers = get_layers_from_vllm_config(config, Attention)
for _, layer in attn_layers.items():
if not layer.impl.fused_qk_norm_rope_kvcache_supported():
continue
if layer.head_size not in SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS:
logger.warning_once(
"QK Norm+RoPE+KVCache fusion not enabled for a layer: "
"head_size=%d is not supported by the "
"fused_qk_norm_rope_cache_pts_quant_shuffle kernel "
"(supported: %s). Falling back to the unfused path.",
layer.head_size,
SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS,
)
continue
if layer.head_size_v != layer.head_size:
# The fused kernel uses a single head_dim for q/k/v.
logger.warning_once(
"QK Norm+RoPE+KVCache fusion not enabled for a layer: "
"head_size_v=%d differs from head_size=%d, which the fused "
"kernel does not support. Falling back to the unfused path.",
layer.head_size_v,
layer.head_size,
)
continue
for epsilon in [1e-5, 1e-6]:
for neox in [True, False]:
for quant_q in [False, True]:
QkNormRopeKvCachePattern(
layer=layer,
eps=epsilon,
is_neox=neox,
quant_query=quant_q,
).register(self.patterns)
self.dump_patterns(config, self.patterns)
@VllmInductorPass.time_and_log
def __call__(self, graph: fx.Graph) -> None:
self.matched_count = self.patterns.apply(graph)
logger.info(
"QK-Norm+RoPE+KVCache fusion: replaced %s pattern(s) "
"with AITER fused_qk_norm_rope_cache_pts_quant_shuffle",
self.matched_count,
)
def is_applicable_for_range(self, compile_range: Range) -> bool:
return compile_range.end <= self.max_token_num
def uuid(self) -> str:
return VllmInductorPass.hash_source(self, QkNormRopeKvCachePattern)
+6
View File
@@ -38,6 +38,7 @@ if current_platform.is_cuda_alike():
from .fusion.mla_attn_quant_fusion import MLAAttnQuantFusionPass
from .fusion.mla_rope_kvcache_cat_fusion import MLARoPEKVCacheCatFusionPass
from .fusion.qk_norm_rope_fusion import QKNormRoPEFusionPass
from .fusion.qk_norm_rope_kvcache_fusion import QkNormRopeKvCacheFusionPass
from .fusion.rms_quant_fusion import RMSNormQuantFusionPass
from .fusion.rope_kvcache_fusion import RopeKVCacheFusionPass
from .utility.scatter_split_replace import ScatterSplitReplacementPass
@@ -171,6 +172,11 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
if rocm_aiter_ops.is_enabled():
self.passes += [RocmAiterSiluMulFp8GroupQuantFusionPass(config)]
if self.pass_config.fuse_qk_norm_rope_kvcache:
self.passes += [SplitCoalescingPass(config)]
self.passes += [ScatterSplitReplacementPass(config)]
self.passes += [QkNormRopeKvCacheFusionPass(config)]
if (
self.pass_config.fuse_mla_dual_rms_norm
and rocm_aiter_ops.is_enabled()
+36 -2
View File
@@ -146,10 +146,16 @@ class PassConfig:
"""Fuse paired q/kv RMS norms in MLA attention."""
fuse_rope_kvcache: bool = None # type: ignore[assignment]
"""Fuse the QK rope + KV cache ops."""
fuse_qk_norm_rope_kvcache: bool = Field(default=None) # type: ignore[assignment]
"""Fuse QK RMSNorm + RoPE + KV cache update into a single AITER HIP
kernel. Supersedes both enable_qk_norm_rope_fusion and fuse_rope_kvcache
for layers that support it. Auto-enabled at O1+ on ROCm for models
with QK-norm (e.g. Qwen3-MoE)."""
rope_kvcache_fusion_max_token_num: int = 256
"""The threshold for ROCm AITER RoPE+KVCache fusion e.g. for small batch decode.
Larger batch sizes e.g. during prefill will use the unfused kernels.
Also applies to the fused QK-Norm+RoPE+KVCache pass.
"""
fi_allreduce_fusion_max_size_mb: float | None = None
@@ -228,6 +234,8 @@ class PassConfig:
"fuse_act_padding",
"fuse_mla_dual_rms_norm",
"fuse_rope_kvcache",
"fuse_qk_norm_rope_kvcache",
"enable_qk_norm_rope_fusion",
"fuse_rope_kvcache_cat_mla",
mode="wrap",
)
@@ -288,6 +296,12 @@ class PassConfig:
"The fusion will be disabled."
)
self.fuse_rope_kvcache = False
if self.fuse_qk_norm_rope_kvcache and not current_platform.is_rocm():
logger.warning_once(
"QK-Norm+RoPE+KVCache fusion requires ROCm with AITER. "
"The fusion will be disabled."
)
self.fuse_qk_norm_rope_kvcache = False
if self.fuse_rope_kvcache_cat_mla and not current_platform.is_cuda_alike():
logger.warning_once(
"MLA KV cache update with RoPE fusion enabled but the "
@@ -302,10 +316,13 @@ class PassConfig:
after all defaults are finalized.
TODO also log the compile ranges for which this is enabled.
"""
fusion_prefixes = ("fuse_", "enable_")
enabled_fusions = [
f.name[len("fuse_") :]
f.name[len(prefix) :]
for f in fields(self) # type: ignore[arg-type]
if getattr(self, f.name) and f.name.startswith("fuse_")
if getattr(self, f.name)
for prefix in fusion_prefixes
if f.name.startswith(prefix)
]
if enabled_fusions:
@@ -947,6 +964,7 @@ class CompilationConfig:
# TODO(zhuhaoran): support rope native forward match and remove this.
# Linked issue: https://github.com/vllm-project/vllm/issues/28042
self.custom_ops.append("+rotary_embedding")
if (
self.pass_config.fuse_rope_kvcache
and "+rotary_embedding" not in self.custom_ops
@@ -955,6 +973,12 @@ class CompilationConfig:
# Linked issue: https://github.com/vllm-project/vllm/issues/28042
self.custom_ops.append("+rotary_embedding")
if (
self.pass_config.fuse_qk_norm_rope_kvcache
and "+rotary_embedding" not in self.custom_ops
):
self.custom_ops.append("+rotary_embedding")
if (
is_torch_equal_or_newer("2.9.0.dev")
and "combo_kernels" not in self.inductor_compile_config
@@ -1137,6 +1161,16 @@ class CompilationConfig:
"to enable RoPE+KV cache fusion."
)
self.pass_config.fuse_rope_kvcache = False
if self.pass_config.fuse_qk_norm_rope_kvcache:
logger.warning_once(
"fuse_qk_norm_rope_kvcache is enabled, but "
"splitting_ops is None and Inductor graph partition "
"is not enabled. Disabling fuse_qk_norm_rope_kvcache. "
"Please either set splitting_ops to an empty list [] "
"or set use_inductor_graph_partition to True "
"to enable QK-Norm+RoPE+KV cache fusion."
)
self.pass_config.fuse_qk_norm_rope_kvcache = False
self.splitting_ops.append("vllm::unified_kv_cache_update")
self.splitting_ops.append("vllm::unified_mla_kv_cache_update")
+32
View File
@@ -193,6 +193,15 @@ def enable_mla_dual_rms_norm_fusion(cfg: "VllmConfig") -> bool:
return rocm_aiter_ops.is_enabled() and check_aiter_fused_qk_rmsnorm()
def enable_qk_norm_rope_kvcache(cfg: "VllmConfig") -> bool:
"""Enable fused QK-norm + RoPE + KV cache update on ROCm with AITER."""
from vllm._aiter_ops import rocm_aiter_ops
if not rocm_aiter_ops.is_enabled():
return False
return cfg.compilation_config.is_custom_op_enabled("rotary_embedding")
OPTIMIZATION_LEVEL_00 = {
"compilation_config": {
"pass_config": {
@@ -205,6 +214,8 @@ OPTIMIZATION_LEVEL_00 = {
"fuse_act_padding": False,
"fuse_mla_dual_rms_norm": False,
"fuse_rope_kvcache": False,
"fuse_qk_norm_rope_kvcache": False,
"enable_qk_norm_rope_fusion": False,
"fuse_rope_kvcache_cat_mla": False,
},
"cudagraph_mode": CUDAGraphMode.NONE,
@@ -226,6 +237,8 @@ OPTIMIZATION_LEVEL_01 = {
"fuse_act_padding": enable_norm_pad_fusion,
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
"fuse_rope_kvcache": False,
"fuse_qk_norm_rope_kvcache": False,
"enable_qk_norm_rope_fusion": False,
"fuse_rope_kvcache_cat_mla": False,
},
"cudagraph_mode": CUDAGraphMode.PIECEWISE,
@@ -247,6 +260,8 @@ OPTIMIZATION_LEVEL_02 = {
"fuse_act_padding": enable_norm_pad_fusion,
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
"fuse_rope_kvcache": enable_rope_kvcache_fusion,
"fuse_qk_norm_rope_kvcache": enable_qk_norm_rope_kvcache,
"enable_qk_norm_rope_fusion": False,
"fuse_rope_kvcache_cat_mla": enable_rope_kvcache_mla_fusion,
},
"cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
@@ -268,6 +283,8 @@ OPTIMIZATION_LEVEL_03 = {
"fuse_act_padding": enable_norm_pad_fusion,
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
"fuse_rope_kvcache": enable_rope_kvcache_fusion,
"fuse_qk_norm_rope_kvcache": enable_qk_norm_rope_kvcache,
"enable_qk_norm_rope_fusion": False,
"fuse_rope_kvcache_cat_mla": enable_rope_kvcache_mla_fusion,
},
"cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
@@ -1971,6 +1988,21 @@ class VllmConfig:
compile_range_end,
)
if compilation_config.pass_config.fuse_qk_norm_rope_kvcache:
max_token_num = (
compilation_config.pass_config.rope_kvcache_fusion_max_token_num
)
if max_token_num is not None:
if compile_range_end is not None and max_token_num < compile_range_end:
computed_compile_ranges_endpoints.append(max_token_num)
else:
logger.debug(
"Max num batched tokens below qk_norm+rope+kvcache "
"fusion threshold, fusion enabled for "
"num_tokens <= %d.",
compile_range_end,
)
if compilation_config.compile_ranges_endpoints is not None:
for x in compilation_config.compile_ranges_endpoints:
assert isinstance(x, int)
@@ -136,10 +136,12 @@ def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) ->
# We also keep q/k/v_scale on host (cpu) memory for attention
# backends that require the scales to be on host instead of on device.
# e.g. Flashinfer
# e.g. Flashinfer & AITER
layer._q_scale_float = 1.0
layer._k_scale_float = 1.0
layer._v_scale_float = 1.0
layer._k_scale_cpu = torch.tensor(1.0, dtype=torch.float32)
layer._v_scale_cpu = torch.tensor(1.0, dtype=torch.float32)
layer._prob_scale_float = 1.0
# Initialize q/k/v range constants used by calc_kv_scales
@@ -586,6 +588,8 @@ class Attention(nn.Module, AttentionLayerBase):
self._q_scale_float = self._q_scale.item()
self._k_scale_float = self._k_scale.item()
self._v_scale_float = self._v_scale.item()
self._k_scale_cpu.fill_(self._k_scale_float)
self._v_scale_cpu.fill_(self._v_scale_float)
# We only calculate the scales once
self.calculate_kv_scales = False
@@ -1024,6 +1024,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self._q_scale_float = self._q_scale.item()
self._k_scale_float = self._k_scale.item()
self._v_scale_float = self._v_scale.item()
self._k_scale_cpu.fill_(self._k_scale_float)
self._v_scale_cpu.fill_(self._v_scale_float)
self.calculate_kv_scales = False
def get_attn_backend(self) -> type[AttentionBackend]:
@@ -1200,6 +1200,10 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
layer._v_scale_float = _to_scalar(layer.v_scale)
layer._q_scale_float = _to_scalar(layer.q_scale)
# Sync host (cpu) scale copies read by AITER fused kernels.
layer._k_scale_cpu.fill_(layer._k_scale_float)
layer._v_scale_cpu.fill_(layer._v_scale_float)
# Discard all placeholders.
del layer.k_scale
del layer.v_scale
@@ -144,6 +144,9 @@ class BaseKVCacheMethod(QuantizeMethodBase):
layer._v_scale.copy_(v_scale)
layer._k_scale_float = k_scale
layer._v_scale_float = v_scale
# Sync host (cpu) scale copies read by AITER fused kernels.
layer._k_scale_cpu.fill_(k_scale)
layer._v_scale_cpu.fill_(v_scale)
if k_scale == 1.0 and v_scale == 1.0 and "e5m2" not in layer.kv_cache_dtype:
logger.warning_once(
"Using KV cache scaling factor 1.0 for fp8_e4m3. "
+33
View File
@@ -757,7 +757,9 @@ class AttentionMetadataBuilder(ABC, Generic[M]):
class AttentionLayer(Protocol):
_q_scale: torch.Tensor
_k_scale: torch.Tensor
_k_scale_cpu: torch.Tensor
_v_scale: torch.Tensor
_v_scale_cpu: torch.Tensor
_q_scale_float: float
_k_scale_float: float
_v_scale_float: float
@@ -923,6 +925,14 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
"""
return False
def fused_qk_norm_rope_kvcache_supported(self):
"""
Does this attention implementation support fused QKNorm+RoPE+KVCache fusion.
This is used by the QkNormRopeKvCachePattern to only fuse the QKNorm ops
with the RoPE ops and the KV cache update for implementations that support it.
"""
return False
def fused_rope_kvcache_supported(self):
"""
Does this attention implementation support RoPE+KVCache fusion.
@@ -931,6 +941,29 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
"""
return False
def do_qk_norm_rope_kvcache_update(
self,
layer: AttentionLayer,
qkv: torch.Tensor,
q_out: torch.Tensor,
k_out: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
kv_cache: torch.Tensor,
layer_slot_mapping: torch.Tensor,
):
"""
If `fused_qk_norm_rope_kvcache_supported` returns True, this method
will be called by the fused custom op. Applies QK-norm + RoPE and
writes K/V to the KV cache. Results are written to the pre-allocated
q_out and k_out tensors; V is split from QKV at the graph level.
"""
raise NotImplementedError
def do_rope_and_kv_cache_update(
self,
layer: AttentionLayer,
@@ -1449,6 +1449,47 @@ class AiterFlashAttentionImpl(AttentionImpl):
and not rocm_aiter_ops.is_shuffle_kv_cache_enabled()
)
def fused_qk_norm_rope_kvcache_supported(self):
return rocm_aiter_ops.is_enabled()
def do_qk_norm_rope_kvcache_update(
self,
layer: AttentionLayer,
qkv: torch.Tensor,
q_out: torch.Tensor,
k_out: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
kv_cache: torch.Tensor,
layer_slot_mapping: torch.Tensor,
):
key_cache, value_cache = kv_cache.unbind(1)
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
qkv=qkv,
q_weight=q_weight,
k_weight=k_weight,
cos_sin_cache=cos_sin_cache,
positions=positions,
num_heads_q=self.num_heads,
num_heads_k=self.num_kv_heads,
head_dim=self.head_size,
is_neox=is_neox,
rms_norm_eps=rms_norm_eps,
q_out=q_out,
k_out=k_out,
key_cache=key_cache,
value_cache=value_cache,
slot_mapping=layer_slot_mapping,
k_scale=layer._k_scale_cpu,
v_scale=layer._v_scale_cpu,
kv_cache_dtype=self.kv_cache_dtype,
use_shuffle_layout=rocm_aiter_ops.is_shuffle_kv_cache_enabled(),
)
def do_rope_and_kv_cache_update(
self,
layer: AttentionLayer,
@@ -311,6 +311,50 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
def fused_rope_kvcache_supported(self):
return rocm_aiter_ops.is_enabled()
def fused_qk_norm_rope_kvcache_supported(self):
return rocm_aiter_ops.is_enabled()
def do_qk_norm_rope_kvcache_update(
self,
layer: AttentionLayer,
qkv: torch.Tensor,
q_out: torch.Tensor,
k_out: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
kv_cache: torch.Tensor,
layer_slot_mapping: torch.Tensor,
):
# _split_kv_cache picks the unbind dim per layout (incl. the K/V-first
# encoder-decoder path). unified reads NHD, so never write the shuffle
# layout here.
key_cache, value_cache = self._split_kv_cache(kv_cache)
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
qkv=qkv,
q_weight=q_weight,
k_weight=k_weight,
cos_sin_cache=cos_sin_cache,
positions=positions,
num_heads_q=self.num_heads,
num_heads_k=self.num_kv_heads,
head_dim=self.head_size,
is_neox=is_neox,
rms_norm_eps=rms_norm_eps,
q_out=q_out,
k_out=k_out,
key_cache=key_cache,
value_cache=value_cache,
slot_mapping=layer_slot_mapping,
k_scale=layer._k_scale_cpu,
v_scale=layer._v_scale_cpu,
kv_cache_dtype=self.kv_cache_dtype,
use_shuffle_layout=False,
)
def do_rope_and_kv_cache_update(
self,
layer: AttentionLayer,