[Model][Hardware][AMD]: Part 1/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_AITER_FA, and ROCM_AITER_UNIFIED_ATTN (#42749)

Signed-off-by: Jack Hu <Jack.Hu@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Jack Hu
2026-07-16 17:39:04 -05:00
committed by GitHub
co-authored by Douglas Lehr
parent ab0a20d151
commit 2cab53ddee
15 changed files with 1361 additions and 4 deletions
+122
View File
@@ -2458,6 +2458,128 @@ class rocm_aiter_ops:
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y)
return y
@staticmethod
def fused_qk_norm_rope_and_cache(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
num_heads_q: int,
num_heads_k: int,
num_heads_v: int,
head_dim: int,
is_neox: bool,
rms_norm_eps: float,
q_out: torch.Tensor,
k_cache: torch.Tensor,
v_cache: torch.Tensor,
slot_mapping: torch.Tensor,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
k_out: torch.Tensor | None,
v_out: torch.Tensor | None,
return_kv: bool,
use_shuffle_layout: bool,
block_size: int,
x: int,
rotary_dim: int = 0,
):
from aiter.ops.fused_qk_norm_rope_cache_quant import (
fused_qk_norm_rope_cache_pts_quant_shuffle,
)
fused_qk_norm_rope_cache_pts_quant_shuffle(
qkv,
q_weight,
k_weight,
cos_sin_cache,
positions,
qkv.size(0),
num_heads_q,
num_heads_k,
num_heads_v,
head_dim,
is_neox,
rms_norm_eps,
q_out,
k_cache,
v_cache,
slot_mapping,
k_scale,
v_scale,
k_out,
v_out,
return_kv,
use_shuffle_layout,
block_size,
x,
rotary_dim,
)
@staticmethod
def do_qk_norm_rope_kvcache_update(
qkv: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
positions: torch.Tensor,
num_heads_q: int,
num_heads_k: int,
head_dim: int,
is_neox: bool,
rms_norm_eps: float,
q_out: torch.Tensor,
k_out: torch.Tensor,
key_cache: torch.Tensor,
value_cache: torch.Tensor,
slot_mapping: torch.Tensor,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
kv_cache_dtype: str,
use_shuffle_layout: bool,
) -> None:
"""Run the fused QK-norm+RoPE+KV-cache op on already-split k/v caches.
Shared by the AITER FA and unified-attention impls. The caller splits
kv_cache, since the unbind dim depends on the layout (e.g. the unified
encoder-decoder path is K/V-first), and passes use_shuffle_layout
(unified reads NHD and must pass False).
"""
if kv_cache_dtype.startswith("fp8"):
key_cache = key_cache.view(current_platform.fp8_dtype())
value_cache = value_cache.view(current_platform.fp8_dtype())
# Partial-rotary support (e.g. GLM-4.7 applies rotary to only a prefix
# of each head's channel dim).
rotary_dim = cos_sin_cache.shape[-1]
kernel_rotary_dim = 0 if rotary_dim == head_dim else rotary_dim
rocm_aiter_ops.fused_qk_norm_rope_and_cache(
qkv=qkv,
q_weight=q_weight,
k_weight=k_weight,
cos_sin_cache=cos_sin_cache,
positions=positions,
num_heads_q=num_heads_q,
num_heads_k=num_heads_k,
num_heads_v=num_heads_k,
head_dim=head_dim,
is_neox=is_neox,
rms_norm_eps=rms_norm_eps,
q_out=q_out,
k_cache=key_cache,
v_cache=value_cache,
slot_mapping=slot_mapping,
k_scale=k_scale,
v_scale=v_scale,
k_out=k_out,
v_out=None,
return_kv=True,
use_shuffle_layout=use_shuffle_layout,
block_size=key_cache.shape[1],
x=16 // key_cache.element_size(),
rotary_dim=kernel_rotary_dim,
)
@staticmethod
def triton_rope_and_cache(
query: torch.Tensor,
@@ -25,6 +25,11 @@ logger = init_logger(__name__)
FUSED_QK_ROPE_OP = torch.ops._C.fused_qk_norm_rope.default
# Head dimensions supported by csrc/fused_qknorm_rope_kernel.cu's
# launchFusedQKNormRope and launchFusedQKNormRopeNTokenHeads dispatchers.
# Keep in sync with the switch statements in that file.
SUPPORTED_FUSED_QK_NORM_ROPE_HEAD_DIMS: tuple[int, ...] = (64, 128, 256)
P = ParamSpec("P")
@@ -190,7 +195,12 @@ class QkNormRopePattern:
class QKNormRoPEFusionPass(VllmPatternMatcherPass):
"""Fuse Q/K RMSNorm + RoPE into fused_qk_norm_rope when the custom op exists."""
"""Fuse Q/K RMSNorm + RoPE into fused_qk_norm_rope when the custom op exists.
Registers patterns for both standard vLLM ops and ROCm AITER ops
(when AITER is enabled), so the fusion fires regardless of which
RMSNorm/RoPE implementation the graph uses.
"""
@enable_fake_mode
def __init__(self, config: VllmConfig) -> None:
@@ -216,6 +226,17 @@ class QKNormRoPEFusionPass(VllmPatternMatcherPass):
)
return
for layer in attn_layers.values():
if layer.head_size not in SUPPORTED_FUSED_QK_NORM_ROPE_HEAD_DIMS:
logger.warning_once(
"QK Norm+RoPE fusion not enabled: layer head_size=%d is not "
"supported by fused_qk_norm_rope kernel (supported: %s). "
"Falling back to unfused QK norm + RoPE path.",
layer.head_size,
SUPPORTED_FUSED_QK_NORM_ROPE_HEAD_DIMS,
)
return
self._attention_geometries = tuple(
sorted(
{
@@ -0,0 +1,492 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import inspect
from collections.abc import Callable
from typing import ParamSpec
import torch
import torch._inductor.pattern_matcher as pm
from torch import fx
from torch._higher_order_ops.auto_functionalize import auto_functionalized
from torch._inductor.pattern_matcher import PatternMatcherPass
import vllm.ir.ops
from vllm.config import VllmConfig, get_layers_from_vllm_config
from vllm.config.utils import Range
from vllm.logger import init_logger
from vllm.model_executor.layers.attention.attention import (
Attention,
get_attention_context,
)
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
from ..inductor_pass import enable_fake_mode
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
from .matcher_utils import MatcherRotaryEmbedding
from .rms_quant_fusion import empty_bf16, empty_fp32, empty_i64
logger = init_logger(__name__)
P = ParamSpec("P")
# Head sizes the fused kernel fused_qk_norm_rope_cache_pts_quant_shuffle() supports
# Other sizes hard-abort, so skip those layers.
SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS: tuple[int, ...] = (64, 128, 256)
# ---------------------------------------------------------------------------
# Custom op: fused QK-norm + RoPE + KV cache update
# ---------------------------------------------------------------------------
def fused_qk_norm_rope_and_unified_kv_cache_update_impl(
q_out: torch.Tensor,
k_out: torch.Tensor,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
layer_name: str = "",
) -> torch.Tensor:
_, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
if layer_slot_mapping is not None:
attn_layer.impl.do_qk_norm_rope_kvcache_update(
attn_layer,
qkv,
q_out,
k_out,
positions,
q_weight,
k_weight,
rms_norm_eps,
cos_sin_cache,
is_neox,
kv_cache,
layer_slot_mapping,
)
else:
# Profiling/dummy run: define q_out/k_out (consumed by attention).
q_out.zero_()
k_out.zero_()
return torch.empty(0, device=qkv.device, dtype=qkv.dtype)
def fused_qk_norm_rope_and_unified_kv_cache_update_fake(
q_out: torch.Tensor,
k_out: torch.Tensor,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
layer_name: str = "",
) -> torch.Tensor:
return torch.empty(0, device=qkv.device, dtype=qkv.dtype)
direct_register_custom_op(
op_name="fused_qk_norm_rope_and_unified_kv_cache_update",
op_func=fused_qk_norm_rope_and_unified_kv_cache_update_impl,
mutates_args=["q_out", "k_out"],
fake_impl=fused_qk_norm_rope_and_unified_kv_cache_update_fake,
)
# ---------------------------------------------------------------------------
# Pattern: QK-norm + RoPE + unified_kv_cache_update
# ---------------------------------------------------------------------------
class QkNormRopeKvCachePattern:
"""
Match the unfused sequence:
q, k, v = split(qkv, ...)
q = rms_norm(q.view(heads), q_weight).view(flat)
k = rms_norm(k.view(heads), k_weight).view(flat)
q, k = rotary_embedding(positions, q, k, cos_sin_cache, is_neox)
q = q.view(num_heads, head_dim)
k = k.view(num_kv_heads, head_dim)
v = v.view(num_kv_heads, head_dim)
dummy = unified_kv_cache_update(k, v, layer_name)
Replace with:
q_out = empty(...)
k_out = empty(...)
dummy = fused_qk_norm_rope_and_unified_kv_cache_update(
q_out, k_out, qkv, positions, q_weight, k_weight,
eps, cos_sin_cache, is_neox, layer_name)
v = split(qkv, ...)[2].view(num_kv_heads, head_dim)
"""
FUSED_OP = torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default
def __init__(
self,
layer: Attention,
eps: float,
is_neox: bool,
quant_query: bool,
) -> None:
self.layer_name = layer.layer_name
self.num_heads = layer.num_heads
self.num_kv_heads = layer.num_kv_heads
self.head_size = layer.head_size
self.head_size_v = layer.head_size_v
self.eps = eps
self.is_neox = is_neox
self.quant_query = quant_query
self.q_size = self.num_heads * self.head_size
self.k_size = self.num_kv_heads * self.head_size
self.v_size = self.num_kv_heads * self.head_size_v
self.rope_matcher = MatcherRotaryEmbedding(
is_neox=is_neox,
head_size=self.head_size,
num_heads=self.num_heads,
num_kv_heads=self.num_kv_heads,
)
def get_inputs(self) -> list[torch.Tensor]:
T = 5
L = 4096
qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size)
positions = empty_i64(T)
q_weight = empty_bf16(1, self.head_size)
k_weight = empty_bf16(1, self.head_size)
cos_sin_cache = empty_bf16(L, self.head_size)
inputs = [qkv, positions, q_weight, k_weight, cos_sin_cache]
if self.quant_query:
q_scale = empty_fp32(1)
inputs += [q_scale]
return inputs
def pattern_non_fp8_quant_query(
self,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size)
q_normed = vllm.ir.ops.rms_norm(q_by_head, q_weight, self.eps)
q_flat = q_normed.view(-1, self.q_size)
k_by_head = k.view(-1, self.k_size // self.head_size, self.head_size)
k_normed = vllm.ir.ops.rms_norm(k_by_head, k_weight, self.eps)
k_flat = k_normed.view(-1, self.k_size)
q_rope, k_rope = self.rope_matcher(positions, q_flat, k_flat, cos_sin_cache)
q_rope = q_rope.view(-1, self.num_heads, self.head_size)
k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size_v)
dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name)
return dummy, q_rope, k_rope, v
def replacement_non_fp8_quant_query(
self,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q_out = torch.empty(
qkv.shape[0],
self.num_heads,
self.head_size,
device=qkv.device,
dtype=qkv.dtype,
)
k_out = torch.empty(
qkv.shape[0],
self.num_kv_heads,
self.head_size,
device=qkv.device,
dtype=qkv.dtype,
)
_, _, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
v = v.view(qkv.shape[0], self.num_kv_heads, self.head_size_v)
results = auto_functionalized(
self.FUSED_OP,
q_out=q_out,
k_out=k_out,
qkv=qkv,
positions=positions,
q_weight=q_weight,
k_weight=k_weight,
rms_norm_eps=self.eps,
cos_sin_cache=cos_sin_cache,
is_neox=self.is_neox,
layer_name=self.layer_name,
)
return results[0], results[1], results[2], v
def pattern_fp8_quant_query(
self,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
q_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size)
q_normed = vllm.ir.ops.rms_norm(q_by_head, q_weight, self.eps)
q_flat = q_normed.view(-1, self.q_size)
k_by_head = k.view(-1, self.k_size // self.head_size, self.head_size)
k_normed = vllm.ir.ops.rms_norm(k_by_head, k_weight, self.eps)
k_flat = k_normed.view(-1, self.k_size)
q_rope, k_rope = self.rope_matcher(positions, q_flat, k_flat, cos_sin_cache)
# Match the quant-query op Attention.forward inserts (fp8 KV + UNIFIED).
# Explicit auto_functionalized (out=[1]) keeps the quant node in the pattern.
q_out = torch.empty_like(q_rope, dtype=current_platform.fp8_dtype())
q_quant = auto_functionalized(
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
out=q_out,
x=q_rope,
scale=q_scale,
is_dynamic=False,
)
# `scale` is mutable: its copy_ write-back to _q_scale bumps the mutation
# region, so keep q flat (a reshape lands past the barrier and won't match).
q_rope_fp8 = q_quant[1]
q_scale_out = q_quant[2]
k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size)
v = v.view(-1, self.num_kv_heads, self.head_size_v)
dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name)
return dummy, q_rope_fp8, k_rope, v, q_scale_out
def replacement_fp8_quant_query(
self,
qkv: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
cos_sin_cache: torch.Tensor,
q_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
q_out = torch.empty(
qkv.shape[0],
self.num_heads,
self.head_size,
device=qkv.device,
dtype=qkv.dtype,
)
k_out = torch.empty(
qkv.shape[0],
self.num_kv_heads,
self.head_size,
device=qkv.device,
dtype=qkv.dtype,
)
_, _, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
v = v.view(qkv.shape[0], self.num_kv_heads, self.head_size_v)
results = auto_functionalized(
self.FUSED_OP,
q_out=q_out,
k_out=k_out,
qkv=qkv,
positions=positions,
q_weight=q_weight,
k_weight=k_weight,
rms_norm_eps=self.eps,
cos_sin_cache=cos_sin_cache,
is_neox=self.is_neox,
layer_name=self.layer_name,
)
# Re-apply the quant on the kernel's bf16 q_out; fused op does not quant q.
# Same explicit auto_functionalized form as the pattern: [1] = quantized
# q, [2] = scale (returned so the buffer-writeback use is preserved).
q_fp8_flat = results[1].view(-1, self.q_size)
q_fp8_out = torch.empty_like(q_fp8_flat, dtype=current_platform.fp8_dtype())
q_requant = auto_functionalized(
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
out=q_fp8_out,
x=q_fp8_flat,
scale=q_scale,
is_dynamic=False,
)
q_fp8 = q_requant[1] # flat to mirror the pattern (see note above)
q_scale_out = q_requant[2]
return results[0], q_fp8, results[2], v, q_scale_out
@staticmethod
def wrap_trace_fn(
trace_fn: Callable[P, fx.GraphModule],
*process_fx_fns: Callable[[fx.GraphModule], None],
) -> Callable[P, fx.GraphModule]:
def wrapped(*args: P.args, **kwargs: P.kwargs) -> fx.GraphModule:
gm = trace_fn(*args, **kwargs)
for process_fx in process_fx_fns:
process_fx(gm)
return gm
return wrapped
@staticmethod
def fx_view_to_reshape(gm: torch.fx.GraphModule) -> None:
from torch._inductor.fx_passes.post_grad import view_to_reshape
view_to_reshape(gm)
def _register(self, pattern, replacement, pm_pass) -> None:
trace_fn = QkNormRopeKvCachePattern.wrap_trace_fn(
pm.fwd_only,
QkNormRopeKvCachePattern.fx_view_to_reshape,
)
# Pre-build the search pattern with `ignore_types=(int, torch.SymInt)`
# and pass it via `search_fn_pattern=` so torch skips both of its
# internal `fx_to_pattern` calls and treats dynamic-shape SymInts as
# wildcards.
inputs = self.get_inputs()
argnames = [*inspect.signature(pattern).parameters.keys()]
search_gm = trace_fn(pattern, inputs)
search_fn_pattern = pm.fx_to_pattern(
search_gm,
ignore_types=(int, torch.SymInt),
argnames=argnames,
)
pm.register_replacement(
pattern,
replacement,
inputs,
trace_fn,
pm_pass,
search_fn_pattern=search_fn_pattern,
)
def register(self, pm_pass: PatternMatcherPass) -> None:
# make_fx counts `self` in bound-method code params; wrap as plain fns.
# Distinct names per branch so mypy doesn't see one name, two signatures.
if self.quant_query:
def pattern_q(qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale):
return self.pattern_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
)
def replacement_q(
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
):
return self.replacement_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
)
self._register(pattern_q, replacement_q, pm_pass)
else:
def pattern_noq(qkv, positions, q_weight, k_weight, cos_sin_cache):
return self.pattern_non_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache
)
def replacement_noq(qkv, positions, q_weight, k_weight, cos_sin_cache):
return self.replacement_non_fp8_quant_query(
qkv, positions, q_weight, k_weight, cos_sin_cache
)
self._register(pattern_noq, replacement_noq, pm_pass)
# ---------------------------------------------------------------------------
# Pass class
# ---------------------------------------------------------------------------
class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
"""
Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for
attention layers that support the combined operation, eliminating two
separate kernel launches and the intermediate memory traffic.
"""
@enable_fake_mode
def __init__(self, config: VllmConfig) -> None:
super().__init__(config)
self.patterns: PatternMatcherPass = PatternMatcherPass(
pass_name="qk_norm_rope_kvcache_fusion_pass"
)
cc = config.compilation_config
self.max_token_num = cc.pass_config.rope_kvcache_fusion_max_token_num
dtype = config.model_config.dtype
if dtype not in (torch.bfloat16, torch.float16):
logger.warning_once(
"QK Norm+RoPE+KVCache fusion not enabled: unsupported dtype %s", dtype
)
return
attn_layers = get_layers_from_vllm_config(config, Attention)
for _, layer in attn_layers.items():
if not layer.impl.fused_qk_norm_rope_kvcache_supported():
continue
if layer.head_size not in SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS:
logger.warning_once(
"QK Norm+RoPE+KVCache fusion not enabled for a layer: "
"head_size=%d is not supported by the "
"fused_qk_norm_rope_cache_pts_quant_shuffle kernel "
"(supported: %s). Falling back to the unfused path.",
layer.head_size,
SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS,
)
continue
if layer.head_size_v != layer.head_size:
# The fused kernel uses a single head_dim for q/k/v.
logger.warning_once(
"QK Norm+RoPE+KVCache fusion not enabled for a layer: "
"head_size_v=%d differs from head_size=%d, which the fused "
"kernel does not support. Falling back to the unfused path.",
layer.head_size_v,
layer.head_size,
)
continue
for epsilon in [1e-5, 1e-6]:
for neox in [True, False]:
for quant_q in [False, True]:
QkNormRopeKvCachePattern(
layer=layer,
eps=epsilon,
is_neox=neox,
quant_query=quant_q,
).register(self.patterns)
self.dump_patterns(config, self.patterns)
@VllmInductorPass.time_and_log
def __call__(self, graph: fx.Graph) -> None:
self.matched_count = self.patterns.apply(graph)
logger.info(
"QK-Norm+RoPE+KVCache fusion: replaced %s pattern(s) "
"with AITER fused_qk_norm_rope_cache_pts_quant_shuffle",
self.matched_count,
)
def is_applicable_for_range(self, compile_range: Range) -> bool:
return compile_range.end <= self.max_token_num
def uuid(self) -> str:
return VllmInductorPass.hash_source(self, QkNormRopeKvCachePattern)
+6
View File
@@ -38,6 +38,7 @@ if current_platform.is_cuda_alike():
from .fusion.mla_attn_quant_fusion import MLAAttnQuantFusionPass
from .fusion.mla_rope_kvcache_cat_fusion import MLARoPEKVCacheCatFusionPass
from .fusion.qk_norm_rope_fusion import QKNormRoPEFusionPass
from .fusion.qk_norm_rope_kvcache_fusion import QkNormRopeKvCacheFusionPass
from .fusion.rms_quant_fusion import RMSNormQuantFusionPass
from .fusion.rope_kvcache_fusion import RopeKVCacheFusionPass
from .utility.scatter_split_replace import ScatterSplitReplacementPass
@@ -171,6 +172,11 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
if rocm_aiter_ops.is_enabled():
self.passes += [RocmAiterSiluMulFp8GroupQuantFusionPass(config)]
if self.pass_config.fuse_qk_norm_rope_kvcache:
self.passes += [SplitCoalescingPass(config)]
self.passes += [ScatterSplitReplacementPass(config)]
self.passes += [QkNormRopeKvCacheFusionPass(config)]
if (
self.pass_config.fuse_mla_dual_rms_norm
and rocm_aiter_ops.is_enabled()
+36 -2
View File
@@ -146,10 +146,16 @@ class PassConfig:
"""Fuse paired q/kv RMS norms in MLA attention."""
fuse_rope_kvcache: bool = None # type: ignore[assignment]
"""Fuse the QK rope + KV cache ops."""
fuse_qk_norm_rope_kvcache: bool = Field(default=None) # type: ignore[assignment]
"""Fuse QK RMSNorm + RoPE + KV cache update into a single AITER HIP
kernel. Supersedes both enable_qk_norm_rope_fusion and fuse_rope_kvcache
for layers that support it. Auto-enabled at O1+ on ROCm for models
with QK-norm (e.g. Qwen3-MoE)."""
rope_kvcache_fusion_max_token_num: int = 256
"""The threshold for ROCm AITER RoPE+KVCache fusion e.g. for small batch decode.
Larger batch sizes e.g. during prefill will use the unfused kernels.
Also applies to the fused QK-Norm+RoPE+KVCache pass.
"""
fi_allreduce_fusion_max_size_mb: float | None = None
@@ -228,6 +234,8 @@ class PassConfig:
"fuse_act_padding",
"fuse_mla_dual_rms_norm",
"fuse_rope_kvcache",
"fuse_qk_norm_rope_kvcache",
"enable_qk_norm_rope_fusion",
"fuse_rope_kvcache_cat_mla",
mode="wrap",
)
@@ -288,6 +296,12 @@ class PassConfig:
"The fusion will be disabled."
)
self.fuse_rope_kvcache = False
if self.fuse_qk_norm_rope_kvcache and not current_platform.is_rocm():
logger.warning_once(
"QK-Norm+RoPE+KVCache fusion requires ROCm with AITER. "
"The fusion will be disabled."
)
self.fuse_qk_norm_rope_kvcache = False
if self.fuse_rope_kvcache_cat_mla and not current_platform.is_cuda_alike():
logger.warning_once(
"MLA KV cache update with RoPE fusion enabled but the "
@@ -302,10 +316,13 @@ class PassConfig:
after all defaults are finalized.
TODO also log the compile ranges for which this is enabled.
"""
fusion_prefixes = ("fuse_", "enable_")
enabled_fusions = [
f.name[len("fuse_") :]
f.name[len(prefix) :]
for f in fields(self) # type: ignore[arg-type]
if getattr(self, f.name) and f.name.startswith("fuse_")
if getattr(self, f.name)
for prefix in fusion_prefixes
if f.name.startswith(prefix)
]
if enabled_fusions:
@@ -947,6 +964,7 @@ class CompilationConfig:
# TODO(zhuhaoran): support rope native forward match and remove this.
# Linked issue: https://github.com/vllm-project/vllm/issues/28042
self.custom_ops.append("+rotary_embedding")
if (
self.pass_config.fuse_rope_kvcache
and "+rotary_embedding" not in self.custom_ops
@@ -955,6 +973,12 @@ class CompilationConfig:
# Linked issue: https://github.com/vllm-project/vllm/issues/28042
self.custom_ops.append("+rotary_embedding")
if (
self.pass_config.fuse_qk_norm_rope_kvcache
and "+rotary_embedding" not in self.custom_ops
):
self.custom_ops.append("+rotary_embedding")
if (
is_torch_equal_or_newer("2.9.0.dev")
and "combo_kernels" not in self.inductor_compile_config
@@ -1137,6 +1161,16 @@ class CompilationConfig:
"to enable RoPE+KV cache fusion."
)
self.pass_config.fuse_rope_kvcache = False
if self.pass_config.fuse_qk_norm_rope_kvcache:
logger.warning_once(
"fuse_qk_norm_rope_kvcache is enabled, but "
"splitting_ops is None and Inductor graph partition "
"is not enabled. Disabling fuse_qk_norm_rope_kvcache. "
"Please either set splitting_ops to an empty list [] "
"or set use_inductor_graph_partition to True "
"to enable QK-Norm+RoPE+KV cache fusion."
)
self.pass_config.fuse_qk_norm_rope_kvcache = False
self.splitting_ops.append("vllm::unified_kv_cache_update")
self.splitting_ops.append("vllm::unified_mla_kv_cache_update")
+32
View File
@@ -193,6 +193,15 @@ def enable_mla_dual_rms_norm_fusion(cfg: "VllmConfig") -> bool:
return rocm_aiter_ops.is_enabled() and check_aiter_fused_qk_rmsnorm()
def enable_qk_norm_rope_kvcache(cfg: "VllmConfig") -> bool:
"""Enable fused QK-norm + RoPE + KV cache update on ROCm with AITER."""
from vllm._aiter_ops import rocm_aiter_ops
if not rocm_aiter_ops.is_enabled():
return False
return cfg.compilation_config.is_custom_op_enabled("rotary_embedding")
OPTIMIZATION_LEVEL_00 = {
"compilation_config": {
"pass_config": {
@@ -205,6 +214,8 @@ OPTIMIZATION_LEVEL_00 = {
"fuse_act_padding": False,
"fuse_mla_dual_rms_norm": False,
"fuse_rope_kvcache": False,
"fuse_qk_norm_rope_kvcache": False,
"enable_qk_norm_rope_fusion": False,
"fuse_rope_kvcache_cat_mla": False,
},
"cudagraph_mode": CUDAGraphMode.NONE,
@@ -226,6 +237,8 @@ OPTIMIZATION_LEVEL_01 = {
"fuse_act_padding": enable_norm_pad_fusion,
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
"fuse_rope_kvcache": False,
"fuse_qk_norm_rope_kvcache": False,
"enable_qk_norm_rope_fusion": False,
"fuse_rope_kvcache_cat_mla": False,
},
"cudagraph_mode": CUDAGraphMode.PIECEWISE,
@@ -247,6 +260,8 @@ OPTIMIZATION_LEVEL_02 = {
"fuse_act_padding": enable_norm_pad_fusion,
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
"fuse_rope_kvcache": enable_rope_kvcache_fusion,
"fuse_qk_norm_rope_kvcache": enable_qk_norm_rope_kvcache,
"enable_qk_norm_rope_fusion": False,
"fuse_rope_kvcache_cat_mla": enable_rope_kvcache_mla_fusion,
},
"cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
@@ -268,6 +283,8 @@ OPTIMIZATION_LEVEL_03 = {
"fuse_act_padding": enable_norm_pad_fusion,
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
"fuse_rope_kvcache": enable_rope_kvcache_fusion,
"fuse_qk_norm_rope_kvcache": enable_qk_norm_rope_kvcache,
"enable_qk_norm_rope_fusion": False,
"fuse_rope_kvcache_cat_mla": enable_rope_kvcache_mla_fusion,
},
"cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
@@ -1971,6 +1988,21 @@ class VllmConfig:
compile_range_end,
)
if compilation_config.pass_config.fuse_qk_norm_rope_kvcache:
max_token_num = (
compilation_config.pass_config.rope_kvcache_fusion_max_token_num
)
if max_token_num is not None:
if compile_range_end is not None and max_token_num < compile_range_end:
computed_compile_ranges_endpoints.append(max_token_num)
else:
logger.debug(
"Max num batched tokens below qk_norm+rope+kvcache "
"fusion threshold, fusion enabled for "
"num_tokens <= %d.",
compile_range_end,
)
if compilation_config.compile_ranges_endpoints is not None:
for x in compilation_config.compile_ranges_endpoints:
assert isinstance(x, int)
@@ -136,10 +136,12 @@ def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) ->
# We also keep q/k/v_scale on host (cpu) memory for attention
# backends that require the scales to be on host instead of on device.
# e.g. Flashinfer
# e.g. Flashinfer & AITER
layer._q_scale_float = 1.0
layer._k_scale_float = 1.0
layer._v_scale_float = 1.0
layer._k_scale_cpu = torch.tensor(1.0, dtype=torch.float32)
layer._v_scale_cpu = torch.tensor(1.0, dtype=torch.float32)
layer._prob_scale_float = 1.0
# Initialize q/k/v range constants used by calc_kv_scales
@@ -586,6 +588,8 @@ class Attention(nn.Module, AttentionLayerBase):
self._q_scale_float = self._q_scale.item()
self._k_scale_float = self._k_scale.item()
self._v_scale_float = self._v_scale.item()
self._k_scale_cpu.fill_(self._k_scale_float)
self._v_scale_cpu.fill_(self._v_scale_float)
# We only calculate the scales once
self.calculate_kv_scales = False
@@ -1024,6 +1024,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
self._q_scale_float = self._q_scale.item()
self._k_scale_float = self._k_scale.item()
self._v_scale_float = self._v_scale.item()
self._k_scale_cpu.fill_(self._k_scale_float)
self._v_scale_cpu.fill_(self._v_scale_float)
self.calculate_kv_scales = False
def get_attn_backend(self) -> type[AttentionBackend]:
@@ -1200,6 +1200,10 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
layer._v_scale_float = _to_scalar(layer.v_scale)
layer._q_scale_float = _to_scalar(layer.q_scale)
# Sync host (cpu) scale copies read by AITER fused kernels.
layer._k_scale_cpu.fill_(layer._k_scale_float)
layer._v_scale_cpu.fill_(layer._v_scale_float)
# Discard all placeholders.
del layer.k_scale
del layer.v_scale
@@ -144,6 +144,9 @@ class BaseKVCacheMethod(QuantizeMethodBase):
layer._v_scale.copy_(v_scale)
layer._k_scale_float = k_scale
layer._v_scale_float = v_scale
# Sync host (cpu) scale copies read by AITER fused kernels.
layer._k_scale_cpu.fill_(k_scale)
layer._v_scale_cpu.fill_(v_scale)
if k_scale == 1.0 and v_scale == 1.0 and "e5m2" not in layer.kv_cache_dtype:
logger.warning_once(
"Using KV cache scaling factor 1.0 for fp8_e4m3. "
+33
View File
@@ -757,7 +757,9 @@ class AttentionMetadataBuilder(ABC, Generic[M]):
class AttentionLayer(Protocol):
_q_scale: torch.Tensor
_k_scale: torch.Tensor
_k_scale_cpu: torch.Tensor
_v_scale: torch.Tensor
_v_scale_cpu: torch.Tensor
_q_scale_float: float
_k_scale_float: float
_v_scale_float: float
@@ -923,6 +925,14 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
"""
return False
def fused_qk_norm_rope_kvcache_supported(self):
"""
Does this attention implementation support fused QKNorm+RoPE+KVCache fusion.
This is used by the QkNormRopeKvCachePattern to only fuse the QKNorm ops
with the RoPE ops and the KV cache update for implementations that support it.
"""
return False
def fused_rope_kvcache_supported(self):
"""
Does this attention implementation support RoPE+KVCache fusion.
@@ -931,6 +941,29 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
"""
return False
def do_qk_norm_rope_kvcache_update(
self,
layer: AttentionLayer,
qkv: torch.Tensor,
q_out: torch.Tensor,
k_out: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
kv_cache: torch.Tensor,
layer_slot_mapping: torch.Tensor,
):
"""
If `fused_qk_norm_rope_kvcache_supported` returns True, this method
will be called by the fused custom op. Applies QK-norm + RoPE and
writes K/V to the KV cache. Results are written to the pre-allocated
q_out and k_out tensors; V is split from QKV at the graph level.
"""
raise NotImplementedError
def do_rope_and_kv_cache_update(
self,
layer: AttentionLayer,
@@ -1449,6 +1449,47 @@ class AiterFlashAttentionImpl(AttentionImpl):
and not rocm_aiter_ops.is_shuffle_kv_cache_enabled()
)
def fused_qk_norm_rope_kvcache_supported(self):
return rocm_aiter_ops.is_enabled()
def do_qk_norm_rope_kvcache_update(
self,
layer: AttentionLayer,
qkv: torch.Tensor,
q_out: torch.Tensor,
k_out: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
kv_cache: torch.Tensor,
layer_slot_mapping: torch.Tensor,
):
key_cache, value_cache = kv_cache.unbind(1)
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
qkv=qkv,
q_weight=q_weight,
k_weight=k_weight,
cos_sin_cache=cos_sin_cache,
positions=positions,
num_heads_q=self.num_heads,
num_heads_k=self.num_kv_heads,
head_dim=self.head_size,
is_neox=is_neox,
rms_norm_eps=rms_norm_eps,
q_out=q_out,
k_out=k_out,
key_cache=key_cache,
value_cache=value_cache,
slot_mapping=layer_slot_mapping,
k_scale=layer._k_scale_cpu,
v_scale=layer._v_scale_cpu,
kv_cache_dtype=self.kv_cache_dtype,
use_shuffle_layout=rocm_aiter_ops.is_shuffle_kv_cache_enabled(),
)
def do_rope_and_kv_cache_update(
self,
layer: AttentionLayer,
@@ -311,6 +311,50 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
def fused_rope_kvcache_supported(self):
return rocm_aiter_ops.is_enabled()
def fused_qk_norm_rope_kvcache_supported(self):
return rocm_aiter_ops.is_enabled()
def do_qk_norm_rope_kvcache_update(
self,
layer: AttentionLayer,
qkv: torch.Tensor,
q_out: torch.Tensor,
k_out: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
kv_cache: torch.Tensor,
layer_slot_mapping: torch.Tensor,
):
# _split_kv_cache picks the unbind dim per layout (incl. the K/V-first
# encoder-decoder path). unified reads NHD, so never write the shuffle
# layout here.
key_cache, value_cache = self._split_kv_cache(kv_cache)
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
qkv=qkv,
q_weight=q_weight,
k_weight=k_weight,
cos_sin_cache=cos_sin_cache,
positions=positions,
num_heads_q=self.num_heads,
num_heads_k=self.num_kv_heads,
head_dim=self.head_size,
is_neox=is_neox,
rms_norm_eps=rms_norm_eps,
q_out=q_out,
k_out=k_out,
key_cache=key_cache,
value_cache=value_cache,
slot_mapping=layer_slot_mapping,
k_scale=layer._k_scale_cpu,
v_scale=layer._v_scale_cpu,
kv_cache_dtype=self.kv_cache_dtype,
use_shuffle_layout=False,
)
def do_rope_and_kv_cache_update(
self,
layer: AttentionLayer,