[Model][Hardware][AMD]: Part 1/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_AITER_FA, and ROCM_AITER_UNIFIED_ATTN (#42749)
Signed-off-by: Jack Hu <Jack.Hu@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
@@ -0,0 +1,478 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.config
|
||||
from tests.compile.backend import TestBackend
|
||||
from tests.v1.attention.utils import BatchSpec, create_common_attn_metadata
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported, rocm_aiter_ops
|
||||
from vllm.compilation.passes.fusion.matcher_utils import ROTARY_OP
|
||||
from vllm.compilation.passes.fusion.qk_norm_rope_kvcache_fusion import (
|
||||
QkNormRopeKvCacheFusionPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.noop_elimination import NoOpEliminationPass
|
||||
from vllm.compilation.passes.utility.post_cleanup import PostCleanupPass
|
||||
from vllm.compilation.passes.utility.scatter_split_replace import (
|
||||
ScatterSplitReplacementPass,
|
||||
)
|
||||
from vllm.compilation.passes.utility.split_coalescing import SplitCoalescingPass
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
CompilationConfig,
|
||||
CompilationMode,
|
||||
ModelConfig,
|
||||
PassConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.forward_context import get_forward_context, set_forward_context
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.rotary_embedding import RotaryEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
CommonAttentionMetadata,
|
||||
)
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec
|
||||
|
||||
INDEX_SELECT_OP = torch.ops.aten.index.Tensor
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
|
||||
class QKNormRoPEKVCacheTestModel(torch.nn.Module):
|
||||
"""Minimal model that reproduces the QK-norm + RoPE + KV cache update
|
||||
pattern matched by QkNormRopeKvCacheFusionPass:
|
||||
|
||||
q, k, v = split(qkv)
|
||||
q = rms_norm(q.view(heads, dim), q_weight).view(flat)
|
||||
k = rms_norm(k.view(heads, dim), k_weight).view(flat)
|
||||
q, k = rotary_emb(positions, q, k)
|
||||
q = q.view(num_heads, head_dim)
|
||||
k = k.view(num_kv_heads, head_dim)
|
||||
v = v.view(num_kv_heads, head_dim)
|
||||
dummy = unified_kv_cache_update(k, v, layer_name)
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
attn_backend: AttentionBackendEnum,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
is_neox: bool,
|
||||
rms_norm_eps: float,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
rotary_dim: int | None = None,
|
||||
prefix: str = "model.layers.0.self_attn.attn",
|
||||
):
|
||||
super().__init__()
|
||||
self.num_heads = num_heads
|
||||
self.num_kv_heads = num_kv_heads
|
||||
self.head_size = head_size
|
||||
self.rotary_dim = rotary_dim if rotary_dim is not None else head_size
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.q_size = num_heads * head_size
|
||||
self.kv_size = num_kv_heads * head_size
|
||||
self.is_neox = is_neox
|
||||
self.dtype = dtype
|
||||
self.device = device
|
||||
self.layer_name = prefix
|
||||
|
||||
self.q_norm = RMSNorm(head_size, eps=rms_norm_eps)
|
||||
self.k_norm = RMSNorm(head_size, eps=rms_norm_eps)
|
||||
|
||||
self.rotary_emb = RotaryEmbedding(
|
||||
head_size,
|
||||
rotary_dim=self.rotary_dim,
|
||||
max_position_embeddings=4096,
|
||||
base=10000,
|
||||
is_neox_style=is_neox,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
|
||||
self.enable_rope_custom_op = self.rotary_emb.enabled()
|
||||
|
||||
self.attn = Attention(
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
scale=1.0 / head_size**0.5,
|
||||
num_kv_heads=num_kv_heads,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=prefix,
|
||||
attn_backend=attn_backend.get_class(),
|
||||
)
|
||||
self.attn_backend: type[AttentionBackend] = self.attn.get_attn_backend()
|
||||
assert not self.attn_backend.forward_includes_kv_cache_update, (
|
||||
f"Attention backend {self.attn_backend} does not support "
|
||||
"fuse_qk_norm_rope_kvcache."
|
||||
)
|
||||
kv_cache_dtype_str = vllm_config.cache_config.cache_dtype
|
||||
self.kv_cache_dtype = (
|
||||
FP8_DTYPE if kv_cache_dtype_str.startswith("fp8") else self.dtype
|
||||
)
|
||||
|
||||
if self.kv_cache_dtype != self.dtype:
|
||||
self.attn._k_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
self.attn._v_scale = torch.tensor(1.0, dtype=torch.float32, device=device)
|
||||
self.attn._k_scale_float = 1.0
|
||||
self.attn._v_scale_float = 1.0
|
||||
else:
|
||||
self.attn._k_scale = self.attn._k_scale.to(device)
|
||||
self.attn._v_scale = self.attn._v_scale.to(device)
|
||||
|
||||
self.builder = self.attn.attn_backend.get_builder_cls()(
|
||||
kv_cache_spec=AttentionSpec(
|
||||
block_size=self.block_size,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
head_size=head_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
),
|
||||
layer_names=[self.attn.layer_name],
|
||||
vllm_config=vllm_config,
|
||||
device=device,
|
||||
)
|
||||
|
||||
def build_attn_metadata(
|
||||
self, batch_size: int, kv_stride_order: tuple[int, ...] | None = None
|
||||
) -> CommonAttentionMetadata:
|
||||
batch_spec = BatchSpec(seq_lens=[1] * batch_size, query_lens=[1] * batch_size)
|
||||
common_attn_metadata = create_common_attn_metadata(
|
||||
batch_spec, self.block_size, self.device, arange_block_indices=True
|
||||
)
|
||||
|
||||
max_blocks = (max(batch_spec.seq_lens) + self.block_size - 1) // self.block_size
|
||||
num_blocks = batch_size * max_blocks
|
||||
|
||||
attn_backend = self.attn.attn_backend
|
||||
kv_cache_shape = attn_backend.get_kv_cache_shape(
|
||||
num_blocks, self.block_size, self.num_kv_heads, self.head_size
|
||||
)
|
||||
# Caller can force a physical layout; else use the backend's.
|
||||
if kv_stride_order is None:
|
||||
try:
|
||||
kv_stride_order = attn_backend.get_kv_cache_stride_order()
|
||||
except (AttributeError, NotImplementedError):
|
||||
kv_stride_order = tuple(range(len(kv_cache_shape)))
|
||||
|
||||
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_stride_order)
|
||||
inv_order = [kv_stride_order.index(i) for i in range(len(kv_stride_order))]
|
||||
|
||||
raw_tensor = torch.zeros(
|
||||
2 * num_blocks * self.block_size * self.num_kv_heads * self.head_size,
|
||||
dtype=self.kv_cache_dtype,
|
||||
device=self.device,
|
||||
)
|
||||
raw_tensor = raw_tensor.view(kv_cache_shape)
|
||||
kv_cache = raw_tensor.permute(*inv_order)
|
||||
|
||||
# Store as a bare tensor (not wrapped in a list) to match production
|
||||
# `bind_kv_cache` behavior. `get_attention_context` returns this
|
||||
# attribute directly to the fused/unfused `do_kv_cache_update` impls,
|
||||
# which call `kv_cache.unbind(0)` and therefore require a tensor.
|
||||
self.attn.kv_cache = kv_cache
|
||||
|
||||
attn_metadata = self.builder.build(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
|
||||
return attn_metadata
|
||||
|
||||
def forward(
|
||||
self, qkv: torch.Tensor, positions: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
qkv = qkv.clone()
|
||||
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
|
||||
|
||||
# QK-norm: RMSNorm on per-head Q and K
|
||||
q = q.view(-1, self.num_heads, self.head_size)
|
||||
q = self.q_norm(q)
|
||||
q = q.view(-1, self.q_size)
|
||||
|
||||
k = k.view(-1, self.num_kv_heads, self.head_size)
|
||||
k = self.k_norm(k)
|
||||
k = k.view(-1, self.kv_size)
|
||||
|
||||
# RoPE
|
||||
q, k = self.rotary_emb(positions, q, k)
|
||||
|
||||
# Mirror Attention.forward: quant-query impls consume an fp8 q.
|
||||
if (
|
||||
self.kv_cache_dtype != self.dtype
|
||||
and self.attn.impl.supports_quant_query_input
|
||||
):
|
||||
q_fp8 = torch.empty_like(q, dtype=FP8_DTYPE)
|
||||
torch.ops.vllm.rocm_aiter_per_tensor_quant(
|
||||
q_fp8, q, self.attn._q_scale, False
|
||||
)
|
||||
q = q_fp8
|
||||
|
||||
# Final views + KV cache update
|
||||
q = q.view(-1, self.num_heads, self.head_size)
|
||||
k = k.view(-1, self.num_kv_heads, self.head_size)
|
||||
v = v.view(-1, self.num_kv_heads, self.head_size)
|
||||
kv_cache_dummy_dep = torch.ops.vllm.unified_kv_cache_update(
|
||||
k, v, self.layer_name
|
||||
)
|
||||
return q, k, v, kv_cache_dummy_dep
|
||||
|
||||
def ops_in_model_before(self) -> list[torch._ops.OpOverload]:
|
||||
ops: list[torch._ops.OpOverload] = []
|
||||
# RoPE is not yet IR-migrated, so its custom op still surfaces
|
||||
# directly in the graph based on `enable_rope_custom_op`.
|
||||
if self.enable_rope_custom_op:
|
||||
if rocm_aiter_ops.is_triton_rotary_embed_enabled():
|
||||
ops.append(torch.ops.vllm.rocm_aiter_triton_rotary_embedding.default)
|
||||
else:
|
||||
ops.append(ROTARY_OP)
|
||||
else:
|
||||
ops.append(INDEX_SELECT_OP)
|
||||
ops.append(torch.ops.vllm.unified_kv_cache_update.default)
|
||||
return ops
|
||||
|
||||
def ops_in_model_after(self) -> list[torch._ops.OpOverload]:
|
||||
return [torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default]
|
||||
|
||||
|
||||
def _run_qk_norm_rope_kvcache_fusion_test(
|
||||
*,
|
||||
attn_backend: AttentionBackendEnum,
|
||||
enable_aiter_triton_rope: bool,
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
block_size: int,
|
||||
is_neox: bool,
|
||||
use_shuffle_kv_layout: str,
|
||||
kv_stride_order: tuple[int, ...],
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
rms_norm_eps: float,
|
||||
custom_op: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
device = os.environ.get("VLLM_TEST_CUDA_DEVICE", "cuda")
|
||||
torch.set_default_device(device)
|
||||
torch.set_default_dtype(dtype)
|
||||
torch.manual_seed(0)
|
||||
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(dtype=dtype),
|
||||
cache_config=CacheConfig(
|
||||
block_size=block_size,
|
||||
cache_dtype=kv_cache_dtype,
|
||||
),
|
||||
compilation_config=CompilationConfig(
|
||||
mode=CompilationMode.VLLM_COMPILE,
|
||||
custom_ops=[custom_op],
|
||||
pass_config=PassConfig(
|
||||
fuse_qk_norm_rope_kvcache=True,
|
||||
eliminate_noops=True,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
with vllm.config.set_current_vllm_config(vllm_config), monkeypatch.context() as m:
|
||||
m.setenv("VLLM_ROCM_USE_AITER", "1")
|
||||
m.setenv(
|
||||
"VLLM_ROCM_USE_AITER_TRITON_ROPE",
|
||||
"1" if enable_aiter_triton_rope else "0",
|
||||
)
|
||||
m.setenv("VLLM_ROCM_SHUFFLE_KV_CACHE_LAYOUT", use_shuffle_kv_layout)
|
||||
rocm_aiter_ops.refresh_env_variables()
|
||||
|
||||
model = QKNormRoPEKVCacheTestModel(
|
||||
vllm_config=vllm_config,
|
||||
attn_backend=attn_backend,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
rotary_dim=rotary_dim,
|
||||
is_neox=is_neox,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
dtype=dtype,
|
||||
device=torch.get_default_device(),
|
||||
)
|
||||
|
||||
fusion_pass = QkNormRopeKvCacheFusionPass(vllm_config)
|
||||
passes = [
|
||||
NoOpEliminationPass(vllm_config),
|
||||
SplitCoalescingPass(vllm_config),
|
||||
ScatterSplitReplacementPass(vllm_config),
|
||||
fusion_pass,
|
||||
PostCleanupPass(vllm_config),
|
||||
]
|
||||
backend = TestBackend(*passes)
|
||||
|
||||
qkv = torch.randn(
|
||||
num_tokens,
|
||||
num_heads * head_size + 2 * num_kv_heads * head_size,
|
||||
dtype=dtype,
|
||||
)
|
||||
pos = torch.arange(num_tokens, dtype=torch.long)
|
||||
|
||||
qkv_unfused = qkv.clone()
|
||||
pos_unfused = pos.clone()
|
||||
|
||||
# Run unfused (eager) forward
|
||||
with set_forward_context(None, vllm_config):
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model.build_attn_metadata(num_tokens, kv_stride_order)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
q_unfused, k_unfused, v_unfused, dummy = model(qkv_unfused, pos_unfused)
|
||||
attn_layer = forward_context.no_compile_layers[model.layer_name]
|
||||
kv_cache_unfused = attn_layer.kv_cache
|
||||
del dummy
|
||||
|
||||
# Run fused (compiled) forward
|
||||
torch._dynamo.mark_dynamic(qkv, 0)
|
||||
torch._dynamo.mark_dynamic(pos, 0)
|
||||
with set_forward_context(None, vllm_config):
|
||||
model_fused = torch.compile(model, backend=backend)
|
||||
forward_context = get_forward_context()
|
||||
attn_metadata = model_fused.build_attn_metadata(num_tokens, kv_stride_order)
|
||||
forward_context.slot_mapping = {
|
||||
model.layer_name: attn_metadata.slot_mapping
|
||||
}
|
||||
q_fused, k_fused, v_fused, dummy = model_fused(qkv, pos)
|
||||
attn_layer = forward_context.no_compile_layers[model.layer_name]
|
||||
kv_cache_fused = attn_layer.kv_cache
|
||||
del dummy
|
||||
|
||||
assert fusion_pass.matched_count == 1
|
||||
|
||||
backend.check_before_ops(model.ops_in_model_before())
|
||||
backend.check_after_ops(model.ops_in_model_after())
|
||||
|
||||
# Sweep-backed (18.2k pts, PR #42749): native-rope ref worst 7.7e-3 -> 1e-2;
|
||||
# AITER-triton-rope ref is itself approximate (plateau 1.28e-2) -> 2e-2.
|
||||
ATOL, RTOL = (2e-2, 2e-2) if enable_aiter_triton_rope else (1e-2, 1e-2)
|
||||
is_fp8_cache = model.kv_cache_dtype != dtype
|
||||
|
||||
if q_fused.dtype == FP8_DTYPE:
|
||||
# Quant-query path: both q are fp8; compare dequant within 1 fp8 ULP.
|
||||
torch.testing.assert_close(
|
||||
q_unfused.float(), q_fused.float(), atol=1.25e-1, rtol=1.25e-1
|
||||
)
|
||||
else:
|
||||
torch.testing.assert_close(q_unfused, q_fused, atol=ATOL, rtol=RTOL)
|
||||
|
||||
if not is_fp8_cache:
|
||||
# The AITER PTS kernel populates k_out only for non-FP8 caches.
|
||||
# With FP8, the kernel writes quantized K directly to the cache
|
||||
# and may leave k_out uninitialised. In production this is fine
|
||||
# because downstream attention reads K from the cache.
|
||||
torch.testing.assert_close(k_unfused, k_fused, atol=ATOL, rtol=RTOL)
|
||||
|
||||
# Should be bit exact since no processing had been done on v for both paths
|
||||
torch.testing.assert_close(v_unfused, v_fused, atol=0.0, rtol=0.0)
|
||||
|
||||
# fp8 vs triton-rope ref requires loosening tolerance to 1.25e-1.
|
||||
if is_fp8_cache and enable_aiter_triton_rope:
|
||||
cache_atol = cache_rtol = 1.25e-1
|
||||
else:
|
||||
cache_atol, cache_rtol = ATOL, RTOL
|
||||
|
||||
torch.testing.assert_close(
|
||||
kv_cache_unfused[0].float(),
|
||||
kv_cache_fused[0].float(),
|
||||
atol=cache_atol,
|
||||
rtol=cache_rtol,
|
||||
)
|
||||
|
||||
|
||||
_FUSION_CONFIGS = [
|
||||
# Full rotary, both neox styles (the original coverage).
|
||||
pytest.param(64, 8, 64, 64, True, id="full-neox"),
|
||||
pytest.param(64, 8, 64, 64, False, id="full-non_neox"),
|
||||
# GLM-4.5/4.6/4.7 (glm4_moe.py:275 partial_rotary_factor=0.5, neox-style)
|
||||
pytest.param(32, 8, 128, 64, True, id="glm4_moe"),
|
||||
# GLM-4 dense (glm4.py:97,124 partial_rotary_factor=0.5, non-neox)
|
||||
pytest.param(32, 2, 128, 64, False, id="glm4_dense"),
|
||||
# Moondream3-style small head (head_size=64, rotary_dim=32)
|
||||
pytest.param(16, 2, 64, 32, True, id="partial_small_head"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_heads, num_kv_heads, head_size, rotary_dim, is_neox",
|
||||
_FUSION_CONFIGS,
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"attn_backend",
|
||||
[
|
||||
AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN,
|
||||
AttentionBackendEnum.ROCM_AITER_FA,
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("num_tokens", [5, 16, 2048])
|
||||
@pytest.mark.parametrize("use_shuffle_kv_layout", ["1", "0"])
|
||||
@pytest.mark.parametrize(
|
||||
"kv_stride_order",
|
||||
[
|
||||
pytest.param((0, 1, 2, 3, 4), id="block_first"),
|
||||
pytest.param((1, 0, 2, 3, 4), id="kv_first"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("enable_aiter_triton_rope", [True, False])
|
||||
@pytest.mark.parametrize("block_size", [16, 32, 64])
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
|
||||
@pytest.mark.parametrize("rms_norm_eps", [1e-5, 1e-6])
|
||||
@pytest.mark.parametrize("custom_op", ["+rotary_embedding", "+rms_norm"])
|
||||
@pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="Only test on ROCm with AITER installed and supported",
|
||||
)
|
||||
def test_qk_norm_rope_kvcache_fusion(
|
||||
num_tokens: int,
|
||||
num_heads: int,
|
||||
num_kv_heads: int,
|
||||
head_size: int,
|
||||
rotary_dim: int,
|
||||
is_neox: bool,
|
||||
attn_backend: AttentionBackendEnum,
|
||||
enable_aiter_triton_rope: bool,
|
||||
use_shuffle_kv_layout: str,
|
||||
kv_stride_order: tuple[int, ...],
|
||||
block_size: int,
|
||||
dtype: torch.dtype,
|
||||
kv_cache_dtype: str,
|
||||
rms_norm_eps: float,
|
||||
custom_op: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
if (
|
||||
attn_backend == AttentionBackendEnum.ROCM_AITER_UNIFIED_ATTN
|
||||
and use_shuffle_kv_layout == "1"
|
||||
):
|
||||
pytest.skip("ROCM_AITER_UNIFIED_ATTN is NHD-only; shuffle env is ignored")
|
||||
_run_qk_norm_rope_kvcache_fusion_test(
|
||||
attn_backend=attn_backend,
|
||||
enable_aiter_triton_rope=enable_aiter_triton_rope,
|
||||
num_tokens=num_tokens,
|
||||
num_heads=num_heads,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
rotary_dim=rotary_dim,
|
||||
block_size=block_size,
|
||||
is_neox=is_neox,
|
||||
use_shuffle_kv_layout=use_shuffle_kv_layout,
|
||||
kv_stride_order=kv_stride_order,
|
||||
dtype=dtype,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
custom_op=custom_op,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
@@ -17,6 +17,10 @@ from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.kernels.linear.scaled_mm import (
|
||||
MarlinFP8ScaledMMLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.layers.attention.attention import (
|
||||
Attention,
|
||||
set_default_quant_scales,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe import FusedMoE
|
||||
from vllm.model_executor.layers.quantization.fp8 import (
|
||||
Fp8Config,
|
||||
@@ -24,6 +28,7 @@ from vllm.model_executor.layers.quantization.fp8 import (
|
||||
Fp8LinearMethod,
|
||||
Fp8MoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import (
|
||||
Fp8PerTensorOnlineLinearMethod,
|
||||
)
|
||||
@@ -422,6 +427,42 @@ def test_fp8_reloading(
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["checkpoint", "runtime_calc"])
|
||||
def test_kv_cache_scale_sync_to_host_copies(source):
|
||||
"""Test device-to-host sync of the k/v quantization scales, for both the
|
||||
checkpoint-load and runtime-calc paths that produce them.
|
||||
"""
|
||||
layer = torch.nn.Module()
|
||||
set_default_quant_scales(layer, register_buffer=True)
|
||||
layer.kv_cache_dtype = "fp8"
|
||||
|
||||
if source == "checkpoint":
|
||||
# Scales come from the checkpoint, so runtime calc is disabled.
|
||||
layer.calculate_kv_scales = False
|
||||
method = BaseKVCacheMethod(quant_config=None)
|
||||
method.create_weights(layer)
|
||||
# 0.3 stays != 1.0 even after the fp8_fnuz x2 rescale.
|
||||
checkpoint_scale = torch.tensor(0.3, dtype=torch.float32)
|
||||
layer.k_scale.weight_loader(layer.k_scale, checkpoint_scale)
|
||||
layer.v_scale.weight_loader(layer.v_scale, checkpoint_scale)
|
||||
method.process_weights_after_loading(layer)
|
||||
else:
|
||||
# First forward computes distinct, non-unity scales from live k/v.
|
||||
layer.calculate_kv_scales = True
|
||||
query = torch.full((4, 8), 10.0)
|
||||
key = torch.full((4, 8), 60.0)
|
||||
value = torch.full((4, 8), 50.0)
|
||||
Attention.calc_kv_scales(layer, query, key, value)
|
||||
|
||||
assert layer._k_scale_float != 1.0
|
||||
assert layer._v_scale_float != 1.0
|
||||
# Host copy must mirror both the float and the device scale tensor.
|
||||
assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale_float)
|
||||
assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale_float)
|
||||
assert layer._k_scale_cpu.item() == pytest.approx(layer._k_scale.item())
|
||||
assert layer._v_scale_cpu.item() == pytest.approx(layer._v_scale.item())
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
|
||||
@@ -2458,6 +2458,128 @@ class rocm_aiter_ops:
|
||||
gemm_afp4wfp4(x_q, weight, x_s, weight_scale.T, out_dtype, y)
|
||||
return y
|
||||
|
||||
@staticmethod
|
||||
def fused_qk_norm_rope_and_cache(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads_q: int,
|
||||
num_heads_k: int,
|
||||
num_heads_v: int,
|
||||
head_dim: int,
|
||||
is_neox: bool,
|
||||
rms_norm_eps: float,
|
||||
q_out: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
v_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
k_out: torch.Tensor | None,
|
||||
v_out: torch.Tensor | None,
|
||||
return_kv: bool,
|
||||
use_shuffle_layout: bool,
|
||||
block_size: int,
|
||||
x: int,
|
||||
rotary_dim: int = 0,
|
||||
):
|
||||
from aiter.ops.fused_qk_norm_rope_cache_quant import (
|
||||
fused_qk_norm_rope_cache_pts_quant_shuffle,
|
||||
)
|
||||
|
||||
fused_qk_norm_rope_cache_pts_quant_shuffle(
|
||||
qkv,
|
||||
q_weight,
|
||||
k_weight,
|
||||
cos_sin_cache,
|
||||
positions,
|
||||
qkv.size(0),
|
||||
num_heads_q,
|
||||
num_heads_k,
|
||||
num_heads_v,
|
||||
head_dim,
|
||||
is_neox,
|
||||
rms_norm_eps,
|
||||
q_out,
|
||||
k_cache,
|
||||
v_cache,
|
||||
slot_mapping,
|
||||
k_scale,
|
||||
v_scale,
|
||||
k_out,
|
||||
v_out,
|
||||
return_kv,
|
||||
use_shuffle_layout,
|
||||
block_size,
|
||||
x,
|
||||
rotary_dim,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def do_qk_norm_rope_kvcache_update(
|
||||
qkv: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
num_heads_q: int,
|
||||
num_heads_k: int,
|
||||
head_dim: int,
|
||||
is_neox: bool,
|
||||
rms_norm_eps: float,
|
||||
q_out: torch.Tensor,
|
||||
k_out: torch.Tensor,
|
||||
key_cache: torch.Tensor,
|
||||
value_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
kv_cache_dtype: str,
|
||||
use_shuffle_layout: bool,
|
||||
) -> None:
|
||||
"""Run the fused QK-norm+RoPE+KV-cache op on already-split k/v caches.
|
||||
|
||||
Shared by the AITER FA and unified-attention impls. The caller splits
|
||||
kv_cache, since the unbind dim depends on the layout (e.g. the unified
|
||||
encoder-decoder path is K/V-first), and passes use_shuffle_layout
|
||||
(unified reads NHD and must pass False).
|
||||
"""
|
||||
if kv_cache_dtype.startswith("fp8"):
|
||||
key_cache = key_cache.view(current_platform.fp8_dtype())
|
||||
value_cache = value_cache.view(current_platform.fp8_dtype())
|
||||
# Partial-rotary support (e.g. GLM-4.7 applies rotary to only a prefix
|
||||
# of each head's channel dim).
|
||||
rotary_dim = cos_sin_cache.shape[-1]
|
||||
kernel_rotary_dim = 0 if rotary_dim == head_dim else rotary_dim
|
||||
rocm_aiter_ops.fused_qk_norm_rope_and_cache(
|
||||
qkv=qkv,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
positions=positions,
|
||||
num_heads_q=num_heads_q,
|
||||
num_heads_k=num_heads_k,
|
||||
num_heads_v=num_heads_k,
|
||||
head_dim=head_dim,
|
||||
is_neox=is_neox,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
q_out=q_out,
|
||||
k_cache=key_cache,
|
||||
v_cache=value_cache,
|
||||
slot_mapping=slot_mapping,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
k_out=k_out,
|
||||
v_out=None,
|
||||
return_kv=True,
|
||||
use_shuffle_layout=use_shuffle_layout,
|
||||
block_size=key_cache.shape[1],
|
||||
x=16 // key_cache.element_size(),
|
||||
rotary_dim=kernel_rotary_dim,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def triton_rope_and_cache(
|
||||
query: torch.Tensor,
|
||||
|
||||
@@ -25,6 +25,11 @@ logger = init_logger(__name__)
|
||||
|
||||
FUSED_QK_ROPE_OP = torch.ops._C.fused_qk_norm_rope.default
|
||||
|
||||
# Head dimensions supported by csrc/fused_qknorm_rope_kernel.cu's
|
||||
# launchFusedQKNormRope and launchFusedQKNormRopeNTokenHeads dispatchers.
|
||||
# Keep in sync with the switch statements in that file.
|
||||
SUPPORTED_FUSED_QK_NORM_ROPE_HEAD_DIMS: tuple[int, ...] = (64, 128, 256)
|
||||
|
||||
P = ParamSpec("P")
|
||||
|
||||
|
||||
@@ -190,7 +195,12 @@ class QkNormRopePattern:
|
||||
|
||||
|
||||
class QKNormRoPEFusionPass(VllmPatternMatcherPass):
|
||||
"""Fuse Q/K RMSNorm + RoPE into fused_qk_norm_rope when the custom op exists."""
|
||||
"""Fuse Q/K RMSNorm + RoPE into fused_qk_norm_rope when the custom op exists.
|
||||
|
||||
Registers patterns for both standard vLLM ops and ROCm AITER ops
|
||||
(when AITER is enabled), so the fusion fires regardless of which
|
||||
RMSNorm/RoPE implementation the graph uses.
|
||||
"""
|
||||
|
||||
@enable_fake_mode
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
@@ -216,6 +226,17 @@ class QKNormRoPEFusionPass(VllmPatternMatcherPass):
|
||||
)
|
||||
return
|
||||
|
||||
for layer in attn_layers.values():
|
||||
if layer.head_size not in SUPPORTED_FUSED_QK_NORM_ROPE_HEAD_DIMS:
|
||||
logger.warning_once(
|
||||
"QK Norm+RoPE fusion not enabled: layer head_size=%d is not "
|
||||
"supported by fused_qk_norm_rope kernel (supported: %s). "
|
||||
"Falling back to unfused QK norm + RoPE path.",
|
||||
layer.head_size,
|
||||
SUPPORTED_FUSED_QK_NORM_ROPE_HEAD_DIMS,
|
||||
)
|
||||
return
|
||||
|
||||
self._attention_geometries = tuple(
|
||||
sorted(
|
||||
{
|
||||
|
||||
@@ -0,0 +1,492 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import inspect
|
||||
from collections.abc import Callable
|
||||
from typing import ParamSpec
|
||||
|
||||
import torch
|
||||
import torch._inductor.pattern_matcher as pm
|
||||
from torch import fx
|
||||
from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
from torch._inductor.pattern_matcher import PatternMatcherPass
|
||||
|
||||
import vllm.ir.ops
|
||||
from vllm.config import VllmConfig, get_layers_from_vllm_config
|
||||
from vllm.config.utils import Range
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.attention import (
|
||||
Attention,
|
||||
get_attention_context,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
|
||||
from ..inductor_pass import enable_fake_mode
|
||||
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
|
||||
from .matcher_utils import MatcherRotaryEmbedding
|
||||
from .rms_quant_fusion import empty_bf16, empty_fp32, empty_i64
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
P = ParamSpec("P")
|
||||
|
||||
# Head sizes the fused kernel fused_qk_norm_rope_cache_pts_quant_shuffle() supports
|
||||
# Other sizes hard-abort, so skip those layers.
|
||||
SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS: tuple[int, ...] = (64, 128, 256)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Custom op: fused QK-norm + RoPE + KV cache update
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def fused_qk_norm_rope_and_unified_kv_cache_update_impl(
|
||||
q_out: torch.Tensor,
|
||||
k_out: torch.Tensor,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
layer_name: str = "",
|
||||
) -> torch.Tensor:
|
||||
_, attn_layer, kv_cache, layer_slot_mapping = get_attention_context(layer_name)
|
||||
if layer_slot_mapping is not None:
|
||||
attn_layer.impl.do_qk_norm_rope_kvcache_update(
|
||||
attn_layer,
|
||||
qkv,
|
||||
q_out,
|
||||
k_out,
|
||||
positions,
|
||||
q_weight,
|
||||
k_weight,
|
||||
rms_norm_eps,
|
||||
cos_sin_cache,
|
||||
is_neox,
|
||||
kv_cache,
|
||||
layer_slot_mapping,
|
||||
)
|
||||
else:
|
||||
# Profiling/dummy run: define q_out/k_out (consumed by attention).
|
||||
q_out.zero_()
|
||||
k_out.zero_()
|
||||
|
||||
return torch.empty(0, device=qkv.device, dtype=qkv.dtype)
|
||||
|
||||
|
||||
def fused_qk_norm_rope_and_unified_kv_cache_update_fake(
|
||||
q_out: torch.Tensor,
|
||||
k_out: torch.Tensor,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
layer_name: str = "",
|
||||
) -> torch.Tensor:
|
||||
return torch.empty(0, device=qkv.device, dtype=qkv.dtype)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="fused_qk_norm_rope_and_unified_kv_cache_update",
|
||||
op_func=fused_qk_norm_rope_and_unified_kv_cache_update_impl,
|
||||
mutates_args=["q_out", "k_out"],
|
||||
fake_impl=fused_qk_norm_rope_and_unified_kv_cache_update_fake,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pattern: QK-norm + RoPE + unified_kv_cache_update
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class QkNormRopeKvCachePattern:
|
||||
"""
|
||||
Match the unfused sequence:
|
||||
q, k, v = split(qkv, ...)
|
||||
q = rms_norm(q.view(heads), q_weight).view(flat)
|
||||
k = rms_norm(k.view(heads), k_weight).view(flat)
|
||||
q, k = rotary_embedding(positions, q, k, cos_sin_cache, is_neox)
|
||||
q = q.view(num_heads, head_dim)
|
||||
k = k.view(num_kv_heads, head_dim)
|
||||
v = v.view(num_kv_heads, head_dim)
|
||||
dummy = unified_kv_cache_update(k, v, layer_name)
|
||||
|
||||
Replace with:
|
||||
q_out = empty(...)
|
||||
k_out = empty(...)
|
||||
dummy = fused_qk_norm_rope_and_unified_kv_cache_update(
|
||||
q_out, k_out, qkv, positions, q_weight, k_weight,
|
||||
eps, cos_sin_cache, is_neox, layer_name)
|
||||
v = split(qkv, ...)[2].view(num_kv_heads, head_dim)
|
||||
"""
|
||||
|
||||
FUSED_OP = torch.ops.vllm.fused_qk_norm_rope_and_unified_kv_cache_update.default
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
layer: Attention,
|
||||
eps: float,
|
||||
is_neox: bool,
|
||||
quant_query: bool,
|
||||
) -> None:
|
||||
self.layer_name = layer.layer_name
|
||||
self.num_heads = layer.num_heads
|
||||
self.num_kv_heads = layer.num_kv_heads
|
||||
self.head_size = layer.head_size
|
||||
self.head_size_v = layer.head_size_v
|
||||
self.eps = eps
|
||||
self.is_neox = is_neox
|
||||
self.quant_query = quant_query
|
||||
|
||||
self.q_size = self.num_heads * self.head_size
|
||||
self.k_size = self.num_kv_heads * self.head_size
|
||||
self.v_size = self.num_kv_heads * self.head_size_v
|
||||
|
||||
self.rope_matcher = MatcherRotaryEmbedding(
|
||||
is_neox=is_neox,
|
||||
head_size=self.head_size,
|
||||
num_heads=self.num_heads,
|
||||
num_kv_heads=self.num_kv_heads,
|
||||
)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
T = 5
|
||||
L = 4096
|
||||
qkv = empty_bf16(T, self.q_size + self.k_size + self.v_size)
|
||||
positions = empty_i64(T)
|
||||
q_weight = empty_bf16(1, self.head_size)
|
||||
k_weight = empty_bf16(1, self.head_size)
|
||||
cos_sin_cache = empty_bf16(L, self.head_size)
|
||||
inputs = [qkv, positions, q_weight, k_weight, cos_sin_cache]
|
||||
if self.quant_query:
|
||||
q_scale = empty_fp32(1)
|
||||
inputs += [q_scale]
|
||||
return inputs
|
||||
|
||||
def pattern_non_fp8_quant_query(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
||||
q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size)
|
||||
q_normed = vllm.ir.ops.rms_norm(q_by_head, q_weight, self.eps)
|
||||
q_flat = q_normed.view(-1, self.q_size)
|
||||
|
||||
k_by_head = k.view(-1, self.k_size // self.head_size, self.head_size)
|
||||
k_normed = vllm.ir.ops.rms_norm(k_by_head, k_weight, self.eps)
|
||||
k_flat = k_normed.view(-1, self.k_size)
|
||||
|
||||
q_rope, k_rope = self.rope_matcher(positions, q_flat, k_flat, cos_sin_cache)
|
||||
|
||||
q_rope = q_rope.view(-1, self.num_heads, self.head_size)
|
||||
k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size)
|
||||
v = v.view(-1, self.num_kv_heads, self.head_size_v)
|
||||
dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name)
|
||||
return dummy, q_rope, k_rope, v
|
||||
|
||||
def replacement_non_fp8_quant_query(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
q_out = torch.empty(
|
||||
qkv.shape[0],
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
device=qkv.device,
|
||||
dtype=qkv.dtype,
|
||||
)
|
||||
k_out = torch.empty(
|
||||
qkv.shape[0],
|
||||
self.num_kv_heads,
|
||||
self.head_size,
|
||||
device=qkv.device,
|
||||
dtype=qkv.dtype,
|
||||
)
|
||||
_, _, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
||||
v = v.view(qkv.shape[0], self.num_kv_heads, self.head_size_v)
|
||||
results = auto_functionalized(
|
||||
self.FUSED_OP,
|
||||
q_out=q_out,
|
||||
k_out=k_out,
|
||||
qkv=qkv,
|
||||
positions=positions,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
rms_norm_eps=self.eps,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
layer_name=self.layer_name,
|
||||
)
|
||||
return results[0], results[1], results[2], v
|
||||
|
||||
def pattern_fp8_quant_query(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
q, k, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
||||
q_by_head = q.view(-1, self.q_size // self.head_size, self.head_size)
|
||||
q_normed = vllm.ir.ops.rms_norm(q_by_head, q_weight, self.eps)
|
||||
q_flat = q_normed.view(-1, self.q_size)
|
||||
|
||||
k_by_head = k.view(-1, self.k_size // self.head_size, self.head_size)
|
||||
k_normed = vllm.ir.ops.rms_norm(k_by_head, k_weight, self.eps)
|
||||
k_flat = k_normed.view(-1, self.k_size)
|
||||
|
||||
q_rope, k_rope = self.rope_matcher(positions, q_flat, k_flat, cos_sin_cache)
|
||||
# Match the quant-query op Attention.forward inserts (fp8 KV + UNIFIED).
|
||||
# Explicit auto_functionalized (out=[1]) keeps the quant node in the pattern.
|
||||
q_out = torch.empty_like(q_rope, dtype=current_platform.fp8_dtype())
|
||||
q_quant = auto_functionalized(
|
||||
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
|
||||
out=q_out,
|
||||
x=q_rope,
|
||||
scale=q_scale,
|
||||
is_dynamic=False,
|
||||
)
|
||||
# `scale` is mutable: its copy_ write-back to _q_scale bumps the mutation
|
||||
# region, so keep q flat (a reshape lands past the barrier and won't match).
|
||||
q_rope_fp8 = q_quant[1]
|
||||
q_scale_out = q_quant[2]
|
||||
|
||||
k_rope = k_rope.view(-1, self.num_kv_heads, self.head_size)
|
||||
v = v.view(-1, self.num_kv_heads, self.head_size_v)
|
||||
dummy = torch.ops.vllm.unified_kv_cache_update(k_rope, v, self.layer_name)
|
||||
return dummy, q_rope_fp8, k_rope, v, q_scale_out
|
||||
|
||||
def replacement_fp8_quant_query(
|
||||
self,
|
||||
qkv: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
q_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
q_out = torch.empty(
|
||||
qkv.shape[0],
|
||||
self.num_heads,
|
||||
self.head_size,
|
||||
device=qkv.device,
|
||||
dtype=qkv.dtype,
|
||||
)
|
||||
k_out = torch.empty(
|
||||
qkv.shape[0],
|
||||
self.num_kv_heads,
|
||||
self.head_size,
|
||||
device=qkv.device,
|
||||
dtype=qkv.dtype,
|
||||
)
|
||||
_, _, v = qkv.split([self.q_size, self.k_size, self.v_size], dim=-1)
|
||||
v = v.view(qkv.shape[0], self.num_kv_heads, self.head_size_v)
|
||||
results = auto_functionalized(
|
||||
self.FUSED_OP,
|
||||
q_out=q_out,
|
||||
k_out=k_out,
|
||||
qkv=qkv,
|
||||
positions=positions,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
rms_norm_eps=self.eps,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
is_neox=self.is_neox,
|
||||
layer_name=self.layer_name,
|
||||
)
|
||||
# Re-apply the quant on the kernel's bf16 q_out; fused op does not quant q.
|
||||
# Same explicit auto_functionalized form as the pattern: [1] = quantized
|
||||
# q, [2] = scale (returned so the buffer-writeback use is preserved).
|
||||
q_fp8_flat = results[1].view(-1, self.q_size)
|
||||
q_fp8_out = torch.empty_like(q_fp8_flat, dtype=current_platform.fp8_dtype())
|
||||
q_requant = auto_functionalized(
|
||||
torch.ops.vllm.rocm_aiter_per_tensor_quant.default,
|
||||
out=q_fp8_out,
|
||||
x=q_fp8_flat,
|
||||
scale=q_scale,
|
||||
is_dynamic=False,
|
||||
)
|
||||
q_fp8 = q_requant[1] # flat to mirror the pattern (see note above)
|
||||
q_scale_out = q_requant[2]
|
||||
return results[0], q_fp8, results[2], v, q_scale_out
|
||||
|
||||
@staticmethod
|
||||
def wrap_trace_fn(
|
||||
trace_fn: Callable[P, fx.GraphModule],
|
||||
*process_fx_fns: Callable[[fx.GraphModule], None],
|
||||
) -> Callable[P, fx.GraphModule]:
|
||||
def wrapped(*args: P.args, **kwargs: P.kwargs) -> fx.GraphModule:
|
||||
gm = trace_fn(*args, **kwargs)
|
||||
for process_fx in process_fx_fns:
|
||||
process_fx(gm)
|
||||
|
||||
return gm
|
||||
|
||||
return wrapped
|
||||
|
||||
@staticmethod
|
||||
def fx_view_to_reshape(gm: torch.fx.GraphModule) -> None:
|
||||
from torch._inductor.fx_passes.post_grad import view_to_reshape
|
||||
|
||||
view_to_reshape(gm)
|
||||
|
||||
def _register(self, pattern, replacement, pm_pass) -> None:
|
||||
trace_fn = QkNormRopeKvCachePattern.wrap_trace_fn(
|
||||
pm.fwd_only,
|
||||
QkNormRopeKvCachePattern.fx_view_to_reshape,
|
||||
)
|
||||
|
||||
# Pre-build the search pattern with `ignore_types=(int, torch.SymInt)`
|
||||
# and pass it via `search_fn_pattern=` so torch skips both of its
|
||||
# internal `fx_to_pattern` calls and treats dynamic-shape SymInts as
|
||||
# wildcards.
|
||||
inputs = self.get_inputs()
|
||||
argnames = [*inspect.signature(pattern).parameters.keys()]
|
||||
search_gm = trace_fn(pattern, inputs)
|
||||
search_fn_pattern = pm.fx_to_pattern(
|
||||
search_gm,
|
||||
ignore_types=(int, torch.SymInt),
|
||||
argnames=argnames,
|
||||
)
|
||||
|
||||
pm.register_replacement(
|
||||
pattern,
|
||||
replacement,
|
||||
inputs,
|
||||
trace_fn,
|
||||
pm_pass,
|
||||
search_fn_pattern=search_fn_pattern,
|
||||
)
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
# make_fx counts `self` in bound-method code params; wrap as plain fns.
|
||||
# Distinct names per branch so mypy doesn't see one name, two signatures.
|
||||
if self.quant_query:
|
||||
|
||||
def pattern_q(qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale):
|
||||
return self.pattern_fp8_quant_query(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
|
||||
)
|
||||
|
||||
def replacement_q(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
|
||||
):
|
||||
return self.replacement_fp8_quant_query(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache, q_scale
|
||||
)
|
||||
|
||||
self._register(pattern_q, replacement_q, pm_pass)
|
||||
else:
|
||||
|
||||
def pattern_noq(qkv, positions, q_weight, k_weight, cos_sin_cache):
|
||||
return self.pattern_non_fp8_quant_query(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache
|
||||
)
|
||||
|
||||
def replacement_noq(qkv, positions, q_weight, k_weight, cos_sin_cache):
|
||||
return self.replacement_non_fp8_quant_query(
|
||||
qkv, positions, q_weight, k_weight, cos_sin_cache
|
||||
)
|
||||
|
||||
self._register(pattern_noq, replacement_noq, pm_pass)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pass class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
|
||||
"""
|
||||
Fuse QK-norm + RoPE + KV cache update into a single AITER HIP kernel.
|
||||
|
||||
Supersedes both QKNormRoPEFusionPass and RopeKVCacheFusionPass for
|
||||
attention layers that support the combined operation, eliminating two
|
||||
separate kernel launches and the intermediate memory traffic.
|
||||
"""
|
||||
|
||||
@enable_fake_mode
|
||||
def __init__(self, config: VllmConfig) -> None:
|
||||
super().__init__(config)
|
||||
|
||||
self.patterns: PatternMatcherPass = PatternMatcherPass(
|
||||
pass_name="qk_norm_rope_kvcache_fusion_pass"
|
||||
)
|
||||
|
||||
cc = config.compilation_config
|
||||
self.max_token_num = cc.pass_config.rope_kvcache_fusion_max_token_num
|
||||
|
||||
dtype = config.model_config.dtype
|
||||
if dtype not in (torch.bfloat16, torch.float16):
|
||||
logger.warning_once(
|
||||
"QK Norm+RoPE+KVCache fusion not enabled: unsupported dtype %s", dtype
|
||||
)
|
||||
return
|
||||
|
||||
attn_layers = get_layers_from_vllm_config(config, Attention)
|
||||
|
||||
for _, layer in attn_layers.items():
|
||||
if not layer.impl.fused_qk_norm_rope_kvcache_supported():
|
||||
continue
|
||||
if layer.head_size not in SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS:
|
||||
logger.warning_once(
|
||||
"QK Norm+RoPE+KVCache fusion not enabled for a layer: "
|
||||
"head_size=%d is not supported by the "
|
||||
"fused_qk_norm_rope_cache_pts_quant_shuffle kernel "
|
||||
"(supported: %s). Falling back to the unfused path.",
|
||||
layer.head_size,
|
||||
SUPPORTED_FUSED_QK_NORM_ROPE_KVCACHE_HEAD_DIMS,
|
||||
)
|
||||
continue
|
||||
if layer.head_size_v != layer.head_size:
|
||||
# The fused kernel uses a single head_dim for q/k/v.
|
||||
logger.warning_once(
|
||||
"QK Norm+RoPE+KVCache fusion not enabled for a layer: "
|
||||
"head_size_v=%d differs from head_size=%d, which the fused "
|
||||
"kernel does not support. Falling back to the unfused path.",
|
||||
layer.head_size_v,
|
||||
layer.head_size,
|
||||
)
|
||||
continue
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
for neox in [True, False]:
|
||||
for quant_q in [False, True]:
|
||||
QkNormRopeKvCachePattern(
|
||||
layer=layer,
|
||||
eps=epsilon,
|
||||
is_neox=neox,
|
||||
quant_query=quant_q,
|
||||
).register(self.patterns)
|
||||
|
||||
self.dump_patterns(config, self.patterns)
|
||||
|
||||
@VllmInductorPass.time_and_log
|
||||
def __call__(self, graph: fx.Graph) -> None:
|
||||
self.matched_count = self.patterns.apply(graph)
|
||||
logger.info(
|
||||
"QK-Norm+RoPE+KVCache fusion: replaced %s pattern(s) "
|
||||
"with AITER fused_qk_norm_rope_cache_pts_quant_shuffle",
|
||||
self.matched_count,
|
||||
)
|
||||
|
||||
def is_applicable_for_range(self, compile_range: Range) -> bool:
|
||||
return compile_range.end <= self.max_token_num
|
||||
|
||||
def uuid(self) -> str:
|
||||
return VllmInductorPass.hash_source(self, QkNormRopeKvCachePattern)
|
||||
@@ -38,6 +38,7 @@ if current_platform.is_cuda_alike():
|
||||
from .fusion.mla_attn_quant_fusion import MLAAttnQuantFusionPass
|
||||
from .fusion.mla_rope_kvcache_cat_fusion import MLARoPEKVCacheCatFusionPass
|
||||
from .fusion.qk_norm_rope_fusion import QKNormRoPEFusionPass
|
||||
from .fusion.qk_norm_rope_kvcache_fusion import QkNormRopeKvCacheFusionPass
|
||||
from .fusion.rms_quant_fusion import RMSNormQuantFusionPass
|
||||
from .fusion.rope_kvcache_fusion import RopeKVCacheFusionPass
|
||||
from .utility.scatter_split_replace import ScatterSplitReplacementPass
|
||||
@@ -171,6 +172,11 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
self.passes += [RocmAiterSiluMulFp8GroupQuantFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_qk_norm_rope_kvcache:
|
||||
self.passes += [SplitCoalescingPass(config)]
|
||||
self.passes += [ScatterSplitReplacementPass(config)]
|
||||
self.passes += [QkNormRopeKvCacheFusionPass(config)]
|
||||
|
||||
if (
|
||||
self.pass_config.fuse_mla_dual_rms_norm
|
||||
and rocm_aiter_ops.is_enabled()
|
||||
|
||||
@@ -146,10 +146,16 @@ class PassConfig:
|
||||
"""Fuse paired q/kv RMS norms in MLA attention."""
|
||||
fuse_rope_kvcache: bool = None # type: ignore[assignment]
|
||||
"""Fuse the QK rope + KV cache ops."""
|
||||
fuse_qk_norm_rope_kvcache: bool = Field(default=None) # type: ignore[assignment]
|
||||
"""Fuse QK RMSNorm + RoPE + KV cache update into a single AITER HIP
|
||||
kernel. Supersedes both enable_qk_norm_rope_fusion and fuse_rope_kvcache
|
||||
for layers that support it. Auto-enabled at O1+ on ROCm for models
|
||||
with QK-norm (e.g. Qwen3-MoE)."""
|
||||
|
||||
rope_kvcache_fusion_max_token_num: int = 256
|
||||
"""The threshold for ROCm AITER RoPE+KVCache fusion e.g. for small batch decode.
|
||||
Larger batch sizes e.g. during prefill will use the unfused kernels.
|
||||
Also applies to the fused QK-Norm+RoPE+KVCache pass.
|
||||
"""
|
||||
|
||||
fi_allreduce_fusion_max_size_mb: float | None = None
|
||||
@@ -228,6 +234,8 @@ class PassConfig:
|
||||
"fuse_act_padding",
|
||||
"fuse_mla_dual_rms_norm",
|
||||
"fuse_rope_kvcache",
|
||||
"fuse_qk_norm_rope_kvcache",
|
||||
"enable_qk_norm_rope_fusion",
|
||||
"fuse_rope_kvcache_cat_mla",
|
||||
mode="wrap",
|
||||
)
|
||||
@@ -288,6 +296,12 @@ class PassConfig:
|
||||
"The fusion will be disabled."
|
||||
)
|
||||
self.fuse_rope_kvcache = False
|
||||
if self.fuse_qk_norm_rope_kvcache and not current_platform.is_rocm():
|
||||
logger.warning_once(
|
||||
"QK-Norm+RoPE+KVCache fusion requires ROCm with AITER. "
|
||||
"The fusion will be disabled."
|
||||
)
|
||||
self.fuse_qk_norm_rope_kvcache = False
|
||||
if self.fuse_rope_kvcache_cat_mla and not current_platform.is_cuda_alike():
|
||||
logger.warning_once(
|
||||
"MLA KV cache update with RoPE fusion enabled but the "
|
||||
@@ -302,10 +316,13 @@ class PassConfig:
|
||||
after all defaults are finalized.
|
||||
TODO also log the compile ranges for which this is enabled.
|
||||
"""
|
||||
fusion_prefixes = ("fuse_", "enable_")
|
||||
enabled_fusions = [
|
||||
f.name[len("fuse_") :]
|
||||
f.name[len(prefix) :]
|
||||
for f in fields(self) # type: ignore[arg-type]
|
||||
if getattr(self, f.name) and f.name.startswith("fuse_")
|
||||
if getattr(self, f.name)
|
||||
for prefix in fusion_prefixes
|
||||
if f.name.startswith(prefix)
|
||||
]
|
||||
|
||||
if enabled_fusions:
|
||||
@@ -947,6 +964,7 @@ class CompilationConfig:
|
||||
# TODO(zhuhaoran): support rope native forward match and remove this.
|
||||
# Linked issue: https://github.com/vllm-project/vllm/issues/28042
|
||||
self.custom_ops.append("+rotary_embedding")
|
||||
|
||||
if (
|
||||
self.pass_config.fuse_rope_kvcache
|
||||
and "+rotary_embedding" not in self.custom_ops
|
||||
@@ -955,6 +973,12 @@ class CompilationConfig:
|
||||
# Linked issue: https://github.com/vllm-project/vllm/issues/28042
|
||||
self.custom_ops.append("+rotary_embedding")
|
||||
|
||||
if (
|
||||
self.pass_config.fuse_qk_norm_rope_kvcache
|
||||
and "+rotary_embedding" not in self.custom_ops
|
||||
):
|
||||
self.custom_ops.append("+rotary_embedding")
|
||||
|
||||
if (
|
||||
is_torch_equal_or_newer("2.9.0.dev")
|
||||
and "combo_kernels" not in self.inductor_compile_config
|
||||
@@ -1137,6 +1161,16 @@ class CompilationConfig:
|
||||
"to enable RoPE+KV cache fusion."
|
||||
)
|
||||
self.pass_config.fuse_rope_kvcache = False
|
||||
if self.pass_config.fuse_qk_norm_rope_kvcache:
|
||||
logger.warning_once(
|
||||
"fuse_qk_norm_rope_kvcache is enabled, but "
|
||||
"splitting_ops is None and Inductor graph partition "
|
||||
"is not enabled. Disabling fuse_qk_norm_rope_kvcache. "
|
||||
"Please either set splitting_ops to an empty list [] "
|
||||
"or set use_inductor_graph_partition to True "
|
||||
"to enable QK-Norm+RoPE+KV cache fusion."
|
||||
)
|
||||
self.pass_config.fuse_qk_norm_rope_kvcache = False
|
||||
self.splitting_ops.append("vllm::unified_kv_cache_update")
|
||||
self.splitting_ops.append("vllm::unified_mla_kv_cache_update")
|
||||
|
||||
|
||||
@@ -193,6 +193,15 @@ def enable_mla_dual_rms_norm_fusion(cfg: "VllmConfig") -> bool:
|
||||
return rocm_aiter_ops.is_enabled() and check_aiter_fused_qk_rmsnorm()
|
||||
|
||||
|
||||
def enable_qk_norm_rope_kvcache(cfg: "VllmConfig") -> bool:
|
||||
"""Enable fused QK-norm + RoPE + KV cache update on ROCm with AITER."""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
if not rocm_aiter_ops.is_enabled():
|
||||
return False
|
||||
return cfg.compilation_config.is_custom_op_enabled("rotary_embedding")
|
||||
|
||||
|
||||
OPTIMIZATION_LEVEL_00 = {
|
||||
"compilation_config": {
|
||||
"pass_config": {
|
||||
@@ -205,6 +214,8 @@ OPTIMIZATION_LEVEL_00 = {
|
||||
"fuse_act_padding": False,
|
||||
"fuse_mla_dual_rms_norm": False,
|
||||
"fuse_rope_kvcache": False,
|
||||
"fuse_qk_norm_rope_kvcache": False,
|
||||
"enable_qk_norm_rope_fusion": False,
|
||||
"fuse_rope_kvcache_cat_mla": False,
|
||||
},
|
||||
"cudagraph_mode": CUDAGraphMode.NONE,
|
||||
@@ -226,6 +237,8 @@ OPTIMIZATION_LEVEL_01 = {
|
||||
"fuse_act_padding": enable_norm_pad_fusion,
|
||||
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
|
||||
"fuse_rope_kvcache": False,
|
||||
"fuse_qk_norm_rope_kvcache": False,
|
||||
"enable_qk_norm_rope_fusion": False,
|
||||
"fuse_rope_kvcache_cat_mla": False,
|
||||
},
|
||||
"cudagraph_mode": CUDAGraphMode.PIECEWISE,
|
||||
@@ -247,6 +260,8 @@ OPTIMIZATION_LEVEL_02 = {
|
||||
"fuse_act_padding": enable_norm_pad_fusion,
|
||||
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
|
||||
"fuse_rope_kvcache": enable_rope_kvcache_fusion,
|
||||
"fuse_qk_norm_rope_kvcache": enable_qk_norm_rope_kvcache,
|
||||
"enable_qk_norm_rope_fusion": False,
|
||||
"fuse_rope_kvcache_cat_mla": enable_rope_kvcache_mla_fusion,
|
||||
},
|
||||
"cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
@@ -268,6 +283,8 @@ OPTIMIZATION_LEVEL_03 = {
|
||||
"fuse_act_padding": enable_norm_pad_fusion,
|
||||
"fuse_mla_dual_rms_norm": enable_mla_dual_rms_norm_fusion,
|
||||
"fuse_rope_kvcache": enable_rope_kvcache_fusion,
|
||||
"fuse_qk_norm_rope_kvcache": enable_qk_norm_rope_kvcache,
|
||||
"enable_qk_norm_rope_fusion": False,
|
||||
"fuse_rope_kvcache_cat_mla": enable_rope_kvcache_mla_fusion,
|
||||
},
|
||||
"cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
|
||||
@@ -1971,6 +1988,21 @@ class VllmConfig:
|
||||
compile_range_end,
|
||||
)
|
||||
|
||||
if compilation_config.pass_config.fuse_qk_norm_rope_kvcache:
|
||||
max_token_num = (
|
||||
compilation_config.pass_config.rope_kvcache_fusion_max_token_num
|
||||
)
|
||||
if max_token_num is not None:
|
||||
if compile_range_end is not None and max_token_num < compile_range_end:
|
||||
computed_compile_ranges_endpoints.append(max_token_num)
|
||||
else:
|
||||
logger.debug(
|
||||
"Max num batched tokens below qk_norm+rope+kvcache "
|
||||
"fusion threshold, fusion enabled for "
|
||||
"num_tokens <= %d.",
|
||||
compile_range_end,
|
||||
)
|
||||
|
||||
if compilation_config.compile_ranges_endpoints is not None:
|
||||
for x in compilation_config.compile_ranges_endpoints:
|
||||
assert isinstance(x, int)
|
||||
|
||||
@@ -136,10 +136,12 @@ def set_default_quant_scales(layer: nn.Module, register_buffer: bool = False) ->
|
||||
|
||||
# We also keep q/k/v_scale on host (cpu) memory for attention
|
||||
# backends that require the scales to be on host instead of on device.
|
||||
# e.g. Flashinfer
|
||||
# e.g. Flashinfer & AITER
|
||||
layer._q_scale_float = 1.0
|
||||
layer._k_scale_float = 1.0
|
||||
layer._v_scale_float = 1.0
|
||||
layer._k_scale_cpu = torch.tensor(1.0, dtype=torch.float32)
|
||||
layer._v_scale_cpu = torch.tensor(1.0, dtype=torch.float32)
|
||||
layer._prob_scale_float = 1.0
|
||||
|
||||
# Initialize q/k/v range constants used by calc_kv_scales
|
||||
@@ -586,6 +588,8 @@ class Attention(nn.Module, AttentionLayerBase):
|
||||
self._q_scale_float = self._q_scale.item()
|
||||
self._k_scale_float = self._k_scale.item()
|
||||
self._v_scale_float = self._v_scale.item()
|
||||
self._k_scale_cpu.fill_(self._k_scale_float)
|
||||
self._v_scale_cpu.fill_(self._v_scale_float)
|
||||
# We only calculate the scales once
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
|
||||
@@ -1024,6 +1024,8 @@ class MLAAttention(nn.Module, AttentionLayerBase):
|
||||
self._q_scale_float = self._q_scale.item()
|
||||
self._k_scale_float = self._k_scale.item()
|
||||
self._v_scale_float = self._v_scale.item()
|
||||
self._k_scale_cpu.fill_(self._k_scale_float)
|
||||
self._v_scale_cpu.fill_(self._v_scale_float)
|
||||
self.calculate_kv_scales = False
|
||||
|
||||
def get_attn_backend(self) -> type[AttentionBackend]:
|
||||
|
||||
@@ -1200,6 +1200,10 @@ class CompressedTensorsKVCacheMethod(BaseKVCacheMethod):
|
||||
layer._v_scale_float = _to_scalar(layer.v_scale)
|
||||
layer._q_scale_float = _to_scalar(layer.q_scale)
|
||||
|
||||
# Sync host (cpu) scale copies read by AITER fused kernels.
|
||||
layer._k_scale_cpu.fill_(layer._k_scale_float)
|
||||
layer._v_scale_cpu.fill_(layer._v_scale_float)
|
||||
|
||||
# Discard all placeholders.
|
||||
del layer.k_scale
|
||||
del layer.v_scale
|
||||
|
||||
@@ -144,6 +144,9 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
layer._v_scale.copy_(v_scale)
|
||||
layer._k_scale_float = k_scale
|
||||
layer._v_scale_float = v_scale
|
||||
# Sync host (cpu) scale copies read by AITER fused kernels.
|
||||
layer._k_scale_cpu.fill_(k_scale)
|
||||
layer._v_scale_cpu.fill_(v_scale)
|
||||
if k_scale == 1.0 and v_scale == 1.0 and "e5m2" not in layer.kv_cache_dtype:
|
||||
logger.warning_once(
|
||||
"Using KV cache scaling factor 1.0 for fp8_e4m3. "
|
||||
|
||||
@@ -757,7 +757,9 @@ class AttentionMetadataBuilder(ABC, Generic[M]):
|
||||
class AttentionLayer(Protocol):
|
||||
_q_scale: torch.Tensor
|
||||
_k_scale: torch.Tensor
|
||||
_k_scale_cpu: torch.Tensor
|
||||
_v_scale: torch.Tensor
|
||||
_v_scale_cpu: torch.Tensor
|
||||
_q_scale_float: float
|
||||
_k_scale_float: float
|
||||
_v_scale_float: float
|
||||
@@ -923,6 +925,14 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
|
||||
"""
|
||||
return False
|
||||
|
||||
def fused_qk_norm_rope_kvcache_supported(self):
|
||||
"""
|
||||
Does this attention implementation support fused QKNorm+RoPE+KVCache fusion.
|
||||
This is used by the QkNormRopeKvCachePattern to only fuse the QKNorm ops
|
||||
with the RoPE ops and the KV cache update for implementations that support it.
|
||||
"""
|
||||
return False
|
||||
|
||||
def fused_rope_kvcache_supported(self):
|
||||
"""
|
||||
Does this attention implementation support RoPE+KVCache fusion.
|
||||
@@ -931,6 +941,29 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
|
||||
"""
|
||||
return False
|
||||
|
||||
def do_qk_norm_rope_kvcache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
qkv: torch.Tensor,
|
||||
q_out: torch.Tensor,
|
||||
k_out: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_slot_mapping: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
If `fused_qk_norm_rope_kvcache_supported` returns True, this method
|
||||
will be called by the fused custom op. Applies QK-norm + RoPE and
|
||||
writes K/V to the KV cache. Results are written to the pre-allocated
|
||||
q_out and k_out tensors; V is split from QKV at the graph level.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def do_rope_and_kv_cache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
|
||||
@@ -1449,6 +1449,47 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
and not rocm_aiter_ops.is_shuffle_kv_cache_enabled()
|
||||
)
|
||||
|
||||
def fused_qk_norm_rope_kvcache_supported(self):
|
||||
return rocm_aiter_ops.is_enabled()
|
||||
|
||||
def do_qk_norm_rope_kvcache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
qkv: torch.Tensor,
|
||||
q_out: torch.Tensor,
|
||||
k_out: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_slot_mapping: torch.Tensor,
|
||||
):
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
|
||||
qkv=qkv,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
positions=positions,
|
||||
num_heads_q=self.num_heads,
|
||||
num_heads_k=self.num_kv_heads,
|
||||
head_dim=self.head_size,
|
||||
is_neox=is_neox,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
q_out=q_out,
|
||||
k_out=k_out,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
slot_mapping=layer_slot_mapping,
|
||||
k_scale=layer._k_scale_cpu,
|
||||
v_scale=layer._v_scale_cpu,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
use_shuffle_layout=rocm_aiter_ops.is_shuffle_kv_cache_enabled(),
|
||||
)
|
||||
|
||||
def do_rope_and_kv_cache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
|
||||
@@ -311,6 +311,50 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
def fused_rope_kvcache_supported(self):
|
||||
return rocm_aiter_ops.is_enabled()
|
||||
|
||||
def fused_qk_norm_rope_kvcache_supported(self):
|
||||
return rocm_aiter_ops.is_enabled()
|
||||
|
||||
def do_qk_norm_rope_kvcache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
qkv: torch.Tensor,
|
||||
q_out: torch.Tensor,
|
||||
k_out: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_slot_mapping: torch.Tensor,
|
||||
):
|
||||
# _split_kv_cache picks the unbind dim per layout (incl. the K/V-first
|
||||
# encoder-decoder path). unified reads NHD, so never write the shuffle
|
||||
# layout here.
|
||||
key_cache, value_cache = self._split_kv_cache(kv_cache)
|
||||
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
|
||||
qkv=qkv,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
positions=positions,
|
||||
num_heads_q=self.num_heads,
|
||||
num_heads_k=self.num_kv_heads,
|
||||
head_dim=self.head_size,
|
||||
is_neox=is_neox,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
q_out=q_out,
|
||||
k_out=k_out,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
slot_mapping=layer_slot_mapping,
|
||||
k_scale=layer._k_scale_cpu,
|
||||
v_scale=layer._v_scale_cpu,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
use_shuffle_layout=False,
|
||||
)
|
||||
|
||||
def do_rope_and_kv_cache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
|
||||
Reference in New Issue
Block a user