[Model][Hardware][AMD][Kernel] Part 2/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_ATTN (#42755)
Signed-off-by: Jack Hu <Jack.Hu@amd.com> Signed-off-by: Jack Hu <jingxujaster@hotmail.com> Signed-off-by: Jack Hu <jack.hu@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
+24
-9
@@ -11,6 +11,7 @@ import torch.distributed as dist
|
||||
from torch._ops import OpOverload
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import direct_register_custom_op
|
||||
@@ -3887,17 +3888,29 @@ class rocm_aiter_ops:
|
||||
v_scale: torch.Tensor,
|
||||
kv_cache_dtype: str,
|
||||
use_shuffle_layout: bool,
|
||||
return_kv: bool,
|
||||
) -> None:
|
||||
"""Run the fused QK-norm+RoPE+KV-cache op on already-split k/v caches.
|
||||
|
||||
Shared by the AITER FA and unified-attention impls. The caller splits
|
||||
Shared by the AITER FA, ROCM ATTN, and unified-attention impls. The caller splits
|
||||
kv_cache, since the unbind dim depends on the layout (e.g. the unified
|
||||
encoder-decoder path is K/V-first), and passes use_shuffle_layout
|
||||
(unified reads NHD and must pass False).
|
||||
(unified reads NHD and must pass False). k_scale/v_scale are the
|
||||
per-tensor CPU scalars. return_kv says whether the impl consumes
|
||||
k_out (FA and ROCM_ATTN prefill do; unified reads K from the cache and
|
||||
passes False, so the kernel skips that store). For fp8 caches the
|
||||
kernel emits k_out in the cache dtype, so a returned k_out is
|
||||
converted to the activation dtype.
|
||||
"""
|
||||
if kv_cache_dtype.startswith("fp8"):
|
||||
key_cache = key_cache.view(current_platform.fp8_dtype())
|
||||
value_cache = value_cache.view(current_platform.fp8_dtype())
|
||||
is_fp8_cache = kv_cache_dtype.startswith("fp8")
|
||||
dequant_k_out = return_kv and is_fp8_cache
|
||||
kernel_k_out = k_out
|
||||
if is_fp8_cache:
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
key_cache = key_cache.view(fp8_dtype)
|
||||
value_cache = value_cache.view(fp8_dtype)
|
||||
if dequant_k_out:
|
||||
kernel_k_out = torch.empty_like(k_out, dtype=fp8_dtype)
|
||||
# Partial-rotary support (e.g. GLM-4.7 applies rotary to only a prefix
|
||||
# of each head's channel dim).
|
||||
rotary_dim = cos_sin_cache.shape[-1]
|
||||
@@ -3920,14 +3933,16 @@ class rocm_aiter_ops:
|
||||
slot_mapping=slot_mapping,
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
k_out=k_out,
|
||||
k_out=kernel_k_out,
|
||||
v_out=None,
|
||||
return_kv=True,
|
||||
return_kv=return_kv,
|
||||
use_shuffle_layout=use_shuffle_layout,
|
||||
block_size=key_cache.shape[1],
|
||||
x=16 // key_cache.element_size(),
|
||||
rotary_dim=kernel_rotary_dim,
|
||||
)
|
||||
if dequant_k_out:
|
||||
ops.convert_fp8(k_out, kernel_k_out, float(k_scale), kv_cache_dtype)
|
||||
|
||||
@staticmethod
|
||||
def fused_qk_norm_mrope_and_cache(
|
||||
@@ -4506,8 +4521,8 @@ class rocm_aiter_ops:
|
||||
scale: float,
|
||||
K_QScale_hip: torch.Tensor,
|
||||
V_QScale_hip: torch.Tensor,
|
||||
K_QScale_asm: torch.Tensor,
|
||||
V_QScale_asm: torch.Tensor,
|
||||
K_QScale_asm: torch.Tensor | None,
|
||||
V_QScale_asm: torch.Tensor | None,
|
||||
out_: torch.Tensor,
|
||||
kv_cache_dtype: str,
|
||||
):
|
||||
|
||||
@@ -122,6 +122,7 @@ def paged_attention_rocm(
|
||||
v_scale: torch.Tensor,
|
||||
fp8_out_scale: torch.Tensor | None = None,
|
||||
mfma_type: str = "fp8" if envs.VLLM_ROCM_FP8_MFMA_PAGE_ATTN else "f16",
|
||||
use_interleaved_v_cache: bool = False,
|
||||
) -> None:
|
||||
torch.ops._rocm_C.paged_attention(
|
||||
out,
|
||||
@@ -144,6 +145,7 @@ def paged_attention_rocm(
|
||||
v_scale,
|
||||
fp8_out_scale,
|
||||
mfma_type,
|
||||
use_interleaved_v_cache,
|
||||
)
|
||||
|
||||
|
||||
@@ -2554,6 +2556,7 @@ def reshape_and_cache(
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
interleaved_v: bool = False,
|
||||
) -> None:
|
||||
torch.ops._C_cache_ops.reshape_and_cache(
|
||||
key,
|
||||
@@ -2564,6 +2567,7 @@ def reshape_and_cache(
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
interleaved_v,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -893,6 +893,7 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
|
||||
layer.head_size,
|
||||
)
|
||||
continue
|
||||
layer.impl.set_interleaved_v_cache()
|
||||
if supports_rope:
|
||||
for epsilon in [1e-5, 1e-6]:
|
||||
for neox in [True, False]:
|
||||
@@ -903,7 +904,6 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
|
||||
is_neox=neox,
|
||||
quant_query=quant_q,
|
||||
).register(self.patterns)
|
||||
|
||||
if supports_mrope:
|
||||
for section, is_interleaved in self.mrope_configs:
|
||||
rotary_dim = 2 * sum(section)
|
||||
|
||||
@@ -531,11 +531,6 @@ class FusedMoEExperts(ABC):
|
||||
def is_monolithic() -> bool:
|
||||
raise NotImplementedError("Implemented by subclasses.")
|
||||
|
||||
@staticmethod
|
||||
def _supports_hash_routing() -> bool:
|
||||
"""Whether a monolithic kernel implements hash routing."""
|
||||
return False
|
||||
|
||||
@property
|
||||
def expects_unquantized_inputs(self) -> bool:
|
||||
"""Whether or not the PrepareFinalize should defer input quantization
|
||||
|
||||
@@ -950,6 +950,16 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
|
||||
"""Whether this implementation supports fused QKNorm+MRoPE+KVCache."""
|
||||
return False
|
||||
|
||||
def set_interleaved_v_cache(self):
|
||||
"""Opt this implementation into the interleaved V-cache layout for all of
|
||||
its cache writers and readers. QkNormRopeKvCacheFusionPass calls this
|
||||
for every eligible layer when it is constructed, i.e. before any KV
|
||||
cache is written, so fused and unfused paths stay consistent whether or
|
||||
not the pattern matches. Default is a no-op; backends whose kernels
|
||||
read an interleaved V-cache under this fusion (e.g. ROCM_ATTN) override.
|
||||
"""
|
||||
return False
|
||||
|
||||
def fused_rope_kvcache_supported(self):
|
||||
"""Does this attention implementation support RoPE+KVCache fusion.
|
||||
This is used by the RopeKVCacheFusionPass to only fuse the RoPE ops
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Attention layer with AiterFlashAttention."""
|
||||
|
||||
from dataclasses import dataclass, replace
|
||||
from dataclasses import dataclass, field, replace
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
@@ -152,8 +152,8 @@ if current_platform.is_rocm():
|
||||
k_reg = tl.load(key_cache_ptr_offset + k_reg_offset)
|
||||
v_reg = tl.load(value_cache_ptr_offset + v_reg_offset)
|
||||
if DEQUANT:
|
||||
k_scale = 1.0
|
||||
v_scale = 1.0
|
||||
k_scale = tl.load(k_scale_ptr)
|
||||
v_scale = tl.load(v_scale_ptr)
|
||||
k_reg = k_reg.to(tl.float32) * k_scale
|
||||
v_reg = v_reg.to(tl.float32) * v_scale
|
||||
tl.store(key_ptr_offset + col_offsets, k_reg)
|
||||
@@ -270,8 +270,8 @@ if current_platform.is_rocm():
|
||||
k_val = tl.load(key_ptr + src_offset_k + offset)
|
||||
v_val = tl.load(value_ptr + src_offset_v + offset)
|
||||
if QUANT:
|
||||
k_scale = 1.0
|
||||
v_scale = 1.0
|
||||
k_scale = tl.load(k_scale_ptr)
|
||||
v_scale = tl.load(v_scale_ptr)
|
||||
k_dtype = key_cache_ptr.type.element_ty
|
||||
v_dtype = value_cache_ptr.type.element_ty
|
||||
k_val = (k_val.to(tl.float32) / k_scale).to(k_dtype)
|
||||
@@ -410,11 +410,48 @@ class AiterFlashAttentionMetadata:
|
||||
kv_sharing_metadata: AiterKVSharingMetadata | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AsmKvScaleTables:
|
||||
"""Dense per-slot scale tables the asm decode kernel reads for an fp8 cache.
|
||||
|
||||
One table per distinct (device, shape, scale), shared by every layer and
|
||||
every metadata builder in the process, so the target and draft models with
|
||||
equal scales reuse a single allocation. Tables are constants: never write
|
||||
to one in place; a per-token or dynamic scale needs its own buffer.
|
||||
"""
|
||||
|
||||
tables: dict[tuple[str, int, int, int, float], torch.Tensor] = field(
|
||||
default_factory=dict
|
||||
)
|
||||
|
||||
def get(
|
||||
self,
|
||||
device: torch.device,
|
||||
num_blocks: int,
|
||||
num_kv_heads: int,
|
||||
block_size: int,
|
||||
scale: float,
|
||||
) -> torch.Tensor:
|
||||
key = (str(device), num_blocks, num_kv_heads, block_size, scale)
|
||||
table = self.tables.get(key)
|
||||
if table is None:
|
||||
table = torch.full(
|
||||
(num_blocks, num_kv_heads, block_size),
|
||||
scale,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
self.tables[key] = table
|
||||
return table
|
||||
|
||||
|
||||
class AiterFlashAttentionMetadataBuilder(
|
||||
AttentionMetadataBuilder[AiterFlashAttentionMetadata]
|
||||
):
|
||||
_cudagraph_support = AttentionCGSupport.UNIFORM_BATCH
|
||||
|
||||
asm_kv_scale_tables: ClassVar[AsmKvScaleTables] = AsmKvScaleTables()
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
kv_cache_spec: AttentionSpec,
|
||||
@@ -466,7 +503,8 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
dtype=self.model_config.dtype,
|
||||
device=device,
|
||||
)
|
||||
self.scale = torch.tensor([1.0], dtype=torch.float, device=self.device)
|
||||
self.k_scale: dict[str, torch.Tensor] | None = None
|
||||
self.v_scale: dict[str, torch.Tensor] | None = None
|
||||
self.kv_sharing_workspace = (
|
||||
torch.empty(
|
||||
(
|
||||
@@ -488,6 +526,38 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
common_prefix_len=0, common_attn_metadata=common_attn_metadata
|
||||
)
|
||||
|
||||
def _build_asm_kv_scale_tables(self) -> None:
|
||||
# Only for fp8 shuffle kv cache: the asm decode kernel reads a dense
|
||||
# [num_blocks, num_kv_heads, block_size] scale table per layer. Map each
|
||||
# layer to the shared table for its scale once the KV cache is bound.
|
||||
if (
|
||||
self.k_scale is not None
|
||||
or not rocm_aiter_ops.is_shuffle_kv_cache_enabled()
|
||||
or not is_quantized_kv_cache(self.vllm_config.cache_config.cache_dtype)
|
||||
):
|
||||
return
|
||||
attn_layers = self.vllm_config.compilation_config.static_forward_context
|
||||
k_scale_tables: dict[str, torch.Tensor] = {}
|
||||
v_scale_tables: dict[str, torch.Tensor] = {}
|
||||
for layer_name in self.layer_names:
|
||||
layer = attn_layers[layer_name]
|
||||
num_blocks = layer.kv_cache.shape[0]
|
||||
k_scale_tables[layer_name] = self.asm_kv_scale_tables.get(
|
||||
self.device,
|
||||
num_blocks,
|
||||
self.num_heads_kv,
|
||||
self.block_size,
|
||||
layer._k_scale_float,
|
||||
)
|
||||
v_scale_tables[layer_name] = self.asm_kv_scale_tables.get(
|
||||
self.device,
|
||||
num_blocks,
|
||||
self.num_heads_kv,
|
||||
self.block_size,
|
||||
layer._v_scale_float,
|
||||
)
|
||||
self.k_scale, self.v_scale = k_scale_tables, v_scale_tables
|
||||
|
||||
def build(
|
||||
self,
|
||||
common_prefix_len: int,
|
||||
@@ -499,24 +569,7 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
common_attn_metadata,
|
||||
decode_threshold=self.reorder_batch_threshold,
|
||||
)
|
||||
# Allocate scales for fp8 shuffle kv cache with shuffle_kv_cache enabled
|
||||
if (
|
||||
rocm_aiter_ops.is_shuffle_kv_cache_enabled()
|
||||
and self.scale.numel() == 1
|
||||
and is_quantized_kv_cache(self.vllm_config.cache_config.cache_dtype)
|
||||
):
|
||||
# Size the scales from a layer this builder owns. The draft model
|
||||
# runs its own builder over its own KV cache, so the first layer of
|
||||
# the whole config can carry an unrelated block count.
|
||||
kv_cache_shape = self.vllm_config.compilation_config.static_forward_context[
|
||||
self.layer_names[0]
|
||||
].kv_cache.shape
|
||||
num_blocks = kv_cache_shape[0]
|
||||
self.scale = torch.ones(
|
||||
[num_blocks, self.num_heads_kv, self.block_size],
|
||||
dtype=torch.float32,
|
||||
device=self.device,
|
||||
)
|
||||
self._build_asm_kv_scale_tables()
|
||||
(
|
||||
num_decodes,
|
||||
num_extends,
|
||||
@@ -709,8 +762,8 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
decode_metadata=decode_metadata,
|
||||
prefill_metadata=prefill_metadata,
|
||||
extend_metadata=extend_metadata,
|
||||
k_scale=self.scale,
|
||||
v_scale=self.scale,
|
||||
k_scale=self.k_scale,
|
||||
v_scale=self.v_scale,
|
||||
kv_sharing_metadata=kv_sharing_metadata,
|
||||
)
|
||||
return attn_metadata
|
||||
@@ -726,6 +779,7 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
skip split_decodes_prefills_and_extends() and avoid all .cpu() /
|
||||
.item() calls that would otherwise break CUDA graph capture.
|
||||
"""
|
||||
self._build_asm_kv_scale_tables()
|
||||
num_reqs = common_attn_metadata.num_reqs
|
||||
num_tokens = common_attn_metadata.num_actual_tokens
|
||||
|
||||
@@ -749,8 +803,8 @@ class AiterFlashAttentionMetadataBuilder(
|
||||
decode_metadata=decode_metadata,
|
||||
prefill_metadata=None,
|
||||
extend_metadata=None,
|
||||
k_scale=self.scale,
|
||||
v_scale=self.scale,
|
||||
k_scale=self.k_scale,
|
||||
v_scale=self.v_scale,
|
||||
)
|
||||
|
||||
|
||||
@@ -1249,9 +1303,6 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
extend_outputs = output[extend_tokens_slice]
|
||||
k_scale = layer._k_scale
|
||||
v_scale = layer._v_scale
|
||||
if rocm_aiter_ops.is_shuffle_kv_cache_enabled():
|
||||
k_scale = attn_metadata.k_scale
|
||||
v_scale = attn_metadata.v_scale
|
||||
self.extend_forward(
|
||||
attn_metadata=attn_metadata,
|
||||
query=extend_queries,
|
||||
@@ -1428,16 +1479,13 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
device=query.device,
|
||||
)
|
||||
max_logits = torch.empty_like(exp_sums)
|
||||
k_qscale = (
|
||||
layer._k_scale
|
||||
if attn_metadata.k_scale is None
|
||||
else attn_metadata.k_scale
|
||||
)
|
||||
v_qscale = (
|
||||
layer._v_scale
|
||||
if attn_metadata.v_scale is None
|
||||
else attn_metadata.v_scale
|
||||
)
|
||||
# asm indexes this layer's dense scale buffer from build();
|
||||
# the HIP kernel reads the per-tensor scale directly.
|
||||
k_qscale_asm = v_qscale_asm = None
|
||||
if attn_metadata.k_scale is not None:
|
||||
assert attn_metadata.v_scale is not None
|
||||
k_qscale_asm = attn_metadata.k_scale[layer.layer_name]
|
||||
v_qscale_asm = attn_metadata.v_scale[layer.layer_name]
|
||||
rocm_aiter_ops.paged_attention_common(
|
||||
Q=query[:num_decode_tokens],
|
||||
K=new_key_cache,
|
||||
@@ -1452,10 +1500,10 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
0
|
||||
),
|
||||
scale=self.scale,
|
||||
K_QScale_hip=k_qscale,
|
||||
V_QScale_hip=v_qscale,
|
||||
K_QScale_asm=k_qscale,
|
||||
V_QScale_asm=v_qscale,
|
||||
K_QScale_hip=layer._k_scale,
|
||||
V_QScale_hip=layer._v_scale,
|
||||
K_QScale_asm=k_qscale_asm,
|
||||
V_QScale_asm=v_qscale_asm,
|
||||
out_=output[:num_decode_tokens],
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
@@ -1542,9 +1590,6 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
# the reshape_and_cache_flash op uses the slot_mapping's shape
|
||||
# to determine the number of actual tokens.
|
||||
if rocm_aiter_ops.is_shuffle_kv_cache_enabled():
|
||||
# We may calculate per token quant scale in
|
||||
# reshape_and_cache_shuffle_triton which might differ from
|
||||
# vllm's style when shuffle layout is used.
|
||||
k_scale = layer._k_scale
|
||||
v_scale = layer._v_scale
|
||||
assert k_scale is not None and v_scale is not None, (
|
||||
@@ -1581,12 +1626,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
)
|
||||
|
||||
def fused_qk_norm_rope_kvcache_supported(self):
|
||||
# Only fuse when shuffle layout is off; the shuffle write path uses a
|
||||
# dedicated cache update, mirroring fused_rope_kvcache_supported.
|
||||
return (
|
||||
rocm_aiter_ops.is_enabled()
|
||||
and not rocm_aiter_ops.is_shuffle_kv_cache_enabled()
|
||||
)
|
||||
return rocm_aiter_ops.is_enabled()
|
||||
|
||||
def do_qk_norm_rope_kvcache_update(
|
||||
self,
|
||||
@@ -1624,6 +1664,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
|
||||
v_scale=layer._v_scale_cpu,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
use_shuffle_layout=rocm_aiter_ops.is_shuffle_kv_cache_enabled(),
|
||||
return_kv=True,
|
||||
)
|
||||
|
||||
def do_rope_and_kv_cache_update(
|
||||
|
||||
@@ -413,6 +413,11 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
def fused_qk_norm_mrope_kvcache_supported(self) -> bool:
|
||||
return IS_AITER_FOUND
|
||||
|
||||
def set_interleaved_v_cache(self):
|
||||
# No-op override of RocmAttentionImpl: this backend
|
||||
# decodes with the AITER triton unified-attention kernel
|
||||
pass
|
||||
|
||||
def do_qk_norm_rope_kvcache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
@@ -449,6 +454,7 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
|
||||
v_scale=layer._v_scale_cpu,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
use_shuffle_layout=self._use_shuffle_kv,
|
||||
return_kv=False,
|
||||
)
|
||||
|
||||
def do_qk_norm_mrope_kvcache_update(
|
||||
|
||||
@@ -292,6 +292,8 @@ class RocmAttentionImpl(AttentionImpl):
|
||||
f"num_heads: {num_heads}."
|
||||
)
|
||||
|
||||
self._use_interleaved_v_cache = False
|
||||
|
||||
def _forward_encoder_attention(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
@@ -450,6 +452,7 @@ class RocmAttentionImpl(AttentionImpl):
|
||||
sm_scale=self.scale,
|
||||
output_scale=output_scale,
|
||||
sinks=self.sinks,
|
||||
use_interleaved_v_cache=self._use_interleaved_v_cache,
|
||||
causal=attn_metadata.causal,
|
||||
)
|
||||
|
||||
@@ -475,6 +478,12 @@ class RocmAttentionImpl(AttentionImpl):
|
||||
block_size = value_cache.shape[3]
|
||||
has_native_layout = has_native_kv_cache_layout(key_cache, value_cache)
|
||||
|
||||
# Match the fused writer's interleaved V-cache when fusion is active.
|
||||
interleaved_v = self._use_interleaved_v_cache
|
||||
interleaved_v_pack_factor = (
|
||||
(16 // value_cache.element_size()) if interleaved_v else 0
|
||||
)
|
||||
|
||||
if block_size in (16, 32) and has_native_layout:
|
||||
# Normal 16, 32 with contiguous blocks: use vLLM native HIP C++ logic.
|
||||
PagedAttention.write_to_paged_cache(
|
||||
@@ -486,6 +495,7 @@ class RocmAttentionImpl(AttentionImpl):
|
||||
self.kv_cache_dtype,
|
||||
layer._k_scale,
|
||||
layer._v_scale,
|
||||
interleaved_v,
|
||||
)
|
||||
else:
|
||||
# Non-standard blocks and hybrid attention/Mamba layouts need the
|
||||
@@ -501,8 +511,61 @@ class RocmAttentionImpl(AttentionImpl):
|
||||
self.kv_cache_dtype,
|
||||
layer._k_scale,
|
||||
layer._v_scale,
|
||||
interleaved_v_pack_factor,
|
||||
)
|
||||
|
||||
def fused_qk_norm_rope_kvcache_supported(self):
|
||||
return rocm_aiter_ops.is_enabled()
|
||||
|
||||
def set_interleaved_v_cache(self):
|
||||
# Decode/prefill kernels must read the interleaved V-cache the AITER
|
||||
# fused write emits when use_shuffle_layout=True.
|
||||
self._use_interleaved_v_cache = True
|
||||
|
||||
def do_qk_norm_rope_kvcache_update(
|
||||
self,
|
||||
layer: AttentionLayer,
|
||||
qkv: torch.Tensor,
|
||||
q_out: torch.Tensor,
|
||||
k_out: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
q_weight: torch.Tensor,
|
||||
k_weight: torch.Tensor,
|
||||
rms_norm_eps: float,
|
||||
cos_sin_cache: torch.Tensor,
|
||||
is_neox: bool,
|
||||
kv_cache: torch.Tensor,
|
||||
layer_slot_mapping: torch.Tensor,
|
||||
):
|
||||
# kv_cache is logical [num_blocks, 2, block_size, num_kv_heads * head_size]
|
||||
# (see forward); the AITER shuffle writer takes one (B, N, H, hs) per side.
|
||||
key_cache, value_cache = kv_cache.unflatten(
|
||||
-1, (self.num_kv_heads, self.head_size)
|
||||
).unbind(1)
|
||||
use_shuffle_layout = self._use_interleaved_v_cache
|
||||
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
|
||||
qkv=qkv,
|
||||
q_weight=q_weight,
|
||||
k_weight=k_weight,
|
||||
cos_sin_cache=cos_sin_cache,
|
||||
positions=positions,
|
||||
num_heads_q=self.num_heads,
|
||||
num_heads_k=self.num_kv_heads,
|
||||
head_dim=self.head_size,
|
||||
is_neox=is_neox,
|
||||
rms_norm_eps=rms_norm_eps,
|
||||
q_out=q_out,
|
||||
k_out=k_out,
|
||||
key_cache=key_cache,
|
||||
value_cache=value_cache,
|
||||
slot_mapping=layer_slot_mapping,
|
||||
k_scale=layer._k_scale_cpu,
|
||||
v_scale=layer._v_scale_cpu,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
use_shuffle_layout=use_shuffle_layout,
|
||||
return_kv=True,
|
||||
)
|
||||
|
||||
def fused_rope_kvcache_supported(self):
|
||||
return rocm_aiter_ops.is_enabled()
|
||||
|
||||
|
||||
@@ -84,6 +84,7 @@ def kernel_paged_attention_2d(
|
||||
query_start_len_ptr, # [num_seqs+1]
|
||||
USE_SINKS: tl.constexpr, # bool
|
||||
USE_FP8: tl.constexpr,
|
||||
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
|
||||
FP8_MIN: tl.constexpr = float8_info.min,
|
||||
FP8_MAX: tl.constexpr = float8_info.max,
|
||||
):
|
||||
@@ -170,13 +171,24 @@ def kernel_paged_attention_2d(
|
||||
+ (offs_d[:, None] % x) * stride_k_cache_4
|
||||
)
|
||||
|
||||
# 4D addressing logic of V (Slot is innermost)
|
||||
v_offset = (
|
||||
p_block_idx[:, None] * stride_v_cache_0
|
||||
+ kv_head_idx * stride_v_cache_1
|
||||
+ offs_d[None, :] * stride_v_cache_2
|
||||
+ internal_offsets[:, None] * stride_v_cache_3
|
||||
)
|
||||
# V addressing: standard 4D or interleaved 5D
|
||||
if INTERLEAVED_V_PACK_FACTOR > 0:
|
||||
v_offset = (
|
||||
p_block_idx[:, None] * stride_v_cache_0
|
||||
+ kv_head_idx * stride_v_cache_1
|
||||
+ (internal_offsets[:, None] // INTERLEAVED_V_PACK_FACTOR)
|
||||
* HEAD_SIZE
|
||||
* INTERLEAVED_V_PACK_FACTOR
|
||||
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
|
||||
+ (internal_offsets[:, None] % INTERLEAVED_V_PACK_FACTOR)
|
||||
)
|
||||
else:
|
||||
v_offset = (
|
||||
p_block_idx[:, None] * stride_v_cache_0
|
||||
+ kv_head_idx * stride_v_cache_1
|
||||
+ offs_d[None, :] * stride_v_cache_2
|
||||
+ internal_offsets[:, None] * stride_v_cache_3
|
||||
)
|
||||
|
||||
# Only the final tile can straddle seq_len. Slots >= seq_len are
|
||||
# unwritten KV cache that may hold NaN/garbage; they are score-masked
|
||||
@@ -304,6 +316,7 @@ def chunked_prefill_paged_decode(
|
||||
# Optional tensor for sinks
|
||||
sinks=None,
|
||||
is_block_table_ptr: bool = False,
|
||||
use_interleaved_v_cache: bool = False,
|
||||
causal: bool = True,
|
||||
):
|
||||
if sm_scale is None:
|
||||
@@ -314,6 +327,10 @@ def chunked_prefill_paged_decode(
|
||||
if sliding_window is None or sliding_window <= 0:
|
||||
sliding_window = 0
|
||||
|
||||
interleaved_v_pack_factor = (
|
||||
(16 // value_cache.element_size()) if use_interleaved_v_cache else 0
|
||||
)
|
||||
|
||||
if max_query_len > 1:
|
||||
context_attention_fwd(
|
||||
q=query,
|
||||
@@ -336,6 +353,7 @@ def chunked_prefill_paged_decode(
|
||||
skip_decode=True,
|
||||
fp8_out_scale=output_scale,
|
||||
sinks=sinks,
|
||||
interleaved_v_pack_factor=interleaved_v_pack_factor,
|
||||
causal=causal,
|
||||
)
|
||||
|
||||
@@ -428,6 +446,7 @@ def chunked_prefill_paged_decode(
|
||||
k_scale=k_scale,
|
||||
v_scale=v_scale,
|
||||
fp8_out_scale=output_scale,
|
||||
use_interleaved_v_cache=use_interleaved_v_cache,
|
||||
)
|
||||
else:
|
||||
logger.warning_once(
|
||||
@@ -504,4 +523,5 @@ def chunked_prefill_paged_decode(
|
||||
query_start_len_ptr=query_start_loc,
|
||||
USE_SINKS=sinks is not None,
|
||||
USE_FP8=output_scale is not None,
|
||||
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
|
||||
)
|
||||
|
||||
@@ -38,6 +38,7 @@ class PagedAttention:
|
||||
kv_cache_dtype: str,
|
||||
k_scale: torch.Tensor,
|
||||
v_scale: torch.Tensor,
|
||||
interleaved_v: bool,
|
||||
) -> None:
|
||||
ops.reshape_and_cache(
|
||||
key,
|
||||
@@ -48,4 +49,5 @@ class PagedAttention:
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
interleaved_v,
|
||||
)
|
||||
|
||||
@@ -56,6 +56,8 @@ def _paged_kv_cache_offsets(
|
||||
stride_v_cache_d,
|
||||
stride_v_cache_bl,
|
||||
PHYSICAL_BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_DMODEL: tl.constexpr,
|
||||
INTERLEAVED_V_PACK_FACTOR: tl.constexpr = 0,
|
||||
MASK_BLOCK_TABLE: tl.constexpr = False,
|
||||
):
|
||||
"""Compute paged K/V cache element offsets for a tile of token positions.
|
||||
@@ -65,7 +67,9 @@ def _paged_kv_cache_offsets(
|
||||
tiles), then the physical element offsets are built with the same layout the
|
||||
kernel uses everywhere:
|
||||
K cache: [num_blocks, num_kv_heads, head_size // x, block_size, x]
|
||||
V cache: [num_blocks, num_kv_heads, head_size, block_size]
|
||||
V cache: [num_blocks, num_kv_heads, head_size, block_size], or the
|
||||
interleaved layout [num_blocks, num_kv_heads, block_size // x,
|
||||
head_size, x] when `INTERLEAVED_V_PACK_FACTOR` (= x) > 0.
|
||||
Returns `(off_k, off_v)` with shapes [D, N] and [N, D] respectively.
|
||||
|
||||
When `MASK_BLOCK_TABLE` is set, the block-table load is masked with
|
||||
@@ -94,12 +98,25 @@ def _paged_kv_cache_offsets(
|
||||
+ internal[None, :] * stride_k_cache_bl
|
||||
+ (offs_d[:, None] % x) * stride_k_cache_x
|
||||
)
|
||||
off_v = (
|
||||
bn[:, None] * stride_v_cache_bs
|
||||
+ cur_kv_head * stride_v_cache_h
|
||||
+ offs_d[None, :] * stride_v_cache_d
|
||||
+ internal[:, None] * stride_v_cache_bl
|
||||
)
|
||||
if INTERLEAVED_V_PACK_FACTOR > 0:
|
||||
# Interleaved V-cache: [num_blocks, num_kv_heads, block_size/x,
|
||||
# head_size, x] flattened onto the 4D stride layout.
|
||||
off_v = (
|
||||
bn[:, None] * stride_v_cache_bs
|
||||
+ cur_kv_head * stride_v_cache_h
|
||||
+ (internal[:, None] // INTERLEAVED_V_PACK_FACTOR)
|
||||
* BLOCK_DMODEL
|
||||
* INTERLEAVED_V_PACK_FACTOR
|
||||
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
|
||||
+ (internal[:, None] % INTERLEAVED_V_PACK_FACTOR)
|
||||
)
|
||||
else:
|
||||
off_v = (
|
||||
bn[:, None] * stride_v_cache_bs
|
||||
+ cur_kv_head * stride_v_cache_h
|
||||
+ offs_d[None, :] * stride_v_cache_d
|
||||
+ internal[:, None] * stride_v_cache_bl
|
||||
)
|
||||
return off_k, off_v
|
||||
|
||||
|
||||
@@ -159,6 +176,7 @@ def _fwd_kernel(
|
||||
USE_FP8: tl.constexpr,
|
||||
CAUSAL: tl.constexpr = True,
|
||||
MAX_Q_LEN: tl.constexpr = 0,
|
||||
INTERLEAVED_V_PACK_FACTOR: tl.constexpr = 0,
|
||||
MAX_CTX_LEN: tl.constexpr = 0,
|
||||
FP8_MIN: tl.constexpr = float8_info.min,
|
||||
FP8_MAX: tl.constexpr = float8_info.max,
|
||||
@@ -263,6 +281,8 @@ def _fwd_kernel(
|
||||
stride_v_cache_d,
|
||||
stride_v_cache_bl,
|
||||
PHYSICAL_BLOCK_SIZE,
|
||||
BLOCK_DMODEL,
|
||||
INTERLEAVED_V_PACK_FACTOR,
|
||||
)
|
||||
|
||||
if (
|
||||
@@ -403,6 +423,8 @@ def _fwd_kernel(
|
||||
stride_v_cache_d,
|
||||
stride_v_cache_bl,
|
||||
PHYSICAL_BLOCK_SIZE,
|
||||
BLOCK_DMODEL,
|
||||
INTERLEAVED_V_PACK_FACTOR,
|
||||
MASK_BLOCK_TABLE=True,
|
||||
)
|
||||
k_cur_load = tl.load(
|
||||
@@ -540,6 +562,7 @@ def _fwd_kernel_alibi(
|
||||
BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2
|
||||
BLOCK_N: tl.constexpr,
|
||||
SKIP_DECODE: tl.constexpr,
|
||||
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
|
||||
):
|
||||
# attn_bias[]
|
||||
cur_batch = tl.program_id(0)
|
||||
@@ -608,12 +631,24 @@ def _fwd_kernel_alibi(
|
||||
+ ((start_n + offs_n[None, :]) % block_size) * stride_k_cache_bl
|
||||
+ (offs_d[:, None] % x) * stride_k_cache_x
|
||||
)
|
||||
off_v = (
|
||||
bn[:, None] * stride_v_cache_bs
|
||||
+ cur_kv_head * stride_v_cache_h
|
||||
+ offs_d[None, :] * stride_v_cache_d
|
||||
+ (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl
|
||||
)
|
||||
if INTERLEAVED_V_PACK_FACTOR > 0:
|
||||
v_internal = (start_n + offs_n[:, None]) % block_size
|
||||
off_v = (
|
||||
bn[:, None] * stride_v_cache_bs
|
||||
+ cur_kv_head * stride_v_cache_h
|
||||
+ (v_internal // INTERLEAVED_V_PACK_FACTOR)
|
||||
* BLOCK_DMODEL
|
||||
* INTERLEAVED_V_PACK_FACTOR
|
||||
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
|
||||
+ (v_internal % INTERLEAVED_V_PACK_FACTOR)
|
||||
)
|
||||
else:
|
||||
off_v = (
|
||||
bn[:, None] * stride_v_cache_bs
|
||||
+ cur_kv_head * stride_v_cache_h
|
||||
+ offs_d[None, :] * stride_v_cache_d
|
||||
+ (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl
|
||||
)
|
||||
k_load = tl.load(
|
||||
K_cache + off_k,
|
||||
mask=dim_mask[:, None] & ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
|
||||
@@ -796,6 +831,7 @@ def context_attention_fwd(
|
||||
fp8_out_scale=None,
|
||||
sinks=None,
|
||||
is_block_table_ptr: bool = False,
|
||||
interleaved_v_pack_factor: int = 0,
|
||||
causal: bool = True,
|
||||
):
|
||||
q_dtype_is_f32 = q.dtype is torch.float32
|
||||
@@ -937,6 +973,7 @@ def context_attention_fwd(
|
||||
BLOCK_DMODEL_PADDED=Lk_padded,
|
||||
BLOCK_N=BLOCK,
|
||||
SKIP_DECODE=skip_decode,
|
||||
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
|
||||
num_warps=NUM_WARPS,
|
||||
num_stages=1,
|
||||
)
|
||||
@@ -1010,6 +1047,7 @@ def context_attention_fwd(
|
||||
num_warps=4,
|
||||
num_stages=1,
|
||||
USE_SINKS=sinks is not None,
|
||||
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
|
||||
CAUSAL=causal,
|
||||
KV_FROM_CACHE=kv_from_cache,
|
||||
**extra_kargs,
|
||||
|
||||
@@ -52,6 +52,9 @@ def reshape_and_cache_kernel_flash(
|
||||
block_size: tl.constexpr,
|
||||
x: tl.constexpr,
|
||||
USE_HEAD_MAJOR_LAYOUT: tl.constexpr,
|
||||
# >0 writes V interleaved as [Block, Head, Slot//kx, Dim, kx] (head-major
|
||||
# layout only); 0 keeps the standard [Block, Head, Dim, Slot] V layout.
|
||||
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
|
||||
# FP8 flags
|
||||
FP8_KV_CACHE: tl.constexpr,
|
||||
# tune parameters
|
||||
@@ -76,13 +79,25 @@ def reshape_and_cache_kernel_flash(
|
||||
# Decompose the tile index back into head and dim coordinates.
|
||||
cur_head = tile_pos // head_size
|
||||
cur_dim = tile_pos % head_size
|
||||
# Value addressing (4D): [Block, Head, Dim, Slot]
|
||||
tgt_idx_v = (
|
||||
block_idx * block_stride
|
||||
+ cur_head * head_stride
|
||||
+ cur_dim * dim_stride_v
|
||||
+ block_offset * 1
|
||||
)
|
||||
# Value addressing: standard 4D [Block, Head, Dim, Slot], or the
|
||||
# interleaved 5D [Block, Head, Slot//kx, Dim, kx] the fused AITER
|
||||
# writer emits and the interleaved read path expects.
|
||||
if INTERLEAVED_V_PACK_FACTOR > 0:
|
||||
tgt_idx_v = (
|
||||
block_idx * block_stride
|
||||
+ cur_head * head_stride
|
||||
+ (block_offset // INTERLEAVED_V_PACK_FACTOR)
|
||||
* (head_size * INTERLEAVED_V_PACK_FACTOR)
|
||||
+ cur_dim * INTERLEAVED_V_PACK_FACTOR
|
||||
+ (block_offset % INTERLEAVED_V_PACK_FACTOR)
|
||||
)
|
||||
else:
|
||||
tgt_idx_v = (
|
||||
block_idx * block_stride
|
||||
+ cur_head * head_stride
|
||||
+ cur_dim * dim_stride_v
|
||||
+ block_offset * 1
|
||||
)
|
||||
# Key addressing (5D): [Block, Head, Dim//8, Slot, 8]
|
||||
tgt_idx_k = (
|
||||
block_idx * block_stride
|
||||
@@ -371,11 +386,18 @@ def triton_reshape_and_cache_flash(
|
||||
kv_cache_dtype: str, # "auto", "fp8"
|
||||
k_scale: torch.Tensor, # float32
|
||||
v_scale: torch.Tensor, # float32
|
||||
interleaved_v_pack_factor: int = 0, # >0: interleave V (head-major); 0: standard
|
||||
):
|
||||
num_heads = key.shape[1]
|
||||
head_size = key.shape[2]
|
||||
|
||||
use_head_major_layout = key_cache.ndim == 5
|
||||
if interleaved_v_pack_factor and not use_head_major_layout:
|
||||
raise NotImplementedError(
|
||||
"Interleaved V-cache write is only implemented for the head-major "
|
||||
"(5D) cache layout; got a non-head-major cache with "
|
||||
f"interleaved_v_pack_factor={interleaved_v_pack_factor}."
|
||||
)
|
||||
if use_head_major_layout:
|
||||
block_size = key_cache.shape[3]
|
||||
x = key_cache.shape[4]
|
||||
@@ -452,6 +474,7 @@ def triton_reshape_and_cache_flash(
|
||||
block_size=block_size,
|
||||
x=x,
|
||||
USE_HEAD_MAJOR_LAYOUT=use_head_major_layout,
|
||||
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
|
||||
FP8_KV_CACHE=FP8_KV_CACHE,
|
||||
# autotune parameters
|
||||
TILE_SIZE=TILE_SIZE,
|
||||
|
||||
Reference in New Issue
Block a user