[Model][Hardware][AMD][Kernel] Part 2/2 -> Enable e2e QK Norm + RoPE + KV Cache runtime fusion for Qwen3-30B-A3B on ROCM_ATTN (#42755)

Signed-off-by: Jack Hu <Jack.Hu@amd.com>
Signed-off-by: Jack Hu <jingxujaster@hotmail.com>
Signed-off-by: Jack Hu <jack.hu@amd.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
Jack Hu
2026-10-15 17:10:14 -07:00
committed by GitHub
co-authored by Douglas Lehr
parent af28c171f5
commit ef38e8d006
23 changed files with 701 additions and 205 deletions
+24 -9
View File
@@ -11,6 +11,7 @@ import torch.distributed as dist
from torch._ops import OpOverload
import vllm.envs as envs
from vllm import _custom_ops as ops
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.torch_utils import direct_register_custom_op
@@ -3887,17 +3888,29 @@ class rocm_aiter_ops:
v_scale: torch.Tensor,
kv_cache_dtype: str,
use_shuffle_layout: bool,
return_kv: bool,
) -> None:
"""Run the fused QK-norm+RoPE+KV-cache op on already-split k/v caches.
Shared by the AITER FA and unified-attention impls. The caller splits
Shared by the AITER FA, ROCM ATTN, and unified-attention impls. The caller splits
kv_cache, since the unbind dim depends on the layout (e.g. the unified
encoder-decoder path is K/V-first), and passes use_shuffle_layout
(unified reads NHD and must pass False).
(unified reads NHD and must pass False). k_scale/v_scale are the
per-tensor CPU scalars. return_kv says whether the impl consumes
k_out (FA and ROCM_ATTN prefill do; unified reads K from the cache and
passes False, so the kernel skips that store). For fp8 caches the
kernel emits k_out in the cache dtype, so a returned k_out is
converted to the activation dtype.
"""
if kv_cache_dtype.startswith("fp8"):
key_cache = key_cache.view(current_platform.fp8_dtype())
value_cache = value_cache.view(current_platform.fp8_dtype())
is_fp8_cache = kv_cache_dtype.startswith("fp8")
dequant_k_out = return_kv and is_fp8_cache
kernel_k_out = k_out
if is_fp8_cache:
fp8_dtype = current_platform.fp8_dtype()
key_cache = key_cache.view(fp8_dtype)
value_cache = value_cache.view(fp8_dtype)
if dequant_k_out:
kernel_k_out = torch.empty_like(k_out, dtype=fp8_dtype)
# Partial-rotary support (e.g. GLM-4.7 applies rotary to only a prefix
# of each head's channel dim).
rotary_dim = cos_sin_cache.shape[-1]
@@ -3920,14 +3933,16 @@ class rocm_aiter_ops:
slot_mapping=slot_mapping,
k_scale=k_scale,
v_scale=v_scale,
k_out=k_out,
k_out=kernel_k_out,
v_out=None,
return_kv=True,
return_kv=return_kv,
use_shuffle_layout=use_shuffle_layout,
block_size=key_cache.shape[1],
x=16 // key_cache.element_size(),
rotary_dim=kernel_rotary_dim,
)
if dequant_k_out:
ops.convert_fp8(k_out, kernel_k_out, float(k_scale), kv_cache_dtype)
@staticmethod
def fused_qk_norm_mrope_and_cache(
@@ -4506,8 +4521,8 @@ class rocm_aiter_ops:
scale: float,
K_QScale_hip: torch.Tensor,
V_QScale_hip: torch.Tensor,
K_QScale_asm: torch.Tensor,
V_QScale_asm: torch.Tensor,
K_QScale_asm: torch.Tensor | None,
V_QScale_asm: torch.Tensor | None,
out_: torch.Tensor,
kv_cache_dtype: str,
):
+4
View File
@@ -122,6 +122,7 @@ def paged_attention_rocm(
v_scale: torch.Tensor,
fp8_out_scale: torch.Tensor | None = None,
mfma_type: str = "fp8" if envs.VLLM_ROCM_FP8_MFMA_PAGE_ATTN else "f16",
use_interleaved_v_cache: bool = False,
) -> None:
torch.ops._rocm_C.paged_attention(
out,
@@ -144,6 +145,7 @@ def paged_attention_rocm(
v_scale,
fp8_out_scale,
mfma_type,
use_interleaved_v_cache,
)
@@ -2554,6 +2556,7 @@ def reshape_and_cache(
kv_cache_dtype: str,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
interleaved_v: bool = False,
) -> None:
torch.ops._C_cache_ops.reshape_and_cache(
key,
@@ -2564,6 +2567,7 @@ def reshape_and_cache(
kv_cache_dtype,
k_scale,
v_scale,
interleaved_v,
)
@@ -893,6 +893,7 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
layer.head_size,
)
continue
layer.impl.set_interleaved_v_cache()
if supports_rope:
for epsilon in [1e-5, 1e-6]:
for neox in [True, False]:
@@ -903,7 +904,6 @@ class QkNormRopeKvCacheFusionPass(VllmPatternMatcherPass):
is_neox=neox,
quant_query=quant_q,
).register(self.patterns)
if supports_mrope:
for section, is_interleaved in self.mrope_configs:
rotary_dim = 2 * sum(section)
@@ -531,11 +531,6 @@ class FusedMoEExperts(ABC):
def is_monolithic() -> bool:
raise NotImplementedError("Implemented by subclasses.")
@staticmethod
def _supports_hash_routing() -> bool:
"""Whether a monolithic kernel implements hash routing."""
return False
@property
def expects_unquantized_inputs(self) -> bool:
"""Whether or not the PrepareFinalize should defer input quantization
+10
View File
@@ -950,6 +950,16 @@ class AttentionImpl(AttentionImplBase[T], Generic[T]):
"""Whether this implementation supports fused QKNorm+MRoPE+KVCache."""
return False
def set_interleaved_v_cache(self):
"""Opt this implementation into the interleaved V-cache layout for all of
its cache writers and readers. QkNormRopeKvCacheFusionPass calls this
for every eligible layer when it is constructed, i.e. before any KV
cache is written, so fused and unfused paths stay consistent whether or
not the pattern matches. Default is a no-op; backends whose kernels
read an interleaved V-cache under this fusion (e.g. ROCM_ATTN) override.
"""
return False
def fused_rope_kvcache_supported(self):
"""Does this attention implementation support RoPE+KVCache fusion.
This is used by the RopeKVCacheFusionPass to only fuse the RoPE ops
+95 -54
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Attention layer with AiterFlashAttention."""
from dataclasses import dataclass, replace
from dataclasses import dataclass, field, replace
from typing import ClassVar
import torch
@@ -152,8 +152,8 @@ if current_platform.is_rocm():
k_reg = tl.load(key_cache_ptr_offset + k_reg_offset)
v_reg = tl.load(value_cache_ptr_offset + v_reg_offset)
if DEQUANT:
k_scale = 1.0
v_scale = 1.0
k_scale = tl.load(k_scale_ptr)
v_scale = tl.load(v_scale_ptr)
k_reg = k_reg.to(tl.float32) * k_scale
v_reg = v_reg.to(tl.float32) * v_scale
tl.store(key_ptr_offset + col_offsets, k_reg)
@@ -270,8 +270,8 @@ if current_platform.is_rocm():
k_val = tl.load(key_ptr + src_offset_k + offset)
v_val = tl.load(value_ptr + src_offset_v + offset)
if QUANT:
k_scale = 1.0
v_scale = 1.0
k_scale = tl.load(k_scale_ptr)
v_scale = tl.load(v_scale_ptr)
k_dtype = key_cache_ptr.type.element_ty
v_dtype = value_cache_ptr.type.element_ty
k_val = (k_val.to(tl.float32) / k_scale).to(k_dtype)
@@ -410,11 +410,48 @@ class AiterFlashAttentionMetadata:
kv_sharing_metadata: AiterKVSharingMetadata | None = None
@dataclass
class AsmKvScaleTables:
"""Dense per-slot scale tables the asm decode kernel reads for an fp8 cache.
One table per distinct (device, shape, scale), shared by every layer and
every metadata builder in the process, so the target and draft models with
equal scales reuse a single allocation. Tables are constants: never write
to one in place; a per-token or dynamic scale needs its own buffer.
"""
tables: dict[tuple[str, int, int, int, float], torch.Tensor] = field(
default_factory=dict
)
def get(
self,
device: torch.device,
num_blocks: int,
num_kv_heads: int,
block_size: int,
scale: float,
) -> torch.Tensor:
key = (str(device), num_blocks, num_kv_heads, block_size, scale)
table = self.tables.get(key)
if table is None:
table = torch.full(
(num_blocks, num_kv_heads, block_size),
scale,
dtype=torch.float32,
device=device,
)
self.tables[key] = table
return table
class AiterFlashAttentionMetadataBuilder(
AttentionMetadataBuilder[AiterFlashAttentionMetadata]
):
_cudagraph_support = AttentionCGSupport.UNIFORM_BATCH
asm_kv_scale_tables: ClassVar[AsmKvScaleTables] = AsmKvScaleTables()
def __init__(
self,
kv_cache_spec: AttentionSpec,
@@ -466,7 +503,8 @@ class AiterFlashAttentionMetadataBuilder(
dtype=self.model_config.dtype,
device=device,
)
self.scale = torch.tensor([1.0], dtype=torch.float, device=self.device)
self.k_scale: dict[str, torch.Tensor] | None = None
self.v_scale: dict[str, torch.Tensor] | None = None
self.kv_sharing_workspace = (
torch.empty(
(
@@ -488,6 +526,38 @@ class AiterFlashAttentionMetadataBuilder(
common_prefix_len=0, common_attn_metadata=common_attn_metadata
)
def _build_asm_kv_scale_tables(self) -> None:
# Only for fp8 shuffle kv cache: the asm decode kernel reads a dense
# [num_blocks, num_kv_heads, block_size] scale table per layer. Map each
# layer to the shared table for its scale once the KV cache is bound.
if (
self.k_scale is not None
or not rocm_aiter_ops.is_shuffle_kv_cache_enabled()
or not is_quantized_kv_cache(self.vllm_config.cache_config.cache_dtype)
):
return
attn_layers = self.vllm_config.compilation_config.static_forward_context
k_scale_tables: dict[str, torch.Tensor] = {}
v_scale_tables: dict[str, torch.Tensor] = {}
for layer_name in self.layer_names:
layer = attn_layers[layer_name]
num_blocks = layer.kv_cache.shape[0]
k_scale_tables[layer_name] = self.asm_kv_scale_tables.get(
self.device,
num_blocks,
self.num_heads_kv,
self.block_size,
layer._k_scale_float,
)
v_scale_tables[layer_name] = self.asm_kv_scale_tables.get(
self.device,
num_blocks,
self.num_heads_kv,
self.block_size,
layer._v_scale_float,
)
self.k_scale, self.v_scale = k_scale_tables, v_scale_tables
def build(
self,
common_prefix_len: int,
@@ -499,24 +569,7 @@ class AiterFlashAttentionMetadataBuilder(
common_attn_metadata,
decode_threshold=self.reorder_batch_threshold,
)
# Allocate scales for fp8 shuffle kv cache with shuffle_kv_cache enabled
if (
rocm_aiter_ops.is_shuffle_kv_cache_enabled()
and self.scale.numel() == 1
and is_quantized_kv_cache(self.vllm_config.cache_config.cache_dtype)
):
# Size the scales from a layer this builder owns. The draft model
# runs its own builder over its own KV cache, so the first layer of
# the whole config can carry an unrelated block count.
kv_cache_shape = self.vllm_config.compilation_config.static_forward_context[
self.layer_names[0]
].kv_cache.shape
num_blocks = kv_cache_shape[0]
self.scale = torch.ones(
[num_blocks, self.num_heads_kv, self.block_size],
dtype=torch.float32,
device=self.device,
)
self._build_asm_kv_scale_tables()
(
num_decodes,
num_extends,
@@ -709,8 +762,8 @@ class AiterFlashAttentionMetadataBuilder(
decode_metadata=decode_metadata,
prefill_metadata=prefill_metadata,
extend_metadata=extend_metadata,
k_scale=self.scale,
v_scale=self.scale,
k_scale=self.k_scale,
v_scale=self.v_scale,
kv_sharing_metadata=kv_sharing_metadata,
)
return attn_metadata
@@ -726,6 +779,7 @@ class AiterFlashAttentionMetadataBuilder(
skip split_decodes_prefills_and_extends() and avoid all .cpu() /
.item() calls that would otherwise break CUDA graph capture.
"""
self._build_asm_kv_scale_tables()
num_reqs = common_attn_metadata.num_reqs
num_tokens = common_attn_metadata.num_actual_tokens
@@ -749,8 +803,8 @@ class AiterFlashAttentionMetadataBuilder(
decode_metadata=decode_metadata,
prefill_metadata=None,
extend_metadata=None,
k_scale=self.scale,
v_scale=self.scale,
k_scale=self.k_scale,
v_scale=self.v_scale,
)
@@ -1249,9 +1303,6 @@ class AiterFlashAttentionImpl(AttentionImpl):
extend_outputs = output[extend_tokens_slice]
k_scale = layer._k_scale
v_scale = layer._v_scale
if rocm_aiter_ops.is_shuffle_kv_cache_enabled():
k_scale = attn_metadata.k_scale
v_scale = attn_metadata.v_scale
self.extend_forward(
attn_metadata=attn_metadata,
query=extend_queries,
@@ -1428,16 +1479,13 @@ class AiterFlashAttentionImpl(AttentionImpl):
device=query.device,
)
max_logits = torch.empty_like(exp_sums)
k_qscale = (
layer._k_scale
if attn_metadata.k_scale is None
else attn_metadata.k_scale
)
v_qscale = (
layer._v_scale
if attn_metadata.v_scale is None
else attn_metadata.v_scale
)
# asm indexes this layer's dense scale buffer from build();
# the HIP kernel reads the per-tensor scale directly.
k_qscale_asm = v_qscale_asm = None
if attn_metadata.k_scale is not None:
assert attn_metadata.v_scale is not None
k_qscale_asm = attn_metadata.k_scale[layer.layer_name]
v_qscale_asm = attn_metadata.v_scale[layer.layer_name]
rocm_aiter_ops.paged_attention_common(
Q=query[:num_decode_tokens],
K=new_key_cache,
@@ -1452,10 +1500,10 @@ class AiterFlashAttentionImpl(AttentionImpl):
0
),
scale=self.scale,
K_QScale_hip=k_qscale,
V_QScale_hip=v_qscale,
K_QScale_asm=k_qscale,
V_QScale_asm=v_qscale,
K_QScale_hip=layer._k_scale,
V_QScale_hip=layer._v_scale,
K_QScale_asm=k_qscale_asm,
V_QScale_asm=v_qscale_asm,
out_=output[:num_decode_tokens],
kv_cache_dtype=self.kv_cache_dtype,
)
@@ -1542,9 +1590,6 @@ class AiterFlashAttentionImpl(AttentionImpl):
# the reshape_and_cache_flash op uses the slot_mapping's shape
# to determine the number of actual tokens.
if rocm_aiter_ops.is_shuffle_kv_cache_enabled():
# We may calculate per token quant scale in
# reshape_and_cache_shuffle_triton which might differ from
# vllm's style when shuffle layout is used.
k_scale = layer._k_scale
v_scale = layer._v_scale
assert k_scale is not None and v_scale is not None, (
@@ -1581,12 +1626,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
)
def fused_qk_norm_rope_kvcache_supported(self):
# Only fuse when shuffle layout is off; the shuffle write path uses a
# dedicated cache update, mirroring fused_rope_kvcache_supported.
return (
rocm_aiter_ops.is_enabled()
and not rocm_aiter_ops.is_shuffle_kv_cache_enabled()
)
return rocm_aiter_ops.is_enabled()
def do_qk_norm_rope_kvcache_update(
self,
@@ -1624,6 +1664,7 @@ class AiterFlashAttentionImpl(AttentionImpl):
v_scale=layer._v_scale_cpu,
kv_cache_dtype=self.kv_cache_dtype,
use_shuffle_layout=rocm_aiter_ops.is_shuffle_kv_cache_enabled(),
return_kv=True,
)
def do_rope_and_kv_cache_update(
@@ -413,6 +413,11 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
def fused_qk_norm_mrope_kvcache_supported(self) -> bool:
return IS_AITER_FOUND
def set_interleaved_v_cache(self):
# No-op override of RocmAttentionImpl: this backend
# decodes with the AITER triton unified-attention kernel
pass
def do_qk_norm_rope_kvcache_update(
self,
layer: AttentionLayer,
@@ -449,6 +454,7 @@ class RocmAiterUnifiedAttentionImpl(RocmAttentionImpl):
v_scale=layer._v_scale_cpu,
kv_cache_dtype=self.kv_cache_dtype,
use_shuffle_layout=self._use_shuffle_kv,
return_kv=False,
)
def do_qk_norm_mrope_kvcache_update(
+63
View File
@@ -292,6 +292,8 @@ class RocmAttentionImpl(AttentionImpl):
f"num_heads: {num_heads}."
)
self._use_interleaved_v_cache = False
def _forward_encoder_attention(
self,
query: torch.Tensor,
@@ -450,6 +452,7 @@ class RocmAttentionImpl(AttentionImpl):
sm_scale=self.scale,
output_scale=output_scale,
sinks=self.sinks,
use_interleaved_v_cache=self._use_interleaved_v_cache,
causal=attn_metadata.causal,
)
@@ -475,6 +478,12 @@ class RocmAttentionImpl(AttentionImpl):
block_size = value_cache.shape[3]
has_native_layout = has_native_kv_cache_layout(key_cache, value_cache)
# Match the fused writer's interleaved V-cache when fusion is active.
interleaved_v = self._use_interleaved_v_cache
interleaved_v_pack_factor = (
(16 // value_cache.element_size()) if interleaved_v else 0
)
if block_size in (16, 32) and has_native_layout:
# Normal 16, 32 with contiguous blocks: use vLLM native HIP C++ logic.
PagedAttention.write_to_paged_cache(
@@ -486,6 +495,7 @@ class RocmAttentionImpl(AttentionImpl):
self.kv_cache_dtype,
layer._k_scale,
layer._v_scale,
interleaved_v,
)
else:
# Non-standard blocks and hybrid attention/Mamba layouts need the
@@ -501,8 +511,61 @@ class RocmAttentionImpl(AttentionImpl):
self.kv_cache_dtype,
layer._k_scale,
layer._v_scale,
interleaved_v_pack_factor,
)
def fused_qk_norm_rope_kvcache_supported(self):
return rocm_aiter_ops.is_enabled()
def set_interleaved_v_cache(self):
# Decode/prefill kernels must read the interleaved V-cache the AITER
# fused write emits when use_shuffle_layout=True.
self._use_interleaved_v_cache = True
def do_qk_norm_rope_kvcache_update(
self,
layer: AttentionLayer,
qkv: torch.Tensor,
q_out: torch.Tensor,
k_out: torch.Tensor,
positions: torch.Tensor,
q_weight: torch.Tensor,
k_weight: torch.Tensor,
rms_norm_eps: float,
cos_sin_cache: torch.Tensor,
is_neox: bool,
kv_cache: torch.Tensor,
layer_slot_mapping: torch.Tensor,
):
# kv_cache is logical [num_blocks, 2, block_size, num_kv_heads * head_size]
# (see forward); the AITER shuffle writer takes one (B, N, H, hs) per side.
key_cache, value_cache = kv_cache.unflatten(
-1, (self.num_kv_heads, self.head_size)
).unbind(1)
use_shuffle_layout = self._use_interleaved_v_cache
rocm_aiter_ops.do_qk_norm_rope_kvcache_update(
qkv=qkv,
q_weight=q_weight,
k_weight=k_weight,
cos_sin_cache=cos_sin_cache,
positions=positions,
num_heads_q=self.num_heads,
num_heads_k=self.num_kv_heads,
head_dim=self.head_size,
is_neox=is_neox,
rms_norm_eps=rms_norm_eps,
q_out=q_out,
k_out=k_out,
key_cache=key_cache,
value_cache=value_cache,
slot_mapping=layer_slot_mapping,
k_scale=layer._k_scale_cpu,
v_scale=layer._v_scale_cpu,
kv_cache_dtype=self.kv_cache_dtype,
use_shuffle_layout=use_shuffle_layout,
return_kv=True,
)
def fused_rope_kvcache_supported(self):
return rocm_aiter_ops.is_enabled()
@@ -84,6 +84,7 @@ def kernel_paged_attention_2d(
query_start_len_ptr, # [num_seqs+1]
USE_SINKS: tl.constexpr, # bool
USE_FP8: tl.constexpr,
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
FP8_MIN: tl.constexpr = float8_info.min,
FP8_MAX: tl.constexpr = float8_info.max,
):
@@ -170,13 +171,24 @@ def kernel_paged_attention_2d(
+ (offs_d[:, None] % x) * stride_k_cache_4
)
# 4D addressing logic of V (Slot is innermost)
v_offset = (
p_block_idx[:, None] * stride_v_cache_0
+ kv_head_idx * stride_v_cache_1
+ offs_d[None, :] * stride_v_cache_2
+ internal_offsets[:, None] * stride_v_cache_3
)
# V addressing: standard 4D or interleaved 5D
if INTERLEAVED_V_PACK_FACTOR > 0:
v_offset = (
p_block_idx[:, None] * stride_v_cache_0
+ kv_head_idx * stride_v_cache_1
+ (internal_offsets[:, None] // INTERLEAVED_V_PACK_FACTOR)
* HEAD_SIZE
* INTERLEAVED_V_PACK_FACTOR
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
+ (internal_offsets[:, None] % INTERLEAVED_V_PACK_FACTOR)
)
else:
v_offset = (
p_block_idx[:, None] * stride_v_cache_0
+ kv_head_idx * stride_v_cache_1
+ offs_d[None, :] * stride_v_cache_2
+ internal_offsets[:, None] * stride_v_cache_3
)
# Only the final tile can straddle seq_len. Slots >= seq_len are
# unwritten KV cache that may hold NaN/garbage; they are score-masked
@@ -304,6 +316,7 @@ def chunked_prefill_paged_decode(
# Optional tensor for sinks
sinks=None,
is_block_table_ptr: bool = False,
use_interleaved_v_cache: bool = False,
causal: bool = True,
):
if sm_scale is None:
@@ -314,6 +327,10 @@ def chunked_prefill_paged_decode(
if sliding_window is None or sliding_window <= 0:
sliding_window = 0
interleaved_v_pack_factor = (
(16 // value_cache.element_size()) if use_interleaved_v_cache else 0
)
if max_query_len > 1:
context_attention_fwd(
q=query,
@@ -336,6 +353,7 @@ def chunked_prefill_paged_decode(
skip_decode=True,
fp8_out_scale=output_scale,
sinks=sinks,
interleaved_v_pack_factor=interleaved_v_pack_factor,
causal=causal,
)
@@ -428,6 +446,7 @@ def chunked_prefill_paged_decode(
k_scale=k_scale,
v_scale=v_scale,
fp8_out_scale=output_scale,
use_interleaved_v_cache=use_interleaved_v_cache,
)
else:
logger.warning_once(
@@ -504,4 +523,5 @@ def chunked_prefill_paged_decode(
query_start_len_ptr=query_start_loc,
USE_SINKS=sinks is not None,
USE_FP8=output_scale is not None,
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
)
+2
View File
@@ -38,6 +38,7 @@ class PagedAttention:
kv_cache_dtype: str,
k_scale: torch.Tensor,
v_scale: torch.Tensor,
interleaved_v: bool,
) -> None:
ops.reshape_and_cache(
key,
@@ -48,4 +49,5 @@ class PagedAttention:
kv_cache_dtype,
k_scale,
v_scale,
interleaved_v,
)
+51 -13
View File
@@ -56,6 +56,8 @@ def _paged_kv_cache_offsets(
stride_v_cache_d,
stride_v_cache_bl,
PHYSICAL_BLOCK_SIZE: tl.constexpr,
BLOCK_DMODEL: tl.constexpr,
INTERLEAVED_V_PACK_FACTOR: tl.constexpr = 0,
MASK_BLOCK_TABLE: tl.constexpr = False,
):
"""Compute paged K/V cache element offsets for a tile of token positions.
@@ -65,7 +67,9 @@ def _paged_kv_cache_offsets(
tiles), then the physical element offsets are built with the same layout the
kernel uses everywhere:
K cache: [num_blocks, num_kv_heads, head_size // x, block_size, x]
V cache: [num_blocks, num_kv_heads, head_size, block_size]
V cache: [num_blocks, num_kv_heads, head_size, block_size], or the
interleaved layout [num_blocks, num_kv_heads, block_size // x,
head_size, x] when `INTERLEAVED_V_PACK_FACTOR` (= x) > 0.
Returns `(off_k, off_v)` with shapes [D, N] and [N, D] respectively.
When `MASK_BLOCK_TABLE` is set, the block-table load is masked with
@@ -94,12 +98,25 @@ def _paged_kv_cache_offsets(
+ internal[None, :] * stride_k_cache_bl
+ (offs_d[:, None] % x) * stride_k_cache_x
)
off_v = (
bn[:, None] * stride_v_cache_bs
+ cur_kv_head * stride_v_cache_h
+ offs_d[None, :] * stride_v_cache_d
+ internal[:, None] * stride_v_cache_bl
)
if INTERLEAVED_V_PACK_FACTOR > 0:
# Interleaved V-cache: [num_blocks, num_kv_heads, block_size/x,
# head_size, x] flattened onto the 4D stride layout.
off_v = (
bn[:, None] * stride_v_cache_bs
+ cur_kv_head * stride_v_cache_h
+ (internal[:, None] // INTERLEAVED_V_PACK_FACTOR)
* BLOCK_DMODEL
* INTERLEAVED_V_PACK_FACTOR
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
+ (internal[:, None] % INTERLEAVED_V_PACK_FACTOR)
)
else:
off_v = (
bn[:, None] * stride_v_cache_bs
+ cur_kv_head * stride_v_cache_h
+ offs_d[None, :] * stride_v_cache_d
+ internal[:, None] * stride_v_cache_bl
)
return off_k, off_v
@@ -159,6 +176,7 @@ def _fwd_kernel(
USE_FP8: tl.constexpr,
CAUSAL: tl.constexpr = True,
MAX_Q_LEN: tl.constexpr = 0,
INTERLEAVED_V_PACK_FACTOR: tl.constexpr = 0,
MAX_CTX_LEN: tl.constexpr = 0,
FP8_MIN: tl.constexpr = float8_info.min,
FP8_MAX: tl.constexpr = float8_info.max,
@@ -263,6 +281,8 @@ def _fwd_kernel(
stride_v_cache_d,
stride_v_cache_bl,
PHYSICAL_BLOCK_SIZE,
BLOCK_DMODEL,
INTERLEAVED_V_PACK_FACTOR,
)
if (
@@ -403,6 +423,8 @@ def _fwd_kernel(
stride_v_cache_d,
stride_v_cache_bl,
PHYSICAL_BLOCK_SIZE,
BLOCK_DMODEL,
INTERLEAVED_V_PACK_FACTOR,
MASK_BLOCK_TABLE=True,
)
k_cur_load = tl.load(
@@ -540,6 +562,7 @@ def _fwd_kernel_alibi(
BLOCK_DMODEL_PADDED: tl.constexpr, # head size padded to a power of 2
BLOCK_N: tl.constexpr,
SKIP_DECODE: tl.constexpr,
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
):
# attn_bias[]
cur_batch = tl.program_id(0)
@@ -608,12 +631,24 @@ def _fwd_kernel_alibi(
+ ((start_n + offs_n[None, :]) % block_size) * stride_k_cache_bl
+ (offs_d[:, None] % x) * stride_k_cache_x
)
off_v = (
bn[:, None] * stride_v_cache_bs
+ cur_kv_head * stride_v_cache_h
+ offs_d[None, :] * stride_v_cache_d
+ (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl
)
if INTERLEAVED_V_PACK_FACTOR > 0:
v_internal = (start_n + offs_n[:, None]) % block_size
off_v = (
bn[:, None] * stride_v_cache_bs
+ cur_kv_head * stride_v_cache_h
+ (v_internal // INTERLEAVED_V_PACK_FACTOR)
* BLOCK_DMODEL
* INTERLEAVED_V_PACK_FACTOR
+ offs_d[None, :] * INTERLEAVED_V_PACK_FACTOR
+ (v_internal % INTERLEAVED_V_PACK_FACTOR)
)
else:
off_v = (
bn[:, None] * stride_v_cache_bs
+ cur_kv_head * stride_v_cache_h
+ offs_d[None, :] * stride_v_cache_d
+ (start_n + offs_n[:, None]) % block_size * stride_v_cache_bl
)
k_load = tl.load(
K_cache + off_k,
mask=dim_mask[:, None] & ((start_n + offs_n[None, :]) < cur_batch_ctx_len),
@@ -796,6 +831,7 @@ def context_attention_fwd(
fp8_out_scale=None,
sinks=None,
is_block_table_ptr: bool = False,
interleaved_v_pack_factor: int = 0,
causal: bool = True,
):
q_dtype_is_f32 = q.dtype is torch.float32
@@ -937,6 +973,7 @@ def context_attention_fwd(
BLOCK_DMODEL_PADDED=Lk_padded,
BLOCK_N=BLOCK,
SKIP_DECODE=skip_decode,
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
num_warps=NUM_WARPS,
num_stages=1,
)
@@ -1010,6 +1047,7 @@ def context_attention_fwd(
num_warps=4,
num_stages=1,
USE_SINKS=sinks is not None,
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
CAUSAL=causal,
KV_FROM_CACHE=kv_from_cache,
**extra_kargs,
@@ -52,6 +52,9 @@ def reshape_and_cache_kernel_flash(
block_size: tl.constexpr,
x: tl.constexpr,
USE_HEAD_MAJOR_LAYOUT: tl.constexpr,
# >0 writes V interleaved as [Block, Head, Slot//kx, Dim, kx] (head-major
# layout only); 0 keeps the standard [Block, Head, Dim, Slot] V layout.
INTERLEAVED_V_PACK_FACTOR: tl.constexpr,
# FP8 flags
FP8_KV_CACHE: tl.constexpr,
# tune parameters
@@ -76,13 +79,25 @@ def reshape_and_cache_kernel_flash(
# Decompose the tile index back into head and dim coordinates.
cur_head = tile_pos // head_size
cur_dim = tile_pos % head_size
# Value addressing (4D): [Block, Head, Dim, Slot]
tgt_idx_v = (
block_idx * block_stride
+ cur_head * head_stride
+ cur_dim * dim_stride_v
+ block_offset * 1
)
# Value addressing: standard 4D [Block, Head, Dim, Slot], or the
# interleaved 5D [Block, Head, Slot//kx, Dim, kx] the fused AITER
# writer emits and the interleaved read path expects.
if INTERLEAVED_V_PACK_FACTOR > 0:
tgt_idx_v = (
block_idx * block_stride
+ cur_head * head_stride
+ (block_offset // INTERLEAVED_V_PACK_FACTOR)
* (head_size * INTERLEAVED_V_PACK_FACTOR)
+ cur_dim * INTERLEAVED_V_PACK_FACTOR
+ (block_offset % INTERLEAVED_V_PACK_FACTOR)
)
else:
tgt_idx_v = (
block_idx * block_stride
+ cur_head * head_stride
+ cur_dim * dim_stride_v
+ block_offset * 1
)
# Key addressing (5D): [Block, Head, Dim//8, Slot, 8]
tgt_idx_k = (
block_idx * block_stride
@@ -371,11 +386,18 @@ def triton_reshape_and_cache_flash(
kv_cache_dtype: str, # "auto", "fp8"
k_scale: torch.Tensor, # float32
v_scale: torch.Tensor, # float32
interleaved_v_pack_factor: int = 0, # >0: interleave V (head-major); 0: standard
):
num_heads = key.shape[1]
head_size = key.shape[2]
use_head_major_layout = key_cache.ndim == 5
if interleaved_v_pack_factor and not use_head_major_layout:
raise NotImplementedError(
"Interleaved V-cache write is only implemented for the head-major "
"(5D) cache layout; got a non-head-major cache with "
f"interleaved_v_pack_factor={interleaved_v_pack_factor}."
)
if use_head_major_layout:
block_size = key_cache.shape[3]
x = key_cache.shape[4]
@@ -452,6 +474,7 @@ def triton_reshape_and_cache_flash(
block_size=block_size,
x=x,
USE_HEAD_MAJOR_LAYOUT=use_head_major_layout,
INTERLEAVED_V_PACK_FACTOR=interleaved_v_pack_factor,
FP8_KV_CACHE=FP8_KV_CACHE,
# autotune parameters
TILE_SIZE=TILE_SIZE,