[Kernel][ROCm][Perf] FlyDSL decode-attention kernel for 4-bit TurboQuant KV cache (#47896)

Signed-off-by: root <root@smci355-ccs-aus-m02-09.cs-aus.dcgpu>
Co-authored-by: root <root@smci355-ccs-aus-m02-09.cs-aus.dcgpu>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
aditi-amd
2026-08-11 08:45:39 -07:00
committed by GitHub
co-authored by root Andreas Karatzas mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent d648236199
commit 5426311d91
13 changed files with 6396 additions and 53 deletions
@@ -0,0 +1,261 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright (c) 2025 FlyDSL Project Contributors
"""Correctness test for the FlyDSL TurboQuant 4-bit KV decode kernel.
The kernel is validated against a pure-PyTorch full-precision attention
oracle rather than another kernel: the synthetic KV cache is built from
known centroid indices, K-norms and V-scales/zeros, so we can dequantize
the exact stored values and run fp32 softmax attention as ground truth.
Both sides therefore see identical quantized values; the only difference
is bf16-vs-fp32 accumulation, which is bounded by ``ATOL`` below.
Skipped unless running on ROCm/gfx950 with FlyDSL importable.
"""
import pytest
import torch
from vllm.platforms import current_platform
# Guard the ROCm-only import: `vllm.platforms.rocm` pulls in amdsmi and other
# ROCm-specific modules that are not importable on non-ROCm platforms, so it
# must not be imported at module load time on CUDA/CPU CI.
if not current_platform.is_rocm():
pytest.skip(
"TurboQuant FlyDSL decode only runs on ROCm gfx950.",
allow_module_level=True,
)
from vllm.platforms.rocm import on_gfx950 # noqa: E402
if not on_gfx950():
pytest.skip(
"TurboQuant FlyDSL decode only runs on ROCm gfx950.",
allow_module_level=True,
)
from vllm.v1.attention.ops.flydsl_turboquant_decode import ( # noqa: E402
flydsl_turboquant_decode_attention,
is_flydsl_available,
is_flydsl_gqa6_available,
)
if not is_flydsl_available():
pytest.skip("FlyDSL runtime is not available.", allow_module_level=True)
HEAD_SIZE = 128
N_CENTROIDS = 16
KEY_DATA_BYTES = HEAD_SIZE // 2 # 4-bit packed key nibbles
DATA_BYTES_PER_SLOT = HEAD_SIZE # packed K (64) + packed V (64)
NUM_SOA_FIELDS = 3
SOA_K_NORM, SOA_V_SCALE, SOA_V_ZERO = 0, 1, 2
# bf16-vs-fp32 accumulation tolerance (FlyDSL test_pa.py default).
ATOL = 5e-3
def _build_cache(
num_seqs, num_kv_heads, seq_len, qg, kv_block_size, seed=0xC0FFEE, device="cuda"
):
"""Vectorized builder for a SoA-layout 4-bit KV cache + fp32 ground truth.
Returns ``(centroids, q_bf16, kv_cache_4d, block_table, seq_lens,
K_ref, V_ref)`` where ``K_ref``/``V_ref`` are the dequantized
full-precision keys/values used by the reference attention.
"""
g = torch.Generator(device=device).manual_seed(seed)
hk = num_kv_heads
hq = hk * qg
centroids = (
torch.randn(N_CENTROIDS, generator=g, dtype=torch.float32, device=device) * 0.5
)
q = (
torch.randn(
num_seqs, hq, HEAD_SIZE, generator=g, dtype=torch.float32, device=device
)
* 0.1
)
q_bf16 = q.to(torch.bfloat16)
blocks_per_seq = (seq_len + kv_block_size - 1) // kv_block_size
max_bps = blocks_per_seq + 4
block_table = torch.zeros(num_seqs, max_bps, dtype=torch.int32, device=device)
block_table[:, :blocks_per_seq] = (
torch.arange(
num_seqs * blocks_per_seq, dtype=torch.int32, device=device
).reshape(num_seqs, blocks_per_seq)
+ 1
)
total_blocks = num_seqs * blocks_per_seq + 8
slot_size_aligned = (KEY_DATA_BYTES + 2) + (KEY_DATA_BYTES + 4) # 134
if slot_size_aligned % 2:
slot_size_aligned += 1
bytes_per_block = kv_block_size * hk * slot_size_aligned
k_idx = torch.randint(
0,
N_CENTROIDS,
(num_seqs, hk, seq_len, HEAD_SIZE),
generator=g,
dtype=torch.uint8,
device=device,
)
v_idx = torch.randint(
0,
N_CENTROIDS,
(num_seqs, hk, seq_len, HEAD_SIZE),
generator=g,
dtype=torch.uint8,
device=device,
)
knorm = (
torch.rand(num_seqs, hk, seq_len, generator=g, device=device) * 0.5 + 0.5
).to(torch.float16)
vscale = (
torch.rand(num_seqs, hk, seq_len, generator=g, device=device) * 0.05 + 0.01
).to(torch.float16)
vzero = (
(torch.rand(num_seqs, hk, seq_len, generator=g, device=device) - 0.5) * 0.1
).to(torch.float16)
k_ref = centroids[k_idx.long()].float() * knorm.float().unsqueeze(-1)
v_ref = v_idx.float() * vscale.float().unsqueeze(-1) + vzero.float().unsqueeze(-1)
k_packed = k_idx[..., 0::2] | (k_idx[..., 1::2] << 4)
v_packed = v_idx[..., 0::2] | (v_idx[..., 1::2] << 4)
kv_cache = torch.zeros(
total_blocks, bytes_per_block, dtype=torch.uint8, device=device
)
seq_ax = (
torch.arange(num_seqs, device=device)
.view(num_seqs, 1)
.expand(num_seqs, seq_len)
)
tok_ax = (
torch.arange(seq_len, device=device).view(1, seq_len).expand(num_seqs, seq_len)
)
blk_for_tok = block_table[seq_ax, tok_ax // kv_block_size]
slot_for_tok = tok_ax % kv_block_size
h_ax = torch.arange(hk, device=device)
base_data = (
slot_for_tok.unsqueeze(1) * hk * DATA_BYTES_PER_SLOT
+ h_ax.view(1, hk, 1) * DATA_BYTES_PER_SLOT
)
blk_b = blk_for_tok.unsqueeze(1).expand(num_seqs, hk, seq_len)
dst_base = blk_b * bytes_per_block + base_data
rng_k = torch.arange(KEY_DATA_BYTES, device=device)
rng_v = torch.arange(KEY_DATA_BYTES, device=device) + KEY_DATA_BYTES
k_dst = (dst_base.unsqueeze(-1) + rng_k.view(1, 1, 1, -1)).reshape(-1)
v_dst = (dst_base.unsqueeze(-1) + rng_v.view(1, 1, 1, -1)).reshape(-1)
flat = kv_cache.view(-1)
flat[k_dst] = k_packed.reshape(-1)
flat[v_dst] = v_packed.reshape(-1)
meta_off = kv_block_size * hk * DATA_BYTES_PER_SLOT
meta_view = kv_cache.view(torch.float16).view(total_blocks, -1)
meta_off_hw = meta_off // 2
common = h_ax.view(
1, hk, 1
) * NUM_SOA_FIELDS * kv_block_size + slot_for_tok.unsqueeze(1)
blk_flat = blk_for_tok.unsqueeze(1).expand(num_seqs, hk, seq_len)
knorm_off = meta_off_hw + common + SOA_K_NORM * kv_block_size
vscale_off = meta_off_hw + common + SOA_V_SCALE * kv_block_size
vzero_off = meta_off_hw + common + SOA_V_ZERO * kv_block_size
meta_view[blk_flat.reshape(-1).long(), knorm_off.reshape(-1).long()] = (
knorm.reshape(-1)
)
meta_view[blk_flat.reshape(-1).long(), vscale_off.reshape(-1).long()] = (
vscale.reshape(-1)
)
meta_view[blk_flat.reshape(-1).long(), vzero_off.reshape(-1).long()] = (
vzero.reshape(-1)
)
seq_lens = torch.full((num_seqs,), seq_len, dtype=torch.int32, device=device)
kv_cache_4d = kv_cache.view(total_blocks, kv_block_size, hk, slot_size_aligned)
return (
centroids,
q_bf16,
kv_cache_4d,
block_table,
seq_lens,
k_ref.cpu(),
v_ref.cpu(),
)
def _reference_attention(q_bf16, k_ref, v_ref, seq_lens, scale):
"""fp32 softmax attention on dequantized keys/values (ground truth)."""
num_seqs, hq, d = q_bf16.shape
hk = k_ref.shape[1]
qg = hq // hk
q = q_bf16.float().reshape(num_seqs, hk, qg, d)
out = torch.zeros(num_seqs, hq, d, dtype=torch.float32)
for s in range(num_seqs):
for h in range(hk):
ql = int(seq_lens[s].item())
k = k_ref[s, h, :ql]
v = v_ref[s, h, :ql]
scores = (q[s, h] @ k.T) * scale
m = scores.max(dim=-1, keepdim=True).values
e = torch.exp(scores - m)
p = e / e.sum(dim=-1, keepdim=True)
out[s, h * qg : (h + 1) * qg] = p @ v
return out
@pytest.mark.parametrize("num_seqs", [1, 4, 16])
@pytest.mark.parametrize("seq_len", [256, 1024, 4096])
@pytest.mark.parametrize(
"num_kv_heads,qg",
[
(8, 8), # Qwen2.5-72B class
(8, 16), # Qwen3-32B class
pytest.param(
4,
6, # MiniMax-M2.5 class (GQA-6 sibling kernel)
marks=pytest.mark.skipif(
not is_flydsl_gqa6_available(),
reason="FlyDSL GQA-6 sibling kernel not available",
),
),
],
)
@pytest.mark.parametrize("kv_block_size", [16, 32])
def test_flydsl_matches_reference(num_seqs, seq_len, num_kv_heads, qg, kv_block_size):
"""FlyDSL decode must match fp32 attention on dequantized KV."""
centroids, q_bf16, kv_cache, block_table, seq_lens, k_ref, v_ref = _build_cache(
num_seqs, num_kv_heads, seq_len, qg, kv_block_size
)
scale = 1.0 / (HEAD_SIZE**0.5)
identity = torch.eye(HEAD_SIZE, dtype=torch.float32, device="cuda")
out = flydsl_turboquant_decode_attention(
query=q_bf16,
kv_cache=kv_cache,
block_table=block_table,
seq_lens=seq_lens,
Pi=identity,
centroids=centroids,
scale=scale,
mse_bits=4,
key_packed_size=KEY_DATA_BYTES + 2,
value_quant_bits=4,
value_packed_size=KEY_DATA_BYTES + 4,
key_fp8=False,
norm_correction=False,
PiT=identity.T.contiguous(),
max_seq_len=seq_len,
max_num_kv_splits=32,
sinks=None,
)
ref = _reference_attention(q_bf16.cpu(), k_ref, v_ref, seq_lens.cpu(), scale)
torch.testing.assert_close(out.cpu().float(), ref, atol=ATOL, rtol=0.0)
+23 -8
View File
@@ -623,23 +623,38 @@ class RocmPlatform(Platform):
if selected_backend is not None:
try:
backend_class = selected_backend.get_class()
invalid_reasons = backend_class.validate_configuration(
sel_invalid_reasons = backend_class.validate_configuration(
device_capability=device_capability,
**attn_selector_config._asdict(),
)
except ImportError:
invalid_reasons = ["ImportError"]
if invalid_reasons:
raise ValueError(
f"Selected backend {selected_backend} is not valid for "
f"this configuration. Reason: {invalid_reasons}"
)
else:
sel_invalid_reasons = ["ImportError"]
if not sel_invalid_reasons:
logger.info_once(
"Using %s backend (selected via --attention-backend).",
selected_backend.name,
)
return selected_backend.get_path()
# Only tolerate the mismatch for turboquant_* KV-cache layers:
# boundary layers keep the native dtype (served by the selected
# backend) while turboquant_* layers need TURBOQUANT, so no single
# --attention-backend can serve every layer. For any other dtype
# the explicit selection is genuinely invalid -> fail loud.
kv_dtype = attn_selector_config.kv_cache_dtype
if not (kv_dtype is not None and str(kv_dtype).startswith("turboquant")):
raise ValueError(
f"Selected backend {selected_backend} is not valid for "
f"this configuration. Reason: {sel_invalid_reasons}"
)
# NOTE: pass a str (not the list) -- info_once hashes its args.
logger.info_once(
"Selected backend %s is incompatible with this turboquant "
"layer (%s); using the auto-selected per-layer backend. "
"Reason: %s",
selected_backend.name,
attn_selector_config.attn_type,
str(sel_invalid_reasons),
)
# No selected backend or the selected backend is invalid,
# so we try finding a valid backend.
+349 -45
View File
@@ -16,6 +16,7 @@ Per-head per-position slot layout:
For turboquant_k3v4_nc head_dim=256: [100 bytes key | 512 bytes value] = 612
"""
import contextlib
import functools
import math
from dataclasses import dataclass
@@ -26,6 +27,7 @@ import torch.nn.functional as F
from vllm.config import get_current_vllm_config
from vllm.config.cache import CacheDType
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization.turboquant.centroids import (
get_centroids,
)
@@ -47,6 +49,15 @@ from vllm.v1.attention.backends.fa_utils import (
is_flash_attn_varlen_func_available,
)
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
# FlyDSL TurboQuant decode (AMD gfx950). Auto-selected when FlyDSL is available
# for eligible layers (SoA store + FlyDSL decode + SoA-aware continuation);
# non-gfx950 or ineligible layers use the SoA Triton decode.
from vllm.v1.attention.ops.flydsl_turboquant_decode import (
flydsl_turboquant_decode_attention,
is_flydsl_available,
is_flydsl_gqa6_available,
)
from vllm.v1.attention.ops.triton_turboquant_decode import (
_tq_full_dequant_kv,
_use_fp8_e4b15,
@@ -59,6 +70,8 @@ from vllm.v1.worker.workspace import (
is_workspace_manager_initialized,
)
logger = init_logger(__name__)
_HAS_FLASH_ATTN = is_flash_attn_varlen_func_available()
if _HAS_FLASH_ATTN:
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func
@@ -71,6 +84,24 @@ if _HAS_FLASH_ATTN:
_CONTINUATION_DECODE_THRESHOLD = 128
def _soa_imports():
"""Lazy import of the HIP-free SoA Triton subset (store / dequant / decode).
Kept lazy so the default (FlyDSL-off) path never imports these modules.
"""
from vllm.v1.attention.ops.turboquant_soa.triton_turboquant_decode import (
_tq_full_dequant_kv as soa_dequant,
)
from vllm.v1.attention.ops.turboquant_soa.triton_turboquant_store import (
triton_turboquant_store as soa_store,
)
from vllm.v1.attention.ops.turboquant_soa.triton_turboquant_unified_attention import ( # noqa: E501
triton_turboquant_decode_attention_soa as soa_decode,
)
return soa_store, soa_dequant, soa_decode
def _build_hadamard(d: int, device_str: str) -> torch.Tensor:
"""Orthonormal Hadamard matrix (Sylvester construction), cached per (d, device).
@@ -290,6 +321,10 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
supports_quant_query_input: bool = False
# Lazily populated before cudagraph capture (FlyDSL decode path only).
_arange_cache: torch.Tensor
_cu_2: torch.Tensor
def __init__(
self,
num_heads: int,
@@ -337,6 +372,17 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
vllm_config.attention_config.tq_max_kv_splits_for_cuda_graph
)
# FlyDSL decode state. Auto-enabled on gfx950 when FlyDSL is available.
self.sliding_window = sliding_window
self.sinks = kwargs.get("sinks")
# Cache max_model_len now (config is available at __init__ but NOT
# during CUDA-graph capture when _ensure_on_device is re-entered).
self._max_model_len = vllm_config.model_config.max_model_len
# SoA store is required by the FlyDSL decode/continuation path, so it
# tracks FlyDSL availability (single switch for the whole pipeline).
self._use_flydsl = is_flydsl_available()
self._soa_store = self._use_flydsl
def _flash_attn_varlen(
self,
q: torch.Tensor,
@@ -382,6 +428,44 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
quantizer is symmetric around zero (sign-flipping a coordinate
maps it to the mirror centroid with identical distortion).
"""
if self._soa_store:
# CUDA-graph capture safety for the FlyDSL decode path on ROCm.
# (1) Pre-allocate _arange_cache / _cu_2 BEFORE any capture; lazy
# allocation during graph replay lands in the HIP graph memory
# pool and yields stale addresses (GPU fault / garbage).
# (2) Pre-warm the WorkspaceManager to its max size before capture
# so mid-capture growth cannot invalidate pointers baked into
# already-captured batch sizes.
_max_len = self._max_model_len
_already_ok = (
hasattr(self, "_arange_cache")
and self._arange_cache.device.type == str(device).split(":")[0]
and self._arange_cache.shape[0] >= _max_len + 2
)
if not _already_ok:
self._arange_cache = torch.arange(
0, _max_len + 2, device=device, dtype=torch.int32
)
if not hasattr(self, "_cu_2") or self._cu_2.device != torch.device(device):
self._cu_2 = torch.zeros(2, device=device, dtype=torch.int32)
if (
is_workspace_manager_initialized()
and not current_workspace_manager().is_locked()
):
B_max = self._max_capture_batch_size()
D = self.head_size
Hq = self.num_heads
S = self.max_num_kv_splits
_pre_warm_bytes = (
B_max * Hq * (S * (D + 1) + D) * 4 # fp32 mid_o + fp32 lse
+ B_max * Hq * D * 2 # query-dtype output (bf16 = 2 B)
+ 512 # alignment padding
)
with contextlib.suppress(AssertionError):
current_workspace_manager().get_simultaneous(
((_pre_warm_bytes,), torch.uint8)
)
if not hasattr(layer, "_tq_cached"):
D = self.head_size
@@ -402,6 +486,29 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
layer._tq_midpoints = (c_sorted[:-1] + c_sorted[1:]) / 2
layer._tq_cached = True
def _max_capture_batch_size(self) -> int:
"""Largest decode batch we might see at runtime (for workspace pre-warm).
Take max(cudagraph_capture_sizes, scheduler.max_num_seqs): a forward
pass exceeding the largest captured graph size falls back to eager,
but the workspace is locked and that eager path can still hit batch
sizes up to max_num_seqs. Falls back to 1024 if config is unavailable.
"""
try:
cfg = get_current_vllm_config()
candidates: list[int] = []
sizes = cfg.compilation_config.cudagraph_capture_sizes
if sizes:
candidates.append(int(max(sizes)))
sched = getattr(cfg, "scheduler_config", None)
if sched is not None and getattr(sched, "max_num_seqs", None):
candidates.append(int(sched.max_num_seqs))
if candidates:
return max(candidates)
except Exception: # noqa: BLE001
pass
return 1024
def do_kv_cache_update(
self,
layer: torch.nn.Module,
@@ -590,6 +697,27 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
layer: Any,
):
"""Quantize + store via fused Triton kernel."""
if self._soa_store:
# SoA layout (data region + metadata region separated per block),
# required by the FlyDSL decode kernel. Pure-Triton store; the
# cache tensor shape is identical to the default AoS store, only
# the within-block byte convention differs.
soa_store, _, _ = _soa_imports()
soa_store(
key=key,
value=value,
kv_cache=kv_cache,
slot_mapping=slot_mapping,
PiT=layer._tq_PiT,
midpoints=layer._tq_midpoints,
mse_bits=self.tq_config.key_mse_bits,
key_packed_size=self.tq_config.key_packed_size,
value_quant_bits=self.tq_config.effective_value_quant_bits,
key_fp8=self.tq_config.key_fp8,
centroids=layer._tq_centroids,
norm_correction=self.tq_config.norm_correction,
)
return
triton_turboquant_store(
key,
value,
@@ -722,21 +850,55 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
# Slice from pre-built arange (no kernel launch)
synth_seq_lens = _arange_cache[cached_len + 1 : seq_len + 1]
synth_bt = attn_metadata.block_table[i : i + 1].expand(q_len, -1)
out = triton_turboquant_decode_attention(
query=q_seq,
kv_cache=kv_cache,
block_table=synth_bt,
seq_lens=synth_seq_lens,
Pi=Pi,
centroids=centroids,
scale=self.scale,
mse_bits=self.tq_config.key_mse_bits,
key_packed_size=self.tq_config.key_packed_size,
value_quant_bits=(self.tq_config.effective_value_quant_bits),
key_fp8=self.tq_config.key_fp8,
norm_correction=self.tq_config.norm_correction,
PiT=PiT,
)
if self._soa_store:
# The cache was written in SoA layout (always, for FlyDSL),
# so it MUST be read with the SoA-aware decode. The
# default AoS decode reads k_norm/v_scale/v_zero from
# the wrong offsets in a SoA cache -> garbage
# cached-prefix output ->
# accuracy collapse on every multi-turn / prefix-cached
# (APC) request. Continuation stays on the Triton SoA
# path even when FlyDSL is the main decode kernel
# (FlyDSL is decode-batch only).
out = self._dispatch_decode_soa(
query=q_seq,
kv_cache=kv_cache,
block_table=synth_bt,
seq_lens=synth_seq_lens,
Pi=Pi,
centroids=centroids,
scale=self.scale,
mse_bits=self.tq_config.key_mse_bits,
key_packed_size=self.tq_config.key_packed_size,
value_quant_bits=(
self.tq_config.effective_value_quant_bits
),
value_packed_size=self.tq_config.value_packed_size,
max_seq_len=int(seq_len),
key_fp8=self.tq_config.key_fp8,
norm_correction=self.tq_config.norm_correction,
PiT=PiT,
sinks=self.sinks,
sliding_window=self.sliding_window,
)
else:
out = triton_turboquant_decode_attention(
query=q_seq,
kv_cache=kv_cache,
block_table=synth_bt,
seq_lens=synth_seq_lens,
Pi=Pi,
centroids=centroids,
scale=self.scale,
mse_bits=self.tq_config.key_mse_bits,
key_packed_size=self.tq_config.key_packed_size,
value_quant_bits=(
self.tq_config.effective_value_quant_bits
),
key_fp8=self.tq_config.key_fp8,
norm_correction=self.tq_config.norm_correction,
PiT=PiT,
)
else:
# Large continuation: dequant cached K/V and use
# flash_attn for better throughput.
@@ -801,36 +963,84 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
v_cached = v_buf[:, :, :alloc_len, :]
grid = (alloc_len, 1 * Hk)
_tq_full_dequant_kv[grid](
kv_cache,
block_table,
centroids,
k_cached,
v_cached,
k_cached.stride(0),
k_cached.stride(1),
k_cached.stride(2),
v_cached.stride(0),
v_cached.stride(1),
v_cached.stride(2),
kv_cache.stride(0),
kv_cache.stride(1),
kv_cache.stride(2),
block_table.stride(0),
HEAD_DIM=D,
BLOCK_SIZE=block_size,
NUM_KV_HEADS=Hk,
MSE_BYTES=mse_bytes,
KPS=self.tq_config.key_packed_size,
VQB=self.tq_config.effective_value_quant_bits,
VAL_DATA_BYTES=val_data_bytes,
MSE_BITS=self.tq_config.key_mse_bits,
KEY_FP8=1 if self.tq_config.key_fp8 else 0,
BLOCK_D=BLOCK_D,
NORM_CORRECTION=1 if self.tq_config.norm_correction else 0,
FP8_E4B15=_use_fp8_e4b15(device.index or 0),
num_warps=4,
)
if self._soa_store:
# SoA-aware dequant: read the data/metadata-separated SoA cache
# written by the SoA store. Constants must match the store side.
_, soa_dequant, _ = _soa_imports()
key_fp8 = self.tq_config.key_fp8
key_data_bytes = D if key_fp8 else mse_bytes
data_bytes_per_slot = key_data_bytes + val_data_bytes
meta_region_offset = block_size * Hk * data_bytes_per_slot
num_soa_fields = 2 if key_fp8 else 3
soa_k_norm = 0
soa_v_scale = 0 if key_fp8 else 1
soa_v_zero = 1 if key_fp8 else 2
kv_cache_u16 = kv_cache.view(torch.uint16)
soa_dequant[grid](
kv_cache,
kv_cache_u16,
block_table,
centroids,
k_cached,
v_cached,
k_cached.stride(0),
k_cached.stride(1),
k_cached.stride(2),
v_cached.stride(0),
v_cached.stride(1),
v_cached.stride(2),
kv_cache.stride(0),
block_table.stride(0),
HEAD_DIM=D,
BLOCK_SIZE=block_size,
NUM_KV_HEADS=Hk,
MSE_BYTES=mse_bytes,
VQB=self.tq_config.effective_value_quant_bits,
VAL_DATA_BYTES=val_data_bytes,
MSE_BITS=self.tq_config.key_mse_bits,
KEY_FP8=1 if key_fp8 else 0,
KEY_DATA_BYTES=key_data_bytes,
META_REGION_OFFSET=meta_region_offset,
NUM_SOA_FIELDS=num_soa_fields,
SOA_K_NORM=soa_k_norm,
SOA_V_SCALE=soa_v_scale,
SOA_V_ZERO=soa_v_zero,
BLOCK_D=BLOCK_D,
NORM_CORRECTION=1 if self.tq_config.norm_correction else 0,
FP8_E4B15=_use_fp8_e4b15(device.index or 0),
num_warps=4,
)
else:
_tq_full_dequant_kv[grid](
kv_cache,
block_table,
centroids,
k_cached,
v_cached,
k_cached.stride(0),
k_cached.stride(1),
k_cached.stride(2),
v_cached.stride(0),
v_cached.stride(1),
v_cached.stride(2),
kv_cache.stride(0),
kv_cache.stride(1),
kv_cache.stride(2),
block_table.stride(0),
HEAD_DIM=D,
BLOCK_SIZE=block_size,
NUM_KV_HEADS=Hk,
MSE_BYTES=mse_bytes,
KPS=self.tq_config.key_packed_size,
VQB=self.tq_config.effective_value_quant_bits,
VAL_DATA_BYTES=val_data_bytes,
MSE_BITS=self.tq_config.key_mse_bits,
KEY_FP8=1 if self.tq_config.key_fp8 else 0,
BLOCK_D=BLOCK_D,
NORM_CORRECTION=1 if self.tq_config.norm_correction else 0,
FP8_E4B15=_use_fp8_e4b15(device.index or 0),
num_warps=4,
)
# Inverse-rotate MSE keys back to original space
if not self.tq_config.key_fp8:
@@ -930,6 +1140,82 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
)
)
if self._use_flydsl:
# FlyDSL decode (gfx950, MSE-key, HEAD_SIZE=128, GQA in {6, 8, 16}).
# GQA-6 routes to the MiniMax sibling kernel. Ineligible layers fall
# back to SoA Triton decode.
_gqa = self.num_kv_groups
flydsl_gqa_ok = (_gqa in (8, 16)) or (
_gqa == 6 and is_flydsl_gqa6_available()
)
flydsl_eligible = (
not self.tq_config.key_fp8
and self.tq_config.key_mse_bits == 4
and self.tq_config.effective_value_quant_bits == 4
and self.head_size == 128
and flydsl_gqa_ok
and self.sinks is None
and not (self.sliding_window and self.sliding_window > 0)
)
if flydsl_eligible:
return flydsl_turboquant_decode_attention(
query=query,
kv_cache=kv_cache,
block_table=attn_metadata.block_table,
seq_lens=attn_metadata.seq_lens,
Pi=Pi,
centroids=centroids,
scale=self.scale,
mse_bits=self.tq_config.key_mse_bits,
key_packed_size=self.tq_config.key_packed_size,
value_quant_bits=self.tq_config.effective_value_quant_bits,
value_packed_size=self.tq_config.value_packed_size,
max_seq_len=attn_metadata.max_seq_len,
key_fp8=self.tq_config.key_fp8,
norm_correction=self.tq_config.norm_correction,
PiT=PiT,
mid_o_buf=mid_o_buf,
output_buf=output_buf,
lse_buf=lse_buf,
buf_holder=layer,
max_num_kv_splits=self.max_num_kv_splits,
sinks=self.sinks,
)
logger.warning_once(
"TurboQuant FlyDSL ineligible (key_fp8=%s mse_bits=%s vqb=%s "
"head_size=%s num_kv_groups=%s sinks=%s) -> SoA Triton decode",
self.tq_config.key_fp8,
self.tq_config.key_mse_bits,
self.tq_config.effective_value_quant_bits,
self.head_size,
self.num_kv_groups,
self.sinks is not None,
)
return self._dispatch_decode_soa(
query=query,
kv_cache=kv_cache,
block_table=attn_metadata.block_table,
seq_lens=attn_metadata.seq_lens,
Pi=Pi,
centroids=centroids,
scale=self.scale,
mse_bits=self.tq_config.key_mse_bits,
key_packed_size=self.tq_config.key_packed_size,
value_quant_bits=self.tq_config.effective_value_quant_bits,
value_packed_size=self.tq_config.value_packed_size,
max_seq_len=attn_metadata.max_seq_len,
key_fp8=self.tq_config.key_fp8,
norm_correction=self.tq_config.norm_correction,
PiT=PiT,
mid_o_buf=mid_o_buf,
output_buf=output_buf,
lse_buf=lse_buf,
buf_holder=layer,
max_num_kv_splits=self.max_num_kv_splits,
sinks=self.sinks,
sliding_window=self.sliding_window,
)
result = triton_turboquant_decode_attention(
query=query,
kv_cache=kv_cache,
@@ -951,3 +1237,21 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
max_num_kv_splits=self.max_num_kv_splits,
)
return result
def _dispatch_decode_soa(self, **kwargs):
"""SoA-aware Triton decode — fallback for FlyDSL-ineligible layers.
The SoA decode launcher accepts a subset of the FlyDSL kwargs; filter to
the params it accepts and raise if a *meaningful* (non-None) kwarg would
be silently dropped (so we never mask a real feature gap).
"""
import inspect
_, _, soa_decode = _soa_imports()
accepted = set(inspect.signature(soa_decode).parameters)
dropped = [k for k, v in kwargs.items() if k not in accepted and v is not None]
if dropped:
raise NotImplementedError(
f"SoA decode does not support kwargs {sorted(dropped)}"
)
return soa_decode(**{k: v for k, v in kwargs.items() if k in accepted})
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,823 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# TurboQuant decode (FlyDSL) — MI355X / gfx950 / CDNA4
#
# Decode-only paged attention over a TurboQuant SoA KV cache.
# Targets Qwen-class models: HEAD_SIZE=128, GQA group=8, MSE_BITS=4 K, VQB=4 V,
# N_CENTROIDS=16, BLOCK_SIZE=16, partitioned mode.
#
# === Design ===
# * 1 wave (64 lanes) per CTA — simpler than 4-warp; trades parallelism for
# correctness/debug-ability. A later revision will fan out to 4 warps.
# * Flash-Attention-2 online softmax: rescale running PV/sum per K-tile,
# no QK score storage across tiles, no P-LDS round-trip (qk_acc layout
# matches PV B-operand directly).
# * Centroids loaded once into LDS at kernel start (LDS-resident LUT).
# * K dequant → LDS [token, head_dim] (natural layout)
# * V dequant → LDS [head_dim, token] (TRANSPOSED — required so PV MFMA's
# A=V operand can be loaded as 4-bf16 contiguous via 1 ds_read_b64).
#
# MFMA layouts used:
# QK: mfma(A=K, B=Q, C=qk_acc) → C[m=token, n=query], 4 fp32/lane
# = 4 tokens for SAME query → softmax via
# xor_shuffle(16),(32) reduces per-row
# PV: mfma(A=V_T, B=P, C=acc_pv) → C[m=head_dim, n=query], 4 fp32/lane
# = 4 contiguous head_dim cols for SAME
# query → 1 buffer_store of 8 bytes
# per chunk per lane
from __future__ import annotations
import flydsl.compiler as flyc
import flydsl.expr as fx
from flydsl._mlir import ir
from flydsl._mlir.dialects import scf as _scf
from flydsl.expr import (
arith,
buffer_ops,
const_expr, # noqa: F401 kept for kernel-DSL API parity
gpu,
range_constexpr,
rocdl,
vector,
)
from flydsl.expr.typing import Int32, T # noqa: F401 Int32 via fx.Int32
from flydsl.runtime.device import get_rocm_arch as get_hip_arch
from flydsl.utils.smem_allocator import SmemAllocator, SmemPtr
# === Constants (Qwen-class TQ decode profile) ===============================
HEAD_SIZE = 128
# default; overridable via build_tq_decode_module(kv_block_size=...)
KV_BLOCK_SIZE = 16
# MFMA tile = 16 tokens (do not change without re-deriving MFMA shapes)
TILE_SIZE = 16
N_CENTROIDS = 16
# default for tests; overridable via build_tq_decode_module(query_group_size=...)
QUERY_GROUP_SIZE = 16
WARP_SIZE = 64
NUM_WARPS = 1
BLOCK_THREADS = NUM_WARPS * WARP_SIZE
KV_COMPUTE_BLOCK = 256 # 16 K-tiles × 16 tokens
# TQ SoA layout
NUM_SOA_FIELDS = 3
SOA_K_NORM = 0
SOA_V_SCALE = 1
SOA_V_ZERO = 2
KEY_DATA_BYTES = HEAD_SIZE // 2
VAL_DATA_BYTES = HEAD_SIZE // 2
DATA_BYTES_PER_SLOT = KEY_DATA_BYTES + VAL_DATA_BYTES
# MFMA
MFMA_M = MFMA_N = 16
MFMA_K_BF16_QK = 32 # CDNA4 wide-K (mfma_f32_16x16x32_bf16)
MFMA_K_BF16_PV = 16 # PV's K = token tile = 16
QK_K_CHUNKS = HEAD_SIZE // MFMA_K_BF16_QK # 4
PV_N_CHUNKS = HEAD_SIZE // MFMA_N # 8
# LDS regions
CENTROID_LDS_BYTES = N_CENTROIDS * 4 # 64
Q_LDS_BYTES = QUERY_GROUP_SIZE * HEAD_SIZE * 2 # 4096
KV_TILE_LDS_BYTES = TILE_SIZE * HEAD_SIZE * 2 # 4096
LOG2E = 1.4426950408889634
NEG_INF_VAL = float("-inf")
def _vsplat_mul(vec, scalar):
s = scalar.ir_value() if hasattr(scalar, "ir_value") else scalar
return vec * vector.broadcast(T.f32x4, s)
allocator = None
def build_tq_decode_module(
num_seqs: int,
num_kv_heads: int,
num_partitions: int,
max_blocks_per_seq: int = 512,
softmax_scale: float | None = None,
query_group_size: int = QUERY_GROUP_SIZE,
kv_block_size: int = KV_BLOCK_SIZE,
use_hw_v_transpose: bool = False,
tile_groups_per_partition: int = 1,
):
"""Build a TQ decode kernel module.
``query_group_size`` (= num_query_heads // num_kv_heads) selects the
GQA factor. Supported values: 8 and 16. For 8 the MFMA's 16-row
capacity is half-used; lanes 8..15 compute garbage and are gated out
of all global-memory writes via an OOB-offset trick on the buffer
resource. Q load iterates ``QG // 4`` chunks (= 2 for QG=8, 4 for
QG=16) so we never read past the real Q rows in HBM.
``kv_block_size`` is the number of tokens per vLLM cache block. Must
be a multiple of TILE_SIZE (=16). Supported values: 16 and 32. With
block_size=32, two MFMA K-tiles fit in each block; the kernel walks
block-table entries every ``kv_block_size // TILE_SIZE`` tiles and
re-uses the entry for sub-tiles within the same block. SoA metadata
region grows linearly with block_size.
``use_hw_v_transpose`` (CDNA4 / gfx950 only) replaces the 32-element
scattered ``ds_write_b16`` V-dequant pattern with a row-major
``V[token][head_dim]`` LDS layout written as 4 wide ``ds_write_b128``
per lane, then read back into the MFMA A operand via the hardware
``ds_read_tr16_b64`` transpose. This eliminates ~28 ds_write
instructions per lane per K-tile (32 -> 4) and 8 strided ds_read_b64
per PV chunk (replaces them with 1 hw-transpose read each).
``tile_groups_per_partition`` (default 1) controls the FA-2 split-K
granularity. With value G each partition processes ``G * 16`` K-tiles
= ``G * KV_COMPUTE_BLOCK`` tokens, with the existing 16-tile loop
body iterated G times. The launcher uses this to bound
``num_partitions`` (e.g. cap at 32) for long context: at 32K /
block_size=32 / num_partitions=32, G=5 covers 32*5*256 = 40 960
tokens of worst-case context with grid.z=32 instead of 256.
Behavior at G=1 is bit-identical to the single-partition kernel.
"""
assert query_group_size in (8, 16), (
f"query_group_size must be 8 or 16; got {query_group_size}"
)
assert kv_block_size in (16, 32), (
f"kv_block_size must be 16 or 32; got {kv_block_size}"
)
assert kv_block_size % TILE_SIZE == 0
assert int(tile_groups_per_partition) >= 1, (
f"tile_groups_per_partition must be >= 1; got {tile_groups_per_partition}"
)
USE_HW_TR = bool(use_hw_v_transpose)
TGPP = int(tile_groups_per_partition)
PARTITION_EXTENT_TOKENS = TGPP * KV_COMPUTE_BLOCK
QG = int(query_group_size)
QG_LOAD_ITERS = QG // 4 # 2 for QG=8, 4 for QG=16
OOB_OFFSET = 0x7FFFFFF0 # noqa: F841 ~2GB byte offset; > any plausible buffer
_BS = int(kv_block_size)
_TILES_PER_BLOCK = _BS // TILE_SIZE # 1 for BS=16, 2 for BS=32
global allocator
arch = get_hip_arch()
if softmax_scale is None:
softmax_scale = 1.0 / (HEAD_SIZE**0.5)
_qk_scale = float(softmax_scale)
# --- Strides ---
_Hq = num_kv_heads * QG
_stride_q_seq = _Hq * HEAD_SIZE
_stride_q_head = HEAD_SIZE
_stride_bt_seq = max_blocks_per_seq
_data_region_bytes = _BS * num_kv_heads * DATA_BYTES_PER_SLOT
_meta_region_bytes = num_kv_heads * NUM_SOA_FIELDS * _BS * 2
_stride_cache_block = _data_region_bytes + _meta_region_bytes
_meta_region_offset = _data_region_bytes
_stride_out_part = QG * HEAD_SIZE
_stride_out_head = num_partitions * QG * HEAD_SIZE
_stride_out_seq = num_kv_heads * num_partitions * QG * HEAD_SIZE
_stride_es_seq = num_kv_heads * num_partitions * QG
_stride_ml_seq = _stride_es_seq
# --- LDS layout ---
allocator = SmemAllocator(None, arch=arch, global_sym_name="tq_smem")
centroid_off = 0
allocator.ptr = CENTROID_LDS_BYTES
q_off = allocator.ptr
allocator.ptr += Q_LDS_BYTES
kv_off = allocator.ptr
allocator.ptr += KV_TILE_LDS_BYTES
@flyc.kernel
def tq_decode_kernel(
out_ptr: fx.Tensor,
exp_sums_ptr: fx.Tensor,
max_logits_ptr: fx.Tensor,
query_ptr: fx.Tensor,
kv_cache_ptr: fx.Tensor,
centroids_ptr: fx.Tensor,
block_tables_ptr: fx.Tensor,
seq_lens_ptr: fx.Tensor,
):
# ---- IDs ---------------------------------------------------------
tid = gpu.thread_idx.x
seq = gpu.block_idx.x
kv_h = gpu.block_idx.y
part = gpu.block_idx.z
lane = tid # 0..63
mfma_row = lane & fx.Int32(15) # = query_row (B's N) /
# token (A's M for QK)
mfma_col_grp = lane >> fx.Int32(4) # 0..3, K-group dim
# ---- Buffer resources -------------------------------------------
q_rsrc = buffer_ops.create_buffer_resource(query_ptr, max_size=True)
kv_rsrc = buffer_ops.create_buffer_resource(kv_cache_ptr, max_size=True)
bt_rsrc = buffer_ops.create_buffer_resource(block_tables_ptr, max_size=True)
sl_rsrc = buffer_ops.create_buffer_resource(seq_lens_ptr, max_size=True)
cent_rsrc = buffer_ops.create_buffer_resource(centroids_ptr, max_size=True)
out_rsrc = buffer_ops.create_buffer_resource(out_ptr, max_size=True)
es_rsrc = buffer_ops.create_buffer_resource(exp_sums_ptr, max_size=True)
ml_rsrc = buffer_ops.create_buffer_resource(max_logits_ptr, max_size=True)
# ---- LDS pointers -----------------------------------------------
base = allocator.get_base()
cent_lds = SmemPtr(base, centroid_off, T.f32, shape=(N_CENTROIDS,))
q_lds_i32 = SmemPtr(base, q_off, T.i32, shape=(Q_LDS_BYTES // 4,)).get()
q_lds_i64 = SmemPtr(base, q_off, T.i64, shape=(Q_LDS_BYTES // 8,)).get()
kv_lds_i32 = SmemPtr(base, kv_off, T.i32, shape=(KV_TILE_LDS_BYTES // 4,)).get() # noqa: F841
kv_lds_i64 = SmemPtr(base, kv_off, T.i64, shape=(KV_TILE_LDS_BYTES // 8,)).get()
kv_lds_i16 = SmemPtr(base, kv_off, T.i16, shape=(KV_TILE_LDS_BYTES // 2,)).get()
# ---- Constants ---------------------------------------------------
c_sq = fx.Int32(_stride_q_seq)
c_qh = fx.Int32(_stride_q_head)
c_qg = fx.Int32(QG)
c_bt = fx.Int32(_stride_bt_seq)
c_block = fx.Int32(_stride_cache_block)
c_meta_off = fx.Int32(_meta_region_offset)
c_data_per_slot = fx.Int32(DATA_BYTES_PER_SLOT)
c_kv_heads = fx.Int32(num_kv_heads)
c_keydata = fx.Int32(KEY_DATA_BYTES)
c_w = fx.Int32(WARP_SIZE)
NEG_INF = arith.constant(NEG_INF_VAL, type=T.f32)
ZERO_F = fx.Float32(0.0)
ONE_F = fx.Float32(1.0)
LOG2E_C = arith.constant(LOG2E, type=T.f32)
QK_SCALE = arith.constant(_qk_scale, type=T.f32)
# Helper: unwrap fx wrapper → raw ir.Value (used in STEP F)
def _ival(v):
return v.ir_value() if hasattr(v, "ir_value") else v
# ===== STEP A: Load centroids → LDS (cooperative, race-safe) =====
c_idx_safe = lane & fx.Int32(N_CENTROIDS - 1)
c_val = buffer_ops.buffer_load(cent_rsrc, c_idx_safe, vec_width=1, dtype=T.f32)
cent_lds.store(c_val, [arith.index_cast(T.index, c_idx_safe)])
gpu.barrier()
# ===== STEP B: Load pre-rotated q_rot → row-major LDS ============
# q_rsrc points to the externally-rotated q_rot tensor. Load 8 bf16
# per lane per iter (vec_width=4 i32) in QG_LOAD_ITERS iterations.
for c in range_constexpr(QG_LOAD_ITERS):
row_chunk = lane + fx.Int32(c * WARP_SIZE) # 0..QG*16-1
row = row_chunk >> fx.Int32(4) # 0..QG-1
col_b = row_chunk & fx.Int32(15) # 0..15
col_elem = col_b * fx.Int32(8) # 0..120 in bf16
q_off_elem = seq * c_sq + (kv_h * c_qg + row) * c_qh + col_elem
q_v = buffer_ops.buffer_load(
q_rsrc,
q_off_elem // fx.Int32(2),
vec_width=4,
dtype=T.i32,
)
q_lds_byte = row * fx.Int32(HEAD_SIZE * 2) + col_elem * fx.Int32(2)
vector.store(
q_v,
q_lds_i32,
[arith.index_cast(T.index, q_lds_byte // fx.Int32(4))],
)
gpu.barrier()
# ===== STEP C: Pre-load Q operands for QK_K_CHUNKS = 4 K-chunks ===
# Wide-K MFMA: K_per_chunk = 32, K_per_lane = 32/4 = 8 bf16 per chunk.
# Lane t holds 8 bf16 at row=mfma_row, K-cols
# chunk*32 + mfma_col_grp*8 + 0..7
# Byte addr = mfma_row * HEAD_SIZE*2 + (chunk*32 + col_grp*8)*2
# = mfma_row * 256 + chunk*64 + col_grp*16
# i64 idx = mfma_row * 32 + chunk*8 + col_grp*2 (each i64 = 4 bf16)
# Load 16 bytes (= 8 bf16 = 2 i64) per chunk via vec(2, i64).
q_chunks = []
for chk in range_constexpr(QK_K_CHUNKS):
q_idx_i64 = (
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
+ fx.Int32(chk * 8)
+ mfma_col_grp * fx.Int32(2)
)
qv = vector.load_op(
T.vec(2, T.i64),
q_lds_i64,
[arith.index_cast(T.index, q_idx_i64)],
)
q_chunks.append(vector.bitcast(T.vec(8, T.bf16), qv))
# ===== STEP D: Online softmax + PV state =========================
running_max = NEG_INF
running_sum = ZERO_F
zero_v4 = arith.constant_vector(0.0, T.f32x4)
acc_pv = [zero_v4 for _ in range(PV_N_CHUNKS)]
# ===== STEP E: Sequence-len + partition base ====================
seq_len = buffer_ops.buffer_load(sl_rsrc, seq, vec_width=1, dtype=T.i32)
# ── Bounded num_partitions + internal looping ───────────────────
# With TGPP > 1 this CTA owns a contiguous slice of length
# ``PARTITION_EXTENT_TOKENS`` that is iterated as ``TGPP`` groups
# of 16 K-tiles each. ``partition_start`` is recomputed per
# outer ``tg`` iteration below; the FA-2 online-softmax state
# (running_max, running_sum, acc_pv) accumulates across all
# ``TGPP * 16`` tiles for this CTA — semantically identical to
# processing one large 16*TGPP-tile partition.
partition_base = part * fx.Int32(PARTITION_EXTENT_TOKENS)
# Per-K-tile dequant lane assignment:
# 16 tokens × 64 packed bytes per token = 1024 bytes
# 64 lanes × 16 bytes/lane = 1024 ✓
# Lane t: token = t / 4, sub-chunk = t % 4 (0..3) → 16 bytes contiguous
tok_in_tile = lane >> fx.Int32(2)
chunk_in_tok = lane & fx.Int32(3)
# ``slot`` (= absolute slot index within the cache block, range
# 0.._BS-1) is recomputed per tile inside the K-tile loop below
# so it reflects ``tile_in_block * TILE_SIZE + tok_in_tile``.
c_meta_u16_per_kvh = fx.Int32(NUM_SOA_FIELDS * _BS)
c_knorm_off_u16 = fx.Int32(SOA_K_NORM * _BS)
c_vscale_off_u16 = fx.Int32(SOA_V_SCALE * _BS)
c_vzero_off_u16 = fx.Int32(SOA_V_ZERO * _BS)
# ===== STEP F: K-tile loop =======================================
# With kv_block_size > TILE_SIZE there are _TILES_PER_BLOCK tiles
# per cache block. Tiles within the same block share a block-table
# entry but address different slot ranges within the block.
#
# ── Per-tile block-table OOB redirect ────────────────────────────
# The K-tile loop is unrolled 16 times so every CTA issues 16
# block-table reads regardless of (partition, seq_len). When a
# partition extends past ``seq_len`` (e.g. num_partitions=2 for
# a 256-token sequence — partition 1 covers tokens 256..511, all
# masked out) the per-tile bt offset can land beyond the bt
# allocation. With ``max_size=True`` the descriptor advertises
# 4 GB so the HW returns whatever pre-existing HBM bytes live at
# that offset; the resulting garbage ``phys_block`` is multiplied
# by ``stride_cache_block`` and the subsequent kv_cache read
# jumps into pages that may be unmapped (→ "Memory access fault
# by GPU node") or may decode as NaN bytes (→ NaN segm_out).
# Allocator-pattern dependent, but reproduces deterministically
# at large B (e.g. B=64 seq=256 in the test sweep produces 8 192
# NaN entries even on canonical QG=8).
#
# Fix: when the tile starts at or past ``seq_len`` (so its
# qk_acc is going to be killed by the per-token
# ``kv_tok < seq_len`` mask anyway), redirect the bt read to
# ``bt[seq, 0]`` — always in bounds. The redundant phys_block
# decode + kv_cache read is wasted work, but correctness is
# preserved without changing the kernel's iteration count.
# ===== Runtime-adaptive scf.ForOp w/ iter_args =========
# Runtime-adaptive outer loop: trip count derives from
# actual seq_len, NOT TGPP_max. At cudagraph-capture warmup
# (seq_len=1) only 1 iteration runs; at 32K production
# decode all TGPP iterations run. Kernel binary stays small
# (single body, looped at runtime) — capture time matches
# the legacy 16-tile-only kernel rather than scaling 4x.
#
# FA-2 state (running_max, running_sum, acc_pv[8]) threads
# through scf iter_args; the body reads them at top, runs
# the unchanged 16-tile inner body, and yields the new
# state at bottom. After the loop, results are pulled out
# of for_op.results back into the local Python names so the
# downstream STEP G (output) is unmodified.
c_kcb = fx.Int32(KV_COMPUTE_BLOCK)
c_tgpp = fx.Int32(TGPP)
c_zero_i32 = fx.Int32(0)
c_one_i32 = fx.Int32(1)
# remaining = seq_len - partition_base (signed, may be <=0)
remaining = seq_len - partition_base
in_range = remaining > c_zero_i32
# trip_raw = ceil(remaining / KV_COMPUTE_BLOCK)
# (when in_range is false, the divisor branch produces
# garbage that the select below overrides with zero, so we
# don't pre-clamp)
trip_raw = (remaining + c_kcb - c_one_i32) // c_kcb
trip_clamped = (trip_raw > c_tgpp).select(c_tgpp, trip_raw)
trip_or_zero = in_range.select(trip_clamped, c_zero_i32)
c_zero_idx = arith.constant(0, index=True)
c_one_idx = arith.constant(1, index=True)
trip_idx = arith.index_cast(
T.index,
trip_or_zero.ir_value()
if hasattr(trip_or_zero, "ir_value")
else trip_or_zero,
)
# Initial iter_args list: FA-2 accumulator state.
# Order: running_max, running_sum, acc_pv[0..PV_N_CHUNKS-1].
# NOTE: scf.ForOp requires ir.Value (with .type). The DSL
# constants ZERO_F / ONE_F are fx.Float32 (Numeric) wrappers,
# not raw ir.Value, so we unwrap via .ir_value() before
# passing. _ival() is defined once near the top of the kernel body.
_init_iter = [
_ival(running_max),
_ival(running_sum),
*[_ival(p) for p in acc_pv],
]
_for_op = _scf.ForOp(
c_zero_idx,
trip_idx,
c_one_idx,
_init_iter,
)
_for_ip = ir.InsertionPoint(_for_op.body)
_for_ip.__enter__()
try:
tg_idx = _for_op.induction_variable
tg_i32 = fx.Int32(arith.index_cast(T.i32, tg_idx))
partition_start = partition_base + tg_i32 * c_kcb
bt_seq_base = seq * c_bt + (partition_start // fx.Int32(_BS))
running_max = _for_op.inner_iter_args[0]
running_sum = _for_op.inner_iter_args[1]
acc_pv = list(_for_op.inner_iter_args[2:])
for n_tile in range_constexpr(16):
block_in_part = n_tile // _TILES_PER_BLOCK
tile_in_block = n_tile % _TILES_PER_BLOCK
tile_start_tok = partition_start + fx.Int32(n_tile * TILE_SIZE)
tile_in_seq = tile_start_tok < seq_len
bt_off = bt_seq_base + fx.Int32(block_in_part)
bt_off_safe = tile_in_seq.select(bt_off, seq * c_bt)
phys_block = buffer_ops.buffer_load(
bt_rsrc,
bt_off_safe,
vec_width=1,
dtype=T.i32,
)
block_base = phys_block * c_block
data_region = block_base
meta_region = block_base + c_meta_off
# ``slot`` is the absolute slot index within the cache block
# (range 0.._BS-1). For BS=16 it equals tok_in_tile; for BS=32
# it equals tile_in_block * 16 + tok_in_tile. Used both for
# the K/V data region and SoA metadata region addressing.
slot = fx.Int32(tile_in_block * TILE_SIZE) + tok_in_tile
data_bases_byte = (
data_region
+ slot * (c_kv_heads * c_data_per_slot)
+ kv_h * c_data_per_slot
)
k_byte = data_bases_byte + chunk_in_tok * fx.Int32(16)
k_packed = buffer_ops.buffer_load(
kv_rsrc,
k_byte // fx.Int32(4),
vec_width=4,
dtype=T.i32,
)
# ---- HOISTED: issue V data + meta HBM loads early ---------
# These are async; their s_waitcnt is pushed by the compiler
# past the K dequant + QK MFMA + softmax block, hiding most
# of the V HBM latency behind compute.
v_byte = data_bases_byte + c_keydata + chunk_in_tok * fx.Int32(16)
v_packed = buffer_ops.buffer_load(
kv_rsrc,
v_byte // fx.Int32(4),
vec_width=4,
dtype=T.i32,
)
vscale_u16 = (
meta_region // fx.Int32(2)
+ kv_h * c_meta_u16_per_kvh
+ c_vscale_off_u16
+ slot
)
vzero_u16 = (
meta_region // fx.Int32(2)
+ kv_h * c_meta_u16_per_kvh
+ c_vzero_off_u16
+ slot
)
vscale_raw = buffer_ops.buffer_load(
kv_rsrc,
vscale_u16,
vec_width=1,
dtype=T.i16,
)
vzero_raw = buffer_ops.buffer_load(
kv_rsrc,
vzero_u16,
vec_width=1,
dtype=T.i16,
)
knorm_u16 = (
meta_region // fx.Int32(2)
+ kv_h * c_meta_u16_per_kvh
+ c_knorm_off_u16
+ slot
)
knorm_raw = buffer_ops.buffer_load(
kv_rsrc,
knorm_u16,
vec_width=1,
dtype=T.i16,
)
knorm_f16 = arith.bitcast(T.f16, knorm_raw)
knorm_f32 = arith.extf(T.f32, knorm_f16)
# K dequant → LDS [token, head_dim] (natural)
# Lane writes 32 bf16 (= 4×8) for token=tok_in_tile,
# head_dims chunk_in_tok*32..+31 (4 sub-chunks of 8).
tok_kreg = tok_in_tile * fx.Int32(HEAD_SIZE * 2 // 8)
chunk_kreg = tok_kreg + chunk_in_tok * fx.Int32(8)
for w in range_constexpr(4):
word_i32 = vector.extract(k_packed, static_position=[w])
bf16_elems = []
for n in range_constexpr(8):
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
nibble_idx = arith.index_cast(T.index, nibble)
cent_f32 = cent_lds.load([nibble_idx])
elem_bf16 = arith.trunc_f(T.bf16, cent_f32 * knorm_f32)
bf16_elems.append(elem_bf16)
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
vector.store(
v_i64,
kv_lds_i64,
[arith.index_cast(T.index, chunk_kreg + fx.Int32(w * 2))],
)
gpu.barrier()
# QK MFMA (CDNA4 wide-K): A=K[token, head_dim], B=Q (= Q^T).
# K read: lane t = K[token=mfma_row, head_dim=chunk*32+col_grp*8..+7]
# Same i64×2 indexing as Q.
qk_acc = zero_v4
for chk in range_constexpr(QK_K_CHUNKS):
k_idx_i64 = (
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
+ fx.Int32(chk * 8)
+ mfma_col_grp * fx.Int32(2)
)
kv_load = vector.load_op(
T.vec(2, T.i64),
kv_lds_i64,
[arith.index_cast(T.index, k_idx_i64)],
)
k_op = vector.bitcast(T.vec(8, T.bf16), kv_load)
qk_acc = rocdl.mfma_f32_16x16x32_bf16(
T.f32x4, [k_op, q_chunks[chk], qk_acc, 0, 0, 0]
)
# qk_acc layout: lane t holds C[token=(t/16)*4..+3, query=t%16]
# 4 fp32/lane = 4 different tokens at SAME query_row.
# Scale + mask out-of-context tokens.
qk_acc = _vsplat_mul(qk_acc, QK_SCALE)
for elem in range_constexpr(4):
kv_tok = (
partition_start
+ fx.Int32(n_tile * TILE_SIZE)
+ mfma_col_grp * fx.Int32(4)
+ fx.Int32(elem)
)
in_b = kv_tok < seq_len
v = vector.extract(qk_acc, static_position=[elem])
qk_acc = vector.insert(
in_b.select(v, NEG_INF),
qk_acc,
static_position=[elem],
dynamic_position=[],
)
# FA2 online softmax: per-query-row reduce.
# Per-row max: max over 4 fp32 in lane, then xor-shuffle 16, 32
# (across the 4 col_grps that share same mfma_row).
local_max = vector.reduction(T.f32, "maxnumf", qk_acc)
r1 = local_max.shuffle_xor(fx.Int32(16), c_w)
local_max = local_max.maximumf(r1)
r2 = local_max.shuffle_xor(fx.Int32(32), c_w)
tile_max = local_max.maximumf(r2)
new_max = running_max.maximumf(tile_max)
max_diff = running_max - new_max
safe_diff = (running_max > NEG_INF).select(max_diff, ZERO_F)
scale = (safe_diff * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
running_sum = running_sum * scale
for h in range_constexpr(PV_N_CHUNKS):
acc_pv[h] = _vsplat_mul(acc_pv[h], scale)
running_max = new_max
# Compute probs: p = exp((qk - new_max) * LOG2E)
tile_sum = ZERO_F
for elem in range_constexpr(4):
s = vector.extract(qk_acc, static_position=[elem])
d = s - new_max
d = (new_max > NEG_INF).select(d, NEG_INF)
p = (d * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
tile_sum = tile_sum + p
qk_acc = vector.insert(
p, qk_acc, static_position=[elem], dynamic_position=[]
)
ts1 = tile_sum.shuffle_xor(fx.Int32(16), c_w)
tile_sum = tile_sum + ts1
ts2 = tile_sum.shuffle_xor(fx.Int32(32), c_w)
tile_sum = tile_sum + ts2
running_sum = running_sum + tile_sum
# ---- V dequant → LDS (V/scale/zero raw loads were hoisted) -
# v_packed, vscale_raw, vzero_raw are already in flight from
# the top of this iteration; only the f32 conversion stays
# here so the actual consume site is still close to the use.
vscale_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vscale_raw))
vzero_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vzero_raw))
if USE_HW_TR:
# V dequant → LDS [token][head_dim] (ROW-MAJOR, no transpose).
# Each lane writes 32 contiguous bf16 (one token, head_dims
# chunk_in_tok*32..+31) as 4× ds_write_b128 = 4× vec(2,i64).
# Replaces 32× ds_write_b16 of the legacy transposed path.
v_lds_elem_base = tok_in_tile * fx.Int32(
HEAD_SIZE
) + chunk_in_tok * fx.Int32(32)
for w in range_constexpr(4):
word_i32 = vector.extract(v_packed, static_position=[w])
bf16_elems = []
for n in range_constexpr(8):
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
nibble_f32 = arith.sitofp(T.f32, nibble)
elem_f32 = nibble_f32 * vscale_f32 + vzero_f32
elem_bf16 = arith.trunc_f(T.bf16, elem_f32)
bf16_elems.append(elem_bf16)
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
v_lds_i64_idx = (v_lds_elem_base + fx.Int32(w * 8)) // fx.Int32(
4
)
vector.store(
v_i64,
kv_lds_i64,
[arith.index_cast(T.index, v_lds_i64_idx)],
)
# ── HW V transpose: cross-lane LDS sync ─────────────────────
# ds_read_tr16_b64 below introduces a cross-lane LDS read:
# consumer lane t reads bytes written by source lanes
# 0,4,8,12 (etc.). The compiler's automatic waitcnt insertion
# is per-lane and has no awareness of the HW transpose's
# cross-lane forwarding, so it can sink the next iteration's
# ds_read ahead of these ds_write_b128 ops in the schedule.
#
# Two-part fence:
# 1. sched_barrier(0): MachineScheduler reorder barrier
# (mask=0 → no instruction class may cross). Pure hint,
# generates no runtime instruction. Use the simple
# sched_barrier (NOT sched_group_barrier, which is an
# IGLP marker that requires partner barriers and
# corrupts the schedule when used in isolation).
# 2. s_waitcnt lgkmcnt=0: runtime drain of the LDS write
# queue. vmcnt/expcnt left at no-wait so we don't
# stall on speculatively-issued HBM loads (which can
# include OOB-but-masked buffer_load reads from the
# next K-tile's hoisted V prefetch).
#
# The SW path below does NOT need this because each lane
# only reads cells it itself wrote (no cross-lane dep), and
# the per-lane same-address waitcnt the compiler emits is
# correct.
#
# Without this fence the reordered ds_read produces
# wrong results on high-partition-count configs (large
# padded num_partitions); low-partition configs are
# unaffected.
rocdl.sched_barrier(0)
# encode_waitcnt(vmcnt=63, expcnt=7, lgkmcnt=0)
# = 0xF | (7<<4) | (0<<8) | (3<<14) = 0xC07F
rocdl.s_waitcnt(0xC07F)
else:
# Legacy transposed V_LDS path: 32 ds_write_b16 per lane.
for w in range_constexpr(4):
word_i32 = vector.extract(v_packed, static_position=[w])
for n in range_constexpr(8):
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
nibble_f32 = arith.sitofp(T.f32, nibble)
elem_f32 = nibble_f32 * vscale_f32 + vzero_f32
elem_bf16 = arith.trunc_f(T.bf16, elem_f32)
elem_i16 = arith.bitcast(T.i16, elem_bf16)
head_dim = chunk_in_tok * fx.Int32(32) + fx.Int32(w * 8 + n)
v_idx_i16 = head_dim * fx.Int32(TILE_SIZE) + tok_in_tile
v_vec = vector.from_elements(T.vec(1, T.i16), [elem_i16])
vector.store(
v_vec,
kv_lds_i16,
[arith.index_cast(T.index, v_idx_i16)],
)
gpu.barrier()
# ---- PV MFMA: A=V[head_dim, token], B=P (=qk_acc bf16) -----
# P operand B layout matches qk_acc: lane t holds 4 bf16 at
# K=token=(t/16)*4..+3, N=query=t%16. Just trunc_f to bf16.
p_bf16 = arith.trunc_f(T.vec(4, T.bf16), qk_acc)
p_op = vector.bitcast(T.vec(4, T.i16), p_bf16)
if USE_HW_TR:
# HW-transpose PV path: V_lds is row-major V[token][head_dim].
# MFMA A operand layout: lane t holds 4 bf16 at
# M=head_dim=mfma_row + h*16, K=token=(t/16)*4..+3
# ds_read_tr16_b64 (4-element 16-bit transpose, per-16-lane block):
# result[lane=t, elem=e] = Input[source_lane=e*4 + (t%16)//4,
# col=t%4]
# Per-lane address: token_idx = lane // 4 (covers 0..15 across
# all four 16-lane MFMA blocks), hd_sub = (lane % 4)*4 selects
# the 4-element column window inside the h*16 chunk.
# Total LDS byte offset = kv_off + token_idx*HEAD_SIZE*2
# + (h*16 + hd_sub)*2
token_idx = lane >> fx.Int32(2)
hd_sub = (lane & fx.Int32(3)) * fx.Int32(4)
v_lane_byte = (
fx.Int32(kv_off)
+ token_idx * fx.Int32(HEAD_SIZE * 2)
+ hd_sub * fx.Int32(2)
)
for h in range_constexpr(PV_N_CHUNKS):
v_byte_off = v_lane_byte + fx.Int32(h * 32)
v_byte_i64 = fx.Int64(v_byte_off)
v_ptr = buffer_ops.create_llvm_ptr(
v_byte_i64,
address_space=3,
)
v_op_raw = rocdl.ds_read_tr16_b64(
T.vec(4, T.i16),
v_ptr,
).result
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
T.f32x4, [v_op_raw, p_op, acc_pv[h], 0, 0, 0]
)
else:
# Legacy transposed-LDS PV: 1 ds_read_b64 per chunk.
# Byte addr = (mfma_row + h*16) * TILE_SIZE * 2 + (col_grp*4) * 2
# = (mfma_row + h*16) * 32 + col_grp*8
# i64 idx = (mfma_row + h*16) * 4 + col_grp
for h in range_constexpr(PV_N_CHUNKS):
v_idx_i64 = (mfma_row + fx.Int32(h * 16)) * fx.Int32(
4
) + mfma_col_grp
kv_load = vector.load_op(
T.vec(1, T.i64),
kv_lds_i64,
[arith.index_cast(T.index, v_idx_i64)],
)
v_op = vector.bitcast(T.vec(4, T.i16), kv_load)
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
T.f32x4, [v_op, p_op, acc_pv[h], 0, 0, 0]
)
_scf.YieldOp(
[
_ival(running_max),
_ival(running_sum),
*[_ival(p) for p in acc_pv],
]
)
finally:
_for_ip.__exit__(None, None, None)
# Pull final accumulator values out of the for_op results.
running_max = _for_op.results[0]
running_sum = _for_op.results[1]
acc_pv = list(_for_op.results[2:])
# ===== STEP G: Output ===========================================
# acc_pv[h] layout: lane t holds 4 fp32 at M=head_dim=(t/16)*4..+3 + h*16,
# N=query_row=t%16.
# 4 fp32/lane = 4 contiguous head_dim values for ONE query_row.
safe_sum = (running_sum > ZERO_F).select(running_sum, ONE_F)
rcp = ONE_F / safe_sum
c_os = fx.Int32(_stride_out_seq)
c_oh = fx.Int32(_stride_out_head)
c_op_ = fx.Int32(_stride_out_part)
out_base = seq * c_os + kv_h * c_oh + part * c_op_
# Gate all global stores by mfma_row < QG. For QG=16 the predicate is
# tautological (mfma_row ∈ [0,15]); for QG=8 lanes 8..15 (whose MFMA
# outputs are computational waste) skip the stores entirely.
valid_row_pred = arith.cmpi(
arith.CmpIPredicate.ult,
mfma_row.ir_value() if hasattr(mfma_row, "ir_value") else mfma_row,
arith.constant(QG, type=T.i32),
)
_if = _scf.IfOp(valid_row_pred)
with ir.InsertionPoint(_if.then_block):
for h in range_constexpr(PV_N_CHUNKS):
pv_norm = _vsplat_mul(acc_pv[h], rcp)
pv_bf16 = arith.trunc_f(T.vec(4, T.bf16), pv_norm)
pv_i32x2 = vector.bitcast(T.vec(2, T.i32), pv_bf16)
head_dim_start = fx.Int32(h * 16) + mfma_col_grp * fx.Int32(4)
out_off_elem = (
out_base + mfma_row * fx.Int32(HEAD_SIZE) + head_dim_start
)
buffer_ops.buffer_store(
pv_i32x2,
out_rsrc,
out_off_elem * fx.Int32(2),
offset_is_bytes=True,
)
c_npq = fx.Int32(num_partitions * QG)
ml_off = (
seq * fx.Int32(_stride_ml_seq) + kv_h * c_npq + part * c_qg + mfma_row
)
es_off = (
seq * fx.Int32(_stride_es_seq) + kv_h * c_npq + part * c_qg + mfma_row
)
buffer_ops.buffer_store(running_max, ml_rsrc, ml_off)
buffer_ops.buffer_store(running_sum, es_rsrc, es_off)
_scf.YieldOp([])
return tq_decode_kernel
@@ -0,0 +1,875 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# TurboQuant decode GQA-6 sibling (FlyDSL) — MI355X / gfx950 / CDNA4
#
# Decode-only paged attention over a TurboQuant SoA KV cache.
# Targets MiniMax-M2.5 class models: HEAD_SIZE=128, GQA group=6, MSE_BITS=4 K,
# VQB=4 V, N_CENTROIDS=16, BLOCK_SIZE=16, partitioned mode.
#
# === Sibling-file rationale ===
# Structurally identical to tq_decode.py except:
# * QUERY_GROUP_SIZE default = 6 (was 8)
# * Build-time assert allows query_group_size = 6 only (Qwen sizes go to
# the canonical tq_decode kernel — NOT this file)
# * QG_LOAD_ITERS = (QG + 3) // 4 (round-up; 2 iters for QG=6 loads
# 8 LDS rows of which only 6 carry real Q; rows 6..7 are zero-padded
# by buffer_load OOB protection — gated out by the same mfma_row < QG
# predicate the canonical kernel already uses for QG=8)
#
# All cross-lane operations (xor_shuffle softmax reductions, ds_read_tr16_b64
# HW V transpose) are mfma_row-isolated and thus already GQA-agnostic. The
# kernel is functionally a direct copy of tq_decode.py with the QG knob
# turned to 6 — kept as a sibling per FlyDSL convention to preserve the
# canonical kernel's invariants for its production callers (Qwen2.5-72B,
# Qwen3-32B).
#
# === Design ===
# * 1 wave (64 lanes) per CTA — simpler than 4-warp; trades parallelism for
# correctness/debug-ability. A later revision will fan out to 4 warps.
# * Flash-Attention-2 online softmax: rescale running PV/sum per K-tile,
# no QK score storage across tiles, no P-LDS round-trip (qk_acc layout
# matches PV B-operand directly).
# * Centroids loaded once into LDS at kernel start (LDS-resident LUT).
# * K dequant → LDS [token, head_dim] (natural layout)
# * V dequant → LDS [head_dim, token] (TRANSPOSED — required so PV MFMA's
# A=V operand can be loaded as 4-bf16 contiguous via 1 ds_read_b64).
#
# MFMA layouts used:
# QK: mfma(A=K, B=Q, C=qk_acc) → C[m=token, n=query], 4 fp32/lane
# = 4 tokens for SAME query → softmax via
# xor_shuffle(16),(32) reduces per-row
# PV: mfma(A=V_T, B=P, C=acc_pv) → C[m=head_dim, n=query], 4 fp32/lane
# = 4 contiguous head_dim cols for SAME
# query → 1 buffer_store of 8 bytes
# per chunk per lane
from __future__ import annotations
import flydsl.compiler as flyc
import flydsl.expr as fx
from flydsl._mlir import ir
from flydsl._mlir.dialects import scf as _scf
from flydsl.expr import (
arith,
buffer_ops,
const_expr, # noqa: F401 kept for kernel-DSL API parity
gpu,
range_constexpr,
rocdl,
vector,
)
from flydsl.expr.typing import Int32, T # noqa: F401 Int32 via fx.Int32
from flydsl.runtime.device import get_rocm_arch as get_hip_arch
from flydsl.utils.smem_allocator import SmemAllocator, SmemPtr
# === Constants (MiniMax-M2.5-class TQ decode profile) =======================
HEAD_SIZE = 128
# default; overridable via build_tq_decode_gqa6_module(kv_block_size=...)
KV_BLOCK_SIZE = 16
# MFMA tile = 16 tokens (do not change without re-deriving MFMA shapes)
TILE_SIZE = 16
N_CENTROIDS = 16
QUERY_GROUP_SIZE = 6 # GQA-6 default for MiniMax-M2.5
WARP_SIZE = 64
NUM_WARPS = 1
BLOCK_THREADS = NUM_WARPS * WARP_SIZE
KV_COMPUTE_BLOCK = 256 # 16 K-tiles × 16 tokens
# TQ SoA layout
NUM_SOA_FIELDS = 3
SOA_K_NORM = 0
SOA_V_SCALE = 1
SOA_V_ZERO = 2
KEY_DATA_BYTES = HEAD_SIZE // 2
VAL_DATA_BYTES = HEAD_SIZE // 2
DATA_BYTES_PER_SLOT = KEY_DATA_BYTES + VAL_DATA_BYTES
# MFMA
MFMA_M = MFMA_N = 16
MFMA_K_BF16_QK = 32 # CDNA4 wide-K (mfma_f32_16x16x32_bf16)
MFMA_K_BF16_PV = 16 # PV's K = token tile = 16
QK_K_CHUNKS = HEAD_SIZE // MFMA_K_BF16_QK # 4
PV_N_CHUNKS = HEAD_SIZE // MFMA_N # 8
# LDS regions
# Q region uses the full 16-row footprint (4096 B) rather than a tight
# QG=6 sizing, keeping the LDS layout identical to the canonical kernel.
CENTROID_LDS_BYTES = N_CENTROIDS * 4 # 64
Q_LDS_BYTES = 16 * HEAD_SIZE * 2 # 4096
KV_TILE_LDS_BYTES = TILE_SIZE * HEAD_SIZE * 2 # 4096
LOG2E = 1.4426950408889634
NEG_INF_VAL = float("-inf")
def _vsplat_mul(vec, scalar):
s = scalar.ir_value() if hasattr(scalar, "ir_value") else scalar
return vec * vector.broadcast(T.f32x4, s)
allocator = None
def build_tq_decode_gqa6_module(
num_seqs: int,
num_kv_heads: int,
num_partitions: int,
max_blocks_per_seq: int = 512,
softmax_scale: float | None = None,
query_group_size: int = QUERY_GROUP_SIZE,
kv_block_size: int = KV_BLOCK_SIZE,
use_hw_v_transpose: bool = False,
tile_groups_per_partition: int = 1,
):
"""Build a TQ decode GQA-6 kernel module (MiniMax-M2.5).
``query_group_size`` (= num_query_heads // num_kv_heads) is fixed at 6
for this sibling. The MFMA's 16-row capacity is 6/16 = 37.5% used;
lanes 6..15 compute garbage and are gated out of all global-memory
writes via the same ``mfma_row < QG`` predicate the canonical kernel
uses for QG=8. Q load iterates ``(QG + 3) // 4`` = 2 chunks (loads
8 LDS rows of which rows 6..7 are zero-padded by the buffer-resource
OOB protection — they're never consumed because the gating lanes
don't write).
For GQA-8 or GQA-16 (Qwen models), use the canonical
``build_tq_decode_module`` from ``tq_decode.py`` instead.
``kv_block_size`` is the number of tokens per vLLM cache block. Must
be a multiple of TILE_SIZE (=16). Supported values: 16 and 32.
``use_hw_v_transpose`` (CDNA4 / gfx950 only) — see canonical kernel
docstring for details. Same fence pattern applies here.
"""
assert query_group_size == 6, (
f"build_tq_decode_gqa6_module is GQA-6 only; got "
f"query_group_size={query_group_size}. Use build_tq_decode_module "
f"from tq_decode.py for GQA-8 or GQA-16."
)
assert kv_block_size in (16, 32), (
f"kv_block_size must be 16 or 32; got {kv_block_size}"
)
assert kv_block_size % TILE_SIZE == 0
assert int(tile_groups_per_partition) >= 1, (
f"tile_groups_per_partition must be >= 1; got {tile_groups_per_partition}"
)
USE_HW_TR = bool(use_hw_v_transpose)
TGPP = int(tile_groups_per_partition)
PARTITION_EXTENT_TOKENS = TGPP * KV_COMPUTE_BLOCK
QG = int(query_group_size)
# Round-up: 2 iters for QG=6 → loads 8 LDS rows (rows 6..7 are
# zero-padded by buffer_load OOB protection; never consumed because
# mfma_row >= QG lanes are gated out of all global stores below).
QG_LOAD_ITERS = (QG + 3) // 4
OOB_OFFSET = 0x7FFFFFF0 # noqa: F841 ~2GB byte offset; > any plausible buffer
_BS = int(kv_block_size)
_TILES_PER_BLOCK = _BS // TILE_SIZE # 1 for BS=16, 2 for BS=32
global allocator
arch = get_hip_arch()
if softmax_scale is None:
softmax_scale = 1.0 / (HEAD_SIZE**0.5)
_qk_scale = float(softmax_scale)
# --- Strides ---
_Hq = num_kv_heads * QG
_stride_q_seq = _Hq * HEAD_SIZE
_stride_q_head = HEAD_SIZE
_stride_bt_seq = max_blocks_per_seq
_data_region_bytes = _BS * num_kv_heads * DATA_BYTES_PER_SLOT
_meta_region_bytes = num_kv_heads * NUM_SOA_FIELDS * _BS * 2
_stride_cache_block = _data_region_bytes + _meta_region_bytes
_meta_region_offset = _data_region_bytes
_stride_out_part = QG * HEAD_SIZE
_stride_out_head = num_partitions * QG * HEAD_SIZE
_stride_out_seq = num_kv_heads * num_partitions * QG * HEAD_SIZE
_stride_es_seq = num_kv_heads * num_partitions * QG
_stride_ml_seq = _stride_es_seq
# --- LDS layout ---
allocator = SmemAllocator(None, arch=arch, global_sym_name="tq_gqa6_smem")
centroid_off = 0
allocator.ptr = CENTROID_LDS_BYTES
q_off = allocator.ptr
allocator.ptr += Q_LDS_BYTES
kv_off = allocator.ptr
allocator.ptr += KV_TILE_LDS_BYTES
@flyc.kernel
def tq_decode_gqa6_kernel(
out_ptr: fx.Tensor,
exp_sums_ptr: fx.Tensor,
max_logits_ptr: fx.Tensor,
query_ptr: fx.Tensor,
kv_cache_ptr: fx.Tensor,
centroids_ptr: fx.Tensor,
block_tables_ptr: fx.Tensor,
seq_lens_ptr: fx.Tensor,
):
# ---- IDs ---------------------------------------------------------
tid = gpu.thread_idx.x
seq = gpu.block_idx.x
kv_h = gpu.block_idx.y
part = gpu.block_idx.z
lane = tid # 0..63
mfma_row = lane & fx.Int32(15) # = query_row (B's N) /
# token (A's M for QK)
mfma_col_grp = lane >> fx.Int32(4) # 0..3, K-group dim
# ---- Buffer resources -------------------------------------------
q_rsrc = buffer_ops.create_buffer_resource(query_ptr, max_size=True)
kv_rsrc = buffer_ops.create_buffer_resource(kv_cache_ptr, max_size=True)
bt_rsrc = buffer_ops.create_buffer_resource(block_tables_ptr, max_size=True)
sl_rsrc = buffer_ops.create_buffer_resource(seq_lens_ptr, max_size=True)
cent_rsrc = buffer_ops.create_buffer_resource(centroids_ptr, max_size=True)
out_rsrc = buffer_ops.create_buffer_resource(out_ptr, max_size=True)
es_rsrc = buffer_ops.create_buffer_resource(exp_sums_ptr, max_size=True)
ml_rsrc = buffer_ops.create_buffer_resource(max_logits_ptr, max_size=True)
# ---- LDS pointers -----------------------------------------------
base = allocator.get_base()
cent_lds = SmemPtr(base, centroid_off, T.f32, shape=(N_CENTROIDS,))
q_lds_i32 = SmemPtr(base, q_off, T.i32, shape=(Q_LDS_BYTES // 4,)).get()
q_lds_i64 = SmemPtr(base, q_off, T.i64, shape=(Q_LDS_BYTES // 8,)).get()
kv_lds_i32 = SmemPtr(base, kv_off, T.i32, shape=(KV_TILE_LDS_BYTES // 4,)).get() # noqa: F841
kv_lds_i64 = SmemPtr(base, kv_off, T.i64, shape=(KV_TILE_LDS_BYTES // 8,)).get()
kv_lds_i16 = SmemPtr(base, kv_off, T.i16, shape=(KV_TILE_LDS_BYTES // 2,)).get()
# ---- Constants ---------------------------------------------------
c_sq = fx.Int32(_stride_q_seq)
c_qh = fx.Int32(_stride_q_head)
c_qg = fx.Int32(QG)
c_bt = fx.Int32(_stride_bt_seq)
c_block = fx.Int32(_stride_cache_block)
c_meta_off = fx.Int32(_meta_region_offset)
c_data_per_slot = fx.Int32(DATA_BYTES_PER_SLOT)
c_kv_heads = fx.Int32(num_kv_heads)
c_keydata = fx.Int32(KEY_DATA_BYTES)
c_w = fx.Int32(WARP_SIZE)
NEG_INF = arith.constant(NEG_INF_VAL, type=T.f32)
ZERO_F = fx.Float32(0.0)
ONE_F = fx.Float32(1.0)
LOG2E_C = arith.constant(LOG2E, type=T.f32)
QK_SCALE = arith.constant(_qk_scale, type=T.f32)
# ===== STEP A: Load centroids → LDS (cooperative, race-safe) =====
c_idx_safe = lane & fx.Int32(N_CENTROIDS - 1)
c_val = buffer_ops.buffer_load(cent_rsrc, c_idx_safe, vec_width=1, dtype=T.f32)
cent_lds.store(c_val, [arith.index_cast(T.index, c_idx_safe)])
gpu.barrier()
# ===== STEP B: Load Q → row-major LDS [query_row, head_dim] ======
# Q layout in HBM: [N, Hq, D] bf16, slice = QG rows × D=128 cols × 2B.
# QG=6 → 1536 B real, 96 chunks; QG_LOAD_ITERS=2 issues 128 chunks
# (8 rows × 16 col_b) so lanes whose row ∈ {6, 7} address heads that
# do NOT exist in the Q tensor (Hq = num_kv_heads * 6).
#
# ── Per-lane Q-load OOB redirect ─────────────────────────────────
# Hq = num_kv_heads * QG, so head index = kv_h * QG + row. For QG=6
# the upper rows alias to (kv_h+1)'s real heads (still in-bounds
# for most kv_h), but for the *last* (seq, kv_h) the read tips
# past the Q tensor entirely. With ``max_size=True`` the buffer
# descriptor advertises 4 GB so the HW does no OOB clamping; the
# padding read then touches unmapped pages and the kernel dies
# with "Memory access fault by GPU node" the moment the
# surrounding allocator state leaves the next page unmapped
# (allocator-pattern-dependent → nondeterministic fault).
#
# Fix: redirect OOB lanes (``row >= QG``) to load Q[0,0,col_elem]
# — a byte offset that is always inside the actual Q tensor.
# The redundant load is wasted but harmless; the resulting LDS
# write to row ∈ {6, 7} is overwritten/ignored downstream because
# every global store is gated by the output-time
# ``mfma_row < QG`` predicate (the same gate canonical QG=8 uses
# for its garbage rows 8..15).
#
# NB: A ``mask=`` parameter on ``buffer_load`` substitutes the
# masked-out offset with 0x7FFFFFFF *bytes*; with ``max_size=True``
# this is still inside the 4 GB descriptor and so the HW happily
# dereferences ``q_base + 2 GB`` → another unmapped page → fault.
# The address-redirect below is the only safe option short of
# threading the true Q size through as a runtime kernel arg.
#
# The canonical Qwen kernel does not need this redirect because
# for QG ∈ {8, 16} the (kv_h, row) iteration covers exactly the
# real heads; QG=6 is the only case where
# ``QG_LOAD_ITERS * 4 != QG``.
# STEP B: load pre-rotated q_rot → Q_LDS. QG=6 over-reads
# (QG_LOAD_ITERS * 4 != QG), so rows >= QG redirect to a safe
# in-bounds offset to avoid faulting past the real Q rows.
for c in range_constexpr(QG_LOAD_ITERS):
row_chunk = lane + fx.Int32(c * WARP_SIZE) # 0..127
row = row_chunk >> fx.Int32(4) # 0..7
col_b = row_chunk & fx.Int32(15) # 0..15
col_elem = col_b * fx.Int32(8) # 0..120 in bf16
q_off_real = seq * c_sq + (kv_h * c_qg + row) * c_qh + col_elem
row_in_qg = row < fx.Int32(QG)
q_off_elem = row_in_qg.select(q_off_real, col_elem)
q_v = buffer_ops.buffer_load(
q_rsrc,
q_off_elem // fx.Int32(2),
vec_width=4,
dtype=T.i32,
)
q_lds_byte = row * fx.Int32(HEAD_SIZE * 2) + col_elem * fx.Int32(2)
vector.store(
q_v,
q_lds_i32,
[arith.index_cast(T.index, q_lds_byte // fx.Int32(4))],
)
gpu.barrier()
# ===== STEP C: Pre-load Q operands for QK_K_CHUNKS = 4 K-chunks ===
# Wide-K MFMA: K_per_chunk = 32, K_per_lane = 32/4 = 8 bf16 per chunk.
# Lane t holds 8 bf16 at row=mfma_row, K-cols
# chunk*32 + mfma_col_grp*8 + 0..7
# Byte addr = mfma_row * HEAD_SIZE*2 + (chunk*32 + col_grp*8)*2
# = mfma_row * 256 + chunk*64 + col_grp*16
# i64 idx = mfma_row * 32 + chunk*8 + col_grp*2 (each i64 = 4 bf16)
# Load 16 bytes (= 8 bf16 = 2 i64) per chunk via vec(2, i64).
q_chunks = []
for chk in range_constexpr(QK_K_CHUNKS):
q_idx_i64 = (
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
+ fx.Int32(chk * 8)
+ mfma_col_grp * fx.Int32(2)
)
qv = vector.load_op(
T.vec(2, T.i64),
q_lds_i64,
[arith.index_cast(T.index, q_idx_i64)],
)
q_chunks.append(vector.bitcast(T.vec(8, T.bf16), qv))
# ===== STEP D: Online softmax + PV state =========================
running_max = NEG_INF
running_sum = ZERO_F
zero_v4 = arith.constant_vector(0.0, T.f32x4)
acc_pv = [zero_v4 for _ in range(PV_N_CHUNKS)]
# ===== STEP E: Sequence-len + partition base ====================
seq_len = buffer_ops.buffer_load(sl_rsrc, seq, vec_width=1, dtype=T.i32)
# ── Bounded num_partitions + internal looping ───────────────────
# Mirrors the canonical tq_decode kernel: each CTA owns
# ``PARTITION_EXTENT_TOKENS = TGPP * KV_COMPUTE_BLOCK`` contiguous
# tokens of the seq, processed as TGPP groups of 16 K-tiles each.
# FA-2 online-softmax state accumulates across all TGPP * 16 tiles.
# With TGPP=1 this is bit-identical to the legacy 16-tile body.
partition_base = part * fx.Int32(PARTITION_EXTENT_TOKENS)
# Per-K-tile dequant lane assignment:
# 16 tokens × 64 packed bytes per token = 1024 bytes
# 64 lanes × 16 bytes/lane = 1024 ✓
# Lane t: token = t / 4, sub-chunk = t % 4 (0..3) → 16 bytes contiguous
tok_in_tile = lane >> fx.Int32(2)
chunk_in_tok = lane & fx.Int32(3)
# ``slot`` (= absolute slot index within the cache block, range
# 0.._BS-1) is recomputed per tile inside the K-tile loop below
# so it reflects ``tile_in_block * TILE_SIZE + tok_in_tile``.
c_meta_u16_per_kvh = fx.Int32(NUM_SOA_FIELDS * _BS)
c_knorm_off_u16 = fx.Int32(SOA_K_NORM * _BS)
c_vscale_off_u16 = fx.Int32(SOA_V_SCALE * _BS)
c_vzero_off_u16 = fx.Int32(SOA_V_ZERO * _BS)
# ===== STEP F: K-tile loop =======================================
# With kv_block_size > TILE_SIZE there are _TILES_PER_BLOCK tiles
# per cache block. Tiles within the same block share a block-table
# entry but address different slot ranges within the block.
#
# ── Per-tile block-table OOB redirect ────────────────────────────
# The K-tile loop is unrolled 16 times so every CTA issues 16
# block-table reads regardless of (partition, seq_len). When a
# partition extends past ``seq_len`` (e.g. num_partitions=2 for a
# 256-token sequence — partition 1 covers tokens 256..511, all
# masked out) the per-tile bt offset can land beyond the bt
# allocation. With ``max_size=True`` the descriptor is 4 GB so
# the HW returns whatever pre-existing HBM bytes live at that
# offset; the resulting garbage ``phys_block`` is multiplied by
# ``stride_cache_block`` and the subsequent kv_cache read jumps
# into unmapped pages, killing the whole launch with a
# "Memory access fault by GPU node" exactly as the OOB Q load
# used to (same allocator-pattern-dependent fault; B=4 makes
# it deterministic by adding more colliding CTAs).
#
# Fix: when the tile starts at or past ``seq_len`` (so its
# qk_acc is going to be killed by the per-token
# ``kv_tok < seq_len`` mask anyway), redirect the bt read to
# ``bt[seq, 0]`` — always in bounds. The redundant phys_block
# decode + kv_cache read is wasted work, but correctness is
# preserved without changing the kernel's iteration count.
#
# The canonical Qwen kernel reads the same tiles and is also
# exposed to this issue in principle; in practice it surfaces on
# GQA-6 first because the surrounding allocator state differs
# (extra Q-tail allocation) and shifts the page maps so the bt
# overflow lands on unmapped pages. Applying the same redirect to
# the canonical kernel is a possible follow-up.
# ===== Runtime-adaptive scf.ForOp w/ iter_args =========
# Runtime-adaptive outer loop: trip count derives from
# actual seq_len, NOT TGPP_max. At cudagraph-capture warmup
# (seq_len=1) only 1 iteration runs; at 32K production
# decode all TGPP iterations run. Kernel binary stays small
# (single body, looped at runtime) — capture time matches
# the legacy 16-tile-only kernel rather than scaling 4x.
#
# FA-2 state (running_max, running_sum, acc_pv[8]) threads
# through scf iter_args; the body reads them at top, runs
# the unchanged 16-tile inner body, and yields the new
# state at bottom. After the loop, results are pulled out
# of for_op.results back into the local Python names so the
# downstream STEP G (output) is unmodified.
c_kcb = fx.Int32(KV_COMPUTE_BLOCK)
c_tgpp = fx.Int32(TGPP)
c_zero_i32 = fx.Int32(0)
c_one_i32 = fx.Int32(1)
# remaining = seq_len - partition_base (signed, may be <=0)
remaining = seq_len - partition_base
in_range = remaining > c_zero_i32
# trip_raw = ceil(remaining / KV_COMPUTE_BLOCK)
# (when in_range is false, the divisor branch produces
# garbage that the select below overrides with zero, so we
# don't pre-clamp)
trip_raw = (remaining + c_kcb - c_one_i32) // c_kcb
trip_clamped = (trip_raw > c_tgpp).select(c_tgpp, trip_raw)
trip_or_zero = in_range.select(trip_clamped, c_zero_i32)
c_zero_idx = arith.constant(0, index=True)
c_one_idx = arith.constant(1, index=True)
trip_idx = arith.index_cast(
T.index,
trip_or_zero.ir_value()
if hasattr(trip_or_zero, "ir_value")
else trip_or_zero,
)
# Initial iter_args list: FA-2 accumulator state.
# Order: running_max, running_sum, acc_pv[0..PV_N_CHUNKS-1].
# NOTE: scf.ForOp requires ir.Value (with .type). The DSL
# constants ZERO_F / ONE_F are fx.Float32 (Numeric) wrappers,
# not raw ir.Value, so we unwrap via .ir_value() before
# passing.
def _ival(v):
return v.ir_value() if hasattr(v, "ir_value") else v
_init_iter = [
_ival(running_max),
_ival(running_sum),
*[_ival(p) for p in acc_pv],
]
_for_op = _scf.ForOp(
c_zero_idx,
trip_idx,
c_one_idx,
_init_iter,
)
_for_ip = ir.InsertionPoint(_for_op.body)
_for_ip.__enter__()
try:
tg_idx = _for_op.induction_variable
tg_i32 = fx.Int32(arith.index_cast(T.i32, tg_idx))
partition_start = partition_base + tg_i32 * c_kcb
bt_seq_base = seq * c_bt + (partition_start // fx.Int32(_BS))
running_max = _for_op.inner_iter_args[0]
running_sum = _for_op.inner_iter_args[1]
acc_pv = list(_for_op.inner_iter_args[2:])
for n_tile in range_constexpr(16):
block_in_part = n_tile // _TILES_PER_BLOCK
tile_in_block = n_tile % _TILES_PER_BLOCK
tile_start_tok = partition_start + fx.Int32(n_tile * TILE_SIZE)
tile_in_seq = tile_start_tok < seq_len
bt_off = bt_seq_base + fx.Int32(block_in_part)
bt_off_safe = tile_in_seq.select(bt_off, seq * c_bt)
phys_block = buffer_ops.buffer_load(
bt_rsrc,
bt_off_safe,
vec_width=1,
dtype=T.i32,
)
block_base = phys_block * c_block
data_region = block_base
meta_region = block_base + c_meta_off
# ``slot`` is the absolute slot index within the cache block
# (range 0.._BS-1). For BS=16 it equals tok_in_tile; for BS=32
# it equals tile_in_block * 16 + tok_in_tile. Used both for
# the K/V data region and SoA metadata region addressing.
slot = fx.Int32(tile_in_block * TILE_SIZE) + tok_in_tile
data_bases_byte = (
data_region
+ slot * (c_kv_heads * c_data_per_slot)
+ kv_h * c_data_per_slot
)
k_byte = data_bases_byte + chunk_in_tok * fx.Int32(16)
k_packed = buffer_ops.buffer_load(
kv_rsrc,
k_byte // fx.Int32(4),
vec_width=4,
dtype=T.i32,
)
# ---- HOISTED: issue V data + meta HBM loads early ---------
# These are async; their s_waitcnt is pushed by the compiler
# past the K dequant + QK MFMA + softmax block, hiding most
# of the V HBM latency behind compute.
v_byte = data_bases_byte + c_keydata + chunk_in_tok * fx.Int32(16)
v_packed = buffer_ops.buffer_load(
kv_rsrc,
v_byte // fx.Int32(4),
vec_width=4,
dtype=T.i32,
)
vscale_u16 = (
meta_region // fx.Int32(2)
+ kv_h * c_meta_u16_per_kvh
+ c_vscale_off_u16
+ slot
)
vzero_u16 = (
meta_region // fx.Int32(2)
+ kv_h * c_meta_u16_per_kvh
+ c_vzero_off_u16
+ slot
)
vscale_raw = buffer_ops.buffer_load(
kv_rsrc,
vscale_u16,
vec_width=1,
dtype=T.i16,
)
vzero_raw = buffer_ops.buffer_load(
kv_rsrc,
vzero_u16,
vec_width=1,
dtype=T.i16,
)
knorm_u16 = (
meta_region // fx.Int32(2)
+ kv_h * c_meta_u16_per_kvh
+ c_knorm_off_u16
+ slot
)
knorm_raw = buffer_ops.buffer_load(
kv_rsrc,
knorm_u16,
vec_width=1,
dtype=T.i16,
)
knorm_f16 = arith.bitcast(T.f16, knorm_raw)
knorm_f32 = arith.extf(T.f32, knorm_f16)
# K dequant → LDS [token, head_dim] (natural)
# Lane writes 32 bf16 (= 4×8) for token=tok_in_tile,
# head_dims chunk_in_tok*32..+31 (4 sub-chunks of 8).
tok_kreg = tok_in_tile * fx.Int32(HEAD_SIZE * 2 // 8)
chunk_kreg = tok_kreg + chunk_in_tok * fx.Int32(8)
for w in range_constexpr(4):
word_i32 = vector.extract(k_packed, static_position=[w])
bf16_elems = []
for n in range_constexpr(8):
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
nibble_idx = arith.index_cast(T.index, nibble)
cent_f32 = cent_lds.load([nibble_idx])
elem_bf16 = arith.trunc_f(T.bf16, cent_f32 * knorm_f32)
bf16_elems.append(elem_bf16)
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
vector.store(
v_i64,
kv_lds_i64,
[arith.index_cast(T.index, chunk_kreg + fx.Int32(w * 2))],
)
gpu.barrier()
# QK MFMA (CDNA4 wide-K): A=K[token, head_dim], B=Q (= Q^T).
# K read: lane t = K[token=mfma_row, head_dim=chunk*32+col_grp*8..+7]
# Same i64×2 indexing as Q.
qk_acc = zero_v4
for chk in range_constexpr(QK_K_CHUNKS):
k_idx_i64 = (
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
+ fx.Int32(chk * 8)
+ mfma_col_grp * fx.Int32(2)
)
kv_load = vector.load_op(
T.vec(2, T.i64),
kv_lds_i64,
[arith.index_cast(T.index, k_idx_i64)],
)
k_op = vector.bitcast(T.vec(8, T.bf16), kv_load)
qk_acc = rocdl.mfma_f32_16x16x32_bf16(
T.f32x4, [k_op, q_chunks[chk], qk_acc, 0, 0, 0]
)
# qk_acc layout: lane t holds C[token=(t/16)*4..+3, query=t%16]
# 4 fp32/lane = 4 different tokens at SAME query_row.
# Scale + mask out-of-context tokens.
qk_acc = _vsplat_mul(qk_acc, QK_SCALE)
for elem in range_constexpr(4):
kv_tok = (
partition_start
+ fx.Int32(n_tile * TILE_SIZE)
+ mfma_col_grp * fx.Int32(4)
+ fx.Int32(elem)
)
in_b = kv_tok < seq_len
v = vector.extract(qk_acc, static_position=[elem])
qk_acc = vector.insert(
in_b.select(v, NEG_INF),
qk_acc,
static_position=[elem],
dynamic_position=[],
)
# FA2 online softmax: per-query-row reduce.
# Per-row max: max over 4 fp32 in lane, then xor-shuffle 16, 32
# (across the 4 col_grps that share same mfma_row).
local_max = vector.reduction(T.f32, "maxnumf", qk_acc)
r1 = local_max.shuffle_xor(fx.Int32(16), c_w)
local_max = local_max.maximumf(r1)
r2 = local_max.shuffle_xor(fx.Int32(32), c_w)
tile_max = local_max.maximumf(r2)
new_max = running_max.maximumf(tile_max)
max_diff = running_max - new_max
safe_diff = (running_max > NEG_INF).select(max_diff, ZERO_F)
scale = (safe_diff * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
running_sum = running_sum * scale
for h in range_constexpr(PV_N_CHUNKS):
acc_pv[h] = _vsplat_mul(acc_pv[h], scale)
running_max = new_max
# Compute probs: p = exp((qk - new_max) * LOG2E)
tile_sum = ZERO_F
for elem in range_constexpr(4):
s = vector.extract(qk_acc, static_position=[elem])
d = s - new_max
d = (new_max > NEG_INF).select(d, NEG_INF)
p = (d * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
tile_sum = tile_sum + p
qk_acc = vector.insert(
p, qk_acc, static_position=[elem], dynamic_position=[]
)
ts1 = tile_sum.shuffle_xor(fx.Int32(16), c_w)
tile_sum = tile_sum + ts1
ts2 = tile_sum.shuffle_xor(fx.Int32(32), c_w)
tile_sum = tile_sum + ts2
running_sum = running_sum + tile_sum
# ---- V dequant → LDS (V/scale/zero raw loads were hoisted) -
# v_packed, vscale_raw, vzero_raw are already in flight from
# the top of this iteration; only the f32 conversion stays
# here so the actual consume site is still close to the use.
vscale_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vscale_raw))
vzero_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vzero_raw))
if USE_HW_TR:
# V dequant → LDS [token][head_dim] (ROW-MAJOR, no transpose).
# Each lane writes 32 contiguous bf16 (one token, head_dims
# chunk_in_tok*32..+31) as 4× ds_write_b128 = 4× vec(2,i64).
# Replaces 32× ds_write_b16 of the legacy transposed path.
v_lds_elem_base = tok_in_tile * fx.Int32(
HEAD_SIZE
) + chunk_in_tok * fx.Int32(32)
for w in range_constexpr(4):
word_i32 = vector.extract(v_packed, static_position=[w])
bf16_elems = []
for n in range_constexpr(8):
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
nibble_f32 = arith.sitofp(T.f32, nibble)
elem_f32 = nibble_f32 * vscale_f32 + vzero_f32
elem_bf16 = arith.trunc_f(T.bf16, elem_f32)
bf16_elems.append(elem_bf16)
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
v_lds_i64_idx = (v_lds_elem_base + fx.Int32(w * 8)) // fx.Int32(
4
)
vector.store(
v_i64,
kv_lds_i64,
[arith.index_cast(T.index, v_lds_i64_idx)],
)
# ── HW V transpose: cross-lane LDS sync ─────────────────────
# ds_read_tr16_b64 below introduces a cross-lane LDS read:
# consumer lane t reads bytes written by source lanes
# 0,4,8,12 (etc.). The compiler's automatic waitcnt insertion
# is per-lane and has no awareness of the HW transpose's
# cross-lane forwarding, so it can sink the next iteration's
# ds_read ahead of these ds_write_b128 ops in the schedule.
#
# Two-part fence:
# 1. sched_barrier(0): MachineScheduler reorder barrier
# (mask=0 → no instruction class may cross). Pure hint,
# generates no runtime instruction. Use the simple
# sched_barrier (NOT sched_group_barrier, which is an
# IGLP marker that requires partner barriers and
# corrupts the schedule when used in isolation).
# 2. s_waitcnt lgkmcnt=0: runtime drain of the LDS write
# queue. vmcnt/expcnt left at no-wait so we don't
# stall on speculatively-issued HBM loads (which can
# include OOB-but-masked buffer_load reads from the
# next K-tile's hoisted V prefetch).
#
# The SW path below does NOT need this because each lane
# only reads cells it itself wrote (no cross-lane dep), and
# the per-lane same-address waitcnt the compiler emits is
# correct.
#
# Without this fence the reordered ds_read produces
# wrong results on high-partition-count configs (large
# padded num_partitions); low-partition configs are
# unaffected.
rocdl.sched_barrier(0)
# encode_waitcnt(vmcnt=63, expcnt=7, lgkmcnt=0)
# = 0xF | (7<<4) | (0<<8) | (3<<14) = 0xC07F
rocdl.s_waitcnt(0xC07F)
else:
# Legacy transposed V_LDS path: 32 ds_write_b16 per lane.
for w in range_constexpr(4):
word_i32 = vector.extract(v_packed, static_position=[w])
for n in range_constexpr(8):
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
nibble_f32 = arith.sitofp(T.f32, nibble)
elem_f32 = nibble_f32 * vscale_f32 + vzero_f32
elem_bf16 = arith.trunc_f(T.bf16, elem_f32)
elem_i16 = arith.bitcast(T.i16, elem_bf16)
head_dim = chunk_in_tok * fx.Int32(32) + fx.Int32(w * 8 + n)
v_idx_i16 = head_dim * fx.Int32(TILE_SIZE) + tok_in_tile
v_vec = vector.from_elements(T.vec(1, T.i16), [elem_i16])
vector.store(
v_vec,
kv_lds_i16,
[arith.index_cast(T.index, v_idx_i16)],
)
gpu.barrier()
# ---- PV MFMA: A=V[head_dim, token], B=P (=qk_acc bf16) -----
# P operand B layout matches qk_acc: lane t holds 4 bf16 at
# K=token=(t/16)*4..+3, N=query=t%16. Just trunc_f to bf16.
p_bf16 = arith.trunc_f(T.vec(4, T.bf16), qk_acc)
p_op = vector.bitcast(T.vec(4, T.i16), p_bf16)
if USE_HW_TR:
# HW-transpose PV path: V_lds is row-major V[token][head_dim].
# MFMA A operand layout: lane t holds 4 bf16 at
# M=head_dim=mfma_row + h*16, K=token=(t/16)*4..+3
# ds_read_tr16_b64 (4-element 16-bit transpose, per-16-lane block):
# result[lane=t, elem=e] = Input[source_lane=e*4 + (t%16)//4,
# col=t%4]
# Per-lane address: token_idx = lane // 4 (covers 0..15 across
# all four 16-lane MFMA blocks), hd_sub = (lane % 4)*4 selects
# the 4-element column window inside the h*16 chunk.
# Total LDS byte offset = kv_off + token_idx*HEAD_SIZE*2
# + (h*16 + hd_sub)*2
token_idx = lane >> fx.Int32(2)
hd_sub = (lane & fx.Int32(3)) * fx.Int32(4)
v_lane_byte = (
fx.Int32(kv_off)
+ token_idx * fx.Int32(HEAD_SIZE * 2)
+ hd_sub * fx.Int32(2)
)
for h in range_constexpr(PV_N_CHUNKS):
v_byte_off = v_lane_byte + fx.Int32(h * 32)
v_byte_i64 = fx.Int64(v_byte_off)
v_ptr = buffer_ops.create_llvm_ptr(
v_byte_i64,
address_space=3,
)
v_op_raw = rocdl.ds_read_tr16_b64(
T.vec(4, T.i16),
v_ptr,
).result
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
T.f32x4, [v_op_raw, p_op, acc_pv[h], 0, 0, 0]
)
else:
# Legacy transposed-LDS PV: 1 ds_read_b64 per chunk.
# Byte addr = (mfma_row + h*16) * TILE_SIZE * 2 + (col_grp*4) * 2
# = (mfma_row + h*16) * 32 + col_grp*8
# i64 idx = (mfma_row + h*16) * 4 + col_grp
for h in range_constexpr(PV_N_CHUNKS):
v_idx_i64 = (mfma_row + fx.Int32(h * 16)) * fx.Int32(
4
) + mfma_col_grp
kv_load = vector.load_op(
T.vec(1, T.i64),
kv_lds_i64,
[arith.index_cast(T.index, v_idx_i64)],
)
v_op = vector.bitcast(T.vec(4, T.i16), kv_load)
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
T.f32x4, [v_op, p_op, acc_pv[h], 0, 0, 0]
)
_scf.YieldOp(
[
_ival(running_max),
_ival(running_sum),
*[_ival(p) for p in acc_pv],
]
)
finally:
_for_ip.__exit__(None, None, None)
# Pull final accumulator values out of the for_op results.
running_max = _for_op.results[0]
running_sum = _for_op.results[1]
acc_pv = list(_for_op.results[2:])
# ===== STEP G: Output ===========================================
# acc_pv[h] layout: lane t holds 4 fp32 at M=head_dim=(t/16)*4..+3 + h*16,
# N=query_row=t%16.
# 4 fp32/lane = 4 contiguous head_dim values for ONE query_row.
safe_sum = (running_sum > ZERO_F).select(running_sum, ONE_F)
rcp = ONE_F / safe_sum
c_os = fx.Int32(_stride_out_seq)
c_oh = fx.Int32(_stride_out_head)
c_op_ = fx.Int32(_stride_out_part)
out_base = seq * c_os + kv_h * c_oh + part * c_op_
# Gate all global stores by mfma_row < QG. For QG=16 the predicate is
# tautological (mfma_row ∈ [0,15]); for QG=8 lanes 8..15 (whose MFMA
# outputs are computational waste) skip the stores entirely.
valid_row_pred = arith.cmpi(
arith.CmpIPredicate.ult,
mfma_row.ir_value() if hasattr(mfma_row, "ir_value") else mfma_row,
arith.constant(QG, type=T.i32),
)
_if = _scf.IfOp(valid_row_pred)
with ir.InsertionPoint(_if.then_block):
for h in range_constexpr(PV_N_CHUNKS):
pv_norm = _vsplat_mul(acc_pv[h], rcp)
pv_bf16 = arith.trunc_f(T.vec(4, T.bf16), pv_norm)
pv_i32x2 = vector.bitcast(T.vec(2, T.i32), pv_bf16)
head_dim_start = fx.Int32(h * 16) + mfma_col_grp * fx.Int32(4)
out_off_elem = (
out_base + mfma_row * fx.Int32(HEAD_SIZE) + head_dim_start
)
buffer_ops.buffer_store(
pv_i32x2,
out_rsrc,
out_off_elem * fx.Int32(2),
offset_is_bytes=True,
)
c_npq = fx.Int32(num_partitions * QG)
ml_off = (
seq * fx.Int32(_stride_ml_seq) + kv_h * c_npq + part * c_qg + mfma_row
)
es_off = (
seq * fx.Int32(_stride_es_seq) + kv_h * c_npq + part * c_qg + mfma_row
)
buffer_ops.buffer_store(running_max, ml_rsrc, ml_off)
buffer_ops.buffer_store(running_sum, es_rsrc, es_off)
_scf.YieldOp([])
return tq_decode_gqa6_kernel
@@ -0,0 +1,874 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""FlyDSL TurboQuant decode launcher (vLLM-side).
Drop-in replacement for ``triton_turboquant_decode_attention_soa`` for the
TQ decode profile HEAD_SIZE=128, MSE_BITS=4 K, VQB=4 V, N_CENTROIDS=16,
BLOCK_SIZE in {16, 32}.
Per-GQA kernel dispatch:
* GQA group ∈ {8, 16} → canonical kernels.tq_decode (Qwen-class)
* GQA group == 6 → kernels.tq_decode_gqa6 sibling (MiniMax-M2.5)
Auto-selected on gfx950 when FlyDSL is importable; falls back to the SoA Triton
decode otherwise (wrong arch, missing build tree). The GQA-6 sibling is imported
best-effort: missing it does not affect Qwen GQA-{8,16} paths.
Architecture:
1. Q rotation: ``q_rot = (query.float() @ PiT).bfloat16()`` — the same
launcher-side rocBLAS GEMM the SoA Triton decode uses.
2. FlyDSL kernel writes per-partition outputs into
``[N, Hk, P, QG, D]`` bf16 + ``[N, Hk, P, QG]`` fp32 max/sum buffers.
3. A small Triton reducer combines partitions in the kernel's native layout
(no permute/cast), writing the final ``[N, Hq, D]`` output.
Kernel module is built once per ``(num_kv_heads, num_partitions,
max_blocks_per_seq, scale)`` shape and cached.
"""
from __future__ import annotations
from typing import Any
import torch
from vllm.logger import init_logger
from vllm.triton_utils import tl, triton
logger = init_logger(__name__)
# -- FlyDSL import bootstrap --------------------------------------------------
# FlyDSL is imported directly and must be importable (pip-installed, or on
# PYTHONPATH). Import failures are caught below and fall back to the SoA Triton
# decode, so vLLM startup is unaffected on hosts without FlyDSL.
_FLYDSL_AVAILABLE: bool | None = None
_TQ_MOD: Any = None # kernels.tq_decode module (Qwen GQA-{8,16})
_TQ_MOD_GQA6: Any = None # kernels.tq_decode_gqa6 module (MiniMax GQA-6, optional)
_FLYC: Any = None # flydsl.compiler
_FX: Any = None # flydsl.expr
_TYPING_T: Any = None
_CC: Any = None # CompilationContext
_IR: Any = None
def is_flydsl_available() -> bool:
"""Return True iff on gfx950 and FlyDSL imports + kernel module load.
The GQA-6 sibling kernel (``kernels.tq_decode_gqa6``) is imported
best-effort: if it's missing (older FlyDSL checkout that pre-dates
MiniMax support) the canonical Qwen path stays fully functional and
only GQA-6 dispatches will fail with a clear error at launch time.
"""
global _FLYDSL_AVAILABLE, _TQ_MOD, _TQ_MOD_GQA6
global _FLYC, _FX, _TYPING_T, _CC, _IR
if _FLYDSL_AVAILABLE is not None:
return _FLYDSL_AVAILABLE
try:
# gfx950-only kernel (CDNA4 intrinsics); gate before importing FlyDSL.
from vllm.platforms.rocm import on_gfx950
if not on_gfx950():
_FLYDSL_AVAILABLE = False
return _FLYDSL_AVAILABLE
import flydsl.compiler as flyc # noqa: F401
import flydsl.expr as fx # noqa: F401
from flydsl._mlir import ir
from flydsl.compiler.kernel_function import CompilationContext
from flydsl.expr.typing import T # noqa: F401
# Kernel is vendored in-tree (ships with vLLM); FlyDSL provides only
# the compiler/runtime framework (imported above).
from vllm.v1.attention.ops.flydsl_kernels import tq_decode as tq_mod
_FLYC = flyc
_FX = fx
_TYPING_T = T
_CC = CompilationContext
_IR = ir
_TQ_MOD = tq_mod
_FLYDSL_AVAILABLE = True
logger.info_once("FlyDSL TQ decode launcher: available")
except Exception as ex: # noqa: BLE001
_FLYDSL_AVAILABLE = False
logger.warning_once(
"FlyDSL TQ decode launcher: unavailable (%s). "
"Falling back to SoA Triton decode.",
ex,
)
return _FLYDSL_AVAILABLE
# Best-effort GQA-6 sibling import (does NOT gate Qwen availability).
try:
from vllm.v1.attention.ops.flydsl_kernels import (
tq_decode_gqa6 as tq_mod_gqa6,
)
_TQ_MOD_GQA6 = tq_mod_gqa6
logger.info_once("FlyDSL TQ decode GQA-6 sibling: available (MiniMax-class)")
except Exception as ex: # noqa: BLE001
_TQ_MOD_GQA6 = None
logger.info_once(
"FlyDSL TQ decode GQA-6 sibling: not available (%s); "
"GQA-6 models will fall back to SoA Triton decode.",
ex,
)
return _FLYDSL_AVAILABLE
def is_flydsl_gqa6_available() -> bool:
"""True iff the optional GQA-6 sibling kernel module loaded.
Used by the eligibility gate in turboquant_attn.py to decide whether
a layer with num_kv_groups==6 can run on FlyDSL or must fall back
to the SoA Triton decode.
"""
if _FLYDSL_AVAILABLE is None:
is_flydsl_available()
return _TQ_MOD_GQA6 is not None
# -- Kernel module cache -------------------------------------------------------
# Compiling a FlyDSL kernel is expensive; cache by the constexprs that
# parameterize stride math: num_kv_heads, num_partitions, max_blocks_per_seq,
# scale. ``num_seqs`` is irrelevant to the kernel body so we omit it.
_KERN_CACHE: dict[tuple, Any] = {}
# -- One-shot log guards ------------------------------------------------------
# Replaces vllm `logger.{info,warning}_once` (which hashes the format string
# every call to dedup) with a true zero-overhead bool guard.
_LOG_INVOKED_ONCE: bool = False
_LOG_SINKS_WARNED: bool = False
_LOG_NORM_WARNED: bool = False
def _detect_max_capture_B() -> int:
"""Probe vLLM compilation config for the largest cudagraph capture size.
Returns max(cudagraph_capture_sizes) if available, else falls back to
512 (vLLM's default cap). The segm-pool auto-grows if a larger batch is
seen (see ``_SegmBufPool.get``).
Used to size the segm-pool bucket once, so that all distinct B's
(whether captured in a graph, or hit eagerly in mixed batches) share
a single allocation per shape — eliminating the per-B growth that
previously dominated dense-capture VRAM cost.
"""
try:
from vllm.config import get_current_vllm_config
cfg = get_current_vllm_config()
sizes = cfg.compilation_config.cudagraph_capture_sizes
if sizes:
return int(max(sizes))
except Exception: # noqa: BLE001
pass
return 512
class _SegmBufPool:
"""Single-bucket buffer pool for segm_out/segm_max/segm_sum/output.
Each unique shape signature (Hk, Hq, num_partitions, QG, D, device,
dtype) is backed by ONE allocation sized at ``max_B`` (= max captured
cudagraph batch size, or env override). Per-call ``get(B, ...)``
returns ``[:B]`` views — same data_ptr base, narrower N dim. The
kernel grid is ``(B, ...)`` so it only ever writes the first B rows
of segm_out / segm_max / segm_sum; downstream readers (Triton
reducer + caller) only consume the first B rows of output.
Why a single bucket: cudagraph records launch pointers at capture
time. With per-B allocations, every captured size triggered a fresh
allocation, so total VRAM scaled with the number of captured sizes.
With single-bucket, the pool tops out at one max_B-sized buffer per
shape, shared across ALL captured sizes and ALL eager mixed-batch
calls. The data_ptr is stable from first allocation onward, so
cudagraph capture is happy across all captured B's.
Eliminates per-decode-step ``cudaMalloc`` (× 4) plus the device-side
memset kernels behind ``torch.full(-inf)`` and ``torch.zeros``.
The kernel always writes the FULL ``[B, Hk, P, QG, D]`` slice and
every (n, kv_h, p, qg) position of segm_max / segm_sum (it stores
``-inf`` and ``0`` for empty partitions itself), so uninitialized
bytes from the pool are safe to reuse for any B ≤ B_bucket.
"""
__slots__ = ("_bufs", "_max_B")
def __init__(self) -> None:
self._bufs: dict[tuple, dict[str, torch.Tensor]] = {}
self._max_B: int | None = None # lazy-init on first get()
def get(
self,
B: int,
Hk: int,
Hq: int,
num_partitions: int,
QG: int,
D: int,
device: torch.device,
q_dtype: torch.dtype,
) -> dict[str, torch.Tensor]:
if self._max_B is None:
self._max_B = _detect_max_capture_B()
# B may exceed the detected max if user set max-num-seqs higher
# than the largest captured size — still safe, we'll grow once.
# Note: growing AFTER cudagraph capture would invalidate captured
# pointers, so this should only happen during warmup or in
# always-eager configs.
B_bucket = max(self._max_B, int(B))
key = (
int(Hk),
int(Hq),
int(num_partitions),
int(QG),
int(D),
str(device),
q_dtype,
)
bufs = self._bufs.get(key)
if bufs is None or bufs["segm_out"].shape[0] < B_bucket:
if bufs is not None and bufs["segm_out"].shape[0] < B_bucket:
# First-time grow: warn so user knows cudagraphs may be
# invalidated. This should only happen during warmup or in
# always-eager configs (a batch larger than any captured
# cudagraph size).
logger.warning_once(
"FlyDSL _SegmBufPool: growing bucket from %d to %d "
"(B=%d). If you see this AFTER cudagraph warmup, the "
"previously-captured graphs hold stale pointers and "
"will GPU-fault (max cudagraph capture size < B).",
bufs["segm_out"].shape[0],
B_bucket,
B,
)
bufs = {
"segm_out": torch.empty(
(B_bucket, Hk, num_partitions, QG, D),
dtype=torch.bfloat16,
device=device,
),
"segm_max": torch.empty(
(B_bucket, Hk, num_partitions, QG),
dtype=torch.float32,
device=device,
),
"segm_sum": torch.empty(
(B_bucket, Hk, num_partitions, QG),
dtype=torch.float32,
device=device,
),
"output": torch.empty(
(B_bucket, Hq, D),
dtype=q_dtype,
device=device,
),
# Stable pre-allocated buffer for the rotated query
# tensor. On ROCm, fresh torch.empty / matmul-result allocations
# after HIP graph capture can land in the graph memory pool and
# cause GPU memory faults when passed as kernel arguments in the
# eager mixed-batch path. By pooling q_rot here (same pattern as
# segm_out/output above), the data_ptr is stable from first
# allocation onward — same for all captured B's and eager calls.
"q_rot": torch.empty(
(B_bucket, Hq, D),
dtype=q_dtype,
device=device,
),
# q_float: fp32 copy of query (bf16→fp32 for mm input).
# Pooled to avoid fresh allocations post-capture.
"q_float": torch.empty(
(B_bucket, Hq, D),
dtype=torch.float32,
device=device,
),
# q_rot_fp32: fp32 output of the rotation mm (before bf16 cast).
# Using torch.mm(..., out=this) eliminates the fresh mm-result
# allocation that would otherwise land in the HIP graph pool.
"q_rot_fp32": torch.empty(
(B_bucket, Hq, D),
dtype=torch.float32,
device=device,
),
}
self._bufs[key] = bufs
self._max_B = B_bucket
logger.info_once(
"FlyDSL _SegmBufPool: allocated single-bucket "
"shape=(Hk=%d, Hq=%d, P=%d, QG=%d, D=%d, dtype=%s) "
"B_bucket=%d (covers all captured + eager B's). "
"VRAM = %.1f MiB / shape.",
Hk,
Hq,
num_partitions,
QG,
D,
q_dtype,
B_bucket,
sum(t.numel() * t.element_size() for t in bufs.values()) / (1 << 20),
)
# Return [:B] views — same data_ptr base, narrower N dim.
return {
"segm_out": bufs["segm_out"][:B],
"segm_max": bufs["segm_max"][:B],
"segm_sum": bufs["segm_sum"][:B],
"output": bufs["output"][:B],
"q_rot": bufs["q_rot"][:B],
"q_float": bufs["q_float"][:B],
"q_rot_fp32": bufs["q_rot_fp32"][:B],
}
def stats(self) -> dict[str, int]:
return {
"shapes": len(self._bufs),
"max_B": int(self._max_B or 0),
"bytes": sum(
sum(t.numel() * t.element_size() for t in d.values())
for d in self._bufs.values()
),
}
_SEGM_POOL = _SegmBufPool()
_HW_TR_CACHED: bool | None = None
def _hw_tr_enabled() -> bool:
"""Resolve the HW V-transpose build flag.
ON for gfx950+ (ds_read_tr16_b64 is bit-exact vs baseline and faster
on Qwen-class shapes), off elsewhere. Resolved from the GPU arch.
"""
global _HW_TR_CACHED
if _HW_TR_CACHED is not None:
return _HW_TR_CACHED
try:
from flydsl.runtime.device import get_rocm_arch as _arch
a = str(_arch() or "")
_HW_TR_CACHED = a.startswith("gfx950")
except Exception: # noqa: BLE001
_HW_TR_CACHED = False
if _HW_TR_CACHED:
logger.info_once("FlyDSL TQ: HW V transpose ON (default for gfx950+)")
else:
logger.info_once("FlyDSL TQ: HW V transpose OFF")
return _HW_TR_CACHED
_GET_KERNEL_STATS = {"hits": 0, "misses": 0, "build_total_s": 0.0}
def _get_kernel(
num_kv_heads: int,
num_partitions: int,
max_blocks_per_seq: int,
scale: float,
query_group_size: int,
kv_block_size: int,
use_hw_v_transpose: bool = False,
num_seqs_hint: int = 1,
tile_groups_per_partition: int = 1,
):
# ``num_seqs_hint`` (= runtime B) is forwarded to the build for shape
# awareness; it does NOT participate in the cache key because the
# kernel body uses gpu.block_idx.x (= seq index at runtime) and is
# GQA-symmetric across B at build time.
#
# ``tile_groups_per_partition`` IS in the cache key — each
# value compiles a different unrolled K-tile loop body.
key = (
num_kv_heads,
int(num_partitions),
int(max_blocks_per_seq),
round(float(scale), 8),
int(query_group_size),
int(kv_block_size),
bool(use_hw_v_transpose),
int(tile_groups_per_partition),
)
cached = _KERN_CACHE.get(key)
if cached is not None:
_GET_KERNEL_STATS["hits"] += 1
return cached
assert is_flydsl_available()
# Time the FlyDSL build path so we can quantify cudagraph capture cost.
import time as _t
_build_t0 = _t.perf_counter()
# Per-GQA dispatch: GQA-6 (MiniMax-M2.5) lives in the sibling module
# tq_decode_gqa6 to keep the Qwen kernel's invariants untouched.
# GQA-{8,16} (Qwen) keep using the canonical tq_decode kernel.
qg = int(query_group_size)
if qg == 6:
if _TQ_MOD_GQA6 is None:
raise RuntimeError(
"FlyDSL TQ GQA-6 sibling kernel "
"(vllm.v1.attention.ops.flydsl_kernels.tq_decode_gqa6) "
"failed to import; check that FlyDSL is installed/importable."
)
kmod = _TQ_MOD_GQA6
kfn = kmod.build_tq_decode_gqa6_module(
num_seqs=int(num_seqs_hint),
num_kv_heads=num_kv_heads,
num_partitions=num_partitions,
max_blocks_per_seq=max_blocks_per_seq,
softmax_scale=float(scale),
query_group_size=qg,
kv_block_size=int(kv_block_size),
use_hw_v_transpose=bool(use_hw_v_transpose),
tile_groups_per_partition=int(tile_groups_per_partition),
)
else:
kmod = _TQ_MOD
kfn = kmod.build_tq_decode_module(
num_seqs=int(num_seqs_hint),
num_kv_heads=num_kv_heads,
num_partitions=num_partitions,
max_blocks_per_seq=max_blocks_per_seq,
softmax_scale=float(scale),
query_group_size=qg,
kv_block_size=int(kv_block_size),
use_hw_v_transpose=bool(use_hw_v_transpose),
tile_groups_per_partition=int(tile_groups_per_partition),
)
al = kmod.allocator
block_threads = kmod.BLOCK_THREADS
flyc = _FLYC
fx = _FX
T = _TYPING_T
CompilationContext = _CC
ir_mod = _IR
@flyc.jit
def _launch(
out,
es,
ml,
q,
kvc,
cents,
bt,
sl,
gx: fx.Int32, # type: ignore[name-defined]
gy: fx.Int32, # type: ignore[name-defined]
gz: fx.Int32, # type: ignore[name-defined]
stream: fx.Stream, # type: ignore[name-defined]
):
# Re-finalize the LDS allocator for this launch (idempotent per build).
al.finalized = False
ctx = CompilationContext.get_current()
with ir_mod.InsertionPoint(ctx.gpu_module_body):
al.finalize()
from flydsl.expr import arith
grid_x = arith.index_cast(T.index, gx.ir_value())
grid_y = arith.index_cast(T.index, gy.ir_value())
grid_z = arith.index_cast(T.index, gz.ir_value())
kfn(out, es, ml, q, kvc, cents, bt, sl).launch(
grid=(grid_x, grid_y, grid_z),
block=(block_threads, 1, 1),
stream=stream,
)
_KERN_CACHE[key] = _launch
_build_dt = _t.perf_counter() - _build_t0
_GET_KERNEL_STATS["misses"] += 1
_GET_KERNEL_STATS["build_total_s"] += _build_dt
logger.info(
"FlyDSL _get_kernel BUILD #%d dt=%.2fs (cumulative=%.1fs) key=%s",
_GET_KERNEL_STATS["misses"],
_build_dt,
_GET_KERNEL_STATS["build_total_s"],
key,
)
return _launch
# -- Partition reducer (Triton, native kernel layout) -------------------------
# Reduces the FA2 split-KV partials produced by the FlyDSL kernel.
# segm_out [N, Hk, P, QG, D] bf16
# segm_max [N, Hk, P, QG] fp32
# segm_sum [N, Hk, P, QG] fp32
# → output [N, Hq=Hk*QG, D] of any (bf16/fp16/fp32) dtype.
@triton.jit
def _reduce_partitions(
output_ptr, # [N, Hq, D] in OUT_DTYPE
segm_out_ptr, # [N, Hk, P, QG, D] bf16
segm_max_ptr, # [N, Hk, P, QG] fp32
segm_sum_ptr, # [N, Hk, P, QG] fp32
out_stride_n: tl.int64, # stride on N axis (in elems of OUT_DTYPE)
out_stride_h: tl.int64, # stride on Hq axis
NUM_KV_HEADS: tl.constexpr,
QG: tl.constexpr,
NUM_PARTS: tl.constexpr,
HEAD_SIZE: tl.constexpr,
):
# grid = (N, Hq)
n = tl.program_id(0)
hq = tl.program_id(1)
kv_h = hq // QG
qg = hq % QG
# Per-partition pointers for this (n, kv_h, qg) row.
# Strides in elems of source dtype.
msum_base = n * (NUM_KV_HEADS * NUM_PARTS * QG) + kv_h * (NUM_PARTS * QG) + qg
so_base = (
n * (NUM_KV_HEADS * NUM_PARTS * QG * HEAD_SIZE)
+ kv_h * (NUM_PARTS * QG * HEAD_SIZE)
+ qg * HEAD_SIZE
)
p_off = tl.arange(0, NUM_PARTS)
d_off = tl.arange(0, HEAD_SIZE)
# Load per-partition max/sum.
m_idx = msum_base + p_off * QG
seg_max = tl.load(segm_max_ptr + m_idx)
seg_sum = tl.load(segm_sum_ptr + m_idx)
# Mask out empty partitions (max == -inf).
valid = seg_max > float("-inf")
overall_max = tl.max(tl.where(valid, seg_max, float("-inf")))
# Rescale exp sums.
rescale = tl.where(valid, tl.exp(seg_max - overall_max), 0.0)
seg_sum_rescaled = seg_sum * rescale
overall_sum = tl.sum(seg_sum_rescaled)
# Load segment outputs and combine.
so_idx = so_base + p_off[:, None] * (QG * HEAD_SIZE) + d_off[None, :]
seg_out = tl.load(segm_out_ptr + so_idx).to(tl.float32)
# acc = sum_p( seg_out_p * (seg_sum_p * rescale_p) ) / overall_sum
weighted = seg_out * seg_sum_rescaled[:, None]
acc_sum = tl.sum(weighted, axis=0)
acc = tl.where(overall_sum > 0.0, acc_sum / overall_sum, 0.0)
# Store.
out_off = n * out_stride_n + hq * out_stride_h + d_off
tl.store(output_ptr + out_off, acc.to(output_ptr.dtype.element_ty))
# -- Public launcher ----------------------------------------------------------
def flydsl_turboquant_decode_attention(
query: torch.Tensor, # [B, Hq, D] bf16/fp16
kv_cache: torch.Tensor, # [num_blocks, BS, Hk, slot_size_aligned]
block_table: torch.Tensor, # [B, max_blocks_per_seq] int32
seq_lens: torch.Tensor, # [B] int32
Pi: torch.Tensor, # [D, D] fp32
centroids: torch.Tensor, # [N_CENTROIDS] fp32
scale: float,
mse_bits: int,
key_packed_size: int,
value_quant_bits: int,
value_packed_size: int,
key_fp8: bool = False,
norm_correction: bool = False,
PiT: torch.Tensor | None = None,
max_seq_len: int = 0,
mid_o_buf: Any = None,
output_buf: torch.Tensor | None = None,
lse_buf: Any = None,
buf_holder: Any = None,
max_num_kv_splits: int = 32,
sinks: torch.Tensor | None = None,
) -> torch.Tensor:
"""SoA-decode-compatible launcher backed by the FlyDSL decode kernel.
Constraints:
* key_fp8 == False
* mse_bits == 4
* value_quant_bits == 4
* centroids.numel() == 16
* D == 128
* block_size in {16, 32}
* Hq // Hk in {6, 8, 16}
- 8/16 → canonical tq_decode kernel (Qwen2.5-72B / Qwen3-32B)
- 6 → tq_decode_gqa6 sibling kernel (MiniMax-M2.5)
Sinks are NYI and silently ignored if set. norm_correction is honored
implicitly via the pre-folded stored K-norm (see footer comment).
"""
del mid_o_buf, lse_buf, key_packed_size, value_packed_size
if not is_flydsl_available():
raise RuntimeError(
"FlyDSL decode requested but FlyDSL is not available (needs gfx950 "
"+ importable FlyDSL); the SoA Triton decode is the fallback."
)
assert not key_fp8, "FlyDSL supports MSE-key path only"
assert mse_bits == 4, f"FlyDSL expects mse_bits=4, got {mse_bits}"
assert value_quant_bits == 4, (
f"FlyDSL expects value_quant_bits=4, got {value_quant_bits}"
)
B, Hq, D = query.shape
Hk = kv_cache.shape[2]
block_size = kv_cache.shape[1]
QG = Hq // Hk
assert D == _TQ_MOD.HEAD_SIZE
assert block_size in (16, 32), (
f"FlyDSL supports kv_block_size 16 or 32, got {block_size}"
)
assert QG in (6, 8, 16), f"FlyDSL supports GQA factor 6, 8 or 16, got {QG}"
if QG == 6 and _TQ_MOD_GQA6 is None:
raise RuntimeError(
"FlyDSL launcher: GQA-6 requested (MiniMax-class) but the "
"tq_decode_gqa6 sibling module is not available. Update your "
"FlyDSL checkout (must include kernels/tq_decode_gqa6.py); "
"otherwise the SoA Triton decode is the fallback."
)
assert centroids.numel() == _TQ_MOD.N_CENTROIDS, (
f"centroids.numel={centroids.numel()} != {_TQ_MOD.N_CENTROIDS}"
)
# ---- Per-layer cache for PiT_f32 + contiguous centroids ---
# PiT and centroids are model constants (set once at layer warmup).
# Avoid the per-decode-step transpose+cast and `.contiguous()` no-op
# check by stashing the pre-cooked tensors on ``buf_holder`` (= layer).
PiT_f32: torch.Tensor
centroids_c: torch.Tensor
if buf_holder is not None:
PiT_f32 = getattr(buf_holder, "_tq_PiT_f32", None)
if PiT_f32 is None:
_PiT_src = PiT if PiT is not None else Pi.T.contiguous()
PiT_f32 = (
_PiT_src
if _PiT_src.dtype == torch.float32
else _PiT_src.to(torch.float32)
)
buf_holder._tq_PiT_f32 = PiT_f32
centroids_c = getattr(buf_holder, "_tq_centroids_c", None)
if centroids_c is None:
centroids_c = centroids.contiguous()
buf_holder._tq_centroids_c = centroids_c
else:
# Defensive fallback (unit tests may pass ``buf_holder=None``).
_PiT_src = PiT if PiT is not None else Pi.T.contiguous()
PiT_f32 = (
_PiT_src if _PiT_src.dtype == torch.float32 else _PiT_src.to(torch.float32)
)
centroids_c = centroids.contiguous()
# ---- Pooled q_rot (stable data_ptr post-capture) ------
# On ROCm, fresh tensor allocations made AFTER HIP graph capture can land
# in the graph memory pool and cause GPU memory faults when passed as kernel
# pointer arguments in the eager mixed-batch path. We pool q_rot and the
# fp32 intermediate the same way segm_out/output are pooled: one max-B
# allocation per shape, reused across all B's.
#
# Note: pool_bufs is computed BELOW after num_partitions is determined;
# q_rot needs pool_bufs["q_float"] and pool_bufs["q_rot"]. We compute the
# rotation inline after the pool is fetched (see the pooled-buffers
# section below).
# ---- Partition count (FA2 split-KV) ----------------------------------
#
# FlyDSL runs INSIDE FULL cudagraph (TurboQuantMetadataBuilder._cudagraph_
# support = UNIFORM_BATCH). That means the gridDim baked at capture
# time MUST equal the gridDim at replay — the kernel launch parameters
# are recorded into the captured graph. So `num_partitions` (which
# becomes gridDim.z) MUST be derived from a stable source that produces
# the same value at capture and at runtime.
#
# We use the worst-case context length implied by the block table's
# allocation (block_table.shape[1] * block_size) as that source. It's
# bounded by `max_model_len` (vLLM allocates the block table for the
# configured max), and it's identical at capture and at runtime
# because `block_table.shape[1]` is fixed once the model is loaded.
# Per-tile OOB redirects + the masked FA-2 reducer ensure that for
# short sequences the extra partitions contribute zero to the output.
#
# We always size from the worst case (block_table.shape[1] * block_size),
# never from the per-step actual max_seq_len: a per-step size would make
# the captured gridDim mismatch runtime and GPU-fault under cudagraph.
kv_compute_block = _TQ_MOD.KV_COMPUTE_BLOCK
worst_case_max_seq_len = int(block_table.shape[1]) * int(block_size)
if max_seq_len <= 0:
max_seq_len = worst_case_max_seq_len
sizing_max_seq_len = worst_case_max_seq_len
# ── Bounded num_partitions + internal tile-group looping ──
# The kernel previously hardcoded one partition = 256 tokens (=
# KV_COMPUTE_BLOCK), forcing num_partitions to scale linearly with
# max_model_len, which made the cudagraph capture cost grow
# prohibitively at long context. The new kernel takes
# ``tile_groups_per_partition`` (TGPP); each CTA processes
# ``TGPP * 16`` K-tiles = ``TGPP * KV_COMPUTE_BLOCK`` tokens with the
# FA-2 online-softmax state accumulating across all of them. This
# mirrors what the SoA Triton decode / HIP SoA-fusion already do.
#
# Strategy: cap num_partitions at ``MAX_PARTITIONS`` (default 32),
# then derive TGPP so that ``num_partitions * TGPP * KV_COMPUTE_BLOCK
# >= sizing_max_seq_len``.
#
# Examples (block_size=32 → worst_case max_bps*32):
# max_model_len=8K: required=32, parts=32, TGPP=1 (no waste)
# max_model_len=16K: required=64, parts=32, TGPP=2 (no waste)
# max_model_len=32K: required=128, parts=32, TGPP=4 (no waste)
# max_model_len=64K: required=256, parts=32, TGPP=8 (no waste)
# max_model_len=128K: required=512, parts=32, TGPP=16
MAX_PARTITIONS = 32
required_num_partitions = (
sizing_max_seq_len + kv_compute_block - 1
) // kv_compute_block
# max_num_kv_splits acts as a parallelism floor (ensure at least this
# many CTAs along grid.z), but is itself capped by MAX_PARTITIONS.
parallelism_floor = min(MAX_PARTITIONS, max(1, max_num_kv_splits))
# Cap num_partitions at MAX_PARTITIONS, but never go below the floor.
num_partitions_actual = max(
parallelism_floor,
min(MAX_PARTITIONS, required_num_partitions),
)
# Round up to next power of 2 for the Triton reducer's tl.arange(0, N)
# constraint (Triton requires power-of-2 ≥ 2).
num_partitions = max(2, triton.next_power_of_2(num_partitions_actual))
# Now derive tile_groups_per_partition (TGPP) so total coverage
# ``num_partitions * TGPP * KV_COMPUTE_BLOCK`` is >= sizing_max_seq_len.
# This guarantees no work is dropped at runtime regardless of seq_len.
# Round TGPP up to the next power-of-2 so the JIT cache key is bounded
# to a small set of values (e.g. {1, 2, 4, 8, 16}) — limits the number
# of distinct kernel binaries that need to be compiled across requests.
_tgpp_required = max(
1,
(required_num_partitions + num_partitions - 1) // num_partitions,
)
tile_groups_per_partition = int(triton.next_power_of_2(_tgpp_required))
# ---- Pooled buffers (no per-call cudaMalloc / memset) ----------
# The FlyDSL kernel writes the FULL [B, Hk, P, QG, D] segm_out and the
# FULL [B, Hk, P, QG] segm_max / segm_sum (running_max=-inf, sum=0 for
# empty partitions are stored by the kernel itself), so uninitialized
# buffers from the pool are safe to reuse.
device = query.device
pool_bufs = _SEGM_POOL.get(
B,
Hk,
Hq,
num_partitions,
QG,
D,
device,
query.dtype,
)
segm_out = pool_bufs["segm_out"]
segm_max = pool_bufs["segm_max"]
segm_sum = pool_bufs["segm_sum"]
if output_buf is None:
output = pool_bufs["output"]
else:
output = output_buf[:B] if output_buf.shape[0] != B else output_buf
# ---- q rotation: GEMM q @ PiT via stable pooled buffers -------------
_q_float = pool_bufs["q_float"] # [B, Hq, D] fp32, stable
_q_rot_f32 = pool_bufs["q_rot_fp32"] # [B, Hq, D] fp32, stable mm output
_q_rot_out = pool_bufs["q_rot"] # [B, Hq, D] query.dtype, stable
_q_float.copy_(query) # bf16 → fp32 in-place, no alloc
# mm into stable fp32 buffer via out= to avoid fresh allocations.
# PiT_f32 is cached on buf_holder (stable ptr since first layer warmup).
torch.mm(
_q_float.view(B * Hq, D),
PiT_f32,
out=_q_rot_f32.view(B * Hq, D), # in-place into pool buffer
)
_q_rot_out.copy_(_q_rot_f32) # fp32 → bf16, into stable buf
q_for_kernel = _q_rot_out # stable ptr for kernel launch
# ---- FlyDSL kernel launch -------------------------------------------
max_bps = int(block_table.shape[1])
use_hw_tr = _hw_tr_enabled()
launch = _get_kernel(
Hk,
num_partitions,
max_bps,
scale,
QG,
block_size,
use_hw_v_transpose=use_hw_tr,
num_seqs_hint=int(B),
tile_groups_per_partition=int(tile_groups_per_partition),
)
# Zero-overhead one-shot info log (replaces logger.info_once which
# hashes its format string on every call to dedup).
global _LOG_INVOKED_ONCE, _LOG_SINKS_WARNED, _LOG_NORM_WARNED
if not _LOG_INVOKED_ONCE:
_LOG_INVOKED_ONCE = True
logger.info(
"FlyDSL launcher invoked (UNIFORM_BATCH cudagraph): "
"B=%d Hk=%d Hq=%d D=%d QG=%d num_partitions=%d (actual=%d, "
"cap=%d) TGPP=%d max_bps=%d block_size=%d max_seq_len=%d "
"hw_v_transpose=%s (coverage=%d tokens, worst_case=%d tokens)",
B,
Hk,
Hq,
D,
QG,
num_partitions,
num_partitions_actual,
MAX_PARTITIONS,
tile_groups_per_partition,
max_bps,
int(block_size),
int(max_seq_len),
use_hw_tr,
num_partitions * tile_groups_per_partition * kv_compute_block,
worst_case_max_seq_len,
)
launch(
segm_out,
segm_sum,
segm_max,
q_for_kernel,
kv_cache,
centroids_c,
block_table,
seq_lens,
B,
Hk,
num_partitions,
torch.cuda.current_stream(),
)
# ---- Reduce partitions -> [B, Hq, D] --------------------------------
_reduce_partitions[(B, Hq)](
output_ptr=output,
segm_out_ptr=segm_out,
segm_max_ptr=segm_max,
segm_sum_ptr=segm_sum,
out_stride_n=output.stride(0),
out_stride_h=output.stride(1),
NUM_KV_HEADS=Hk,
QG=QG,
NUM_PARTS=num_partitions,
HEAD_SIZE=D,
)
if sinks is not None and not _LOG_SINKS_WARNED:
_LOG_SINKS_WARNED = True
logger.warning(
"FlyDSL launcher: sinks ignored (NYI). Disable sinks if they "
"are required for correctness."
)
# ── norm_correction is honored IMPLICITLY ───────────────────────────
# When the model was stored with norm_correction=True (the *_nc presets
# turboquant_4bit_nc / k3v4_nc / 3bit_nc), the per-token K-norm scalar
# was pre-folded to ||k_t|| / ||c_t|| at store time by the TurboQuant
# store path. The decode kernel just multiplies `c_vals * stored_knorm`,
# which then equals `(c_vals / ||c_t||) * ||k_t||` — exactly the
# unit-norm-renormalized centroid times the original key norm. The
# Triton decode does the identical multiply, so both decoders honor
# norm_correction equivalently and there is nothing to "do" at decode
# time. The launcher's `norm_correction` arg is only kept for API
# parity; we emit a one-shot INFO log to make the contract explicit.
if norm_correction and not _LOG_NORM_WARNED:
_LOG_NORM_WARNED = True
logger.info(
"FlyDSL launcher: norm_correction honored implicitly via "
"pre-folded stored K-norm (cf. triton_turboquant_store step 3); "
"no decode-time work required, identical to the SoA decode behavior."
)
return output
@@ -0,0 +1,3 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# HIP-free SoA Triton subset for the FlyDSL TurboQuant decode path.
@@ -0,0 +1,643 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton fused TurboQuant decode attention.
Decode path: Triton stage1 (split-KV tiled attention scoring + value
accumulation) + stage2 (log-sum-exp reduction across splits).
Supports FP8 (E4M3) keys, 3-bit and 4-bit uniform quantized values.
"""
import math
from typing import Any
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.attention.ops.triton_decode_attention import (
_fwd_kernel_stage2,
)
_FP8_E4B15: dict[int, int] = {}
def _use_fp8_e4b15(device: int = 0) -> int:
"""Return 1 if device needs fp8e4b15 (Ampere/Ada, SM < 8.9), else 0.
On non-CUDA platforms (e.g. XPU), always returns 0 (use e4nv format).
"""
if device not in _FP8_E4B15:
if current_platform.is_cuda_alike():
cap = torch.cuda.get_device_capability(device)
_FP8_E4B15[device] = 1 if cap < (8, 9) else 0
else:
_FP8_E4B15[device] = 0
return _FP8_E4B15[device]
# ---------------------------------------------------------------------------
# Stage 1: Fused TQ score + value accumulation (BLOCK_KV tiled)
# ---------------------------------------------------------------------------
@triton.jit
def _tq_decode_stage1(
# Precomputed query projection
Q_rot_ptr, # [B, Hq, D] float32
# Compressed KV cache (combined K+V)
KV_cache_ptr, # [num_blocks, block_size, Hk, padded_slot] uint8
KV_cache_u16_ptr, # uint16 view of same storage — Opt#3 SoA loads
# Block table and sequence info
Block_table_ptr, # [B, max_num_blocks] int32
Seq_lens_ptr, # [B] int32
# TQ parameters
Centroids_ptr, # [n_centroids] float32
# Output (intermediate for stage2)
Mid_o_ptr, # [B, Hq, NUM_KV_SPLITS, D+1] float32
# Sink support
Sink_ptr, # [Hq] float32, pre-computed sink attention logits per head
# Strides
stride_qb,
stride_qh, # Q strides: [B, Hq, D]
stride_cache_block: tl.int64, # bytes per block (bs*H*slot_aligned)
stride_bt_b: tl.int64, # block_table stride per batch
stride_mid_b,
stride_mid_h,
stride_mid_s, # mid_o strides
# Constexpr dims
NUM_KV_HEADS: tl.constexpr,
HEAD_DIM: tl.constexpr,
BLOCK_SIZE: tl.constexpr, # KV cache block_size (pages)
NUM_KV_SPLITS: tl.constexpr,
KV_GROUP_SIZE: tl.constexpr, # Hq // Hk
# TQ layout constants
MSE_BITS: tl.constexpr, # 3 or 4
MSE_BYTES: tl.constexpr, # ceil(D * mse_bits / 8)
VQB: tl.constexpr, # value_quant_bits (4 or 8=FP8)
VAL_DATA_BYTES: tl.constexpr, # ceil(D * vqb / 8) or D for FP8
# Opt#3 SoA layout
KEY_DATA_BYTES: tl.constexpr, # MSE_BYTES (MSE) or HEAD_DIM (FP8)
META_REGION_OFFSET: tl.constexpr, # bytes from block start to SoA region
NUM_SOA_FIELDS: tl.constexpr, # 3 (MSE) or 2 (FP8)
SOA_K_NORM: tl.constexpr, # 0 (MSE); unused for FP8
SOA_V_SCALE: tl.constexpr, # 1 (MSE) / 0 (FP8)
SOA_V_ZERO: tl.constexpr, # 2 (MSE) / 1 (FP8)
# Score constants
ATTN_SCALE: tl.constexpr, # 1/sqrt(D)
# Block tile sizes
BLOCK_D: tl.constexpr, # next_power_of_2(HEAD_DIM)
BLOCK_KV: tl.constexpr, # tokens per tile (16)
KEY_FP8: tl.constexpr, # 1 if K is stored as FP8
NORM_CORRECTION: tl.constexpr = 0, # 1 = re-normalize centroids
FP8_E4B15: tl.constexpr = 0, # 1 = use e4b15 (Ampere/Ada), 0 = e4nv (Hopper+)
USE_SINKS: tl.constexpr = 0, # 1 = use sink tokens for attention anchoring
):
bid = tl.program_id(0) # batch index
hid = tl.program_id(1) # q_head index
sid = tl.program_id(2) # kv_split index
kv_head = hid // KV_GROUP_SIZE
# Sequence length for this batch
seq_len = tl.load(Seq_lens_ptr + bid)
# KV split range
split_len = tl.cdiv(seq_len, NUM_KV_SPLITS)
split_start = split_len * sid
split_end = tl.minimum(split_start + split_len, seq_len)
if split_start >= split_end:
return
# Dimension offsets
d_offs = tl.arange(0, BLOCK_D)
d_mask = d_offs < HEAD_DIM
kv_range = tl.arange(0, BLOCK_KV)
# Load query vector: q_rot — [BLOCK_D] float32
q_base = bid * stride_qb + hid * stride_qh
q_rot = tl.load(Q_rot_ptr + q_base + d_offs, mask=d_mask, other=0.0).to(tl.float32)
# Precompute byte/bit index vectors for MSE gather loads
if not KEY_FP8:
mse_bit_off = d_offs * MSE_BITS
mse_byte_idx = mse_bit_off // 8
mse_bit_shift = mse_bit_off % 8
mse_mask = (1 << MSE_BITS) - 1
# Precompute value bit/byte index vectors (loop-invariant)
if VQB == 3:
val_bit_off = d_offs * 3
val_byte_idx = val_bit_off // 8
val_bit_shift = val_bit_off % 8
# Online softmax accumulators
# Sink tokens provide a pre-computed attention bias that should be
# included in the softmax normalization. For the first split (sid==0),
# we initialize m_prev with the sink logit. L is always 1.0 for all
# splits to match unified attention semantics.
if USE_SINKS:
if sid == 0:
m_prev = tl.load(Sink_ptr + hid).to(tl.float32)
l_prev = 1.0
else:
m_prev = -float("inf")
l_prev = 0.0
else:
m_prev = -float("inf")
l_prev = 0.0
acc = tl.zeros([BLOCK_D], dtype=tl.float32)
bt_base = bid * stride_bt_b
# ================================================================
# TILED LOOP: process BLOCK_KV tokens per iteration
# ================================================================
for start_n in range(split_start, split_end, BLOCK_KV):
kv_offs = start_n + kv_range
kv_mask = kv_offs < split_end
page_idx = kv_offs // BLOCK_SIZE
page_off = kv_offs % BLOCK_SIZE
block_nums = tl.load(
Block_table_ptr + bt_base + page_idx,
mask=kv_mask,
other=0,
).to(tl.int64)
# Opt#3 SoA addressing: data region then metadata region per block.
slot_within_block = page_off.to(tl.int64)
block_base = block_nums * stride_cache_block
DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES
data_bases = (
block_base
+ slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT)
+ tl.cast(kv_head, tl.int64) * DATA_BYTES_PER_SLOT
)
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast(
kv_head, tl.int64
) * (NUM_SOA_FIELDS * BLOCK_SIZE)
knorm_u16_addrs = (
head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block
)
vscale_u16_addrs = (
head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block
)
vzero_u16_addrs = (
head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block
)
# ============================================================
# COMPUTE ATTENTION SCORES: [BLOCK_KV]
# ============================================================
if KEY_FP8:
k_addrs = data_bases[:, None] + d_offs[None, :]
k_raw = tl.load(
KV_cache_ptr + k_addrs,
mask=kv_mask[:, None] & d_mask[None, :],
other=0,
)
if FP8_E4B15:
k_float = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32)
else:
k_float = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32)
scores = (
tl.sum(
tl.where(d_mask[None, :], q_rot[None, :] * k_float, 0.0),
axis=1,
)
* ATTN_SCALE
)
scores = tl.where(kv_mask, scores, -float("inf"))
else:
# MSE unpack + norms
mse_addrs0 = data_bases[:, None] + mse_byte_idx[None, :]
mse_raw0 = tl.load(
KV_cache_ptr + mse_addrs0,
mask=kv_mask[:, None] & d_mask[None, :],
other=0,
).to(tl.int32)
mse_raw1 = tl.load(
KV_cache_ptr + mse_addrs0 + 1,
mask=kv_mask[:, None] & d_mask[None, :],
other=0,
).to(tl.int32)
raw16 = mse_raw0 | (mse_raw1 << 8)
mse_idx = (raw16 >> mse_bit_shift[None, :]) & mse_mask
# Centroid gather + dot product
c_vals = tl.load(
Centroids_ptr + mse_idx,
mask=kv_mask[:, None] & d_mask[None, :],
other=0.0,
)
# Opt#1: norm-correction (1/||c_vec||) is pre-folded into the
# stored per-token scalar by the MSE store kernel when
# NORM_CORRECTION=1, so we skip the per-tile sum+sqrt+divide.
# NORM_CORRECTION is kept in the signature for API parity; it is
# implicit in the stored norm value now.
term1 = tl.sum(
tl.where(d_mask[None, :], q_rot[None, :] * c_vals, 0.0),
axis=1,
)
# Opt#3: K-norms live in the per-block SoA region. Single u16
# load per token; contiguous across tokens within a block.
norm_u16 = tl.load(
KV_cache_u16_ptr + knorm_u16_addrs, mask=kv_mask, other=0
)
vec_norms = norm_u16.to(tl.float16, bitcast=True).to(tl.float32)
scores = vec_norms * term1 * ATTN_SCALE
scores = tl.where(kv_mask, scores, -float("inf"))
# ============================================================
# ONLINE SOFTMAX UPDATE (block-level)
# ============================================================
n_e_max = tl.maximum(tl.max(scores, 0), m_prev)
re_scale = tl.exp(m_prev - n_e_max)
p = tl.exp(scores - n_e_max)
# ============================================================
# VALUE LOAD + DEQUANTIZE: [BLOCK_KV, BLOCK_D]
# Opt#3: V data immediately follows K data in the slot (no
# interleaved metadata). Scale/zero come from the SoA metadata
# region as single u16 loads per field per token.
# ============================================================
val_bases = data_bases + KEY_DATA_BYTES
if VQB == 3:
val_addrs0 = val_bases[:, None] + val_byte_idx[None, :]
val_raw0 = tl.load(
KV_cache_ptr + val_addrs0,
mask=kv_mask[:, None] & d_mask[None, :],
other=0,
).to(tl.int32)
val_raw1 = tl.load(
KV_cache_ptr + val_addrs0 + 1,
mask=kv_mask[:, None] & d_mask[None, :],
other=0,
).to(tl.int32)
raw16 = val_raw0 | (val_raw1 << 8)
v_idx = ((raw16 >> val_bit_shift[None, :]) & 0x7).to(tl.float32)
else: # VQB == 4
vb_idx = d_offs // 2
vb_shift = (d_offs % 2) * 4
val_addrs = val_bases[:, None] + vb_idx[None, :]
val_raw = tl.load(
KV_cache_ptr + val_addrs,
mask=kv_mask[:, None] & d_mask[None, :],
other=0,
).to(tl.int32)
v_idx = ((val_raw >> vb_shift[None, :]) & 0xF).to(tl.float32)
scale_u16 = tl.load(KV_cache_u16_ptr + vscale_u16_addrs, mask=kv_mask, other=0)
zero_u16 = tl.load(KV_cache_u16_ptr + vzero_u16_addrs, mask=kv_mask, other=0)
v_scales = scale_u16.to(tl.float16, bitcast=True).to(tl.float32)
v_zeros = zero_u16.to(tl.float16, bitcast=True).to(tl.float32)
values = v_idx * v_scales[:, None] + v_zeros[:, None]
# ============================================================
# WEIGHTED VALUE ACCUMULATION
# ============================================================
acc = acc * re_scale + tl.sum(p[:, None] * values, 0)
l_prev = l_prev * re_scale + tl.sum(p, 0)
m_prev = n_e_max
# Store partial result
out_base = bid * stride_mid_b + hid * stride_mid_h + sid * stride_mid_s
safe_l = tl.where(l_prev > 0.0, l_prev, 1.0)
tl.store(Mid_o_ptr + out_base + d_offs, acc / safe_l, mask=d_mask)
lse = m_prev + tl.log(safe_l)
tl.store(Mid_o_ptr + out_base + HEAD_DIM, lse)
# ---------------------------------------------------------------------------
# Pre-dequant kernel: Bulk dequant K (MSE+norms) and V to fp16
# ---------------------------------------------------------------------------
@triton.jit
def _tq_full_dequant_kv(
KV_cache_ptr,
KV_cache_u16_ptr,
Block_table_ptr,
Centroids_ptr,
K_out_ptr, # [B, Hk, max_seq, D] float16
V_out_ptr, # [B, Hk, max_seq, D] float16
stride_ko_b: tl.int64,
stride_ko_h: tl.int64,
stride_ko_s: tl.int64,
stride_vo_b: tl.int64,
stride_vo_h: tl.int64,
stride_vo_s: tl.int64,
stride_cache_block: tl.int64,
stride_bt_b: tl.int64,
HEAD_DIM: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
NUM_KV_HEADS: tl.constexpr,
MSE_BYTES: tl.constexpr,
VQB: tl.constexpr,
VAL_DATA_BYTES: tl.constexpr,
MSE_BITS: tl.constexpr,
KEY_FP8: tl.constexpr,
# Opt#3 SoA layout
KEY_DATA_BYTES: tl.constexpr,
META_REGION_OFFSET: tl.constexpr,
NUM_SOA_FIELDS: tl.constexpr,
SOA_K_NORM: tl.constexpr,
SOA_V_SCALE: tl.constexpr,
SOA_V_ZERO: tl.constexpr,
BLOCK_D: tl.constexpr,
NORM_CORRECTION: tl.constexpr = 0,
FP8_E4B15: tl.constexpr = 0, # 1 = use e4b15 (Ampere/Ada), 0 = e4nv (Hopper+)
):
"""Full dequant: reconstruct K (MSE centroids * norm or FP8) and V to fp16.
Opt#3 SoA layout: data region [bs, H, KD+VD] + per-block SoA metadata.
"""
pos = tl.program_id(0)
bh = tl.program_id(1)
bid = bh // NUM_KV_HEADS
hid = bh % NUM_KV_HEADS
page_idx = pos // BLOCK_SIZE
page_off = pos % BLOCK_SIZE
block_num = tl.load(Block_table_ptr + bid * stride_bt_b + page_idx).to(tl.int64)
block_base = block_num * stride_cache_block
slot_within_block = tl.cast(page_off, tl.int64)
DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES
data_base = (
block_base
+ slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT)
+ tl.cast(hid, tl.int64) * DATA_BYTES_PER_SLOT
)
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast(
hid, tl.int64
) * (NUM_SOA_FIELDS * BLOCK_SIZE)
knorm_u16_addr = head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block
vscale_u16_addr = head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block
vzero_u16_addr = head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block
d_offs = tl.arange(0, BLOCK_D)
d_mask = d_offs < HEAD_DIM
# === K dequant ===
ko_base = bid * stride_ko_b + hid * stride_ko_h + pos * stride_ko_s
if KEY_FP8:
k_raw = tl.load(KV_cache_ptr + data_base + d_offs, mask=d_mask, other=0)
if FP8_E4B15:
k_recon = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32)
else:
k_recon = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32)
tl.store(K_out_ptr + ko_base + d_offs, k_recon.to(tl.float16), mask=d_mask)
else:
mse_bit_off = d_offs * MSE_BITS
mse_byte_idx = mse_bit_off // 8
mse_bit_shift = mse_bit_off % 8
mse_umask = (1 << MSE_BITS) - 1
mse_raw0 = tl.load(
KV_cache_ptr + data_base + mse_byte_idx, mask=d_mask, other=0
).to(tl.int32)
mse_raw1 = tl.load(
KV_cache_ptr + data_base + mse_byte_idx + 1, mask=d_mask, other=0
).to(tl.int32)
raw16_key = mse_raw0 | (mse_raw1 << 8)
mse_idx = (raw16_key >> mse_bit_shift) & mse_umask
k_mse = tl.load(Centroids_ptr + mse_idx, mask=d_mask, other=0.0)
# Opt#1: norm-correction is pre-folded into the stored scalar.
# Opt#3: K-norm lives in the SoA region — single u16 load.
norm_u16 = tl.load(KV_cache_u16_ptr + knorm_u16_addr)
vec_norm = norm_u16.to(tl.float16, bitcast=True).to(tl.float32)
k_recon = vec_norm * k_mse
tl.store(K_out_ptr + ko_base + d_offs, k_recon.to(tl.float16), mask=d_mask)
# === V dequant ===
val_base = data_base + KEY_DATA_BYTES
if VQB == 4:
vb_idx = d_offs // 2
vb_shift = (d_offs % 2) * 4
val_raw = tl.load(KV_cache_ptr + val_base + vb_idx, mask=d_mask, other=0).to(
tl.int32
)
v_idx = ((val_raw >> vb_shift) & 0xF).to(tl.float32)
elif VQB == 3:
val_bit_off = d_offs * 3
val_byte_idx = val_bit_off // 8
val_bit_shift = val_bit_off % 8
val_raw0 = tl.load(
KV_cache_ptr + val_base + val_byte_idx, mask=d_mask, other=0
).to(tl.int32)
val_raw1 = tl.load(
KV_cache_ptr + val_base + val_byte_idx + 1, mask=d_mask, other=0
).to(tl.int32)
raw16_val = val_raw0 | (val_raw1 << 8)
v_idx = ((raw16_val >> val_bit_shift) & 0x7).to(tl.float32)
else:
v_idx = tl.zeros([BLOCK_D], dtype=tl.float32)
# Opt#3: V scale/zero single-u16 loads from SoA region.
sc_u16 = tl.load(KV_cache_u16_ptr + vscale_u16_addr)
zr_u16 = tl.load(KV_cache_u16_ptr + vzero_u16_addr)
v_scale = sc_u16.to(tl.float16, bitcast=True).to(tl.float32)
v_zero = zr_u16.to(tl.float16, bitcast=True).to(tl.float32)
v_vals = v_idx * v_scale + v_zero
vo_base = bid * stride_vo_b + hid * stride_vo_h + pos * stride_vo_s
tl.store(V_out_ptr + vo_base + d_offs, v_vals.to(tl.float16), mask=d_mask)
# ---------------------------------------------------------------------------
# Stage 2: Reuse from triton_decode_attention.py
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Launcher — cached constants + fused GEMM
# ---------------------------------------------------------------------------
_layout_cache: dict = {}
def _get_layout(D, mse_bits, value_quant_bits, key_packed_size):
"""Get cached layout constants."""
key = (D, mse_bits, value_quant_bits, key_packed_size)
cfg = _layout_cache.get(key)
if cfg is None:
val_data_bytes = math.ceil(D * value_quant_bits / 8)
cfg = {
"mse_bytes": math.ceil(D * mse_bits / 8),
"val_data_bytes": val_data_bytes,
"mse_bits": mse_bits,
"n_centroids": 2**mse_bits,
"BLOCK_D": triton.next_power_of_2(D),
}
_layout_cache[key] = cfg
return cfg
def triton_turboquant_decode_attention(
query: torch.Tensor, # [B, Hq, D] — original query
kv_cache: torch.Tensor, # [num_blocks, block_size, Hk, padded_slot] uint8
block_table: torch.Tensor, # [B, max_num_blocks] int32
seq_lens: torch.Tensor, # [B] int32
Pi: torch.Tensor, # [D, D] float32
centroids: torch.Tensor, # [n_centroids] float32
scale: float,
mse_bits: int,
key_packed_size: int,
value_quant_bits: int,
key_fp8: bool = False,
norm_correction: bool = False,
PiT: torch.Tensor | None = None, # [D, D] pre-computed Pi.T contiguous
# Pre-allocated buffers (optional, avoids per-call allocation)
mid_o_buf: torch.Tensor | None = None,
output_buf: torch.Tensor | None = None,
lse_buf: torch.Tensor | None = None,
buf_holder: Any = None,
max_num_kv_splits: int = 32, # fixed split count (must be constant for cudagraph)
sinks: torch.Tensor | None = None, # [Hq] float32, pre-computed sink logits
) -> torch.Tensor:
"""Launch fused TQ decode attention (Triton stage1 + stage2).
Returns: output tensor [B, Hq, D] in query's dtype.
"""
B, Hq, D = query.shape
Hk = kv_cache.shape[2]
block_size = kv_cache.shape[1]
kv_group_size = Hq // Hk
device = query.device
cfg = _get_layout(D, mse_bits, value_quant_bits, key_packed_size)
# Opt#3 SoA layout constants (derived locally; match store-side values).
key_data_bytes = D if key_fp8 else cfg["mse_bytes"]
data_bytes_per_slot = key_data_bytes + cfg["val_data_bytes"]
meta_region_offset = block_size * Hk * data_bytes_per_slot
num_soa_fields = 2 if key_fp8 else 3
soa_k_norm = 0
soa_v_scale = 0 if key_fp8 else 1
soa_v_zero = 1 if key_fp8 else 2
kv_cache_u16 = kv_cache.view(torch.uint16)
# Compute q_rot = q @ Pi.T (rotated query for MSE key scoring)
# FP8 path: pass query directly (float16); kernel casts inline.
# MSE path: still needs external GEMM (cuBLAS), so q_rot is float32.
if key_fp8:
q_rot = query.contiguous()
else:
q_float = query.float()
if PiT is None:
PiT = Pi.T.contiguous()
q_rot = (q_float @ PiT).contiguous()
NUM_KV_SPLITS = max_num_kv_splits
if (
mid_o_buf is not None
and mid_o_buf.shape[0] >= B
and mid_o_buf.shape[2] >= NUM_KV_SPLITS
):
mid_o = mid_o_buf[:B, :Hq, :NUM_KV_SPLITS, :]
else:
mid_o = torch.empty(
B,
Hq,
NUM_KV_SPLITS,
D + 1,
dtype=torch.float32,
device=device,
)
if buf_holder is not None:
buf_holder._tq_mid_o_buf = mid_o
# Stage 1: split-KV tiled attention scoring + value accumulation
fp8_e4b15 = _use_fp8_e4b15(device.index or 0)
BLOCK_KV = 4
grid = (B, Hq, NUM_KV_SPLITS)
# Prepare sink pointer (convert to float32 on device if provided)
use_sinks = sinks is not None
_tq_decode_stage1[grid](
q_rot,
kv_cache,
kv_cache_u16,
block_table,
seq_lens,
centroids,
mid_o,
sinks,
q_rot.stride(0),
q_rot.stride(1),
kv_cache.stride(0),
block_table.stride(0),
mid_o.stride(0),
mid_o.stride(1),
mid_o.stride(2),
NUM_KV_HEADS=Hk,
HEAD_DIM=D,
BLOCK_SIZE=block_size,
NUM_KV_SPLITS=NUM_KV_SPLITS,
KV_GROUP_SIZE=kv_group_size,
MSE_BITS=mse_bits,
MSE_BYTES=cfg["mse_bytes"],
VQB=value_quant_bits,
VAL_DATA_BYTES=cfg["val_data_bytes"],
KEY_DATA_BYTES=key_data_bytes,
META_REGION_OFFSET=meta_region_offset,
NUM_SOA_FIELDS=num_soa_fields,
SOA_K_NORM=soa_k_norm,
SOA_V_SCALE=soa_v_scale,
SOA_V_ZERO=soa_v_zero,
ATTN_SCALE=scale,
BLOCK_D=cfg["BLOCK_D"],
BLOCK_KV=BLOCK_KV,
KEY_FP8=1 if key_fp8 else 0,
NORM_CORRECTION=1 if norm_correction else 0,
FP8_E4B15=fp8_e4b15,
USE_SINKS=1 if use_sinks else 0,
num_warps=1,
num_stages=1,
)
# Stage 2: Reduce across KV splits
if output_buf is not None and output_buf.shape[0] >= B:
output = output_buf[:B, :Hq, :D]
else:
output = torch.empty(B, Hq, D, dtype=torch.float32, device=device)
if buf_holder is not None:
buf_holder._tq_output_buf = output
if lse_buf is not None and lse_buf.shape[0] >= B:
lse = lse_buf[:B, :Hq]
else:
lse = torch.empty(B, Hq, dtype=torch.float32, device=device)
if buf_holder is not None:
buf_holder._tq_lse_buf = lse
grid2 = (B, Hq)
_fwd_kernel_stage2[grid2](
mid_o,
output,
lse,
seq_lens,
mid_o.stride(0),
mid_o.stride(1),
mid_o.stride(2),
output.stride(0),
output.stride(1),
lse.stride(0),
NUM_KV_SPLITS=NUM_KV_SPLITS,
BLOCK_DV=cfg["BLOCK_D"],
Lv=D,
num_warps=4,
num_stages=2,
)
return output.to(query.dtype)
@@ -0,0 +1,669 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Optimized Triton TurboQuant decode attention (v2).
FLUTE-paper optimizations applied:
1. Grouped Q heads: grid over (B, Hk, splits) instead of (B, Hq, splits).
Each program loads BLOCK_M Q heads sharing a KV head into a 2D tile,
enabling tl.dot on tensor cores (MFMA/WMMA) for both Q·K and P·V.
2. Vectorized pair LUT: precompute pair_table[i][j] = (T[i], T[j])
offline. At runtime, extract adjacent index pairs and fetch two
dequantized centroids with a single gather, halving LUT lookups.
3. exp2 instead of exp: scores pre-scaled by log2(e) so the hardware-
native exp2 instruction replaces the more expensive exp.
4. Wider index extraction: for 4-bit MSE, two adjacent 4-bit indices share
a byte. One byte load yields both, eliminating redundant loads.
5. Centroids pre-warmed in L1 at kernel start.
6. BLOCK_KV = TILE_SIZE raised to 16-32 (from 4), reducing loop iterations
and softmax rescaling overhead.
Stage 2 is reused unchanged from triton_decode_attention.py.
"""
import math
from typing import Any
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.attention.ops.triton_decode_attention import _fwd_kernel_stage2
from .triton_turboquant_decode import _use_fp8_e4b15
# ROCm prefers num_stages=1 in attention-like kernels to reduce shared-memory
# pressure (mirrors the pattern used in triton_decode_attention.py and
# triton_turboquant_decode.py).
_is_hip = current_platform.is_rocm()
# On ROCm, bf16 has the same MFMA throughput as fp16 but wider dynamic range
# (8-bit exponent vs 5-bit), which is safer for attention scores. On CUDA,
# fp16 tensor cores may be faster than bf16 for some shapes.
_DOT_DTYPE = tl.bfloat16 if _is_hip else tl.float16
# ---------------------------------------------------------------------------
# Pair LUT construction (FLUTE) — called once at launcher time
# ---------------------------------------------------------------------------
def build_pair_lut(centroids: torch.Tensor) -> torch.Tensor:
"""Build vectorized pair lookup table.
For N centroids, returns a [N, N, 2] float32 tensor where
pair_lut[i, j] = (centroids[i], centroids[j]).
Flattened to [N*N, 2] for kernel indexing: pair_lut[i*N + j].
For 4-bit MSE (N=16): 16*16*2*4 = 2048 bytes — fits in L1/smem.
For 3-bit MSE (N=8): 8*8*2*4 = 512 bytes.
"""
N = centroids.shape[0]
# pair_lut[i,j,0] = centroids[i], pair_lut[i,j,1] = centroids[j]
c = centroids.float()
lut = torch.empty(N, N, 2, dtype=torch.float32, device=centroids.device)
lut[:, :, 0] = c[:, None]
lut[:, :, 1] = c[None, :]
return lut.reshape(N * N, 2).contiguous()
# ---------------------------------------------------------------------------
# Stage 1 v2: Grouped-Q + pair LUT + tl.dot + exp2
# ---------------------------------------------------------------------------
@triton.jit
def _tq_decode_stage1_v2(
Q_rot_ptr,
KV_cache_ptr,
KV_cache_u16_ptr, # uint16 view — Opt#3 SoA loads
Block_table_ptr,
Seq_lens_ptr,
Centroids_ptr,
Pair_lut_ptr,
Mid_o_ptr,
stride_qb,
stride_qh,
stride_cache_block, # bytes per block (bs*H*slot_aligned)
stride_bt_b,
stride_mid_b,
stride_mid_h,
stride_mid_s,
NUM_Q_HEADS: tl.constexpr,
NUM_KV_HEADS: tl.constexpr,
HEAD_DIM: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
PADDED_SLOT: tl.constexpr,
MAX_NUM_BLOCKS: tl.constexpr,
NUM_KV_SPLITS: tl.constexpr,
KV_GROUP_SIZE: tl.constexpr,
MSE_BITS: tl.constexpr,
MSE_BYTES: tl.constexpr,
VQB: tl.constexpr,
VAL_DATA_BYTES: tl.constexpr,
# Opt#3 SoA layout
KEY_DATA_BYTES: tl.constexpr,
META_REGION_OFFSET: tl.constexpr,
NUM_SOA_FIELDS: tl.constexpr,
SOA_K_NORM: tl.constexpr,
SOA_V_SCALE: tl.constexpr,
SOA_V_ZERO: tl.constexpr,
N_CENTROIDS: tl.constexpr,
ATTN_SCALE: tl.constexpr,
BLOCK_D: tl.constexpr,
TILE_SIZE: tl.constexpr,
BLOCK_M: tl.constexpr,
KEY_FP8: tl.constexpr,
NORM_CORRECTION: tl.constexpr = 0,
FP8_E4B15: tl.constexpr = 0,
USE_PAIR_LUT: tl.constexpr = 0,
USE_BF16_DOT: tl.constexpr = 0,
):
# Grid: (B, Hk, NUM_KV_SPLITS) — one program per KV head group
bid = tl.program_id(0)
kv_hid = tl.program_id(1)
sid = tl.program_id(2)
seq_len = tl.load(Seq_lens_ptr + bid)
split_len = tl.cdiv(seq_len, NUM_KV_SPLITS)
split_start = split_len * sid
split_end = tl.minimum(split_start + split_len, seq_len)
if split_start >= split_end:
return
# Dimension and tile offsets
d_offs = tl.arange(0, BLOCK_D)
d_mask = d_offs < HEAD_DIM
m_offs = tl.arange(0, BLOCK_M)
t_offs = tl.arange(0, TILE_SIZE)
# Q head indices for this KV group: [kv_hid * GS, kv_hid * GS + GS)
q_head_offs = kv_hid * KV_GROUP_SIZE + m_offs
q_mask = m_offs < KV_GROUP_SIZE
# Load Q tile: [BLOCK_M, BLOCK_D] float32
q_addrs = bid * stride_qb + q_head_offs[:, None] * stride_qh + d_offs[None, :]
Q = tl.load(
Q_rot_ptr + q_addrs,
mask=q_mask[:, None] & d_mask[None, :],
other=0.0,
)
# Pre-warm centroids in L1
if not KEY_FP8:
_c_warm = tl.load(Centroids_ptr + tl.arange(0, N_CENTROIDS)) # noqa: F841
# Loop-invariant bit-extraction constants for MSE keys
if not KEY_FP8:
if MSE_BITS == 4:
pass # 4-bit byte extraction is computed inline in the tile loop
else:
# Generic bit extraction
mse_bit_off = d_offs * MSE_BITS
mse_byte_idx = mse_bit_off // 8
mse_bit_shift = mse_bit_off % 8
mse_mask_val = (1 << MSE_BITS) - 1
# Loop-invariant bit-extraction for 3-bit values
if VQB == 3:
val_bit_off = d_offs * 3
val_byte_idx = val_bit_off // 8
val_bit_shift = val_bit_off % 8
# exp2 pre-scaling
RCP_LN2: tl.constexpr = 1.4426950408889634
LN2: tl.constexpr = 0.6931471805599453
QK_SCALE = ATTN_SCALE * RCP_LN2
# Online softmax accumulators: [BLOCK_M] for max, [BLOCK_M] for L
M = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32)
L = tl.zeros([BLOCK_M], dtype=tl.float32)
acc = tl.zeros([BLOCK_M, BLOCK_D], dtype=tl.float32)
bt_base = bid * stride_bt_b
# ================================================================
# TILED LOOP: TILE_SIZE KV tokens per iteration
# ================================================================
for start_n in range(split_start, split_end, TILE_SIZE):
kv_offs = start_n + t_offs
kv_mask_1d = kv_offs < split_end
page_idx = kv_offs // BLOCK_SIZE
page_off = kv_offs % BLOCK_SIZE
block_nums = tl.load(
Block_table_ptr + bt_base + page_idx,
mask=kv_mask_1d,
other=0,
).to(tl.int64)
# Opt#3 SoA addressing: data region then per-block SoA metadata.
slot_within_block = page_off.to(tl.int64)
block_base = block_nums * stride_cache_block
DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES
data_bases = (
block_base
+ slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT)
+ tl.cast(kv_hid, tl.int64) * DATA_BYTES_PER_SLOT
)
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast(
kv_hid, tl.int64
) * (NUM_SOA_FIELDS * BLOCK_SIZE)
knorm_u16_addrs = (
head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block
)
vscale_u16_addrs = (
head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block
)
vzero_u16_addrs = (
head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block
)
# ============================================================
# KEY DEQUANT → K_T: [BLOCK_D, TILE_SIZE] for tl.dot(Q, K_T)
# ============================================================
if KEY_FP8:
k_addrs = data_bases[:, None] + d_offs[None, :]
k_raw = tl.load(
KV_cache_ptr + k_addrs,
mask=kv_mask_1d[:, None] & d_mask[None, :],
other=0,
)
if FP8_E4B15:
K_f = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32)
else:
K_f = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32)
# K_f: [TILE_SIZE, BLOCK_D] → transpose for tl.dot
K_T = tl.trans(K_f)
# S: [BLOCK_M, TILE_SIZE] (bf16 dot on ROCm for better dynamic range)
if USE_BF16_DOT:
S = QK_SCALE * tl.dot(Q.to(tl.bfloat16), K_T.to(tl.bfloat16))
else:
S = QK_SCALE * tl.dot(Q.to(tl.float16), K_T.to(tl.float16))
else:
# --- MSE dequantization ---
if MSE_BITS == 4 and USE_PAIR_LUT:
# OPT#1 (load-halving): FLUTE §3.2 pair LUT, but with each
# packed byte loaded EXACTLY ONCE instead of twice (the
# previous code built byte_addrs from d_offs // 2 which has
# the pattern [0,0,1,1,2,2,...], duplicating every byte
# load). We load the packed bytes as a [TILE_SIZE, HALF_D]
# tile, decode both nibbles, then issue a single 3-D gather
# against pair_lut which returns (T[lo], T[hi]) in slots
# 0/1. Reshape [TILE_SIZE, HALF_D, 2] -> [TILE_SIZE, BLOCK_D]
# interleaves the pairs into the expected dim order
# (d=2k -> T[lo_k], d=2k+1 -> T[hi_k]).
HALF_D: tl.constexpr = BLOCK_D // 2
half_offs = tl.arange(0, HALF_D)
# Each byte k covers dims (2k, 2k+1); it is in-range iff
# 2k < HEAD_DIM. HEAD_DIM is always even in our layouts,
# so this is equivalent to k < HEAD_DIM // 2.
byte_mask = (half_offs * 2) < HEAD_DIM
byte_addrs = data_bases[:, None] + half_offs[None, :]
byte_raw = tl.load(
KV_cache_ptr + byte_addrs,
mask=kv_mask_1d[:, None] & byte_mask[None, :],
other=0,
).to(tl.int32)
lo_idx = byte_raw & 0xF # [TILE_SIZE, HALF_D]
hi_idx = (byte_raw >> 4) & 0xF # [TILE_SIZE, HALF_D]
pair_key = lo_idx * N_CENTROIDS + hi_idx # [TILE_SIZE, HALF_D]
# Single 3-D gather: pair_lut[pair_key*2 + {0,1}]
# -> [TILE_SIZE, HALF_D, 2] = (T[lo_k], T[hi_k]) per element.
pair_slot = tl.arange(0, 2)
c_pair = tl.load(
Pair_lut_ptr + pair_key[:, :, None] * 2 + pair_slot[None, None, :],
mask=(kv_mask_1d[:, None, None] & byte_mask[None, :, None]),
other=0.0,
)
# Row-major reshape interleaves pairs:
# out[t, 2k] = c_pair[t, k, 0] = T[lo_k]
# out[t, 2k+1] = c_pair[t, k, 1] = T[hi_k]
c_vals = tl.reshape(c_pair, [TILE_SIZE, BLOCK_D])
elif MSE_BITS == 4:
# 4-bit without pair LUT: exploit byte alignment
half_idx = d_offs // 2
nibble_shift = (d_offs % 2) * 4
mse_addrs = data_bases[:, None] + half_idx[None, :]
mse_raw = tl.load(
KV_cache_ptr + mse_addrs,
mask=kv_mask_1d[:, None] & d_mask[None, :],
other=0,
).to(tl.int32)
mse_idx = (mse_raw >> nibble_shift[None, :]) & 0xF
c_vals = tl.load(
Centroids_ptr + mse_idx,
mask=kv_mask_1d[:, None] & d_mask[None, :],
other=0.0,
)
else:
# Generic 3-bit path: two byte loads → 16-bit → shift/mask
mse_addrs0 = data_bases[:, None] + mse_byte_idx[None, :]
mse_raw0 = tl.load(
KV_cache_ptr + mse_addrs0,
mask=kv_mask_1d[:, None] & d_mask[None, :],
other=0,
).to(tl.int32)
mse_raw1 = tl.load(
KV_cache_ptr + mse_addrs0 + 1,
mask=kv_mask_1d[:, None] & d_mask[None, :],
other=0,
).to(tl.int32)
raw16 = mse_raw0 | (mse_raw1 << 8)
mse_idx = (raw16 >> mse_bit_shift[None, :]) & mse_mask_val
c_vals = tl.load(
Centroids_ptr + mse_idx,
mask=kv_mask_1d[:, None] & d_mask[None, :],
other=0.0,
)
# Opt#1: norm-correction is pre-folded into the stored per-token
# scalar at store time, so no per-tile sum+sqrt+divide here. The
# stored vec_norm carries ||k||/||c_vec||; multiply by c_vals
# below gives the original c_vec/||c_vec|| * ||k||.
# Opt#3: K-norms from per-block SoA region. Single u16 load per
# token; contiguous across tokens within a block.
norm_u16 = tl.load(
KV_cache_u16_ptr + knorm_u16_addrs, mask=kv_mask_1d, other=0
)
vec_norms = norm_u16.to(tl.float16, bitcast=True).to(tl.float32)
# Reconstruct K: K_recon[t, d] = norm[t] * centroid[t, d]
K_recon = c_vals * vec_norms[:, None]
# K_T: [BLOCK_D, TILE_SIZE]
K_T = tl.trans(K_recon)
# S: [BLOCK_M, TILE_SIZE] via tensor core MMA
if USE_BF16_DOT:
S = QK_SCALE * tl.dot(Q.to(tl.bfloat16), K_T.to(tl.bfloat16))
else:
S = QK_SCALE * tl.dot(Q.to(tl.float16), K_T.to(tl.float16))
# Mask out-of-range positions
S = tl.where(kv_mask_1d[None, :], S, float("-inf"))
# ============================================================
# ONLINE SOFTMAX with exp2
# ============================================================
m_j = tl.maximum(M, tl.max(S, axis=1))
m_j = tl.where(m_j > float("-inf"), m_j, 0.0)
P = tl.math.exp2(S - m_j[:, None])
l_j = tl.sum(P, axis=1)
alpha = tl.math.exp2(M - m_j)
acc = acc * alpha[:, None]
L = L * alpha + l_j
M = m_j
# ============================================================
# VALUE DEQUANT → V: [TILE_SIZE, BLOCK_D] for tl.dot(P, V)
# Opt#3: V data lives at data_base + KEY_DATA_BYTES; scale/zero
# move to the SoA metadata region (one u16 load each per token).
# ============================================================
val_bases = data_bases + KEY_DATA_BYTES
if VQB == 3:
val_addrs0 = val_bases[:, None] + val_byte_idx[None, :]
val_raw0 = tl.load(
KV_cache_ptr + val_addrs0,
mask=kv_mask_1d[:, None] & d_mask[None, :],
other=0,
).to(tl.int32)
val_raw1 = tl.load(
KV_cache_ptr + val_addrs0 + 1,
mask=kv_mask_1d[:, None] & d_mask[None, :],
other=0,
).to(tl.int32)
raw16 = val_raw0 | (val_raw1 << 8)
v_idx = ((raw16 >> val_bit_shift[None, :]) & 0x7).to(tl.float32)
else: # VQB == 4
# OPT#3 (load-halving, value path): same pattern as OPT#1 for
# keys -- load each byte exactly once as [TILE_SIZE, HALF_D],
# decode both nibbles, then tl.interleave to [TILE_SIZE, BLOCK_D]
# with pattern [v_lo0, v_hi0, v_lo1, v_hi1, ...].
V_HALF_D: tl.constexpr = BLOCK_D // 2
v_half_offs = tl.arange(0, V_HALF_D)
v_byte_mask = (v_half_offs * 2) < HEAD_DIM
val_addrs = val_bases[:, None] + v_half_offs[None, :]
val_byte = tl.load(
KV_cache_ptr + val_addrs,
mask=kv_mask_1d[:, None] & v_byte_mask[None, :],
other=0,
).to(tl.int32)
v_lo = (val_byte & 0xF).to(tl.float32) # [T, HALF_D]
v_hi = ((val_byte >> 4) & 0xF).to(tl.float32) # [T, HALF_D]
v_idx = tl.interleave(v_lo, v_hi) # [T, BLOCK_D]
scale_u16 = tl.load(
KV_cache_u16_ptr + vscale_u16_addrs, mask=kv_mask_1d, other=0
)
zero_u16 = tl.load(KV_cache_u16_ptr + vzero_u16_addrs, mask=kv_mask_1d, other=0)
v_scales = scale_u16.to(tl.float16, bitcast=True).to(tl.float32)
v_zeros = zero_u16.to(tl.float16, bitcast=True).to(tl.float32)
V = v_idx * v_scales[:, None] + v_zeros[:, None]
# P·V accumulation via tensor core MMA
# P: [BLOCK_M, TILE_SIZE], V: [TILE_SIZE, BLOCK_D]
if USE_BF16_DOT:
acc += tl.dot(P.to(tl.bfloat16), V.to(tl.bfloat16))
else:
acc += tl.dot(P.to(tl.float16), V.to(tl.float16))
# ================================================================
# EPILOGUE: Store per-Q-head partial results for stage2
# ================================================================
# acc: [BLOCK_M, BLOCK_D], L: [BLOCK_M], M: [BLOCK_M]
# Stage2 expects per-Q-head: mid_o[B, Hq, split, D+1]
safe_L = tl.where(L > 0.0, L, 1.0)
one_over_L = 1.0 / safe_L
acc = acc * one_over_L[:, None]
# Convert M from log2 space back to natural log for stage2 compatibility
lse = M * LN2 + tl.log(safe_L)
# Store each Q head's result using 2D scatter
# q_head_offs: [BLOCK_M] = kv_hid * KV_GROUP_SIZE + m_offs
out_addrs = (
bid * stride_mid_b
+ q_head_offs[:, None] * stride_mid_h
+ sid * stride_mid_s
+ d_offs[None, :]
)
tl.store(
Mid_o_ptr + out_addrs,
acc,
mask=q_mask[:, None] & d_mask[None, :],
)
# Store LSE per Q head
lse_addrs = (
bid * stride_mid_b + q_head_offs * stride_mid_h + sid * stride_mid_s + HEAD_DIM
)
tl.store(Mid_o_ptr + lse_addrs, lse, mask=q_mask)
# ---------------------------------------------------------------------------
# Launcher v2
# ---------------------------------------------------------------------------
_layout_cache: dict = {}
def _get_layout(D, mse_bits, value_quant_bits, key_packed_size):
key = (D, mse_bits, value_quant_bits, key_packed_size)
cfg = _layout_cache.get(key)
if cfg is None:
val_data_bytes = math.ceil(D * value_quant_bits / 8)
cfg = {
"mse_bytes": math.ceil(D * mse_bits / 8),
"val_data_bytes": val_data_bytes,
"mse_bits": mse_bits,
"n_centroids": 2**mse_bits,
"BLOCK_D": triton.next_power_of_2(D),
}
_layout_cache[key] = cfg
return cfg
def _get_pair_lut(centroids: torch.Tensor) -> torch.Tensor:
"""Return a fresh pair-LUT for ``centroids`` on each call.
The LUT is tiny (N*N*2 fp32, e.g. 2KB for 4-bit MSE) so the build cost
is negligible compared to attention. We avoid caching by data_ptr()
because CUDA allocator memory reuse across different centroid tensors
can silently return a stale LUT (subtle correctness bug). If this ever
shows up on a profile, cache by a hash-of-values fingerprint instead.
"""
return build_pair_lut(centroids)
def triton_turboquant_decode_attention_v2(
query: torch.Tensor,
kv_cache: torch.Tensor,
block_table: torch.Tensor,
seq_lens: torch.Tensor,
Pi: torch.Tensor,
centroids: torch.Tensor,
scale: float,
mse_bits: int,
key_packed_size: int,
value_quant_bits: int,
value_packed_size: int,
max_seq_len: int = 0, # unused; kept for backward compatibility
key_fp8: bool = False,
norm_correction: bool = False,
PiT: torch.Tensor | None = None,
# Pre-allocated buffers (optional, required for CUDA-graph stability).
mid_o_buf: torch.Tensor | None = None,
output_buf: torch.Tensor | None = None,
lse_buf: torch.Tensor | None = None,
buf_holder: Any = None,
# Fixed split count — MUST be a compile-time constant across iterations
# for CUDA-graph capture/replay to work. Mirrors the v1 launcher contract.
max_num_kv_splits: int = 32,
) -> torch.Tensor:
"""Launch optimized TQ decode attention (v2 stage1 + shared stage2).
Follows the same buffer-reuse + fixed-grid contract as the v1 launcher
(triton_turboquant_decode_attention) so the backend can capture a CUDA
graph across both versions.
"""
B, Hq, D = query.shape
Hk = kv_cache.shape[2]
block_size = kv_cache.shape[1]
padded_slot = kv_cache.shape[3]
max_num_blocks = block_table.shape[1]
n_centroids = centroids.shape[0]
kv_group_size = Hq // Hk
device = query.device
del max_seq_len # no longer used: splits is fixed via max_num_kv_splits
cfg = _get_layout(D, mse_bits, value_quant_bits, key_packed_size)
# Opt#3 SoA layout constants (match store-side computation).
key_data_bytes = D if key_fp8 else cfg["mse_bytes"]
data_bytes_per_slot = key_data_bytes + cfg["val_data_bytes"]
meta_region_offset = block_size * Hk * data_bytes_per_slot
num_soa_fields = 2 if key_fp8 else 3
soa_k_norm = 0
soa_v_scale = 0 if key_fp8 else 1
soa_v_zero = 1 if key_fp8 else 2
kv_cache_u16 = kv_cache.view(torch.uint16)
# Compute q_rot = q @ Pi.T
if key_fp8:
q_rot = query.float().contiguous()
else:
q_float = query.float()
if PiT is None:
PiT = Pi.T.contiguous()
q_rot = (q_float @ PiT).contiguous()
# BLOCK_M: pad KV_GROUP_SIZE to power of 2, minimum 16 for tensor cores
BLOCK_M = max(16, triton.next_power_of_2(kv_group_size))
# TILE_SIZE (BLOCK_KV): tokens per inner-loop iteration
TILE_SIZE = 16
# Fixed split count — must be constant across calls for cudagraph replay.
NUM_KV_SPLITS = max_num_kv_splits
# --- mid_o buffer reuse (same pattern as v1 launcher) ---
if (
mid_o_buf is not None
and mid_o_buf.shape[0] >= B
and mid_o_buf.shape[2] >= NUM_KV_SPLITS
):
mid_o = mid_o_buf[:B, :Hq, :NUM_KV_SPLITS, :]
else:
mid_o = torch.empty(
B,
Hq,
NUM_KV_SPLITS,
D + 1,
dtype=torch.float32,
device=device,
)
if buf_holder is not None:
buf_holder._tq_mid_o_buf = mid_o
fp8_e4b15 = _use_fp8_e4b15(device.index or 0)
# Build pair LUT for 4-bit MSE (FLUTE §3.2)
use_pair_lut = mse_bits == 4 and not key_fp8
pair_lut = _get_pair_lut(centroids) if use_pair_lut else centroids
# Platform-dependent pipelining depth: ROCm prefers num_stages=1.
stage1_num_stages = 1 if _is_hip else 2
# --- Stage 1: v2 kernel ---
# Grid over KV heads (not Q heads) — each program handles all Q heads
grid = (B, Hk, NUM_KV_SPLITS)
_tq_decode_stage1_v2[grid](
q_rot,
kv_cache,
kv_cache_u16,
block_table,
seq_lens,
centroids,
pair_lut,
mid_o,
q_rot.stride(0),
q_rot.stride(1),
kv_cache.stride(0),
block_table.stride(0),
mid_o.stride(0),
mid_o.stride(1),
mid_o.stride(2),
NUM_Q_HEADS=Hq,
NUM_KV_HEADS=Hk,
HEAD_DIM=D,
BLOCK_SIZE=block_size,
PADDED_SLOT=padded_slot,
MAX_NUM_BLOCKS=max_num_blocks,
NUM_KV_SPLITS=NUM_KV_SPLITS,
KV_GROUP_SIZE=kv_group_size,
MSE_BITS=mse_bits,
MSE_BYTES=cfg["mse_bytes"],
VQB=value_quant_bits,
VAL_DATA_BYTES=cfg["val_data_bytes"],
KEY_DATA_BYTES=key_data_bytes,
META_REGION_OFFSET=meta_region_offset,
NUM_SOA_FIELDS=num_soa_fields,
SOA_K_NORM=soa_k_norm,
SOA_V_SCALE=soa_v_scale,
SOA_V_ZERO=soa_v_zero,
N_CENTROIDS=n_centroids,
ATTN_SCALE=scale,
BLOCK_D=cfg["BLOCK_D"],
TILE_SIZE=TILE_SIZE,
BLOCK_M=BLOCK_M,
KEY_FP8=1 if key_fp8 else 0,
NORM_CORRECTION=1 if norm_correction else 0,
FP8_E4B15=fp8_e4b15,
USE_PAIR_LUT=1 if use_pair_lut else 0,
USE_BF16_DOT=1 if _is_hip else 0,
num_warps=4,
num_stages=stage1_num_stages,
)
# --- output / lse buffer reuse (same pattern as v1 launcher) ---
if output_buf is not None and output_buf.shape[0] >= B:
output = output_buf[:B, :Hq, :D]
else:
output = torch.empty(B, Hq, D, dtype=torch.float32, device=device)
if buf_holder is not None:
buf_holder._tq_output_buf = output
if lse_buf is not None and lse_buf.shape[0] >= B:
lse = lse_buf[:B, :Hq]
else:
lse = torch.empty(B, Hq, dtype=torch.float32, device=device)
if buf_holder is not None:
buf_holder._tq_lse_buf = lse
# --- Stage 2: reduce across KV splits (unchanged) ---
grid2 = (B, Hq)
_fwd_kernel_stage2[grid2](
mid_o,
output,
lse,
seq_lens,
mid_o.stride(0),
mid_o.stride(1),
mid_o.stride(2),
output.stride(0),
output.stride(1),
lse.stride(0),
NUM_KV_SPLITS=NUM_KV_SPLITS,
BLOCK_DV=cfg["BLOCK_D"],
Lv=D,
num_warps=4,
num_stages=2,
)
return output.to(query.dtype)
@@ -0,0 +1,524 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused Triton kernels for TurboQuant KV store.
Two kernels:
1. _tq_fused_store_fp8: FP8 key scatter + value uniform quantization.
2. _tq_fused_store_mse: Fused binary-search bucketize + MSE index
packing + value quantization.
The launcher `triton_turboquant_store` selects the appropriate kernel.
"""
import math
import torch
from vllm.triton_utils import tl, triton
from .triton_turboquant_decode import _use_fp8_e4b15
# ═══════════════════════════════════════════════════════════════════════
# Shared: value uniform quantization + pack + SoA scale/zero store
# ═══════════════════════════════════════════════════════════════════════
@triton.jit
def _store_quantized_value(
Value_ptr,
KV_cache_ptr,
KV_cache_u16_ptr, # Opt#3: u16 view for SoA scale/zero stores
base, # pid * D offset into Value_ptr
data_base, # byte offset into KV_cache_ptr for this slot+head's DATA region
vscale_u16_addr, # u16 element index for V-scale in the SoA region
vzero_u16_addr, # u16 element index for V-zero in the SoA region
d_offs, # tl.arange(0, BLOCK_D)
d_mask, # d_offs < D
D: tl.constexpr,
# Offset (bytes) from data_base to V-data region within the slot's data.
# For MSE keys this equals MSE_BYTES; for FP8 keys it equals D.
V_DATA_OFFSET: tl.constexpr,
VQB: tl.constexpr,
VAL_DATA_BYTES: tl.constexpr,
BLOCK_D: tl.constexpr,
BLOCK_VAL: tl.constexpr,
BLOCK_GRP: tl.constexpr,
):
"""Uniform quantization of values to VQB bits, pack, and store with
scale/zero in the per-block SoA metadata region (Opt#3 layout)."""
if VQB == 3:
val_vec = tl.load(Value_ptr + base + d_offs, mask=d_mask, other=0.0).to(
tl.float32
)
val_min = tl.min(tl.where(d_mask, val_vec, float("inf")), axis=0)
val_max = tl.max(tl.where(d_mask, val_vec, -float("inf")), axis=0)
v_scale = (val_max - val_min) / 7.0
v_scale = tl.where(v_scale > 1e-8, v_scale, 1e-8)
q_vals = tl.minimum(
tl.maximum(((val_vec - val_min) / v_scale + 0.5).to(tl.int32), 0), 7
)
grp_offs = tl.arange(0, BLOCK_GRP)
grp_mask = grp_offs < (D // 8)
q_grp = tl.reshape(q_vals, [BLOCK_GRP, 8])
shifts_3bit = tl.arange(0, 8) * 3
packed_24 = tl.sum(q_grp << shifts_3bit[None, :], axis=1)
b0 = (packed_24 & 0xFF).to(tl.uint8)
b1 = ((packed_24 >> 8) & 0xFF).to(tl.uint8)
b2 = ((packed_24 >> 16) & 0xFF).to(tl.uint8)
tl.store(
KV_cache_ptr + data_base + V_DATA_OFFSET + grp_offs * 3,
b0,
mask=grp_mask,
)
tl.store(
KV_cache_ptr + data_base + V_DATA_OFFSET + grp_offs * 3 + 1,
b1,
mask=grp_mask,
)
tl.store(
KV_cache_ptr + data_base + V_DATA_OFFSET + grp_offs * 3 + 2,
b2,
mask=grp_mask,
)
else: # VQB == 4
val_vec = tl.load(Value_ptr + base + d_offs, mask=d_mask, other=0.0).to(
tl.float32
)
val_min = tl.min(tl.where(d_mask, val_vec, float("inf")), axis=0)
val_max = tl.max(tl.where(d_mask, val_vec, -float("inf")), axis=0)
v_scale = (val_max - val_min) / 15.0
v_scale = tl.where(v_scale > 1e-8, v_scale, 1e-8)
# Quantize all D elements from register (no re-load)
q_all = tl.minimum(
tl.maximum(((val_vec - val_min) / v_scale + 0.5).to(tl.int32), 0), 15
)
q_pairs = tl.reshape(q_all, [BLOCK_D // 2, 2])
shifts_4 = tl.arange(0, 2) * 4
packed_val = tl.sum((q_pairs & 0xF) << shifts_4[None, :], axis=1).to(tl.uint8)
val_offs = tl.arange(0, BLOCK_D // 2)
val_mask = val_offs < VAL_DATA_BYTES
tl.store(
KV_cache_ptr + data_base + V_DATA_OFFSET + val_offs,
packed_val,
mask=val_mask,
)
# ── SoA scale/zero store (Opt#3) ──────────────────────────────────
# Scale and zero each get one fp16 stored at their token's slot index
# within the per-head SoA region. Single u16 store per field vs the
# previous 2×u8 writes per field.
sc_u16 = v_scale.to(tl.float16).to(tl.uint16, bitcast=True)
zr_u16 = val_min.to(tl.float16).to(tl.uint16, bitcast=True)
tl.store(KV_cache_u16_ptr + vscale_u16_addr, sc_u16)
tl.store(KV_cache_u16_ptr + vzero_u16_addr, zr_u16)
# ═══════════════════════════════════════════════════════════════════════
# FP8 key store + value uniform quantization
# ═══════════════════════════════════════════════════════════════════════
@triton.jit
def _tq_fused_store_fp8(
Key_ptr, # [NH, D] float16/bfloat16 — raw keys
Value_ptr, # [NH, D] float16/bfloat16 — raw values
KV_cache_ptr, # [total_bytes] uint8 (flattened view)
KV_cache_u16_ptr, # uint16-aliased view of the same storage (Opt#3)
Slot_mapping_ptr, # [N] int32 — per-token slot indices
# Cache strides (stride_cache_block = bytes per block = bs*H*slot_aligned)
stride_cache_block: tl.constexpr,
# Dimensions
D: tl.constexpr,
H: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
BLOCK_D: tl.constexpr,
# SoA layout (Opt#3): data region is [bs, H, KEY_DATA_BYTES+VAL_DATA_BYTES],
# followed by metadata region [H, NUM_SOA_FIELDS, bs] of fp16.
KEY_DATA_BYTES: tl.constexpr, # FP8: = D
VAL_DATA_BYTES: tl.constexpr,
META_REGION_OFFSET: tl.constexpr, # bytes from block start to SoA region
NUM_SOA_FIELDS: tl.constexpr, # FP8: 2 (V-scale, V-zero)
SOA_V_SCALE: tl.constexpr, # FP8: 0
SOA_V_ZERO: tl.constexpr, # FP8: 1
# Value quantization
VQB: tl.constexpr,
# Packing block sizes
BLOCK_VAL: tl.constexpr,
BLOCK_GRP: tl.constexpr = 16,
FP8_E4B15: tl.constexpr = 0, # 1 = e4b15 (Ampere/Ada), 0 = e4nv (Hopper+)
):
"""FP8 key cast+scatter + value uniform quantization (SoA layout)."""
pid = tl.program_id(0)
token_idx = pid // H
head_idx = pid % H
slot = tl.load(Slot_mapping_ptr + token_idx)
if slot < 0:
return
blk = (slot // BLOCK_SIZE).to(tl.int64)
off = (slot % BLOCK_SIZE).to(tl.int64)
head_idx_i64 = tl.cast(head_idx, tl.int64)
# Block start byte address — stride_cache_block is unchanged (same total
# bytes per block as the AoS layout).
block_base = blk * stride_cache_block
# Data region: per-slot stride = H * DATA_BYTES_PER_SLOT; per-head stride
# within a slot = DATA_BYTES_PER_SLOT.
DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES
data_base = (
block_base
+ off * (H * DATA_BYTES_PER_SLOT)
+ head_idx_i64 * DATA_BYTES_PER_SLOT
)
# SoA metadata region (u16 element indexing): head strip = NUM_SOA_FIELDS
# consecutive bs-long fp16 arrays.
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + head_idx_i64 * (
NUM_SOA_FIELDS * BLOCK_SIZE
)
vscale_u16_addr = head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + off
vzero_u16_addr = head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + off
base = pid * D
# ── FP8 KEY: cast to FP8 in-kernel and store at data_base ────────
d_offs = tl.arange(0, BLOCK_D)
d_mask = d_offs < D
k_vals = tl.load(Key_ptr + base + d_offs, mask=d_mask, other=0.0)
k_fp8 = k_vals.to(tl.float8e4b15) if FP8_E4B15 else k_vals.to(tl.float8e4nv)
k_bytes = k_fp8.to(tl.uint8, bitcast=True)
tl.store(KV_cache_ptr + data_base + d_offs, k_bytes, mask=d_mask)
# ── VALUE QUANTIZE + PACK + SoA scale/zero store ─────────────────
_store_quantized_value(
Value_ptr,
KV_cache_ptr,
KV_cache_u16_ptr,
base,
data_base,
vscale_u16_addr,
vzero_u16_addr,
d_offs,
d_mask,
D=D,
V_DATA_OFFSET=KEY_DATA_BYTES, # V follows K in the data region
VQB=VQB,
VAL_DATA_BYTES=VAL_DATA_BYTES,
BLOCK_D=BLOCK_D,
BLOCK_VAL=BLOCK_VAL,
BLOCK_GRP=BLOCK_GRP,
)
# ═══════════════════════════════════════════════════════════════════════
# Fused MSE store: bucketize + MSE index pack + norm store + value pack
# (eliminates 4 PyTorch kernel launches per layer vs pack-only kernel)
# ═══════════════════════════════════════════════════════════════════════
@triton.jit
def _tq_fused_store_mse(
# Post-rotation inputs
Y_ptr, # [NH, D] float32 — rotated normalized keys (x_hat @ PiT)
Norms_ptr, # [NH] float32 — key vector norms (||k||)
Value_ptr, # [NH, D] float32 — raw values
# Quantization tables
Midpoints_ptr, # [n_centroids-1] float32
Centroids_ptr, # [n_centroids] float32 — used only when NORM_CORRECTION=1
# Cache and indexing
KV_cache_ptr, # [total_bytes] uint8 (flattened view)
KV_cache_u16_ptr, # uint16-aliased view of the same storage (Opt#3)
Slot_mapping_ptr, # [N] int32 — per-token slot indices
# Cache strides (stride_cache_block = bs*H*slot_aligned, unchanged)
stride_cache_block: tl.constexpr,
# Dimensions
D: tl.constexpr,
H: tl.constexpr,
BLOCK_SIZE: tl.constexpr,
BLOCK_D: tl.constexpr,
# SoA layout (Opt#3): data region [bs, H, MSE_BYTES+VAL_DATA_BYTES],
# then metadata region [H, NUM_SOA_FIELDS, bs] of fp16.
MSE_BYTES: tl.constexpr,
VAL_DATA_BYTES: tl.constexpr,
META_REGION_OFFSET: tl.constexpr, # bytes
NUM_SOA_FIELDS: tl.constexpr, # MSE: 3 (k_norm, v_scale, v_zero)
SOA_K_NORM: tl.constexpr, # MSE: 0
SOA_V_SCALE: tl.constexpr, # MSE: 1
SOA_V_ZERO: tl.constexpr, # MSE: 2
# Value quantization
VQB: tl.constexpr,
# Packing block sizes
BLOCK_VAL: tl.constexpr,
# MSE params
MSE_BITS: tl.constexpr,
N_CENTROIDS: tl.constexpr,
# When 1, fold the per-token centroid-vector norm ||c_t|| into the
# stored scalar: store ||k_t|| / ||c_t|| instead of ||k_t||. This
# removes the per-tile sum+sqrt+divide cost from the attention kernel's
# K-load path. Load-side math becomes a plain multiply K = c * stored.
NORM_CORRECTION: tl.constexpr = 0,
BLOCK_GRP: tl.constexpr = 16,
):
"""Fused MSE quantize + pack + SoA-metadata store (Opt#3 layout).
Binary-search bucketize → MSE-index pack → K-norm store (SoA) → value
uniform quantize + pack → V scale/zero store (SoA), in one kernel.
"""
pid = tl.program_id(0)
token_idx = pid // H
head_idx = pid % H
slot = tl.load(Slot_mapping_ptr + token_idx)
if slot < 0:
return
blk = (slot // BLOCK_SIZE).to(tl.int64)
off = (slot % BLOCK_SIZE).to(tl.int64)
head_idx_i64 = tl.cast(head_idx, tl.int64)
# Block start byte address — stride unchanged from AoS (same total
# bytes per block); layout inside the block differs.
block_base = blk * stride_cache_block
# Data region offset (packed K indices + packed V data).
DATA_BYTES_PER_SLOT: tl.constexpr = MSE_BYTES + VAL_DATA_BYTES
data_base = (
block_base
+ off * (H * DATA_BYTES_PER_SLOT)
+ head_idx_i64 * DATA_BYTES_PER_SLOT
)
# SoA metadata addresses (u16 element indexing).
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + head_idx_i64 * (
NUM_SOA_FIELDS * BLOCK_SIZE
)
knorm_u16_addr = head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + off
vscale_u16_addr = head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + off
vzero_u16_addr = head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + off
base = pid * D
d_offs = tl.arange(0, BLOCK_D)
d_mask = d_offs < D
# ── 1. BINARY SEARCH BUCKETIZE ────────────────────────────────────
y_vec = tl.load(Y_ptr + base + d_offs, mask=d_mask, other=0.0)
lo = tl.zeros([BLOCK_D], dtype=tl.int32)
hi = tl.full([BLOCK_D], N_CENTROIDS - 1, dtype=tl.int32)
for _ in range(MSE_BITS):
mid = (lo + hi) >> 1
safe_mid = tl.minimum(mid, N_CENTROIDS - 2)
mid_val = tl.load(Midpoints_ptr + safe_mid, mask=d_mask, other=0.0)
lo = tl.where(y_vec >= mid_val, mid + 1, lo)
hi = tl.where(y_vec >= mid_val, hi, mid)
idx = tl.minimum(lo, N_CENTROIDS - 1)
# ── 2. PACK MSE INDICES into data region at data_base[0:MSE_BYTES] ──
if MSE_BITS == 4:
idx_pairs = tl.reshape(idx, [BLOCK_D // 2, 2])
shifts_4 = tl.arange(0, 2) * 4
packed = tl.sum((idx_pairs & 0xF) << shifts_4[None, :], axis=1).to(tl.uint8)
mse_offs = tl.arange(0, BLOCK_D // 2)
mse_mask = mse_offs < MSE_BYTES
tl.store(KV_cache_ptr + data_base + mse_offs, packed, mask=mse_mask)
elif MSE_BITS == 3:
grp_offs = tl.arange(0, BLOCK_GRP)
grp_mask = grp_offs < (D // 8)
idx_grp = tl.reshape(idx, [BLOCK_GRP, 8])
shifts_3 = tl.arange(0, 8) * 3
packed_24 = tl.sum((idx_grp & 0x7) << shifts_3[None, :], axis=1)
b0 = (packed_24 & 0xFF).to(tl.uint8)
b1 = ((packed_24 >> 8) & 0xFF).to(tl.uint8)
b2 = ((packed_24 >> 16) & 0xFF).to(tl.uint8)
tl.store(KV_cache_ptr + data_base + grp_offs * 3, b0, mask=grp_mask)
tl.store(KV_cache_ptr + data_base + grp_offs * 3 + 1, b1, mask=grp_mask)
tl.store(KV_cache_ptr + data_base + grp_offs * 3 + 2, b2, mask=grp_mask)
# ── 3. STORE K-norm in SoA region (single u16 store) ─────────────
# Norm-correction (Opt#1) still folds 1/||c_vec|| into the stored value;
# only the destination moved to the SoA K-norm array.
vn_f32 = tl.load(Norms_ptr + pid) # ||k_t||, fp32
if NORM_CORRECTION:
safe_idx = tl.minimum(tl.maximum(idx, 0), N_CENTROIDS - 1)
c_vals = tl.load(Centroids_ptr + safe_idx, mask=d_mask, other=0.0)
c_norm_sq = tl.sum(tl.where(d_mask, c_vals * c_vals, 0.0), axis=0)
c_inv_norm = 1.0 / tl.sqrt(c_norm_sq + 1e-16)
vn_f32 = vn_f32 * c_inv_norm
vn_u16 = vn_f32.to(tl.float16).to(tl.uint16, bitcast=True)
tl.store(KV_cache_u16_ptr + knorm_u16_addr, vn_u16)
# ── 4. VALUE QUANTIZE + PACK + SoA scale/zero store ──────────────
_store_quantized_value(
Value_ptr,
KV_cache_ptr,
KV_cache_u16_ptr,
base,
data_base,
vscale_u16_addr,
vzero_u16_addr,
d_offs,
d_mask,
D=D,
V_DATA_OFFSET=MSE_BYTES, # V follows K indices in the data region
VQB=VQB,
VAL_DATA_BYTES=VAL_DATA_BYTES,
BLOCK_D=BLOCK_D,
BLOCK_VAL=BLOCK_VAL,
BLOCK_GRP=BLOCK_GRP,
)
# ═══════════════════════════════════════════════════════════════════════
# Launcher
# ═══════════════════════════════════════════════════════════════════════
def triton_turboquant_store(
key: torch.Tensor, # [N, H, D] — raw keys (post-RoPE)
value: torch.Tensor, # [N, H, D] — raw values
kv_cache: torch.Tensor, # [num_blocks, block_size, Hk, padded_slot] uint8
slot_mapping: torch.Tensor, # [N] int32
PiT: torch.Tensor, # [D, D] float32
midpoints: torch.Tensor, # [n_centroids-1] float32
mse_bits: int,
key_packed_size: int,
value_quant_bits: int,
key_fp8: bool = False,
centroids: torch.Tensor
| None = None, # [n_centroids] float32 — required when norm_correction=True
norm_correction: bool = False,
):
"""Launch TQ store kernel (FP8 or MSE path).
When ``norm_correction=True`` (default off; set by MSE norm-correction
presets), the centroid-vector norm ||c_t|| is folded into the stored
per-token scalar at store time (stored value = ||k_t|| / ||c_t||). The
load-side attention kernel no longer computes per-tile sum+sqrt+divide,
saving ~26% of TQ overhead on gpt-oss decode (ctx=4096). ``centroids``
must be provided in this mode.
"""
if norm_correction and not key_fp8:
assert centroids is not None, (
"norm_correction=True requires centroids tensor; got None. "
"Pass the same [n_centroids] fp32 tensor used to derive midpoints."
)
N, H, D = key.shape
NH = N * H
block_size = kv_cache.shape[1]
BLOCK_D = triton.next_power_of_2(D)
mse_bytes = math.ceil(D * mse_bits / 8)
n_centroids = 2**mse_bits
val_data_bytes = math.ceil(D * value_quant_bits / 8)
BLOCK_VAL = triton.next_power_of_2(val_data_bytes)
# Cache strides (element_size=1 for uint8, so stride in bytes = stride()).
# Only stride_cache_block (== bs*H*slot_aligned) is used under Opt#3;
# per-slot and per-head strides are computed from DATA_BYTES_PER_SLOT.
stride_block = kv_cache.stride(0)
block_grp = triton.next_power_of_2(D // 8) if D >= 8 else 1
# ── Opt#3 SoA layout constants (derived locally so the launcher API
# stays unchanged). Invariant: key_packed_size + value_packed_size
# == slot_size_aligned == (key_data + val_data) + (k_meta + v_meta).
key_data_bytes = D if key_fp8 else mse_bytes
data_bytes_per_slot = key_data_bytes + val_data_bytes
# Data region occupies [0, block_size * H * data_bytes_per_slot) bytes
# at the start of each block; metadata region starts at that offset.
meta_region_offset = block_size * H * data_bytes_per_slot
num_soa_fields = 2 if key_fp8 else 3
soa_v_scale = 0 if key_fp8 else 1
soa_v_zero = 1 if key_fp8 else 2
# uint16-aliased view of the same storage — enables single-instruction
# u16 writes for the SoA K-norm / V-scale / V-zero stores. The cache
# bytes are always contiguous and 2-byte aligned under the Opt#3 layout
# (MSE_BYTES, VAL_DATA_BYTES, meta_region_offset are all even).
kv_cache_u16 = kv_cache.view(torch.uint16)
# ── FP8 PATH: in-kernel FP8 cast + scatter via fp8 kernel ──
if key_fp8:
k_flat = key.reshape(NH, D).contiguous()
v_flat = value.reshape(NH, D).contiguous()
fp8_e4b15 = _use_fp8_e4b15(key.device.index or 0)
grid = (NH,)
_tq_fused_store_fp8[grid](
k_flat,
v_flat,
kv_cache.view(-1),
kv_cache_u16.view(-1),
slot_mapping,
stride_cache_block=stride_block,
D=D,
H=H,
BLOCK_SIZE=block_size,
BLOCK_D=BLOCK_D,
KEY_DATA_BYTES=key_data_bytes,
VAL_DATA_BYTES=val_data_bytes,
META_REGION_OFFSET=meta_region_offset,
NUM_SOA_FIELDS=num_soa_fields,
SOA_V_SCALE=soa_v_scale,
SOA_V_ZERO=soa_v_zero,
VQB=value_quant_bits,
BLOCK_VAL=BLOCK_VAL,
BLOCK_GRP=block_grp,
FP8_E4B15=fp8_e4b15,
num_warps=4,
num_stages=1,
)
return
# ── MSE PATH: external GEMM + fused bucketize/pack kernel ──
# Normalize + rotation GEMM externally (cuBLAS is faster than in-kernel)
k_flat = key.float().reshape(NH, D)
norms = k_flat.norm(dim=1, keepdim=True)
x_hat = k_flat / (norms + 1e-8)
y = x_hat @ PiT
v_flat = value.float().reshape(NH, D)
# When norm_correction=True, the kernel folds 1/||c_t|| into the
# stored per-token K-norm so the load kernel can skip per-tile norm
# math (Opt#1). centroids is only dereferenced inside the kernel when
# NORM_CORRECTION=1; pass midpoints as a harmless placeholder otherwise.
centroids_ptr = centroids if centroids is not None else midpoints
grid = (NH,)
_tq_fused_store_mse[grid](
y,
norms.squeeze(1),
v_flat,
midpoints,
centroids_ptr,
kv_cache.view(-1),
kv_cache_u16.view(-1),
slot_mapping,
stride_cache_block=stride_block,
D=D,
H=H,
BLOCK_SIZE=block_size,
BLOCK_D=BLOCK_D,
MSE_BYTES=mse_bytes,
VAL_DATA_BYTES=val_data_bytes,
META_REGION_OFFSET=meta_region_offset,
NUM_SOA_FIELDS=num_soa_fields,
SOA_K_NORM=0,
SOA_V_SCALE=soa_v_scale,
SOA_V_ZERO=soa_v_zero,
VQB=value_quant_bits,
BLOCK_VAL=BLOCK_VAL,
MSE_BITS=mse_bits,
N_CENTROIDS=n_centroids,
NORM_CORRECTION=1 if norm_correction else 0,
BLOCK_GRP=block_grp,
num_warps=4,
num_stages=1,
)
File diff suppressed because it is too large Load Diff