[Kernel][ROCm][Perf] FlyDSL decode-attention kernel for 4-bit TurboQuant KV cache (#47896)
Signed-off-by: root <root@smci355-ccs-aus-m02-09.cs-aus.dcgpu> Co-authored-by: root <root@smci355-ccs-aus-m02-09.cs-aus.dcgpu> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
root
Andreas Karatzas
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
d648236199
commit
5426311d91
@@ -0,0 +1,261 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Copyright (c) 2025 FlyDSL Project Contributors
|
||||
"""Correctness test for the FlyDSL TurboQuant 4-bit KV decode kernel.
|
||||
|
||||
The kernel is validated against a pure-PyTorch full-precision attention
|
||||
oracle rather than another kernel: the synthetic KV cache is built from
|
||||
known centroid indices, K-norms and V-scales/zeros, so we can dequantize
|
||||
the exact stored values and run fp32 softmax attention as ground truth.
|
||||
Both sides therefore see identical quantized values; the only difference
|
||||
is bf16-vs-fp32 accumulation, which is bounded by ``ATOL`` below.
|
||||
|
||||
Skipped unless running on ROCm/gfx950 with FlyDSL importable.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
# Guard the ROCm-only import: `vllm.platforms.rocm` pulls in amdsmi and other
|
||||
# ROCm-specific modules that are not importable on non-ROCm platforms, so it
|
||||
# must not be imported at module load time on CUDA/CPU CI.
|
||||
if not current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
"TurboQuant FlyDSL decode only runs on ROCm gfx950.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.platforms.rocm import on_gfx950 # noqa: E402
|
||||
|
||||
if not on_gfx950():
|
||||
pytest.skip(
|
||||
"TurboQuant FlyDSL decode only runs on ROCm gfx950.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
from vllm.v1.attention.ops.flydsl_turboquant_decode import ( # noqa: E402
|
||||
flydsl_turboquant_decode_attention,
|
||||
is_flydsl_available,
|
||||
is_flydsl_gqa6_available,
|
||||
)
|
||||
|
||||
if not is_flydsl_available():
|
||||
pytest.skip("FlyDSL runtime is not available.", allow_module_level=True)
|
||||
|
||||
HEAD_SIZE = 128
|
||||
N_CENTROIDS = 16
|
||||
KEY_DATA_BYTES = HEAD_SIZE // 2 # 4-bit packed key nibbles
|
||||
DATA_BYTES_PER_SLOT = HEAD_SIZE # packed K (64) + packed V (64)
|
||||
NUM_SOA_FIELDS = 3
|
||||
SOA_K_NORM, SOA_V_SCALE, SOA_V_ZERO = 0, 1, 2
|
||||
|
||||
# bf16-vs-fp32 accumulation tolerance (FlyDSL test_pa.py default).
|
||||
ATOL = 5e-3
|
||||
|
||||
|
||||
def _build_cache(
|
||||
num_seqs, num_kv_heads, seq_len, qg, kv_block_size, seed=0xC0FFEE, device="cuda"
|
||||
):
|
||||
"""Vectorized builder for a SoA-layout 4-bit KV cache + fp32 ground truth.
|
||||
|
||||
Returns ``(centroids, q_bf16, kv_cache_4d, block_table, seq_lens,
|
||||
K_ref, V_ref)`` where ``K_ref``/``V_ref`` are the dequantized
|
||||
full-precision keys/values used by the reference attention.
|
||||
"""
|
||||
g = torch.Generator(device=device).manual_seed(seed)
|
||||
hk = num_kv_heads
|
||||
hq = hk * qg
|
||||
|
||||
centroids = (
|
||||
torch.randn(N_CENTROIDS, generator=g, dtype=torch.float32, device=device) * 0.5
|
||||
)
|
||||
q = (
|
||||
torch.randn(
|
||||
num_seqs, hq, HEAD_SIZE, generator=g, dtype=torch.float32, device=device
|
||||
)
|
||||
* 0.1
|
||||
)
|
||||
q_bf16 = q.to(torch.bfloat16)
|
||||
|
||||
blocks_per_seq = (seq_len + kv_block_size - 1) // kv_block_size
|
||||
max_bps = blocks_per_seq + 4
|
||||
block_table = torch.zeros(num_seqs, max_bps, dtype=torch.int32, device=device)
|
||||
block_table[:, :blocks_per_seq] = (
|
||||
torch.arange(
|
||||
num_seqs * blocks_per_seq, dtype=torch.int32, device=device
|
||||
).reshape(num_seqs, blocks_per_seq)
|
||||
+ 1
|
||||
)
|
||||
total_blocks = num_seqs * blocks_per_seq + 8
|
||||
|
||||
slot_size_aligned = (KEY_DATA_BYTES + 2) + (KEY_DATA_BYTES + 4) # 134
|
||||
if slot_size_aligned % 2:
|
||||
slot_size_aligned += 1
|
||||
bytes_per_block = kv_block_size * hk * slot_size_aligned
|
||||
|
||||
k_idx = torch.randint(
|
||||
0,
|
||||
N_CENTROIDS,
|
||||
(num_seqs, hk, seq_len, HEAD_SIZE),
|
||||
generator=g,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
v_idx = torch.randint(
|
||||
0,
|
||||
N_CENTROIDS,
|
||||
(num_seqs, hk, seq_len, HEAD_SIZE),
|
||||
generator=g,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
knorm = (
|
||||
torch.rand(num_seqs, hk, seq_len, generator=g, device=device) * 0.5 + 0.5
|
||||
).to(torch.float16)
|
||||
vscale = (
|
||||
torch.rand(num_seqs, hk, seq_len, generator=g, device=device) * 0.05 + 0.01
|
||||
).to(torch.float16)
|
||||
vzero = (
|
||||
(torch.rand(num_seqs, hk, seq_len, generator=g, device=device) - 0.5) * 0.1
|
||||
).to(torch.float16)
|
||||
|
||||
k_ref = centroids[k_idx.long()].float() * knorm.float().unsqueeze(-1)
|
||||
v_ref = v_idx.float() * vscale.float().unsqueeze(-1) + vzero.float().unsqueeze(-1)
|
||||
|
||||
k_packed = k_idx[..., 0::2] | (k_idx[..., 1::2] << 4)
|
||||
v_packed = v_idx[..., 0::2] | (v_idx[..., 1::2] << 4)
|
||||
|
||||
kv_cache = torch.zeros(
|
||||
total_blocks, bytes_per_block, dtype=torch.uint8, device=device
|
||||
)
|
||||
|
||||
seq_ax = (
|
||||
torch.arange(num_seqs, device=device)
|
||||
.view(num_seqs, 1)
|
||||
.expand(num_seqs, seq_len)
|
||||
)
|
||||
tok_ax = (
|
||||
torch.arange(seq_len, device=device).view(1, seq_len).expand(num_seqs, seq_len)
|
||||
)
|
||||
blk_for_tok = block_table[seq_ax, tok_ax // kv_block_size]
|
||||
slot_for_tok = tok_ax % kv_block_size
|
||||
|
||||
h_ax = torch.arange(hk, device=device)
|
||||
base_data = (
|
||||
slot_for_tok.unsqueeze(1) * hk * DATA_BYTES_PER_SLOT
|
||||
+ h_ax.view(1, hk, 1) * DATA_BYTES_PER_SLOT
|
||||
)
|
||||
blk_b = blk_for_tok.unsqueeze(1).expand(num_seqs, hk, seq_len)
|
||||
dst_base = blk_b * bytes_per_block + base_data
|
||||
rng_k = torch.arange(KEY_DATA_BYTES, device=device)
|
||||
rng_v = torch.arange(KEY_DATA_BYTES, device=device) + KEY_DATA_BYTES
|
||||
k_dst = (dst_base.unsqueeze(-1) + rng_k.view(1, 1, 1, -1)).reshape(-1)
|
||||
v_dst = (dst_base.unsqueeze(-1) + rng_v.view(1, 1, 1, -1)).reshape(-1)
|
||||
flat = kv_cache.view(-1)
|
||||
flat[k_dst] = k_packed.reshape(-1)
|
||||
flat[v_dst] = v_packed.reshape(-1)
|
||||
|
||||
meta_off = kv_block_size * hk * DATA_BYTES_PER_SLOT
|
||||
meta_view = kv_cache.view(torch.float16).view(total_blocks, -1)
|
||||
meta_off_hw = meta_off // 2
|
||||
common = h_ax.view(
|
||||
1, hk, 1
|
||||
) * NUM_SOA_FIELDS * kv_block_size + slot_for_tok.unsqueeze(1)
|
||||
blk_flat = blk_for_tok.unsqueeze(1).expand(num_seqs, hk, seq_len)
|
||||
knorm_off = meta_off_hw + common + SOA_K_NORM * kv_block_size
|
||||
vscale_off = meta_off_hw + common + SOA_V_SCALE * kv_block_size
|
||||
vzero_off = meta_off_hw + common + SOA_V_ZERO * kv_block_size
|
||||
meta_view[blk_flat.reshape(-1).long(), knorm_off.reshape(-1).long()] = (
|
||||
knorm.reshape(-1)
|
||||
)
|
||||
meta_view[blk_flat.reshape(-1).long(), vscale_off.reshape(-1).long()] = (
|
||||
vscale.reshape(-1)
|
||||
)
|
||||
meta_view[blk_flat.reshape(-1).long(), vzero_off.reshape(-1).long()] = (
|
||||
vzero.reshape(-1)
|
||||
)
|
||||
|
||||
seq_lens = torch.full((num_seqs,), seq_len, dtype=torch.int32, device=device)
|
||||
kv_cache_4d = kv_cache.view(total_blocks, kv_block_size, hk, slot_size_aligned)
|
||||
return (
|
||||
centroids,
|
||||
q_bf16,
|
||||
kv_cache_4d,
|
||||
block_table,
|
||||
seq_lens,
|
||||
k_ref.cpu(),
|
||||
v_ref.cpu(),
|
||||
)
|
||||
|
||||
|
||||
def _reference_attention(q_bf16, k_ref, v_ref, seq_lens, scale):
|
||||
"""fp32 softmax attention on dequantized keys/values (ground truth)."""
|
||||
num_seqs, hq, d = q_bf16.shape
|
||||
hk = k_ref.shape[1]
|
||||
qg = hq // hk
|
||||
q = q_bf16.float().reshape(num_seqs, hk, qg, d)
|
||||
out = torch.zeros(num_seqs, hq, d, dtype=torch.float32)
|
||||
for s in range(num_seqs):
|
||||
for h in range(hk):
|
||||
ql = int(seq_lens[s].item())
|
||||
k = k_ref[s, h, :ql]
|
||||
v = v_ref[s, h, :ql]
|
||||
scores = (q[s, h] @ k.T) * scale
|
||||
m = scores.max(dim=-1, keepdim=True).values
|
||||
e = torch.exp(scores - m)
|
||||
p = e / e.sum(dim=-1, keepdim=True)
|
||||
out[s, h * qg : (h + 1) * qg] = p @ v
|
||||
return out
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_seqs", [1, 4, 16])
|
||||
@pytest.mark.parametrize("seq_len", [256, 1024, 4096])
|
||||
@pytest.mark.parametrize(
|
||||
"num_kv_heads,qg",
|
||||
[
|
||||
(8, 8), # Qwen2.5-72B class
|
||||
(8, 16), # Qwen3-32B class
|
||||
pytest.param(
|
||||
4,
|
||||
6, # MiniMax-M2.5 class (GQA-6 sibling kernel)
|
||||
marks=pytest.mark.skipif(
|
||||
not is_flydsl_gqa6_available(),
|
||||
reason="FlyDSL GQA-6 sibling kernel not available",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("kv_block_size", [16, 32])
|
||||
def test_flydsl_matches_reference(num_seqs, seq_len, num_kv_heads, qg, kv_block_size):
|
||||
"""FlyDSL decode must match fp32 attention on dequantized KV."""
|
||||
centroids, q_bf16, kv_cache, block_table, seq_lens, k_ref, v_ref = _build_cache(
|
||||
num_seqs, num_kv_heads, seq_len, qg, kv_block_size
|
||||
)
|
||||
scale = 1.0 / (HEAD_SIZE**0.5)
|
||||
identity = torch.eye(HEAD_SIZE, dtype=torch.float32, device="cuda")
|
||||
|
||||
out = flydsl_turboquant_decode_attention(
|
||||
query=q_bf16,
|
||||
kv_cache=kv_cache,
|
||||
block_table=block_table,
|
||||
seq_lens=seq_lens,
|
||||
Pi=identity,
|
||||
centroids=centroids,
|
||||
scale=scale,
|
||||
mse_bits=4,
|
||||
key_packed_size=KEY_DATA_BYTES + 2,
|
||||
value_quant_bits=4,
|
||||
value_packed_size=KEY_DATA_BYTES + 4,
|
||||
key_fp8=False,
|
||||
norm_correction=False,
|
||||
PiT=identity.T.contiguous(),
|
||||
max_seq_len=seq_len,
|
||||
max_num_kv_splits=32,
|
||||
sinks=None,
|
||||
)
|
||||
|
||||
ref = _reference_attention(q_bf16.cpu(), k_ref, v_ref, seq_lens.cpu(), scale)
|
||||
|
||||
torch.testing.assert_close(out.cpu().float(), ref, atol=ATOL, rtol=0.0)
|
||||
+23
-8
@@ -623,23 +623,38 @@ class RocmPlatform(Platform):
|
||||
if selected_backend is not None:
|
||||
try:
|
||||
backend_class = selected_backend.get_class()
|
||||
invalid_reasons = backend_class.validate_configuration(
|
||||
sel_invalid_reasons = backend_class.validate_configuration(
|
||||
device_capability=device_capability,
|
||||
**attn_selector_config._asdict(),
|
||||
)
|
||||
except ImportError:
|
||||
invalid_reasons = ["ImportError"]
|
||||
if invalid_reasons:
|
||||
raise ValueError(
|
||||
f"Selected backend {selected_backend} is not valid for "
|
||||
f"this configuration. Reason: {invalid_reasons}"
|
||||
)
|
||||
else:
|
||||
sel_invalid_reasons = ["ImportError"]
|
||||
if not sel_invalid_reasons:
|
||||
logger.info_once(
|
||||
"Using %s backend (selected via --attention-backend).",
|
||||
selected_backend.name,
|
||||
)
|
||||
return selected_backend.get_path()
|
||||
# Only tolerate the mismatch for turboquant_* KV-cache layers:
|
||||
# boundary layers keep the native dtype (served by the selected
|
||||
# backend) while turboquant_* layers need TURBOQUANT, so no single
|
||||
# --attention-backend can serve every layer. For any other dtype
|
||||
# the explicit selection is genuinely invalid -> fail loud.
|
||||
kv_dtype = attn_selector_config.kv_cache_dtype
|
||||
if not (kv_dtype is not None and str(kv_dtype).startswith("turboquant")):
|
||||
raise ValueError(
|
||||
f"Selected backend {selected_backend} is not valid for "
|
||||
f"this configuration. Reason: {sel_invalid_reasons}"
|
||||
)
|
||||
# NOTE: pass a str (not the list) -- info_once hashes its args.
|
||||
logger.info_once(
|
||||
"Selected backend %s is incompatible with this turboquant "
|
||||
"layer (%s); using the auto-selected per-layer backend. "
|
||||
"Reason: %s",
|
||||
selected_backend.name,
|
||||
attn_selector_config.attn_type,
|
||||
str(sel_invalid_reasons),
|
||||
)
|
||||
|
||||
# No selected backend or the selected backend is invalid,
|
||||
# so we try finding a valid backend.
|
||||
|
||||
@@ -16,6 +16,7 @@ Per-head per-position slot layout:
|
||||
For turboquant_k3v4_nc head_dim=256: [100 bytes key | 512 bytes value] = 612
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
@@ -26,6 +27,7 @@ import torch.nn.functional as F
|
||||
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.config.cache import CacheDType
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.quantization.turboquant.centroids import (
|
||||
get_centroids,
|
||||
)
|
||||
@@ -47,6 +49,15 @@ from vllm.v1.attention.backends.fa_utils import (
|
||||
is_flash_attn_varlen_func_available,
|
||||
)
|
||||
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
|
||||
|
||||
# FlyDSL TurboQuant decode (AMD gfx950). Auto-selected when FlyDSL is available
|
||||
# for eligible layers (SoA store + FlyDSL decode + SoA-aware continuation);
|
||||
# non-gfx950 or ineligible layers use the SoA Triton decode.
|
||||
from vllm.v1.attention.ops.flydsl_turboquant_decode import (
|
||||
flydsl_turboquant_decode_attention,
|
||||
is_flydsl_available,
|
||||
is_flydsl_gqa6_available,
|
||||
)
|
||||
from vllm.v1.attention.ops.triton_turboquant_decode import (
|
||||
_tq_full_dequant_kv,
|
||||
_use_fp8_e4b15,
|
||||
@@ -59,6 +70,8 @@ from vllm.v1.worker.workspace import (
|
||||
is_workspace_manager_initialized,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_HAS_FLASH_ATTN = is_flash_attn_varlen_func_available()
|
||||
if _HAS_FLASH_ATTN:
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func
|
||||
@@ -71,6 +84,24 @@ if _HAS_FLASH_ATTN:
|
||||
_CONTINUATION_DECODE_THRESHOLD = 128
|
||||
|
||||
|
||||
def _soa_imports():
|
||||
"""Lazy import of the HIP-free SoA Triton subset (store / dequant / decode).
|
||||
|
||||
Kept lazy so the default (FlyDSL-off) path never imports these modules.
|
||||
"""
|
||||
from vllm.v1.attention.ops.turboquant_soa.triton_turboquant_decode import (
|
||||
_tq_full_dequant_kv as soa_dequant,
|
||||
)
|
||||
from vllm.v1.attention.ops.turboquant_soa.triton_turboquant_store import (
|
||||
triton_turboquant_store as soa_store,
|
||||
)
|
||||
from vllm.v1.attention.ops.turboquant_soa.triton_turboquant_unified_attention import ( # noqa: E501
|
||||
triton_turboquant_decode_attention_soa as soa_decode,
|
||||
)
|
||||
|
||||
return soa_store, soa_dequant, soa_decode
|
||||
|
||||
|
||||
def _build_hadamard(d: int, device_str: str) -> torch.Tensor:
|
||||
"""Orthonormal Hadamard matrix (Sylvester construction), cached per (d, device).
|
||||
|
||||
@@ -290,6 +321,10 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
|
||||
|
||||
supports_quant_query_input: bool = False
|
||||
|
||||
# Lazily populated before cudagraph capture (FlyDSL decode path only).
|
||||
_arange_cache: torch.Tensor
|
||||
_cu_2: torch.Tensor
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
@@ -337,6 +372,17 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
|
||||
vllm_config.attention_config.tq_max_kv_splits_for_cuda_graph
|
||||
)
|
||||
|
||||
# FlyDSL decode state. Auto-enabled on gfx950 when FlyDSL is available.
|
||||
self.sliding_window = sliding_window
|
||||
self.sinks = kwargs.get("sinks")
|
||||
# Cache max_model_len now (config is available at __init__ but NOT
|
||||
# during CUDA-graph capture when _ensure_on_device is re-entered).
|
||||
self._max_model_len = vllm_config.model_config.max_model_len
|
||||
# SoA store is required by the FlyDSL decode/continuation path, so it
|
||||
# tracks FlyDSL availability (single switch for the whole pipeline).
|
||||
self._use_flydsl = is_flydsl_available()
|
||||
self._soa_store = self._use_flydsl
|
||||
|
||||
def _flash_attn_varlen(
|
||||
self,
|
||||
q: torch.Tensor,
|
||||
@@ -382,6 +428,44 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
|
||||
quantizer is symmetric around zero (sign-flipping a coordinate
|
||||
maps it to the mirror centroid with identical distortion).
|
||||
"""
|
||||
if self._soa_store:
|
||||
# CUDA-graph capture safety for the FlyDSL decode path on ROCm.
|
||||
# (1) Pre-allocate _arange_cache / _cu_2 BEFORE any capture; lazy
|
||||
# allocation during graph replay lands in the HIP graph memory
|
||||
# pool and yields stale addresses (GPU fault / garbage).
|
||||
# (2) Pre-warm the WorkspaceManager to its max size before capture
|
||||
# so mid-capture growth cannot invalidate pointers baked into
|
||||
# already-captured batch sizes.
|
||||
_max_len = self._max_model_len
|
||||
_already_ok = (
|
||||
hasattr(self, "_arange_cache")
|
||||
and self._arange_cache.device.type == str(device).split(":")[0]
|
||||
and self._arange_cache.shape[0] >= _max_len + 2
|
||||
)
|
||||
if not _already_ok:
|
||||
self._arange_cache = torch.arange(
|
||||
0, _max_len + 2, device=device, dtype=torch.int32
|
||||
)
|
||||
if not hasattr(self, "_cu_2") or self._cu_2.device != torch.device(device):
|
||||
self._cu_2 = torch.zeros(2, device=device, dtype=torch.int32)
|
||||
if (
|
||||
is_workspace_manager_initialized()
|
||||
and not current_workspace_manager().is_locked()
|
||||
):
|
||||
B_max = self._max_capture_batch_size()
|
||||
D = self.head_size
|
||||
Hq = self.num_heads
|
||||
S = self.max_num_kv_splits
|
||||
_pre_warm_bytes = (
|
||||
B_max * Hq * (S * (D + 1) + D) * 4 # fp32 mid_o + fp32 lse
|
||||
+ B_max * Hq * D * 2 # query-dtype output (bf16 = 2 B)
|
||||
+ 512 # alignment padding
|
||||
)
|
||||
with contextlib.suppress(AssertionError):
|
||||
current_workspace_manager().get_simultaneous(
|
||||
((_pre_warm_bytes,), torch.uint8)
|
||||
)
|
||||
|
||||
if not hasattr(layer, "_tq_cached"):
|
||||
D = self.head_size
|
||||
|
||||
@@ -402,6 +486,29 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
|
||||
layer._tq_midpoints = (c_sorted[:-1] + c_sorted[1:]) / 2
|
||||
layer._tq_cached = True
|
||||
|
||||
def _max_capture_batch_size(self) -> int:
|
||||
"""Largest decode batch we might see at runtime (for workspace pre-warm).
|
||||
|
||||
Take max(cudagraph_capture_sizes, scheduler.max_num_seqs): a forward
|
||||
pass exceeding the largest captured graph size falls back to eager,
|
||||
but the workspace is locked and that eager path can still hit batch
|
||||
sizes up to max_num_seqs. Falls back to 1024 if config is unavailable.
|
||||
"""
|
||||
try:
|
||||
cfg = get_current_vllm_config()
|
||||
candidates: list[int] = []
|
||||
sizes = cfg.compilation_config.cudagraph_capture_sizes
|
||||
if sizes:
|
||||
candidates.append(int(max(sizes)))
|
||||
sched = getattr(cfg, "scheduler_config", None)
|
||||
if sched is not None and getattr(sched, "max_num_seqs", None):
|
||||
candidates.append(int(sched.max_num_seqs))
|
||||
if candidates:
|
||||
return max(candidates)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return 1024
|
||||
|
||||
def do_kv_cache_update(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -590,6 +697,27 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
|
||||
layer: Any,
|
||||
):
|
||||
"""Quantize + store via fused Triton kernel."""
|
||||
if self._soa_store:
|
||||
# SoA layout (data region + metadata region separated per block),
|
||||
# required by the FlyDSL decode kernel. Pure-Triton store; the
|
||||
# cache tensor shape is identical to the default AoS store, only
|
||||
# the within-block byte convention differs.
|
||||
soa_store, _, _ = _soa_imports()
|
||||
soa_store(
|
||||
key=key,
|
||||
value=value,
|
||||
kv_cache=kv_cache,
|
||||
slot_mapping=slot_mapping,
|
||||
PiT=layer._tq_PiT,
|
||||
midpoints=layer._tq_midpoints,
|
||||
mse_bits=self.tq_config.key_mse_bits,
|
||||
key_packed_size=self.tq_config.key_packed_size,
|
||||
value_quant_bits=self.tq_config.effective_value_quant_bits,
|
||||
key_fp8=self.tq_config.key_fp8,
|
||||
centroids=layer._tq_centroids,
|
||||
norm_correction=self.tq_config.norm_correction,
|
||||
)
|
||||
return
|
||||
triton_turboquant_store(
|
||||
key,
|
||||
value,
|
||||
@@ -722,21 +850,55 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
|
||||
# Slice from pre-built arange (no kernel launch)
|
||||
synth_seq_lens = _arange_cache[cached_len + 1 : seq_len + 1]
|
||||
synth_bt = attn_metadata.block_table[i : i + 1].expand(q_len, -1)
|
||||
out = triton_turboquant_decode_attention(
|
||||
query=q_seq,
|
||||
kv_cache=kv_cache,
|
||||
block_table=synth_bt,
|
||||
seq_lens=synth_seq_lens,
|
||||
Pi=Pi,
|
||||
centroids=centroids,
|
||||
scale=self.scale,
|
||||
mse_bits=self.tq_config.key_mse_bits,
|
||||
key_packed_size=self.tq_config.key_packed_size,
|
||||
value_quant_bits=(self.tq_config.effective_value_quant_bits),
|
||||
key_fp8=self.tq_config.key_fp8,
|
||||
norm_correction=self.tq_config.norm_correction,
|
||||
PiT=PiT,
|
||||
)
|
||||
if self._soa_store:
|
||||
# The cache was written in SoA layout (always, for FlyDSL),
|
||||
# so it MUST be read with the SoA-aware decode. The
|
||||
# default AoS decode reads k_norm/v_scale/v_zero from
|
||||
# the wrong offsets in a SoA cache -> garbage
|
||||
# cached-prefix output ->
|
||||
# accuracy collapse on every multi-turn / prefix-cached
|
||||
# (APC) request. Continuation stays on the Triton SoA
|
||||
# path even when FlyDSL is the main decode kernel
|
||||
# (FlyDSL is decode-batch only).
|
||||
out = self._dispatch_decode_soa(
|
||||
query=q_seq,
|
||||
kv_cache=kv_cache,
|
||||
block_table=synth_bt,
|
||||
seq_lens=synth_seq_lens,
|
||||
Pi=Pi,
|
||||
centroids=centroids,
|
||||
scale=self.scale,
|
||||
mse_bits=self.tq_config.key_mse_bits,
|
||||
key_packed_size=self.tq_config.key_packed_size,
|
||||
value_quant_bits=(
|
||||
self.tq_config.effective_value_quant_bits
|
||||
),
|
||||
value_packed_size=self.tq_config.value_packed_size,
|
||||
max_seq_len=int(seq_len),
|
||||
key_fp8=self.tq_config.key_fp8,
|
||||
norm_correction=self.tq_config.norm_correction,
|
||||
PiT=PiT,
|
||||
sinks=self.sinks,
|
||||
sliding_window=self.sliding_window,
|
||||
)
|
||||
else:
|
||||
out = triton_turboquant_decode_attention(
|
||||
query=q_seq,
|
||||
kv_cache=kv_cache,
|
||||
block_table=synth_bt,
|
||||
seq_lens=synth_seq_lens,
|
||||
Pi=Pi,
|
||||
centroids=centroids,
|
||||
scale=self.scale,
|
||||
mse_bits=self.tq_config.key_mse_bits,
|
||||
key_packed_size=self.tq_config.key_packed_size,
|
||||
value_quant_bits=(
|
||||
self.tq_config.effective_value_quant_bits
|
||||
),
|
||||
key_fp8=self.tq_config.key_fp8,
|
||||
norm_correction=self.tq_config.norm_correction,
|
||||
PiT=PiT,
|
||||
)
|
||||
else:
|
||||
# Large continuation: dequant cached K/V and use
|
||||
# flash_attn for better throughput.
|
||||
@@ -801,36 +963,84 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
|
||||
v_cached = v_buf[:, :, :alloc_len, :]
|
||||
|
||||
grid = (alloc_len, 1 * Hk)
|
||||
_tq_full_dequant_kv[grid](
|
||||
kv_cache,
|
||||
block_table,
|
||||
centroids,
|
||||
k_cached,
|
||||
v_cached,
|
||||
k_cached.stride(0),
|
||||
k_cached.stride(1),
|
||||
k_cached.stride(2),
|
||||
v_cached.stride(0),
|
||||
v_cached.stride(1),
|
||||
v_cached.stride(2),
|
||||
kv_cache.stride(0),
|
||||
kv_cache.stride(1),
|
||||
kv_cache.stride(2),
|
||||
block_table.stride(0),
|
||||
HEAD_DIM=D,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_KV_HEADS=Hk,
|
||||
MSE_BYTES=mse_bytes,
|
||||
KPS=self.tq_config.key_packed_size,
|
||||
VQB=self.tq_config.effective_value_quant_bits,
|
||||
VAL_DATA_BYTES=val_data_bytes,
|
||||
MSE_BITS=self.tq_config.key_mse_bits,
|
||||
KEY_FP8=1 if self.tq_config.key_fp8 else 0,
|
||||
BLOCK_D=BLOCK_D,
|
||||
NORM_CORRECTION=1 if self.tq_config.norm_correction else 0,
|
||||
FP8_E4B15=_use_fp8_e4b15(device.index or 0),
|
||||
num_warps=4,
|
||||
)
|
||||
if self._soa_store:
|
||||
# SoA-aware dequant: read the data/metadata-separated SoA cache
|
||||
# written by the SoA store. Constants must match the store side.
|
||||
_, soa_dequant, _ = _soa_imports()
|
||||
key_fp8 = self.tq_config.key_fp8
|
||||
key_data_bytes = D if key_fp8 else mse_bytes
|
||||
data_bytes_per_slot = key_data_bytes + val_data_bytes
|
||||
meta_region_offset = block_size * Hk * data_bytes_per_slot
|
||||
num_soa_fields = 2 if key_fp8 else 3
|
||||
soa_k_norm = 0
|
||||
soa_v_scale = 0 if key_fp8 else 1
|
||||
soa_v_zero = 1 if key_fp8 else 2
|
||||
kv_cache_u16 = kv_cache.view(torch.uint16)
|
||||
soa_dequant[grid](
|
||||
kv_cache,
|
||||
kv_cache_u16,
|
||||
block_table,
|
||||
centroids,
|
||||
k_cached,
|
||||
v_cached,
|
||||
k_cached.stride(0),
|
||||
k_cached.stride(1),
|
||||
k_cached.stride(2),
|
||||
v_cached.stride(0),
|
||||
v_cached.stride(1),
|
||||
v_cached.stride(2),
|
||||
kv_cache.stride(0),
|
||||
block_table.stride(0),
|
||||
HEAD_DIM=D,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_KV_HEADS=Hk,
|
||||
MSE_BYTES=mse_bytes,
|
||||
VQB=self.tq_config.effective_value_quant_bits,
|
||||
VAL_DATA_BYTES=val_data_bytes,
|
||||
MSE_BITS=self.tq_config.key_mse_bits,
|
||||
KEY_FP8=1 if key_fp8 else 0,
|
||||
KEY_DATA_BYTES=key_data_bytes,
|
||||
META_REGION_OFFSET=meta_region_offset,
|
||||
NUM_SOA_FIELDS=num_soa_fields,
|
||||
SOA_K_NORM=soa_k_norm,
|
||||
SOA_V_SCALE=soa_v_scale,
|
||||
SOA_V_ZERO=soa_v_zero,
|
||||
BLOCK_D=BLOCK_D,
|
||||
NORM_CORRECTION=1 if self.tq_config.norm_correction else 0,
|
||||
FP8_E4B15=_use_fp8_e4b15(device.index or 0),
|
||||
num_warps=4,
|
||||
)
|
||||
else:
|
||||
_tq_full_dequant_kv[grid](
|
||||
kv_cache,
|
||||
block_table,
|
||||
centroids,
|
||||
k_cached,
|
||||
v_cached,
|
||||
k_cached.stride(0),
|
||||
k_cached.stride(1),
|
||||
k_cached.stride(2),
|
||||
v_cached.stride(0),
|
||||
v_cached.stride(1),
|
||||
v_cached.stride(2),
|
||||
kv_cache.stride(0),
|
||||
kv_cache.stride(1),
|
||||
kv_cache.stride(2),
|
||||
block_table.stride(0),
|
||||
HEAD_DIM=D,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_KV_HEADS=Hk,
|
||||
MSE_BYTES=mse_bytes,
|
||||
KPS=self.tq_config.key_packed_size,
|
||||
VQB=self.tq_config.effective_value_quant_bits,
|
||||
VAL_DATA_BYTES=val_data_bytes,
|
||||
MSE_BITS=self.tq_config.key_mse_bits,
|
||||
KEY_FP8=1 if self.tq_config.key_fp8 else 0,
|
||||
BLOCK_D=BLOCK_D,
|
||||
NORM_CORRECTION=1 if self.tq_config.norm_correction else 0,
|
||||
FP8_E4B15=_use_fp8_e4b15(device.index or 0),
|
||||
num_warps=4,
|
||||
)
|
||||
|
||||
# Inverse-rotate MSE keys back to original space
|
||||
if not self.tq_config.key_fp8:
|
||||
@@ -930,6 +1140,82 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
|
||||
)
|
||||
)
|
||||
|
||||
if self._use_flydsl:
|
||||
# FlyDSL decode (gfx950, MSE-key, HEAD_SIZE=128, GQA in {6, 8, 16}).
|
||||
# GQA-6 routes to the MiniMax sibling kernel. Ineligible layers fall
|
||||
# back to SoA Triton decode.
|
||||
_gqa = self.num_kv_groups
|
||||
flydsl_gqa_ok = (_gqa in (8, 16)) or (
|
||||
_gqa == 6 and is_flydsl_gqa6_available()
|
||||
)
|
||||
flydsl_eligible = (
|
||||
not self.tq_config.key_fp8
|
||||
and self.tq_config.key_mse_bits == 4
|
||||
and self.tq_config.effective_value_quant_bits == 4
|
||||
and self.head_size == 128
|
||||
and flydsl_gqa_ok
|
||||
and self.sinks is None
|
||||
and not (self.sliding_window and self.sliding_window > 0)
|
||||
)
|
||||
if flydsl_eligible:
|
||||
return flydsl_turboquant_decode_attention(
|
||||
query=query,
|
||||
kv_cache=kv_cache,
|
||||
block_table=attn_metadata.block_table,
|
||||
seq_lens=attn_metadata.seq_lens,
|
||||
Pi=Pi,
|
||||
centroids=centroids,
|
||||
scale=self.scale,
|
||||
mse_bits=self.tq_config.key_mse_bits,
|
||||
key_packed_size=self.tq_config.key_packed_size,
|
||||
value_quant_bits=self.tq_config.effective_value_quant_bits,
|
||||
value_packed_size=self.tq_config.value_packed_size,
|
||||
max_seq_len=attn_metadata.max_seq_len,
|
||||
key_fp8=self.tq_config.key_fp8,
|
||||
norm_correction=self.tq_config.norm_correction,
|
||||
PiT=PiT,
|
||||
mid_o_buf=mid_o_buf,
|
||||
output_buf=output_buf,
|
||||
lse_buf=lse_buf,
|
||||
buf_holder=layer,
|
||||
max_num_kv_splits=self.max_num_kv_splits,
|
||||
sinks=self.sinks,
|
||||
)
|
||||
logger.warning_once(
|
||||
"TurboQuant FlyDSL ineligible (key_fp8=%s mse_bits=%s vqb=%s "
|
||||
"head_size=%s num_kv_groups=%s sinks=%s) -> SoA Triton decode",
|
||||
self.tq_config.key_fp8,
|
||||
self.tq_config.key_mse_bits,
|
||||
self.tq_config.effective_value_quant_bits,
|
||||
self.head_size,
|
||||
self.num_kv_groups,
|
||||
self.sinks is not None,
|
||||
)
|
||||
return self._dispatch_decode_soa(
|
||||
query=query,
|
||||
kv_cache=kv_cache,
|
||||
block_table=attn_metadata.block_table,
|
||||
seq_lens=attn_metadata.seq_lens,
|
||||
Pi=Pi,
|
||||
centroids=centroids,
|
||||
scale=self.scale,
|
||||
mse_bits=self.tq_config.key_mse_bits,
|
||||
key_packed_size=self.tq_config.key_packed_size,
|
||||
value_quant_bits=self.tq_config.effective_value_quant_bits,
|
||||
value_packed_size=self.tq_config.value_packed_size,
|
||||
max_seq_len=attn_metadata.max_seq_len,
|
||||
key_fp8=self.tq_config.key_fp8,
|
||||
norm_correction=self.tq_config.norm_correction,
|
||||
PiT=PiT,
|
||||
mid_o_buf=mid_o_buf,
|
||||
output_buf=output_buf,
|
||||
lse_buf=lse_buf,
|
||||
buf_holder=layer,
|
||||
max_num_kv_splits=self.max_num_kv_splits,
|
||||
sinks=self.sinks,
|
||||
sliding_window=self.sliding_window,
|
||||
)
|
||||
|
||||
result = triton_turboquant_decode_attention(
|
||||
query=query,
|
||||
kv_cache=kv_cache,
|
||||
@@ -951,3 +1237,21 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]):
|
||||
max_num_kv_splits=self.max_num_kv_splits,
|
||||
)
|
||||
return result
|
||||
|
||||
def _dispatch_decode_soa(self, **kwargs):
|
||||
"""SoA-aware Triton decode — fallback for FlyDSL-ineligible layers.
|
||||
|
||||
The SoA decode launcher accepts a subset of the FlyDSL kwargs; filter to
|
||||
the params it accepts and raise if a *meaningful* (non-None) kwarg would
|
||||
be silently dropped (so we never mask a real feature gap).
|
||||
"""
|
||||
import inspect
|
||||
|
||||
_, _, soa_decode = _soa_imports()
|
||||
accepted = set(inspect.signature(soa_decode).parameters)
|
||||
dropped = [k for k, v in kwargs.items() if k not in accepted and v is not None]
|
||||
if dropped:
|
||||
raise NotImplementedError(
|
||||
f"SoA decode does not support kwargs {sorted(dropped)}"
|
||||
)
|
||||
return soa_decode(**{k: v for k, v in kwargs.items() if k in accepted})
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,823 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# TurboQuant decode (FlyDSL) — MI355X / gfx950 / CDNA4
|
||||
#
|
||||
# Decode-only paged attention over a TurboQuant SoA KV cache.
|
||||
# Targets Qwen-class models: HEAD_SIZE=128, GQA group=8, MSE_BITS=4 K, VQB=4 V,
|
||||
# N_CENTROIDS=16, BLOCK_SIZE=16, partitioned mode.
|
||||
#
|
||||
# === Design ===
|
||||
# * 1 wave (64 lanes) per CTA — simpler than 4-warp; trades parallelism for
|
||||
# correctness/debug-ability. A later revision will fan out to 4 warps.
|
||||
# * Flash-Attention-2 online softmax: rescale running PV/sum per K-tile,
|
||||
# no QK score storage across tiles, no P-LDS round-trip (qk_acc layout
|
||||
# matches PV B-operand directly).
|
||||
# * Centroids loaded once into LDS at kernel start (LDS-resident LUT).
|
||||
# * K dequant → LDS [token, head_dim] (natural layout)
|
||||
# * V dequant → LDS [head_dim, token] (TRANSPOSED — required so PV MFMA's
|
||||
# A=V operand can be loaded as 4-bf16 contiguous via 1 ds_read_b64).
|
||||
#
|
||||
# MFMA layouts used:
|
||||
# QK: mfma(A=K, B=Q, C=qk_acc) → C[m=token, n=query], 4 fp32/lane
|
||||
# = 4 tokens for SAME query → softmax via
|
||||
# xor_shuffle(16),(32) reduces per-row
|
||||
# PV: mfma(A=V_T, B=P, C=acc_pv) → C[m=head_dim, n=query], 4 fp32/lane
|
||||
# = 4 contiguous head_dim cols for SAME
|
||||
# query → 1 buffer_store of 8 bytes
|
||||
# per chunk per lane
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import flydsl.compiler as flyc
|
||||
import flydsl.expr as fx
|
||||
from flydsl._mlir import ir
|
||||
from flydsl._mlir.dialects import scf as _scf
|
||||
from flydsl.expr import (
|
||||
arith,
|
||||
buffer_ops,
|
||||
const_expr, # noqa: F401 kept for kernel-DSL API parity
|
||||
gpu,
|
||||
range_constexpr,
|
||||
rocdl,
|
||||
vector,
|
||||
)
|
||||
from flydsl.expr.typing import Int32, T # noqa: F401 Int32 via fx.Int32
|
||||
from flydsl.runtime.device import get_rocm_arch as get_hip_arch
|
||||
from flydsl.utils.smem_allocator import SmemAllocator, SmemPtr
|
||||
|
||||
# === Constants (Qwen-class TQ decode profile) ===============================
|
||||
HEAD_SIZE = 128
|
||||
# default; overridable via build_tq_decode_module(kv_block_size=...)
|
||||
KV_BLOCK_SIZE = 16
|
||||
# MFMA tile = 16 tokens (do not change without re-deriving MFMA shapes)
|
||||
TILE_SIZE = 16
|
||||
N_CENTROIDS = 16
|
||||
# default for tests; overridable via build_tq_decode_module(query_group_size=...)
|
||||
QUERY_GROUP_SIZE = 16
|
||||
WARP_SIZE = 64
|
||||
NUM_WARPS = 1
|
||||
BLOCK_THREADS = NUM_WARPS * WARP_SIZE
|
||||
KV_COMPUTE_BLOCK = 256 # 16 K-tiles × 16 tokens
|
||||
|
||||
# TQ SoA layout
|
||||
NUM_SOA_FIELDS = 3
|
||||
SOA_K_NORM = 0
|
||||
SOA_V_SCALE = 1
|
||||
SOA_V_ZERO = 2
|
||||
KEY_DATA_BYTES = HEAD_SIZE // 2
|
||||
VAL_DATA_BYTES = HEAD_SIZE // 2
|
||||
DATA_BYTES_PER_SLOT = KEY_DATA_BYTES + VAL_DATA_BYTES
|
||||
|
||||
# MFMA
|
||||
MFMA_M = MFMA_N = 16
|
||||
MFMA_K_BF16_QK = 32 # CDNA4 wide-K (mfma_f32_16x16x32_bf16)
|
||||
MFMA_K_BF16_PV = 16 # PV's K = token tile = 16
|
||||
QK_K_CHUNKS = HEAD_SIZE // MFMA_K_BF16_QK # 4
|
||||
PV_N_CHUNKS = HEAD_SIZE // MFMA_N # 8
|
||||
|
||||
# LDS regions
|
||||
CENTROID_LDS_BYTES = N_CENTROIDS * 4 # 64
|
||||
Q_LDS_BYTES = QUERY_GROUP_SIZE * HEAD_SIZE * 2 # 4096
|
||||
KV_TILE_LDS_BYTES = TILE_SIZE * HEAD_SIZE * 2 # 4096
|
||||
|
||||
LOG2E = 1.4426950408889634
|
||||
NEG_INF_VAL = float("-inf")
|
||||
|
||||
|
||||
def _vsplat_mul(vec, scalar):
|
||||
s = scalar.ir_value() if hasattr(scalar, "ir_value") else scalar
|
||||
return vec * vector.broadcast(T.f32x4, s)
|
||||
|
||||
|
||||
allocator = None
|
||||
|
||||
|
||||
def build_tq_decode_module(
|
||||
num_seqs: int,
|
||||
num_kv_heads: int,
|
||||
num_partitions: int,
|
||||
max_blocks_per_seq: int = 512,
|
||||
softmax_scale: float | None = None,
|
||||
query_group_size: int = QUERY_GROUP_SIZE,
|
||||
kv_block_size: int = KV_BLOCK_SIZE,
|
||||
use_hw_v_transpose: bool = False,
|
||||
tile_groups_per_partition: int = 1,
|
||||
):
|
||||
"""Build a TQ decode kernel module.
|
||||
|
||||
``query_group_size`` (= num_query_heads // num_kv_heads) selects the
|
||||
GQA factor. Supported values: 8 and 16. For 8 the MFMA's 16-row
|
||||
capacity is half-used; lanes 8..15 compute garbage and are gated out
|
||||
of all global-memory writes via an OOB-offset trick on the buffer
|
||||
resource. Q load iterates ``QG // 4`` chunks (= 2 for QG=8, 4 for
|
||||
QG=16) so we never read past the real Q rows in HBM.
|
||||
|
||||
``kv_block_size`` is the number of tokens per vLLM cache block. Must
|
||||
be a multiple of TILE_SIZE (=16). Supported values: 16 and 32. With
|
||||
block_size=32, two MFMA K-tiles fit in each block; the kernel walks
|
||||
block-table entries every ``kv_block_size // TILE_SIZE`` tiles and
|
||||
re-uses the entry for sub-tiles within the same block. SoA metadata
|
||||
region grows linearly with block_size.
|
||||
|
||||
``use_hw_v_transpose`` (CDNA4 / gfx950 only) replaces the 32-element
|
||||
scattered ``ds_write_b16`` V-dequant pattern with a row-major
|
||||
``V[token][head_dim]`` LDS layout written as 4 wide ``ds_write_b128``
|
||||
per lane, then read back into the MFMA A operand via the hardware
|
||||
``ds_read_tr16_b64`` transpose. This eliminates ~28 ds_write
|
||||
instructions per lane per K-tile (32 -> 4) and 8 strided ds_read_b64
|
||||
per PV chunk (replaces them with 1 hw-transpose read each).
|
||||
|
||||
``tile_groups_per_partition`` (default 1) controls the FA-2 split-K
|
||||
granularity. With value G each partition processes ``G * 16`` K-tiles
|
||||
= ``G * KV_COMPUTE_BLOCK`` tokens, with the existing 16-tile loop
|
||||
body iterated G times. The launcher uses this to bound
|
||||
``num_partitions`` (e.g. cap at 32) for long context: at 32K /
|
||||
block_size=32 / num_partitions=32, G=5 covers 32*5*256 = 40 960
|
||||
tokens of worst-case context with grid.z=32 instead of 256.
|
||||
Behavior at G=1 is bit-identical to the single-partition kernel.
|
||||
"""
|
||||
assert query_group_size in (8, 16), (
|
||||
f"query_group_size must be 8 or 16; got {query_group_size}"
|
||||
)
|
||||
assert kv_block_size in (16, 32), (
|
||||
f"kv_block_size must be 16 or 32; got {kv_block_size}"
|
||||
)
|
||||
assert kv_block_size % TILE_SIZE == 0
|
||||
assert int(tile_groups_per_partition) >= 1, (
|
||||
f"tile_groups_per_partition must be >= 1; got {tile_groups_per_partition}"
|
||||
)
|
||||
USE_HW_TR = bool(use_hw_v_transpose)
|
||||
TGPP = int(tile_groups_per_partition)
|
||||
PARTITION_EXTENT_TOKENS = TGPP * KV_COMPUTE_BLOCK
|
||||
|
||||
QG = int(query_group_size)
|
||||
QG_LOAD_ITERS = QG // 4 # 2 for QG=8, 4 for QG=16
|
||||
OOB_OFFSET = 0x7FFFFFF0 # noqa: F841 ~2GB byte offset; > any plausible buffer
|
||||
|
||||
_BS = int(kv_block_size)
|
||||
_TILES_PER_BLOCK = _BS // TILE_SIZE # 1 for BS=16, 2 for BS=32
|
||||
|
||||
global allocator
|
||||
arch = get_hip_arch()
|
||||
|
||||
if softmax_scale is None:
|
||||
softmax_scale = 1.0 / (HEAD_SIZE**0.5)
|
||||
_qk_scale = float(softmax_scale)
|
||||
|
||||
# --- Strides ---
|
||||
_Hq = num_kv_heads * QG
|
||||
_stride_q_seq = _Hq * HEAD_SIZE
|
||||
_stride_q_head = HEAD_SIZE
|
||||
_stride_bt_seq = max_blocks_per_seq
|
||||
|
||||
_data_region_bytes = _BS * num_kv_heads * DATA_BYTES_PER_SLOT
|
||||
_meta_region_bytes = num_kv_heads * NUM_SOA_FIELDS * _BS * 2
|
||||
_stride_cache_block = _data_region_bytes + _meta_region_bytes
|
||||
_meta_region_offset = _data_region_bytes
|
||||
|
||||
_stride_out_part = QG * HEAD_SIZE
|
||||
_stride_out_head = num_partitions * QG * HEAD_SIZE
|
||||
_stride_out_seq = num_kv_heads * num_partitions * QG * HEAD_SIZE
|
||||
_stride_es_seq = num_kv_heads * num_partitions * QG
|
||||
_stride_ml_seq = _stride_es_seq
|
||||
|
||||
# --- LDS layout ---
|
||||
allocator = SmemAllocator(None, arch=arch, global_sym_name="tq_smem")
|
||||
centroid_off = 0
|
||||
allocator.ptr = CENTROID_LDS_BYTES
|
||||
q_off = allocator.ptr
|
||||
allocator.ptr += Q_LDS_BYTES
|
||||
kv_off = allocator.ptr
|
||||
allocator.ptr += KV_TILE_LDS_BYTES
|
||||
|
||||
@flyc.kernel
|
||||
def tq_decode_kernel(
|
||||
out_ptr: fx.Tensor,
|
||||
exp_sums_ptr: fx.Tensor,
|
||||
max_logits_ptr: fx.Tensor,
|
||||
query_ptr: fx.Tensor,
|
||||
kv_cache_ptr: fx.Tensor,
|
||||
centroids_ptr: fx.Tensor,
|
||||
block_tables_ptr: fx.Tensor,
|
||||
seq_lens_ptr: fx.Tensor,
|
||||
):
|
||||
# ---- IDs ---------------------------------------------------------
|
||||
tid = gpu.thread_idx.x
|
||||
seq = gpu.block_idx.x
|
||||
kv_h = gpu.block_idx.y
|
||||
part = gpu.block_idx.z
|
||||
lane = tid # 0..63
|
||||
mfma_row = lane & fx.Int32(15) # = query_row (B's N) /
|
||||
# token (A's M for QK)
|
||||
mfma_col_grp = lane >> fx.Int32(4) # 0..3, K-group dim
|
||||
|
||||
# ---- Buffer resources -------------------------------------------
|
||||
q_rsrc = buffer_ops.create_buffer_resource(query_ptr, max_size=True)
|
||||
kv_rsrc = buffer_ops.create_buffer_resource(kv_cache_ptr, max_size=True)
|
||||
bt_rsrc = buffer_ops.create_buffer_resource(block_tables_ptr, max_size=True)
|
||||
sl_rsrc = buffer_ops.create_buffer_resource(seq_lens_ptr, max_size=True)
|
||||
cent_rsrc = buffer_ops.create_buffer_resource(centroids_ptr, max_size=True)
|
||||
out_rsrc = buffer_ops.create_buffer_resource(out_ptr, max_size=True)
|
||||
es_rsrc = buffer_ops.create_buffer_resource(exp_sums_ptr, max_size=True)
|
||||
ml_rsrc = buffer_ops.create_buffer_resource(max_logits_ptr, max_size=True)
|
||||
|
||||
# ---- LDS pointers -----------------------------------------------
|
||||
base = allocator.get_base()
|
||||
cent_lds = SmemPtr(base, centroid_off, T.f32, shape=(N_CENTROIDS,))
|
||||
q_lds_i32 = SmemPtr(base, q_off, T.i32, shape=(Q_LDS_BYTES // 4,)).get()
|
||||
q_lds_i64 = SmemPtr(base, q_off, T.i64, shape=(Q_LDS_BYTES // 8,)).get()
|
||||
kv_lds_i32 = SmemPtr(base, kv_off, T.i32, shape=(KV_TILE_LDS_BYTES // 4,)).get() # noqa: F841
|
||||
kv_lds_i64 = SmemPtr(base, kv_off, T.i64, shape=(KV_TILE_LDS_BYTES // 8,)).get()
|
||||
kv_lds_i16 = SmemPtr(base, kv_off, T.i16, shape=(KV_TILE_LDS_BYTES // 2,)).get()
|
||||
|
||||
# ---- Constants ---------------------------------------------------
|
||||
c_sq = fx.Int32(_stride_q_seq)
|
||||
c_qh = fx.Int32(_stride_q_head)
|
||||
c_qg = fx.Int32(QG)
|
||||
c_bt = fx.Int32(_stride_bt_seq)
|
||||
c_block = fx.Int32(_stride_cache_block)
|
||||
c_meta_off = fx.Int32(_meta_region_offset)
|
||||
c_data_per_slot = fx.Int32(DATA_BYTES_PER_SLOT)
|
||||
c_kv_heads = fx.Int32(num_kv_heads)
|
||||
c_keydata = fx.Int32(KEY_DATA_BYTES)
|
||||
c_w = fx.Int32(WARP_SIZE)
|
||||
|
||||
NEG_INF = arith.constant(NEG_INF_VAL, type=T.f32)
|
||||
ZERO_F = fx.Float32(0.0)
|
||||
ONE_F = fx.Float32(1.0)
|
||||
LOG2E_C = arith.constant(LOG2E, type=T.f32)
|
||||
QK_SCALE = arith.constant(_qk_scale, type=T.f32)
|
||||
|
||||
# Helper: unwrap fx wrapper → raw ir.Value (used in STEP F)
|
||||
def _ival(v):
|
||||
return v.ir_value() if hasattr(v, "ir_value") else v
|
||||
|
||||
# ===== STEP A: Load centroids → LDS (cooperative, race-safe) =====
|
||||
c_idx_safe = lane & fx.Int32(N_CENTROIDS - 1)
|
||||
c_val = buffer_ops.buffer_load(cent_rsrc, c_idx_safe, vec_width=1, dtype=T.f32)
|
||||
cent_lds.store(c_val, [arith.index_cast(T.index, c_idx_safe)])
|
||||
gpu.barrier()
|
||||
|
||||
# ===== STEP B: Load pre-rotated q_rot → row-major LDS ============
|
||||
# q_rsrc points to the externally-rotated q_rot tensor. Load 8 bf16
|
||||
# per lane per iter (vec_width=4 i32) in QG_LOAD_ITERS iterations.
|
||||
for c in range_constexpr(QG_LOAD_ITERS):
|
||||
row_chunk = lane + fx.Int32(c * WARP_SIZE) # 0..QG*16-1
|
||||
row = row_chunk >> fx.Int32(4) # 0..QG-1
|
||||
col_b = row_chunk & fx.Int32(15) # 0..15
|
||||
col_elem = col_b * fx.Int32(8) # 0..120 in bf16
|
||||
q_off_elem = seq * c_sq + (kv_h * c_qg + row) * c_qh + col_elem
|
||||
q_v = buffer_ops.buffer_load(
|
||||
q_rsrc,
|
||||
q_off_elem // fx.Int32(2),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
q_lds_byte = row * fx.Int32(HEAD_SIZE * 2) + col_elem * fx.Int32(2)
|
||||
vector.store(
|
||||
q_v,
|
||||
q_lds_i32,
|
||||
[arith.index_cast(T.index, q_lds_byte // fx.Int32(4))],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# ===== STEP C: Pre-load Q operands for QK_K_CHUNKS = 4 K-chunks ===
|
||||
# Wide-K MFMA: K_per_chunk = 32, K_per_lane = 32/4 = 8 bf16 per chunk.
|
||||
# Lane t holds 8 bf16 at row=mfma_row, K-cols
|
||||
# chunk*32 + mfma_col_grp*8 + 0..7
|
||||
# Byte addr = mfma_row * HEAD_SIZE*2 + (chunk*32 + col_grp*8)*2
|
||||
# = mfma_row * 256 + chunk*64 + col_grp*16
|
||||
# i64 idx = mfma_row * 32 + chunk*8 + col_grp*2 (each i64 = 4 bf16)
|
||||
# Load 16 bytes (= 8 bf16 = 2 i64) per chunk via vec(2, i64).
|
||||
q_chunks = []
|
||||
for chk in range_constexpr(QK_K_CHUNKS):
|
||||
q_idx_i64 = (
|
||||
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
+ fx.Int32(chk * 8)
|
||||
+ mfma_col_grp * fx.Int32(2)
|
||||
)
|
||||
qv = vector.load_op(
|
||||
T.vec(2, T.i64),
|
||||
q_lds_i64,
|
||||
[arith.index_cast(T.index, q_idx_i64)],
|
||||
)
|
||||
q_chunks.append(vector.bitcast(T.vec(8, T.bf16), qv))
|
||||
|
||||
# ===== STEP D: Online softmax + PV state =========================
|
||||
running_max = NEG_INF
|
||||
running_sum = ZERO_F
|
||||
zero_v4 = arith.constant_vector(0.0, T.f32x4)
|
||||
acc_pv = [zero_v4 for _ in range(PV_N_CHUNKS)]
|
||||
|
||||
# ===== STEP E: Sequence-len + partition base ====================
|
||||
seq_len = buffer_ops.buffer_load(sl_rsrc, seq, vec_width=1, dtype=T.i32)
|
||||
# ── Bounded num_partitions + internal looping ───────────────────
|
||||
# With TGPP > 1 this CTA owns a contiguous slice of length
|
||||
# ``PARTITION_EXTENT_TOKENS`` that is iterated as ``TGPP`` groups
|
||||
# of 16 K-tiles each. ``partition_start`` is recomputed per
|
||||
# outer ``tg`` iteration below; the FA-2 online-softmax state
|
||||
# (running_max, running_sum, acc_pv) accumulates across all
|
||||
# ``TGPP * 16`` tiles for this CTA — semantically identical to
|
||||
# processing one large 16*TGPP-tile partition.
|
||||
partition_base = part * fx.Int32(PARTITION_EXTENT_TOKENS)
|
||||
|
||||
# Per-K-tile dequant lane assignment:
|
||||
# 16 tokens × 64 packed bytes per token = 1024 bytes
|
||||
# 64 lanes × 16 bytes/lane = 1024 ✓
|
||||
# Lane t: token = t / 4, sub-chunk = t % 4 (0..3) → 16 bytes contiguous
|
||||
tok_in_tile = lane >> fx.Int32(2)
|
||||
chunk_in_tok = lane & fx.Int32(3)
|
||||
# ``slot`` (= absolute slot index within the cache block, range
|
||||
# 0.._BS-1) is recomputed per tile inside the K-tile loop below
|
||||
# so it reflects ``tile_in_block * TILE_SIZE + tok_in_tile``.
|
||||
|
||||
c_meta_u16_per_kvh = fx.Int32(NUM_SOA_FIELDS * _BS)
|
||||
c_knorm_off_u16 = fx.Int32(SOA_K_NORM * _BS)
|
||||
c_vscale_off_u16 = fx.Int32(SOA_V_SCALE * _BS)
|
||||
c_vzero_off_u16 = fx.Int32(SOA_V_ZERO * _BS)
|
||||
|
||||
# ===== STEP F: K-tile loop =======================================
|
||||
# With kv_block_size > TILE_SIZE there are _TILES_PER_BLOCK tiles
|
||||
# per cache block. Tiles within the same block share a block-table
|
||||
# entry but address different slot ranges within the block.
|
||||
#
|
||||
# ── Per-tile block-table OOB redirect ────────────────────────────
|
||||
# The K-tile loop is unrolled 16 times so every CTA issues 16
|
||||
# block-table reads regardless of (partition, seq_len). When a
|
||||
# partition extends past ``seq_len`` (e.g. num_partitions=2 for
|
||||
# a 256-token sequence — partition 1 covers tokens 256..511, all
|
||||
# masked out) the per-tile bt offset can land beyond the bt
|
||||
# allocation. With ``max_size=True`` the descriptor advertises
|
||||
# 4 GB so the HW returns whatever pre-existing HBM bytes live at
|
||||
# that offset; the resulting garbage ``phys_block`` is multiplied
|
||||
# by ``stride_cache_block`` and the subsequent kv_cache read
|
||||
# jumps into pages that may be unmapped (→ "Memory access fault
|
||||
# by GPU node") or may decode as NaN bytes (→ NaN segm_out).
|
||||
# Allocator-pattern dependent, but reproduces deterministically
|
||||
# at large B (e.g. B=64 seq=256 in the test sweep produces 8 192
|
||||
# NaN entries even on canonical QG=8).
|
||||
#
|
||||
# Fix: when the tile starts at or past ``seq_len`` (so its
|
||||
# qk_acc is going to be killed by the per-token
|
||||
# ``kv_tok < seq_len`` mask anyway), redirect the bt read to
|
||||
# ``bt[seq, 0]`` — always in bounds. The redundant phys_block
|
||||
# decode + kv_cache read is wasted work, but correctness is
|
||||
# preserved without changing the kernel's iteration count.
|
||||
# ===== Runtime-adaptive scf.ForOp w/ iter_args =========
|
||||
# Runtime-adaptive outer loop: trip count derives from
|
||||
# actual seq_len, NOT TGPP_max. At cudagraph-capture warmup
|
||||
# (seq_len=1) only 1 iteration runs; at 32K production
|
||||
# decode all TGPP iterations run. Kernel binary stays small
|
||||
# (single body, looped at runtime) — capture time matches
|
||||
# the legacy 16-tile-only kernel rather than scaling 4x.
|
||||
#
|
||||
# FA-2 state (running_max, running_sum, acc_pv[8]) threads
|
||||
# through scf iter_args; the body reads them at top, runs
|
||||
# the unchanged 16-tile inner body, and yields the new
|
||||
# state at bottom. After the loop, results are pulled out
|
||||
# of for_op.results back into the local Python names so the
|
||||
# downstream STEP G (output) is unmodified.
|
||||
c_kcb = fx.Int32(KV_COMPUTE_BLOCK)
|
||||
c_tgpp = fx.Int32(TGPP)
|
||||
c_zero_i32 = fx.Int32(0)
|
||||
c_one_i32 = fx.Int32(1)
|
||||
# remaining = seq_len - partition_base (signed, may be <=0)
|
||||
remaining = seq_len - partition_base
|
||||
in_range = remaining > c_zero_i32
|
||||
# trip_raw = ceil(remaining / KV_COMPUTE_BLOCK)
|
||||
# (when in_range is false, the divisor branch produces
|
||||
# garbage that the select below overrides with zero, so we
|
||||
# don't pre-clamp)
|
||||
trip_raw = (remaining + c_kcb - c_one_i32) // c_kcb
|
||||
trip_clamped = (trip_raw > c_tgpp).select(c_tgpp, trip_raw)
|
||||
trip_or_zero = in_range.select(trip_clamped, c_zero_i32)
|
||||
c_zero_idx = arith.constant(0, index=True)
|
||||
c_one_idx = arith.constant(1, index=True)
|
||||
trip_idx = arith.index_cast(
|
||||
T.index,
|
||||
trip_or_zero.ir_value()
|
||||
if hasattr(trip_or_zero, "ir_value")
|
||||
else trip_or_zero,
|
||||
)
|
||||
# Initial iter_args list: FA-2 accumulator state.
|
||||
# Order: running_max, running_sum, acc_pv[0..PV_N_CHUNKS-1].
|
||||
# NOTE: scf.ForOp requires ir.Value (with .type). The DSL
|
||||
# constants ZERO_F / ONE_F are fx.Float32 (Numeric) wrappers,
|
||||
# not raw ir.Value, so we unwrap via .ir_value() before
|
||||
# passing. _ival() is defined once near the top of the kernel body.
|
||||
_init_iter = [
|
||||
_ival(running_max),
|
||||
_ival(running_sum),
|
||||
*[_ival(p) for p in acc_pv],
|
||||
]
|
||||
_for_op = _scf.ForOp(
|
||||
c_zero_idx,
|
||||
trip_idx,
|
||||
c_one_idx,
|
||||
_init_iter,
|
||||
)
|
||||
_for_ip = ir.InsertionPoint(_for_op.body)
|
||||
_for_ip.__enter__()
|
||||
try:
|
||||
tg_idx = _for_op.induction_variable
|
||||
tg_i32 = fx.Int32(arith.index_cast(T.i32, tg_idx))
|
||||
partition_start = partition_base + tg_i32 * c_kcb
|
||||
bt_seq_base = seq * c_bt + (partition_start // fx.Int32(_BS))
|
||||
running_max = _for_op.inner_iter_args[0]
|
||||
running_sum = _for_op.inner_iter_args[1]
|
||||
acc_pv = list(_for_op.inner_iter_args[2:])
|
||||
for n_tile in range_constexpr(16):
|
||||
block_in_part = n_tile // _TILES_PER_BLOCK
|
||||
tile_in_block = n_tile % _TILES_PER_BLOCK
|
||||
tile_start_tok = partition_start + fx.Int32(n_tile * TILE_SIZE)
|
||||
tile_in_seq = tile_start_tok < seq_len
|
||||
bt_off = bt_seq_base + fx.Int32(block_in_part)
|
||||
bt_off_safe = tile_in_seq.select(bt_off, seq * c_bt)
|
||||
phys_block = buffer_ops.buffer_load(
|
||||
bt_rsrc,
|
||||
bt_off_safe,
|
||||
vec_width=1,
|
||||
dtype=T.i32,
|
||||
)
|
||||
block_base = phys_block * c_block
|
||||
data_region = block_base
|
||||
meta_region = block_base + c_meta_off
|
||||
|
||||
# ``slot`` is the absolute slot index within the cache block
|
||||
# (range 0.._BS-1). For BS=16 it equals tok_in_tile; for BS=32
|
||||
# it equals tile_in_block * 16 + tok_in_tile. Used both for
|
||||
# the K/V data region and SoA metadata region addressing.
|
||||
slot = fx.Int32(tile_in_block * TILE_SIZE) + tok_in_tile
|
||||
data_bases_byte = (
|
||||
data_region
|
||||
+ slot * (c_kv_heads * c_data_per_slot)
|
||||
+ kv_h * c_data_per_slot
|
||||
)
|
||||
k_byte = data_bases_byte + chunk_in_tok * fx.Int32(16)
|
||||
k_packed = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
k_byte // fx.Int32(4),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
|
||||
# ---- HOISTED: issue V data + meta HBM loads early ---------
|
||||
# These are async; their s_waitcnt is pushed by the compiler
|
||||
# past the K dequant + QK MFMA + softmax block, hiding most
|
||||
# of the V HBM latency behind compute.
|
||||
v_byte = data_bases_byte + c_keydata + chunk_in_tok * fx.Int32(16)
|
||||
v_packed = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
v_byte // fx.Int32(4),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
vscale_u16 = (
|
||||
meta_region // fx.Int32(2)
|
||||
+ kv_h * c_meta_u16_per_kvh
|
||||
+ c_vscale_off_u16
|
||||
+ slot
|
||||
)
|
||||
vzero_u16 = (
|
||||
meta_region // fx.Int32(2)
|
||||
+ kv_h * c_meta_u16_per_kvh
|
||||
+ c_vzero_off_u16
|
||||
+ slot
|
||||
)
|
||||
vscale_raw = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
vscale_u16,
|
||||
vec_width=1,
|
||||
dtype=T.i16,
|
||||
)
|
||||
vzero_raw = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
vzero_u16,
|
||||
vec_width=1,
|
||||
dtype=T.i16,
|
||||
)
|
||||
|
||||
knorm_u16 = (
|
||||
meta_region // fx.Int32(2)
|
||||
+ kv_h * c_meta_u16_per_kvh
|
||||
+ c_knorm_off_u16
|
||||
+ slot
|
||||
)
|
||||
knorm_raw = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
knorm_u16,
|
||||
vec_width=1,
|
||||
dtype=T.i16,
|
||||
)
|
||||
knorm_f16 = arith.bitcast(T.f16, knorm_raw)
|
||||
knorm_f32 = arith.extf(T.f32, knorm_f16)
|
||||
|
||||
# K dequant → LDS [token, head_dim] (natural)
|
||||
# Lane writes 32 bf16 (= 4×8) for token=tok_in_tile,
|
||||
# head_dims chunk_in_tok*32..+31 (4 sub-chunks of 8).
|
||||
tok_kreg = tok_in_tile * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
chunk_kreg = tok_kreg + chunk_in_tok * fx.Int32(8)
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(k_packed, static_position=[w])
|
||||
bf16_elems = []
|
||||
for n in range_constexpr(8):
|
||||
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
|
||||
nibble_idx = arith.index_cast(T.index, nibble)
|
||||
cent_f32 = cent_lds.load([nibble_idx])
|
||||
elem_bf16 = arith.trunc_f(T.bf16, cent_f32 * knorm_f32)
|
||||
bf16_elems.append(elem_bf16)
|
||||
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
|
||||
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
|
||||
vector.store(
|
||||
v_i64,
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, chunk_kreg + fx.Int32(w * 2))],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# QK MFMA (CDNA4 wide-K): A=K[token, head_dim], B=Q (= Q^T).
|
||||
# K read: lane t = K[token=mfma_row, head_dim=chunk*32+col_grp*8..+7]
|
||||
# Same i64×2 indexing as Q.
|
||||
qk_acc = zero_v4
|
||||
for chk in range_constexpr(QK_K_CHUNKS):
|
||||
k_idx_i64 = (
|
||||
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
+ fx.Int32(chk * 8)
|
||||
+ mfma_col_grp * fx.Int32(2)
|
||||
)
|
||||
kv_load = vector.load_op(
|
||||
T.vec(2, T.i64),
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, k_idx_i64)],
|
||||
)
|
||||
k_op = vector.bitcast(T.vec(8, T.bf16), kv_load)
|
||||
qk_acc = rocdl.mfma_f32_16x16x32_bf16(
|
||||
T.f32x4, [k_op, q_chunks[chk], qk_acc, 0, 0, 0]
|
||||
)
|
||||
|
||||
# qk_acc layout: lane t holds C[token=(t/16)*4..+3, query=t%16]
|
||||
# 4 fp32/lane = 4 different tokens at SAME query_row.
|
||||
|
||||
# Scale + mask out-of-context tokens.
|
||||
qk_acc = _vsplat_mul(qk_acc, QK_SCALE)
|
||||
for elem in range_constexpr(4):
|
||||
kv_tok = (
|
||||
partition_start
|
||||
+ fx.Int32(n_tile * TILE_SIZE)
|
||||
+ mfma_col_grp * fx.Int32(4)
|
||||
+ fx.Int32(elem)
|
||||
)
|
||||
in_b = kv_tok < seq_len
|
||||
v = vector.extract(qk_acc, static_position=[elem])
|
||||
qk_acc = vector.insert(
|
||||
in_b.select(v, NEG_INF),
|
||||
qk_acc,
|
||||
static_position=[elem],
|
||||
dynamic_position=[],
|
||||
)
|
||||
|
||||
# FA2 online softmax: per-query-row reduce.
|
||||
# Per-row max: max over 4 fp32 in lane, then xor-shuffle 16, 32
|
||||
# (across the 4 col_grps that share same mfma_row).
|
||||
local_max = vector.reduction(T.f32, "maxnumf", qk_acc)
|
||||
r1 = local_max.shuffle_xor(fx.Int32(16), c_w)
|
||||
local_max = local_max.maximumf(r1)
|
||||
r2 = local_max.shuffle_xor(fx.Int32(32), c_w)
|
||||
tile_max = local_max.maximumf(r2)
|
||||
|
||||
new_max = running_max.maximumf(tile_max)
|
||||
max_diff = running_max - new_max
|
||||
safe_diff = (running_max > NEG_INF).select(max_diff, ZERO_F)
|
||||
scale = (safe_diff * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
|
||||
running_sum = running_sum * scale
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
acc_pv[h] = _vsplat_mul(acc_pv[h], scale)
|
||||
running_max = new_max
|
||||
|
||||
# Compute probs: p = exp((qk - new_max) * LOG2E)
|
||||
tile_sum = ZERO_F
|
||||
for elem in range_constexpr(4):
|
||||
s = vector.extract(qk_acc, static_position=[elem])
|
||||
d = s - new_max
|
||||
d = (new_max > NEG_INF).select(d, NEG_INF)
|
||||
p = (d * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
|
||||
tile_sum = tile_sum + p
|
||||
qk_acc = vector.insert(
|
||||
p, qk_acc, static_position=[elem], dynamic_position=[]
|
||||
)
|
||||
|
||||
ts1 = tile_sum.shuffle_xor(fx.Int32(16), c_w)
|
||||
tile_sum = tile_sum + ts1
|
||||
ts2 = tile_sum.shuffle_xor(fx.Int32(32), c_w)
|
||||
tile_sum = tile_sum + ts2
|
||||
running_sum = running_sum + tile_sum
|
||||
|
||||
# ---- V dequant → LDS (V/scale/zero raw loads were hoisted) -
|
||||
# v_packed, vscale_raw, vzero_raw are already in flight from
|
||||
# the top of this iteration; only the f32 conversion stays
|
||||
# here so the actual consume site is still close to the use.
|
||||
vscale_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vscale_raw))
|
||||
vzero_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vzero_raw))
|
||||
|
||||
if USE_HW_TR:
|
||||
# V dequant → LDS [token][head_dim] (ROW-MAJOR, no transpose).
|
||||
# Each lane writes 32 contiguous bf16 (one token, head_dims
|
||||
# chunk_in_tok*32..+31) as 4× ds_write_b128 = 4× vec(2,i64).
|
||||
# Replaces 32× ds_write_b16 of the legacy transposed path.
|
||||
v_lds_elem_base = tok_in_tile * fx.Int32(
|
||||
HEAD_SIZE
|
||||
) + chunk_in_tok * fx.Int32(32)
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(v_packed, static_position=[w])
|
||||
bf16_elems = []
|
||||
for n in range_constexpr(8):
|
||||
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
|
||||
nibble_f32 = arith.sitofp(T.f32, nibble)
|
||||
elem_f32 = nibble_f32 * vscale_f32 + vzero_f32
|
||||
elem_bf16 = arith.trunc_f(T.bf16, elem_f32)
|
||||
bf16_elems.append(elem_bf16)
|
||||
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
|
||||
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
|
||||
v_lds_i64_idx = (v_lds_elem_base + fx.Int32(w * 8)) // fx.Int32(
|
||||
4
|
||||
)
|
||||
vector.store(
|
||||
v_i64,
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, v_lds_i64_idx)],
|
||||
)
|
||||
# ── HW V transpose: cross-lane LDS sync ─────────────────────
|
||||
# ds_read_tr16_b64 below introduces a cross-lane LDS read:
|
||||
# consumer lane t reads bytes written by source lanes
|
||||
# 0,4,8,12 (etc.). The compiler's automatic waitcnt insertion
|
||||
# is per-lane and has no awareness of the HW transpose's
|
||||
# cross-lane forwarding, so it can sink the next iteration's
|
||||
# ds_read ahead of these ds_write_b128 ops in the schedule.
|
||||
#
|
||||
# Two-part fence:
|
||||
# 1. sched_barrier(0): MachineScheduler reorder barrier
|
||||
# (mask=0 → no instruction class may cross). Pure hint,
|
||||
# generates no runtime instruction. Use the simple
|
||||
# sched_barrier (NOT sched_group_barrier, which is an
|
||||
# IGLP marker that requires partner barriers and
|
||||
# corrupts the schedule when used in isolation).
|
||||
# 2. s_waitcnt lgkmcnt=0: runtime drain of the LDS write
|
||||
# queue. vmcnt/expcnt left at no-wait so we don't
|
||||
# stall on speculatively-issued HBM loads (which can
|
||||
# include OOB-but-masked buffer_load reads from the
|
||||
# next K-tile's hoisted V prefetch).
|
||||
#
|
||||
# The SW path below does NOT need this because each lane
|
||||
# only reads cells it itself wrote (no cross-lane dep), and
|
||||
# the per-lane same-address waitcnt the compiler emits is
|
||||
# correct.
|
||||
#
|
||||
# Without this fence the reordered ds_read produces
|
||||
# wrong results on high-partition-count configs (large
|
||||
# padded num_partitions); low-partition configs are
|
||||
# unaffected.
|
||||
rocdl.sched_barrier(0)
|
||||
# encode_waitcnt(vmcnt=63, expcnt=7, lgkmcnt=0)
|
||||
# = 0xF | (7<<4) | (0<<8) | (3<<14) = 0xC07F
|
||||
rocdl.s_waitcnt(0xC07F)
|
||||
else:
|
||||
# Legacy transposed V_LDS path: 32 ds_write_b16 per lane.
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(v_packed, static_position=[w])
|
||||
for n in range_constexpr(8):
|
||||
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
|
||||
nibble_f32 = arith.sitofp(T.f32, nibble)
|
||||
elem_f32 = nibble_f32 * vscale_f32 + vzero_f32
|
||||
elem_bf16 = arith.trunc_f(T.bf16, elem_f32)
|
||||
elem_i16 = arith.bitcast(T.i16, elem_bf16)
|
||||
head_dim = chunk_in_tok * fx.Int32(32) + fx.Int32(w * 8 + n)
|
||||
v_idx_i16 = head_dim * fx.Int32(TILE_SIZE) + tok_in_tile
|
||||
v_vec = vector.from_elements(T.vec(1, T.i16), [elem_i16])
|
||||
vector.store(
|
||||
v_vec,
|
||||
kv_lds_i16,
|
||||
[arith.index_cast(T.index, v_idx_i16)],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# ---- PV MFMA: A=V[head_dim, token], B=P (=qk_acc bf16) -----
|
||||
# P operand B layout matches qk_acc: lane t holds 4 bf16 at
|
||||
# K=token=(t/16)*4..+3, N=query=t%16. Just trunc_f to bf16.
|
||||
p_bf16 = arith.trunc_f(T.vec(4, T.bf16), qk_acc)
|
||||
p_op = vector.bitcast(T.vec(4, T.i16), p_bf16)
|
||||
|
||||
if USE_HW_TR:
|
||||
# HW-transpose PV path: V_lds is row-major V[token][head_dim].
|
||||
# MFMA A operand layout: lane t holds 4 bf16 at
|
||||
# M=head_dim=mfma_row + h*16, K=token=(t/16)*4..+3
|
||||
# ds_read_tr16_b64 (4-element 16-bit transpose, per-16-lane block):
|
||||
# result[lane=t, elem=e] = Input[source_lane=e*4 + (t%16)//4,
|
||||
# col=t%4]
|
||||
# Per-lane address: token_idx = lane // 4 (covers 0..15 across
|
||||
# all four 16-lane MFMA blocks), hd_sub = (lane % 4)*4 selects
|
||||
# the 4-element column window inside the h*16 chunk.
|
||||
# Total LDS byte offset = kv_off + token_idx*HEAD_SIZE*2
|
||||
# + (h*16 + hd_sub)*2
|
||||
token_idx = lane >> fx.Int32(2)
|
||||
hd_sub = (lane & fx.Int32(3)) * fx.Int32(4)
|
||||
v_lane_byte = (
|
||||
fx.Int32(kv_off)
|
||||
+ token_idx * fx.Int32(HEAD_SIZE * 2)
|
||||
+ hd_sub * fx.Int32(2)
|
||||
)
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
v_byte_off = v_lane_byte + fx.Int32(h * 32)
|
||||
v_byte_i64 = fx.Int64(v_byte_off)
|
||||
v_ptr = buffer_ops.create_llvm_ptr(
|
||||
v_byte_i64,
|
||||
address_space=3,
|
||||
)
|
||||
v_op_raw = rocdl.ds_read_tr16_b64(
|
||||
T.vec(4, T.i16),
|
||||
v_ptr,
|
||||
).result
|
||||
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
|
||||
T.f32x4, [v_op_raw, p_op, acc_pv[h], 0, 0, 0]
|
||||
)
|
||||
else:
|
||||
# Legacy transposed-LDS PV: 1 ds_read_b64 per chunk.
|
||||
# Byte addr = (mfma_row + h*16) * TILE_SIZE * 2 + (col_grp*4) * 2
|
||||
# = (mfma_row + h*16) * 32 + col_grp*8
|
||||
# i64 idx = (mfma_row + h*16) * 4 + col_grp
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
v_idx_i64 = (mfma_row + fx.Int32(h * 16)) * fx.Int32(
|
||||
4
|
||||
) + mfma_col_grp
|
||||
kv_load = vector.load_op(
|
||||
T.vec(1, T.i64),
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, v_idx_i64)],
|
||||
)
|
||||
v_op = vector.bitcast(T.vec(4, T.i16), kv_load)
|
||||
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
|
||||
T.f32x4, [v_op, p_op, acc_pv[h], 0, 0, 0]
|
||||
)
|
||||
|
||||
_scf.YieldOp(
|
||||
[
|
||||
_ival(running_max),
|
||||
_ival(running_sum),
|
||||
*[_ival(p) for p in acc_pv],
|
||||
]
|
||||
)
|
||||
finally:
|
||||
_for_ip.__exit__(None, None, None)
|
||||
# Pull final accumulator values out of the for_op results.
|
||||
running_max = _for_op.results[0]
|
||||
running_sum = _for_op.results[1]
|
||||
acc_pv = list(_for_op.results[2:])
|
||||
|
||||
# ===== STEP G: Output ===========================================
|
||||
# acc_pv[h] layout: lane t holds 4 fp32 at M=head_dim=(t/16)*4..+3 + h*16,
|
||||
# N=query_row=t%16.
|
||||
# 4 fp32/lane = 4 contiguous head_dim values for ONE query_row.
|
||||
safe_sum = (running_sum > ZERO_F).select(running_sum, ONE_F)
|
||||
rcp = ONE_F / safe_sum
|
||||
|
||||
c_os = fx.Int32(_stride_out_seq)
|
||||
c_oh = fx.Int32(_stride_out_head)
|
||||
c_op_ = fx.Int32(_stride_out_part)
|
||||
out_base = seq * c_os + kv_h * c_oh + part * c_op_
|
||||
|
||||
# Gate all global stores by mfma_row < QG. For QG=16 the predicate is
|
||||
# tautological (mfma_row ∈ [0,15]); for QG=8 lanes 8..15 (whose MFMA
|
||||
# outputs are computational waste) skip the stores entirely.
|
||||
valid_row_pred = arith.cmpi(
|
||||
arith.CmpIPredicate.ult,
|
||||
mfma_row.ir_value() if hasattr(mfma_row, "ir_value") else mfma_row,
|
||||
arith.constant(QG, type=T.i32),
|
||||
)
|
||||
_if = _scf.IfOp(valid_row_pred)
|
||||
with ir.InsertionPoint(_if.then_block):
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
pv_norm = _vsplat_mul(acc_pv[h], rcp)
|
||||
pv_bf16 = arith.trunc_f(T.vec(4, T.bf16), pv_norm)
|
||||
pv_i32x2 = vector.bitcast(T.vec(2, T.i32), pv_bf16)
|
||||
head_dim_start = fx.Int32(h * 16) + mfma_col_grp * fx.Int32(4)
|
||||
out_off_elem = (
|
||||
out_base + mfma_row * fx.Int32(HEAD_SIZE) + head_dim_start
|
||||
)
|
||||
buffer_ops.buffer_store(
|
||||
pv_i32x2,
|
||||
out_rsrc,
|
||||
out_off_elem * fx.Int32(2),
|
||||
offset_is_bytes=True,
|
||||
)
|
||||
|
||||
c_npq = fx.Int32(num_partitions * QG)
|
||||
ml_off = (
|
||||
seq * fx.Int32(_stride_ml_seq) + kv_h * c_npq + part * c_qg + mfma_row
|
||||
)
|
||||
es_off = (
|
||||
seq * fx.Int32(_stride_es_seq) + kv_h * c_npq + part * c_qg + mfma_row
|
||||
)
|
||||
buffer_ops.buffer_store(running_max, ml_rsrc, ml_off)
|
||||
buffer_ops.buffer_store(running_sum, es_rsrc, es_off)
|
||||
_scf.YieldOp([])
|
||||
|
||||
return tq_decode_kernel
|
||||
@@ -0,0 +1,875 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# TurboQuant decode GQA-6 sibling (FlyDSL) — MI355X / gfx950 / CDNA4
|
||||
#
|
||||
# Decode-only paged attention over a TurboQuant SoA KV cache.
|
||||
# Targets MiniMax-M2.5 class models: HEAD_SIZE=128, GQA group=6, MSE_BITS=4 K,
|
||||
# VQB=4 V, N_CENTROIDS=16, BLOCK_SIZE=16, partitioned mode.
|
||||
#
|
||||
# === Sibling-file rationale ===
|
||||
# Structurally identical to tq_decode.py except:
|
||||
# * QUERY_GROUP_SIZE default = 6 (was 8)
|
||||
# * Build-time assert allows query_group_size = 6 only (Qwen sizes go to
|
||||
# the canonical tq_decode kernel — NOT this file)
|
||||
# * QG_LOAD_ITERS = (QG + 3) // 4 (round-up; 2 iters for QG=6 loads
|
||||
# 8 LDS rows of which only 6 carry real Q; rows 6..7 are zero-padded
|
||||
# by buffer_load OOB protection — gated out by the same mfma_row < QG
|
||||
# predicate the canonical kernel already uses for QG=8)
|
||||
#
|
||||
# All cross-lane operations (xor_shuffle softmax reductions, ds_read_tr16_b64
|
||||
# HW V transpose) are mfma_row-isolated and thus already GQA-agnostic. The
|
||||
# kernel is functionally a direct copy of tq_decode.py with the QG knob
|
||||
# turned to 6 — kept as a sibling per FlyDSL convention to preserve the
|
||||
# canonical kernel's invariants for its production callers (Qwen2.5-72B,
|
||||
# Qwen3-32B).
|
||||
#
|
||||
# === Design ===
|
||||
# * 1 wave (64 lanes) per CTA — simpler than 4-warp; trades parallelism for
|
||||
# correctness/debug-ability. A later revision will fan out to 4 warps.
|
||||
# * Flash-Attention-2 online softmax: rescale running PV/sum per K-tile,
|
||||
# no QK score storage across tiles, no P-LDS round-trip (qk_acc layout
|
||||
# matches PV B-operand directly).
|
||||
# * Centroids loaded once into LDS at kernel start (LDS-resident LUT).
|
||||
# * K dequant → LDS [token, head_dim] (natural layout)
|
||||
# * V dequant → LDS [head_dim, token] (TRANSPOSED — required so PV MFMA's
|
||||
# A=V operand can be loaded as 4-bf16 contiguous via 1 ds_read_b64).
|
||||
#
|
||||
# MFMA layouts used:
|
||||
# QK: mfma(A=K, B=Q, C=qk_acc) → C[m=token, n=query], 4 fp32/lane
|
||||
# = 4 tokens for SAME query → softmax via
|
||||
# xor_shuffle(16),(32) reduces per-row
|
||||
# PV: mfma(A=V_T, B=P, C=acc_pv) → C[m=head_dim, n=query], 4 fp32/lane
|
||||
# = 4 contiguous head_dim cols for SAME
|
||||
# query → 1 buffer_store of 8 bytes
|
||||
# per chunk per lane
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import flydsl.compiler as flyc
|
||||
import flydsl.expr as fx
|
||||
from flydsl._mlir import ir
|
||||
from flydsl._mlir.dialects import scf as _scf
|
||||
from flydsl.expr import (
|
||||
arith,
|
||||
buffer_ops,
|
||||
const_expr, # noqa: F401 kept for kernel-DSL API parity
|
||||
gpu,
|
||||
range_constexpr,
|
||||
rocdl,
|
||||
vector,
|
||||
)
|
||||
from flydsl.expr.typing import Int32, T # noqa: F401 Int32 via fx.Int32
|
||||
from flydsl.runtime.device import get_rocm_arch as get_hip_arch
|
||||
from flydsl.utils.smem_allocator import SmemAllocator, SmemPtr
|
||||
|
||||
# === Constants (MiniMax-M2.5-class TQ decode profile) =======================
|
||||
HEAD_SIZE = 128
|
||||
# default; overridable via build_tq_decode_gqa6_module(kv_block_size=...)
|
||||
KV_BLOCK_SIZE = 16
|
||||
# MFMA tile = 16 tokens (do not change without re-deriving MFMA shapes)
|
||||
TILE_SIZE = 16
|
||||
N_CENTROIDS = 16
|
||||
QUERY_GROUP_SIZE = 6 # GQA-6 default for MiniMax-M2.5
|
||||
WARP_SIZE = 64
|
||||
NUM_WARPS = 1
|
||||
BLOCK_THREADS = NUM_WARPS * WARP_SIZE
|
||||
KV_COMPUTE_BLOCK = 256 # 16 K-tiles × 16 tokens
|
||||
|
||||
# TQ SoA layout
|
||||
NUM_SOA_FIELDS = 3
|
||||
SOA_K_NORM = 0
|
||||
SOA_V_SCALE = 1
|
||||
SOA_V_ZERO = 2
|
||||
KEY_DATA_BYTES = HEAD_SIZE // 2
|
||||
VAL_DATA_BYTES = HEAD_SIZE // 2
|
||||
DATA_BYTES_PER_SLOT = KEY_DATA_BYTES + VAL_DATA_BYTES
|
||||
|
||||
# MFMA
|
||||
MFMA_M = MFMA_N = 16
|
||||
MFMA_K_BF16_QK = 32 # CDNA4 wide-K (mfma_f32_16x16x32_bf16)
|
||||
MFMA_K_BF16_PV = 16 # PV's K = token tile = 16
|
||||
QK_K_CHUNKS = HEAD_SIZE // MFMA_K_BF16_QK # 4
|
||||
PV_N_CHUNKS = HEAD_SIZE // MFMA_N # 8
|
||||
|
||||
# LDS regions
|
||||
# Q region uses the full 16-row footprint (4096 B) rather than a tight
|
||||
# QG=6 sizing, keeping the LDS layout identical to the canonical kernel.
|
||||
CENTROID_LDS_BYTES = N_CENTROIDS * 4 # 64
|
||||
Q_LDS_BYTES = 16 * HEAD_SIZE * 2 # 4096
|
||||
KV_TILE_LDS_BYTES = TILE_SIZE * HEAD_SIZE * 2 # 4096
|
||||
|
||||
LOG2E = 1.4426950408889634
|
||||
NEG_INF_VAL = float("-inf")
|
||||
|
||||
|
||||
def _vsplat_mul(vec, scalar):
|
||||
s = scalar.ir_value() if hasattr(scalar, "ir_value") else scalar
|
||||
return vec * vector.broadcast(T.f32x4, s)
|
||||
|
||||
|
||||
allocator = None
|
||||
|
||||
|
||||
def build_tq_decode_gqa6_module(
|
||||
num_seqs: int,
|
||||
num_kv_heads: int,
|
||||
num_partitions: int,
|
||||
max_blocks_per_seq: int = 512,
|
||||
softmax_scale: float | None = None,
|
||||
query_group_size: int = QUERY_GROUP_SIZE,
|
||||
kv_block_size: int = KV_BLOCK_SIZE,
|
||||
use_hw_v_transpose: bool = False,
|
||||
tile_groups_per_partition: int = 1,
|
||||
):
|
||||
"""Build a TQ decode GQA-6 kernel module (MiniMax-M2.5).
|
||||
|
||||
``query_group_size`` (= num_query_heads // num_kv_heads) is fixed at 6
|
||||
for this sibling. The MFMA's 16-row capacity is 6/16 = 37.5% used;
|
||||
lanes 6..15 compute garbage and are gated out of all global-memory
|
||||
writes via the same ``mfma_row < QG`` predicate the canonical kernel
|
||||
uses for QG=8. Q load iterates ``(QG + 3) // 4`` = 2 chunks (loads
|
||||
8 LDS rows of which rows 6..7 are zero-padded by the buffer-resource
|
||||
OOB protection — they're never consumed because the gating lanes
|
||||
don't write).
|
||||
|
||||
For GQA-8 or GQA-16 (Qwen models), use the canonical
|
||||
``build_tq_decode_module`` from ``tq_decode.py`` instead.
|
||||
|
||||
``kv_block_size`` is the number of tokens per vLLM cache block. Must
|
||||
be a multiple of TILE_SIZE (=16). Supported values: 16 and 32.
|
||||
|
||||
``use_hw_v_transpose`` (CDNA4 / gfx950 only) — see canonical kernel
|
||||
docstring for details. Same fence pattern applies here.
|
||||
"""
|
||||
assert query_group_size == 6, (
|
||||
f"build_tq_decode_gqa6_module is GQA-6 only; got "
|
||||
f"query_group_size={query_group_size}. Use build_tq_decode_module "
|
||||
f"from tq_decode.py for GQA-8 or GQA-16."
|
||||
)
|
||||
assert kv_block_size in (16, 32), (
|
||||
f"kv_block_size must be 16 or 32; got {kv_block_size}"
|
||||
)
|
||||
assert kv_block_size % TILE_SIZE == 0
|
||||
assert int(tile_groups_per_partition) >= 1, (
|
||||
f"tile_groups_per_partition must be >= 1; got {tile_groups_per_partition}"
|
||||
)
|
||||
USE_HW_TR = bool(use_hw_v_transpose)
|
||||
TGPP = int(tile_groups_per_partition)
|
||||
PARTITION_EXTENT_TOKENS = TGPP * KV_COMPUTE_BLOCK
|
||||
|
||||
QG = int(query_group_size)
|
||||
# Round-up: 2 iters for QG=6 → loads 8 LDS rows (rows 6..7 are
|
||||
# zero-padded by buffer_load OOB protection; never consumed because
|
||||
# mfma_row >= QG lanes are gated out of all global stores below).
|
||||
QG_LOAD_ITERS = (QG + 3) // 4
|
||||
OOB_OFFSET = 0x7FFFFFF0 # noqa: F841 ~2GB byte offset; > any plausible buffer
|
||||
|
||||
_BS = int(kv_block_size)
|
||||
_TILES_PER_BLOCK = _BS // TILE_SIZE # 1 for BS=16, 2 for BS=32
|
||||
|
||||
global allocator
|
||||
arch = get_hip_arch()
|
||||
|
||||
if softmax_scale is None:
|
||||
softmax_scale = 1.0 / (HEAD_SIZE**0.5)
|
||||
_qk_scale = float(softmax_scale)
|
||||
|
||||
# --- Strides ---
|
||||
_Hq = num_kv_heads * QG
|
||||
_stride_q_seq = _Hq * HEAD_SIZE
|
||||
_stride_q_head = HEAD_SIZE
|
||||
_stride_bt_seq = max_blocks_per_seq
|
||||
|
||||
_data_region_bytes = _BS * num_kv_heads * DATA_BYTES_PER_SLOT
|
||||
_meta_region_bytes = num_kv_heads * NUM_SOA_FIELDS * _BS * 2
|
||||
_stride_cache_block = _data_region_bytes + _meta_region_bytes
|
||||
_meta_region_offset = _data_region_bytes
|
||||
|
||||
_stride_out_part = QG * HEAD_SIZE
|
||||
_stride_out_head = num_partitions * QG * HEAD_SIZE
|
||||
_stride_out_seq = num_kv_heads * num_partitions * QG * HEAD_SIZE
|
||||
_stride_es_seq = num_kv_heads * num_partitions * QG
|
||||
_stride_ml_seq = _stride_es_seq
|
||||
|
||||
# --- LDS layout ---
|
||||
allocator = SmemAllocator(None, arch=arch, global_sym_name="tq_gqa6_smem")
|
||||
centroid_off = 0
|
||||
allocator.ptr = CENTROID_LDS_BYTES
|
||||
q_off = allocator.ptr
|
||||
allocator.ptr += Q_LDS_BYTES
|
||||
kv_off = allocator.ptr
|
||||
allocator.ptr += KV_TILE_LDS_BYTES
|
||||
|
||||
@flyc.kernel
|
||||
def tq_decode_gqa6_kernel(
|
||||
out_ptr: fx.Tensor,
|
||||
exp_sums_ptr: fx.Tensor,
|
||||
max_logits_ptr: fx.Tensor,
|
||||
query_ptr: fx.Tensor,
|
||||
kv_cache_ptr: fx.Tensor,
|
||||
centroids_ptr: fx.Tensor,
|
||||
block_tables_ptr: fx.Tensor,
|
||||
seq_lens_ptr: fx.Tensor,
|
||||
):
|
||||
# ---- IDs ---------------------------------------------------------
|
||||
tid = gpu.thread_idx.x
|
||||
seq = gpu.block_idx.x
|
||||
kv_h = gpu.block_idx.y
|
||||
part = gpu.block_idx.z
|
||||
lane = tid # 0..63
|
||||
mfma_row = lane & fx.Int32(15) # = query_row (B's N) /
|
||||
# token (A's M for QK)
|
||||
mfma_col_grp = lane >> fx.Int32(4) # 0..3, K-group dim
|
||||
|
||||
# ---- Buffer resources -------------------------------------------
|
||||
q_rsrc = buffer_ops.create_buffer_resource(query_ptr, max_size=True)
|
||||
kv_rsrc = buffer_ops.create_buffer_resource(kv_cache_ptr, max_size=True)
|
||||
bt_rsrc = buffer_ops.create_buffer_resource(block_tables_ptr, max_size=True)
|
||||
sl_rsrc = buffer_ops.create_buffer_resource(seq_lens_ptr, max_size=True)
|
||||
cent_rsrc = buffer_ops.create_buffer_resource(centroids_ptr, max_size=True)
|
||||
out_rsrc = buffer_ops.create_buffer_resource(out_ptr, max_size=True)
|
||||
es_rsrc = buffer_ops.create_buffer_resource(exp_sums_ptr, max_size=True)
|
||||
ml_rsrc = buffer_ops.create_buffer_resource(max_logits_ptr, max_size=True)
|
||||
|
||||
# ---- LDS pointers -----------------------------------------------
|
||||
base = allocator.get_base()
|
||||
cent_lds = SmemPtr(base, centroid_off, T.f32, shape=(N_CENTROIDS,))
|
||||
q_lds_i32 = SmemPtr(base, q_off, T.i32, shape=(Q_LDS_BYTES // 4,)).get()
|
||||
q_lds_i64 = SmemPtr(base, q_off, T.i64, shape=(Q_LDS_BYTES // 8,)).get()
|
||||
kv_lds_i32 = SmemPtr(base, kv_off, T.i32, shape=(KV_TILE_LDS_BYTES // 4,)).get() # noqa: F841
|
||||
kv_lds_i64 = SmemPtr(base, kv_off, T.i64, shape=(KV_TILE_LDS_BYTES // 8,)).get()
|
||||
kv_lds_i16 = SmemPtr(base, kv_off, T.i16, shape=(KV_TILE_LDS_BYTES // 2,)).get()
|
||||
|
||||
# ---- Constants ---------------------------------------------------
|
||||
c_sq = fx.Int32(_stride_q_seq)
|
||||
c_qh = fx.Int32(_stride_q_head)
|
||||
c_qg = fx.Int32(QG)
|
||||
c_bt = fx.Int32(_stride_bt_seq)
|
||||
c_block = fx.Int32(_stride_cache_block)
|
||||
c_meta_off = fx.Int32(_meta_region_offset)
|
||||
c_data_per_slot = fx.Int32(DATA_BYTES_PER_SLOT)
|
||||
c_kv_heads = fx.Int32(num_kv_heads)
|
||||
c_keydata = fx.Int32(KEY_DATA_BYTES)
|
||||
c_w = fx.Int32(WARP_SIZE)
|
||||
|
||||
NEG_INF = arith.constant(NEG_INF_VAL, type=T.f32)
|
||||
ZERO_F = fx.Float32(0.0)
|
||||
ONE_F = fx.Float32(1.0)
|
||||
LOG2E_C = arith.constant(LOG2E, type=T.f32)
|
||||
QK_SCALE = arith.constant(_qk_scale, type=T.f32)
|
||||
|
||||
# ===== STEP A: Load centroids → LDS (cooperative, race-safe) =====
|
||||
c_idx_safe = lane & fx.Int32(N_CENTROIDS - 1)
|
||||
c_val = buffer_ops.buffer_load(cent_rsrc, c_idx_safe, vec_width=1, dtype=T.f32)
|
||||
cent_lds.store(c_val, [arith.index_cast(T.index, c_idx_safe)])
|
||||
gpu.barrier()
|
||||
|
||||
# ===== STEP B: Load Q → row-major LDS [query_row, head_dim] ======
|
||||
# Q layout in HBM: [N, Hq, D] bf16, slice = QG rows × D=128 cols × 2B.
|
||||
# QG=6 → 1536 B real, 96 chunks; QG_LOAD_ITERS=2 issues 128 chunks
|
||||
# (8 rows × 16 col_b) so lanes whose row ∈ {6, 7} address heads that
|
||||
# do NOT exist in the Q tensor (Hq = num_kv_heads * 6).
|
||||
#
|
||||
# ── Per-lane Q-load OOB redirect ─────────────────────────────────
|
||||
# Hq = num_kv_heads * QG, so head index = kv_h * QG + row. For QG=6
|
||||
# the upper rows alias to (kv_h+1)'s real heads (still in-bounds
|
||||
# for most kv_h), but for the *last* (seq, kv_h) the read tips
|
||||
# past the Q tensor entirely. With ``max_size=True`` the buffer
|
||||
# descriptor advertises 4 GB so the HW does no OOB clamping; the
|
||||
# padding read then touches unmapped pages and the kernel dies
|
||||
# with "Memory access fault by GPU node" the moment the
|
||||
# surrounding allocator state leaves the next page unmapped
|
||||
# (allocator-pattern-dependent → nondeterministic fault).
|
||||
#
|
||||
# Fix: redirect OOB lanes (``row >= QG``) to load Q[0,0,col_elem]
|
||||
# — a byte offset that is always inside the actual Q tensor.
|
||||
# The redundant load is wasted but harmless; the resulting LDS
|
||||
# write to row ∈ {6, 7} is overwritten/ignored downstream because
|
||||
# every global store is gated by the output-time
|
||||
# ``mfma_row < QG`` predicate (the same gate canonical QG=8 uses
|
||||
# for its garbage rows 8..15).
|
||||
#
|
||||
# NB: A ``mask=`` parameter on ``buffer_load`` substitutes the
|
||||
# masked-out offset with 0x7FFFFFFF *bytes*; with ``max_size=True``
|
||||
# this is still inside the 4 GB descriptor and so the HW happily
|
||||
# dereferences ``q_base + 2 GB`` → another unmapped page → fault.
|
||||
# The address-redirect below is the only safe option short of
|
||||
# threading the true Q size through as a runtime kernel arg.
|
||||
#
|
||||
# The canonical Qwen kernel does not need this redirect because
|
||||
# for QG ∈ {8, 16} the (kv_h, row) iteration covers exactly the
|
||||
# real heads; QG=6 is the only case where
|
||||
# ``QG_LOAD_ITERS * 4 != QG``.
|
||||
# STEP B: load pre-rotated q_rot → Q_LDS. QG=6 over-reads
|
||||
# (QG_LOAD_ITERS * 4 != QG), so rows >= QG redirect to a safe
|
||||
# in-bounds offset to avoid faulting past the real Q rows.
|
||||
for c in range_constexpr(QG_LOAD_ITERS):
|
||||
row_chunk = lane + fx.Int32(c * WARP_SIZE) # 0..127
|
||||
row = row_chunk >> fx.Int32(4) # 0..7
|
||||
col_b = row_chunk & fx.Int32(15) # 0..15
|
||||
col_elem = col_b * fx.Int32(8) # 0..120 in bf16
|
||||
q_off_real = seq * c_sq + (kv_h * c_qg + row) * c_qh + col_elem
|
||||
row_in_qg = row < fx.Int32(QG)
|
||||
q_off_elem = row_in_qg.select(q_off_real, col_elem)
|
||||
q_v = buffer_ops.buffer_load(
|
||||
q_rsrc,
|
||||
q_off_elem // fx.Int32(2),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
q_lds_byte = row * fx.Int32(HEAD_SIZE * 2) + col_elem * fx.Int32(2)
|
||||
vector.store(
|
||||
q_v,
|
||||
q_lds_i32,
|
||||
[arith.index_cast(T.index, q_lds_byte // fx.Int32(4))],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# ===== STEP C: Pre-load Q operands for QK_K_CHUNKS = 4 K-chunks ===
|
||||
# Wide-K MFMA: K_per_chunk = 32, K_per_lane = 32/4 = 8 bf16 per chunk.
|
||||
# Lane t holds 8 bf16 at row=mfma_row, K-cols
|
||||
# chunk*32 + mfma_col_grp*8 + 0..7
|
||||
# Byte addr = mfma_row * HEAD_SIZE*2 + (chunk*32 + col_grp*8)*2
|
||||
# = mfma_row * 256 + chunk*64 + col_grp*16
|
||||
# i64 idx = mfma_row * 32 + chunk*8 + col_grp*2 (each i64 = 4 bf16)
|
||||
# Load 16 bytes (= 8 bf16 = 2 i64) per chunk via vec(2, i64).
|
||||
q_chunks = []
|
||||
for chk in range_constexpr(QK_K_CHUNKS):
|
||||
q_idx_i64 = (
|
||||
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
+ fx.Int32(chk * 8)
|
||||
+ mfma_col_grp * fx.Int32(2)
|
||||
)
|
||||
qv = vector.load_op(
|
||||
T.vec(2, T.i64),
|
||||
q_lds_i64,
|
||||
[arith.index_cast(T.index, q_idx_i64)],
|
||||
)
|
||||
q_chunks.append(vector.bitcast(T.vec(8, T.bf16), qv))
|
||||
|
||||
# ===== STEP D: Online softmax + PV state =========================
|
||||
running_max = NEG_INF
|
||||
running_sum = ZERO_F
|
||||
zero_v4 = arith.constant_vector(0.0, T.f32x4)
|
||||
acc_pv = [zero_v4 for _ in range(PV_N_CHUNKS)]
|
||||
|
||||
# ===== STEP E: Sequence-len + partition base ====================
|
||||
seq_len = buffer_ops.buffer_load(sl_rsrc, seq, vec_width=1, dtype=T.i32)
|
||||
# ── Bounded num_partitions + internal looping ───────────────────
|
||||
# Mirrors the canonical tq_decode kernel: each CTA owns
|
||||
# ``PARTITION_EXTENT_TOKENS = TGPP * KV_COMPUTE_BLOCK`` contiguous
|
||||
# tokens of the seq, processed as TGPP groups of 16 K-tiles each.
|
||||
# FA-2 online-softmax state accumulates across all TGPP * 16 tiles.
|
||||
# With TGPP=1 this is bit-identical to the legacy 16-tile body.
|
||||
partition_base = part * fx.Int32(PARTITION_EXTENT_TOKENS)
|
||||
|
||||
# Per-K-tile dequant lane assignment:
|
||||
# 16 tokens × 64 packed bytes per token = 1024 bytes
|
||||
# 64 lanes × 16 bytes/lane = 1024 ✓
|
||||
# Lane t: token = t / 4, sub-chunk = t % 4 (0..3) → 16 bytes contiguous
|
||||
tok_in_tile = lane >> fx.Int32(2)
|
||||
chunk_in_tok = lane & fx.Int32(3)
|
||||
# ``slot`` (= absolute slot index within the cache block, range
|
||||
# 0.._BS-1) is recomputed per tile inside the K-tile loop below
|
||||
# so it reflects ``tile_in_block * TILE_SIZE + tok_in_tile``.
|
||||
|
||||
c_meta_u16_per_kvh = fx.Int32(NUM_SOA_FIELDS * _BS)
|
||||
c_knorm_off_u16 = fx.Int32(SOA_K_NORM * _BS)
|
||||
c_vscale_off_u16 = fx.Int32(SOA_V_SCALE * _BS)
|
||||
c_vzero_off_u16 = fx.Int32(SOA_V_ZERO * _BS)
|
||||
|
||||
# ===== STEP F: K-tile loop =======================================
|
||||
# With kv_block_size > TILE_SIZE there are _TILES_PER_BLOCK tiles
|
||||
# per cache block. Tiles within the same block share a block-table
|
||||
# entry but address different slot ranges within the block.
|
||||
#
|
||||
# ── Per-tile block-table OOB redirect ────────────────────────────
|
||||
# The K-tile loop is unrolled 16 times so every CTA issues 16
|
||||
# block-table reads regardless of (partition, seq_len). When a
|
||||
# partition extends past ``seq_len`` (e.g. num_partitions=2 for a
|
||||
# 256-token sequence — partition 1 covers tokens 256..511, all
|
||||
# masked out) the per-tile bt offset can land beyond the bt
|
||||
# allocation. With ``max_size=True`` the descriptor is 4 GB so
|
||||
# the HW returns whatever pre-existing HBM bytes live at that
|
||||
# offset; the resulting garbage ``phys_block`` is multiplied by
|
||||
# ``stride_cache_block`` and the subsequent kv_cache read jumps
|
||||
# into unmapped pages, killing the whole launch with a
|
||||
# "Memory access fault by GPU node" exactly as the OOB Q load
|
||||
# used to (same allocator-pattern-dependent fault; B=4 makes
|
||||
# it deterministic by adding more colliding CTAs).
|
||||
#
|
||||
# Fix: when the tile starts at or past ``seq_len`` (so its
|
||||
# qk_acc is going to be killed by the per-token
|
||||
# ``kv_tok < seq_len`` mask anyway), redirect the bt read to
|
||||
# ``bt[seq, 0]`` — always in bounds. The redundant phys_block
|
||||
# decode + kv_cache read is wasted work, but correctness is
|
||||
# preserved without changing the kernel's iteration count.
|
||||
#
|
||||
# The canonical Qwen kernel reads the same tiles and is also
|
||||
# exposed to this issue in principle; in practice it surfaces on
|
||||
# GQA-6 first because the surrounding allocator state differs
|
||||
# (extra Q-tail allocation) and shifts the page maps so the bt
|
||||
# overflow lands on unmapped pages. Applying the same redirect to
|
||||
# the canonical kernel is a possible follow-up.
|
||||
# ===== Runtime-adaptive scf.ForOp w/ iter_args =========
|
||||
# Runtime-adaptive outer loop: trip count derives from
|
||||
# actual seq_len, NOT TGPP_max. At cudagraph-capture warmup
|
||||
# (seq_len=1) only 1 iteration runs; at 32K production
|
||||
# decode all TGPP iterations run. Kernel binary stays small
|
||||
# (single body, looped at runtime) — capture time matches
|
||||
# the legacy 16-tile-only kernel rather than scaling 4x.
|
||||
#
|
||||
# FA-2 state (running_max, running_sum, acc_pv[8]) threads
|
||||
# through scf iter_args; the body reads them at top, runs
|
||||
# the unchanged 16-tile inner body, and yields the new
|
||||
# state at bottom. After the loop, results are pulled out
|
||||
# of for_op.results back into the local Python names so the
|
||||
# downstream STEP G (output) is unmodified.
|
||||
c_kcb = fx.Int32(KV_COMPUTE_BLOCK)
|
||||
c_tgpp = fx.Int32(TGPP)
|
||||
c_zero_i32 = fx.Int32(0)
|
||||
c_one_i32 = fx.Int32(1)
|
||||
# remaining = seq_len - partition_base (signed, may be <=0)
|
||||
remaining = seq_len - partition_base
|
||||
in_range = remaining > c_zero_i32
|
||||
# trip_raw = ceil(remaining / KV_COMPUTE_BLOCK)
|
||||
# (when in_range is false, the divisor branch produces
|
||||
# garbage that the select below overrides with zero, so we
|
||||
# don't pre-clamp)
|
||||
trip_raw = (remaining + c_kcb - c_one_i32) // c_kcb
|
||||
trip_clamped = (trip_raw > c_tgpp).select(c_tgpp, trip_raw)
|
||||
trip_or_zero = in_range.select(trip_clamped, c_zero_i32)
|
||||
c_zero_idx = arith.constant(0, index=True)
|
||||
c_one_idx = arith.constant(1, index=True)
|
||||
trip_idx = arith.index_cast(
|
||||
T.index,
|
||||
trip_or_zero.ir_value()
|
||||
if hasattr(trip_or_zero, "ir_value")
|
||||
else trip_or_zero,
|
||||
)
|
||||
|
||||
# Initial iter_args list: FA-2 accumulator state.
|
||||
# Order: running_max, running_sum, acc_pv[0..PV_N_CHUNKS-1].
|
||||
# NOTE: scf.ForOp requires ir.Value (with .type). The DSL
|
||||
# constants ZERO_F / ONE_F are fx.Float32 (Numeric) wrappers,
|
||||
# not raw ir.Value, so we unwrap via .ir_value() before
|
||||
# passing.
|
||||
def _ival(v):
|
||||
return v.ir_value() if hasattr(v, "ir_value") else v
|
||||
|
||||
_init_iter = [
|
||||
_ival(running_max),
|
||||
_ival(running_sum),
|
||||
*[_ival(p) for p in acc_pv],
|
||||
]
|
||||
_for_op = _scf.ForOp(
|
||||
c_zero_idx,
|
||||
trip_idx,
|
||||
c_one_idx,
|
||||
_init_iter,
|
||||
)
|
||||
_for_ip = ir.InsertionPoint(_for_op.body)
|
||||
_for_ip.__enter__()
|
||||
try:
|
||||
tg_idx = _for_op.induction_variable
|
||||
tg_i32 = fx.Int32(arith.index_cast(T.i32, tg_idx))
|
||||
partition_start = partition_base + tg_i32 * c_kcb
|
||||
bt_seq_base = seq * c_bt + (partition_start // fx.Int32(_BS))
|
||||
running_max = _for_op.inner_iter_args[0]
|
||||
running_sum = _for_op.inner_iter_args[1]
|
||||
acc_pv = list(_for_op.inner_iter_args[2:])
|
||||
for n_tile in range_constexpr(16):
|
||||
block_in_part = n_tile // _TILES_PER_BLOCK
|
||||
tile_in_block = n_tile % _TILES_PER_BLOCK
|
||||
tile_start_tok = partition_start + fx.Int32(n_tile * TILE_SIZE)
|
||||
tile_in_seq = tile_start_tok < seq_len
|
||||
bt_off = bt_seq_base + fx.Int32(block_in_part)
|
||||
bt_off_safe = tile_in_seq.select(bt_off, seq * c_bt)
|
||||
phys_block = buffer_ops.buffer_load(
|
||||
bt_rsrc,
|
||||
bt_off_safe,
|
||||
vec_width=1,
|
||||
dtype=T.i32,
|
||||
)
|
||||
block_base = phys_block * c_block
|
||||
data_region = block_base
|
||||
meta_region = block_base + c_meta_off
|
||||
|
||||
# ``slot`` is the absolute slot index within the cache block
|
||||
# (range 0.._BS-1). For BS=16 it equals tok_in_tile; for BS=32
|
||||
# it equals tile_in_block * 16 + tok_in_tile. Used both for
|
||||
# the K/V data region and SoA metadata region addressing.
|
||||
slot = fx.Int32(tile_in_block * TILE_SIZE) + tok_in_tile
|
||||
data_bases_byte = (
|
||||
data_region
|
||||
+ slot * (c_kv_heads * c_data_per_slot)
|
||||
+ kv_h * c_data_per_slot
|
||||
)
|
||||
k_byte = data_bases_byte + chunk_in_tok * fx.Int32(16)
|
||||
k_packed = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
k_byte // fx.Int32(4),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
|
||||
# ---- HOISTED: issue V data + meta HBM loads early ---------
|
||||
# These are async; their s_waitcnt is pushed by the compiler
|
||||
# past the K dequant + QK MFMA + softmax block, hiding most
|
||||
# of the V HBM latency behind compute.
|
||||
v_byte = data_bases_byte + c_keydata + chunk_in_tok * fx.Int32(16)
|
||||
v_packed = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
v_byte // fx.Int32(4),
|
||||
vec_width=4,
|
||||
dtype=T.i32,
|
||||
)
|
||||
vscale_u16 = (
|
||||
meta_region // fx.Int32(2)
|
||||
+ kv_h * c_meta_u16_per_kvh
|
||||
+ c_vscale_off_u16
|
||||
+ slot
|
||||
)
|
||||
vzero_u16 = (
|
||||
meta_region // fx.Int32(2)
|
||||
+ kv_h * c_meta_u16_per_kvh
|
||||
+ c_vzero_off_u16
|
||||
+ slot
|
||||
)
|
||||
vscale_raw = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
vscale_u16,
|
||||
vec_width=1,
|
||||
dtype=T.i16,
|
||||
)
|
||||
vzero_raw = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
vzero_u16,
|
||||
vec_width=1,
|
||||
dtype=T.i16,
|
||||
)
|
||||
|
||||
knorm_u16 = (
|
||||
meta_region // fx.Int32(2)
|
||||
+ kv_h * c_meta_u16_per_kvh
|
||||
+ c_knorm_off_u16
|
||||
+ slot
|
||||
)
|
||||
knorm_raw = buffer_ops.buffer_load(
|
||||
kv_rsrc,
|
||||
knorm_u16,
|
||||
vec_width=1,
|
||||
dtype=T.i16,
|
||||
)
|
||||
knorm_f16 = arith.bitcast(T.f16, knorm_raw)
|
||||
knorm_f32 = arith.extf(T.f32, knorm_f16)
|
||||
|
||||
# K dequant → LDS [token, head_dim] (natural)
|
||||
# Lane writes 32 bf16 (= 4×8) for token=tok_in_tile,
|
||||
# head_dims chunk_in_tok*32..+31 (4 sub-chunks of 8).
|
||||
tok_kreg = tok_in_tile * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
chunk_kreg = tok_kreg + chunk_in_tok * fx.Int32(8)
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(k_packed, static_position=[w])
|
||||
bf16_elems = []
|
||||
for n in range_constexpr(8):
|
||||
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
|
||||
nibble_idx = arith.index_cast(T.index, nibble)
|
||||
cent_f32 = cent_lds.load([nibble_idx])
|
||||
elem_bf16 = arith.trunc_f(T.bf16, cent_f32 * knorm_f32)
|
||||
bf16_elems.append(elem_bf16)
|
||||
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
|
||||
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
|
||||
vector.store(
|
||||
v_i64,
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, chunk_kreg + fx.Int32(w * 2))],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# QK MFMA (CDNA4 wide-K): A=K[token, head_dim], B=Q (= Q^T).
|
||||
# K read: lane t = K[token=mfma_row, head_dim=chunk*32+col_grp*8..+7]
|
||||
# Same i64×2 indexing as Q.
|
||||
qk_acc = zero_v4
|
||||
for chk in range_constexpr(QK_K_CHUNKS):
|
||||
k_idx_i64 = (
|
||||
mfma_row * fx.Int32(HEAD_SIZE * 2 // 8)
|
||||
+ fx.Int32(chk * 8)
|
||||
+ mfma_col_grp * fx.Int32(2)
|
||||
)
|
||||
kv_load = vector.load_op(
|
||||
T.vec(2, T.i64),
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, k_idx_i64)],
|
||||
)
|
||||
k_op = vector.bitcast(T.vec(8, T.bf16), kv_load)
|
||||
qk_acc = rocdl.mfma_f32_16x16x32_bf16(
|
||||
T.f32x4, [k_op, q_chunks[chk], qk_acc, 0, 0, 0]
|
||||
)
|
||||
|
||||
# qk_acc layout: lane t holds C[token=(t/16)*4..+3, query=t%16]
|
||||
# 4 fp32/lane = 4 different tokens at SAME query_row.
|
||||
|
||||
# Scale + mask out-of-context tokens.
|
||||
qk_acc = _vsplat_mul(qk_acc, QK_SCALE)
|
||||
for elem in range_constexpr(4):
|
||||
kv_tok = (
|
||||
partition_start
|
||||
+ fx.Int32(n_tile * TILE_SIZE)
|
||||
+ mfma_col_grp * fx.Int32(4)
|
||||
+ fx.Int32(elem)
|
||||
)
|
||||
in_b = kv_tok < seq_len
|
||||
v = vector.extract(qk_acc, static_position=[elem])
|
||||
qk_acc = vector.insert(
|
||||
in_b.select(v, NEG_INF),
|
||||
qk_acc,
|
||||
static_position=[elem],
|
||||
dynamic_position=[],
|
||||
)
|
||||
|
||||
# FA2 online softmax: per-query-row reduce.
|
||||
# Per-row max: max over 4 fp32 in lane, then xor-shuffle 16, 32
|
||||
# (across the 4 col_grps that share same mfma_row).
|
||||
local_max = vector.reduction(T.f32, "maxnumf", qk_acc)
|
||||
r1 = local_max.shuffle_xor(fx.Int32(16), c_w)
|
||||
local_max = local_max.maximumf(r1)
|
||||
r2 = local_max.shuffle_xor(fx.Int32(32), c_w)
|
||||
tile_max = local_max.maximumf(r2)
|
||||
|
||||
new_max = running_max.maximumf(tile_max)
|
||||
max_diff = running_max - new_max
|
||||
safe_diff = (running_max > NEG_INF).select(max_diff, ZERO_F)
|
||||
scale = (safe_diff * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
|
||||
running_sum = running_sum * scale
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
acc_pv[h] = _vsplat_mul(acc_pv[h], scale)
|
||||
running_max = new_max
|
||||
|
||||
# Compute probs: p = exp((qk - new_max) * LOG2E)
|
||||
tile_sum = ZERO_F
|
||||
for elem in range_constexpr(4):
|
||||
s = vector.extract(qk_acc, static_position=[elem])
|
||||
d = s - new_max
|
||||
d = (new_max > NEG_INF).select(d, NEG_INF)
|
||||
p = (d * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast)
|
||||
tile_sum = tile_sum + p
|
||||
qk_acc = vector.insert(
|
||||
p, qk_acc, static_position=[elem], dynamic_position=[]
|
||||
)
|
||||
|
||||
ts1 = tile_sum.shuffle_xor(fx.Int32(16), c_w)
|
||||
tile_sum = tile_sum + ts1
|
||||
ts2 = tile_sum.shuffle_xor(fx.Int32(32), c_w)
|
||||
tile_sum = tile_sum + ts2
|
||||
running_sum = running_sum + tile_sum
|
||||
|
||||
# ---- V dequant → LDS (V/scale/zero raw loads were hoisted) -
|
||||
# v_packed, vscale_raw, vzero_raw are already in flight from
|
||||
# the top of this iteration; only the f32 conversion stays
|
||||
# here so the actual consume site is still close to the use.
|
||||
vscale_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vscale_raw))
|
||||
vzero_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vzero_raw))
|
||||
|
||||
if USE_HW_TR:
|
||||
# V dequant → LDS [token][head_dim] (ROW-MAJOR, no transpose).
|
||||
# Each lane writes 32 contiguous bf16 (one token, head_dims
|
||||
# chunk_in_tok*32..+31) as 4× ds_write_b128 = 4× vec(2,i64).
|
||||
# Replaces 32× ds_write_b16 of the legacy transposed path.
|
||||
v_lds_elem_base = tok_in_tile * fx.Int32(
|
||||
HEAD_SIZE
|
||||
) + chunk_in_tok * fx.Int32(32)
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(v_packed, static_position=[w])
|
||||
bf16_elems = []
|
||||
for n in range_constexpr(8):
|
||||
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
|
||||
nibble_f32 = arith.sitofp(T.f32, nibble)
|
||||
elem_f32 = nibble_f32 * vscale_f32 + vzero_f32
|
||||
elem_bf16 = arith.trunc_f(T.bf16, elem_f32)
|
||||
bf16_elems.append(elem_bf16)
|
||||
v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems)
|
||||
v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16)
|
||||
v_lds_i64_idx = (v_lds_elem_base + fx.Int32(w * 8)) // fx.Int32(
|
||||
4
|
||||
)
|
||||
vector.store(
|
||||
v_i64,
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, v_lds_i64_idx)],
|
||||
)
|
||||
# ── HW V transpose: cross-lane LDS sync ─────────────────────
|
||||
# ds_read_tr16_b64 below introduces a cross-lane LDS read:
|
||||
# consumer lane t reads bytes written by source lanes
|
||||
# 0,4,8,12 (etc.). The compiler's automatic waitcnt insertion
|
||||
# is per-lane and has no awareness of the HW transpose's
|
||||
# cross-lane forwarding, so it can sink the next iteration's
|
||||
# ds_read ahead of these ds_write_b128 ops in the schedule.
|
||||
#
|
||||
# Two-part fence:
|
||||
# 1. sched_barrier(0): MachineScheduler reorder barrier
|
||||
# (mask=0 → no instruction class may cross). Pure hint,
|
||||
# generates no runtime instruction. Use the simple
|
||||
# sched_barrier (NOT sched_group_barrier, which is an
|
||||
# IGLP marker that requires partner barriers and
|
||||
# corrupts the schedule when used in isolation).
|
||||
# 2. s_waitcnt lgkmcnt=0: runtime drain of the LDS write
|
||||
# queue. vmcnt/expcnt left at no-wait so we don't
|
||||
# stall on speculatively-issued HBM loads (which can
|
||||
# include OOB-but-masked buffer_load reads from the
|
||||
# next K-tile's hoisted V prefetch).
|
||||
#
|
||||
# The SW path below does NOT need this because each lane
|
||||
# only reads cells it itself wrote (no cross-lane dep), and
|
||||
# the per-lane same-address waitcnt the compiler emits is
|
||||
# correct.
|
||||
#
|
||||
# Without this fence the reordered ds_read produces
|
||||
# wrong results on high-partition-count configs (large
|
||||
# padded num_partitions); low-partition configs are
|
||||
# unaffected.
|
||||
rocdl.sched_barrier(0)
|
||||
# encode_waitcnt(vmcnt=63, expcnt=7, lgkmcnt=0)
|
||||
# = 0xF | (7<<4) | (0<<8) | (3<<14) = 0xC07F
|
||||
rocdl.s_waitcnt(0xC07F)
|
||||
else:
|
||||
# Legacy transposed V_LDS path: 32 ds_write_b16 per lane.
|
||||
for w in range_constexpr(4):
|
||||
word_i32 = vector.extract(v_packed, static_position=[w])
|
||||
for n in range_constexpr(8):
|
||||
nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF)
|
||||
nibble_f32 = arith.sitofp(T.f32, nibble)
|
||||
elem_f32 = nibble_f32 * vscale_f32 + vzero_f32
|
||||
elem_bf16 = arith.trunc_f(T.bf16, elem_f32)
|
||||
elem_i16 = arith.bitcast(T.i16, elem_bf16)
|
||||
head_dim = chunk_in_tok * fx.Int32(32) + fx.Int32(w * 8 + n)
|
||||
v_idx_i16 = head_dim * fx.Int32(TILE_SIZE) + tok_in_tile
|
||||
v_vec = vector.from_elements(T.vec(1, T.i16), [elem_i16])
|
||||
vector.store(
|
||||
v_vec,
|
||||
kv_lds_i16,
|
||||
[arith.index_cast(T.index, v_idx_i16)],
|
||||
)
|
||||
gpu.barrier()
|
||||
|
||||
# ---- PV MFMA: A=V[head_dim, token], B=P (=qk_acc bf16) -----
|
||||
# P operand B layout matches qk_acc: lane t holds 4 bf16 at
|
||||
# K=token=(t/16)*4..+3, N=query=t%16. Just trunc_f to bf16.
|
||||
p_bf16 = arith.trunc_f(T.vec(4, T.bf16), qk_acc)
|
||||
p_op = vector.bitcast(T.vec(4, T.i16), p_bf16)
|
||||
|
||||
if USE_HW_TR:
|
||||
# HW-transpose PV path: V_lds is row-major V[token][head_dim].
|
||||
# MFMA A operand layout: lane t holds 4 bf16 at
|
||||
# M=head_dim=mfma_row + h*16, K=token=(t/16)*4..+3
|
||||
# ds_read_tr16_b64 (4-element 16-bit transpose, per-16-lane block):
|
||||
# result[lane=t, elem=e] = Input[source_lane=e*4 + (t%16)//4,
|
||||
# col=t%4]
|
||||
# Per-lane address: token_idx = lane // 4 (covers 0..15 across
|
||||
# all four 16-lane MFMA blocks), hd_sub = (lane % 4)*4 selects
|
||||
# the 4-element column window inside the h*16 chunk.
|
||||
# Total LDS byte offset = kv_off + token_idx*HEAD_SIZE*2
|
||||
# + (h*16 + hd_sub)*2
|
||||
token_idx = lane >> fx.Int32(2)
|
||||
hd_sub = (lane & fx.Int32(3)) * fx.Int32(4)
|
||||
v_lane_byte = (
|
||||
fx.Int32(kv_off)
|
||||
+ token_idx * fx.Int32(HEAD_SIZE * 2)
|
||||
+ hd_sub * fx.Int32(2)
|
||||
)
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
v_byte_off = v_lane_byte + fx.Int32(h * 32)
|
||||
v_byte_i64 = fx.Int64(v_byte_off)
|
||||
v_ptr = buffer_ops.create_llvm_ptr(
|
||||
v_byte_i64,
|
||||
address_space=3,
|
||||
)
|
||||
v_op_raw = rocdl.ds_read_tr16_b64(
|
||||
T.vec(4, T.i16),
|
||||
v_ptr,
|
||||
).result
|
||||
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
|
||||
T.f32x4, [v_op_raw, p_op, acc_pv[h], 0, 0, 0]
|
||||
)
|
||||
else:
|
||||
# Legacy transposed-LDS PV: 1 ds_read_b64 per chunk.
|
||||
# Byte addr = (mfma_row + h*16) * TILE_SIZE * 2 + (col_grp*4) * 2
|
||||
# = (mfma_row + h*16) * 32 + col_grp*8
|
||||
# i64 idx = (mfma_row + h*16) * 4 + col_grp
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
v_idx_i64 = (mfma_row + fx.Int32(h * 16)) * fx.Int32(
|
||||
4
|
||||
) + mfma_col_grp
|
||||
kv_load = vector.load_op(
|
||||
T.vec(1, T.i64),
|
||||
kv_lds_i64,
|
||||
[arith.index_cast(T.index, v_idx_i64)],
|
||||
)
|
||||
v_op = vector.bitcast(T.vec(4, T.i16), kv_load)
|
||||
acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k(
|
||||
T.f32x4, [v_op, p_op, acc_pv[h], 0, 0, 0]
|
||||
)
|
||||
|
||||
_scf.YieldOp(
|
||||
[
|
||||
_ival(running_max),
|
||||
_ival(running_sum),
|
||||
*[_ival(p) for p in acc_pv],
|
||||
]
|
||||
)
|
||||
finally:
|
||||
_for_ip.__exit__(None, None, None)
|
||||
# Pull final accumulator values out of the for_op results.
|
||||
running_max = _for_op.results[0]
|
||||
running_sum = _for_op.results[1]
|
||||
acc_pv = list(_for_op.results[2:])
|
||||
|
||||
# ===== STEP G: Output ===========================================
|
||||
# acc_pv[h] layout: lane t holds 4 fp32 at M=head_dim=(t/16)*4..+3 + h*16,
|
||||
# N=query_row=t%16.
|
||||
# 4 fp32/lane = 4 contiguous head_dim values for ONE query_row.
|
||||
safe_sum = (running_sum > ZERO_F).select(running_sum, ONE_F)
|
||||
rcp = ONE_F / safe_sum
|
||||
|
||||
c_os = fx.Int32(_stride_out_seq)
|
||||
c_oh = fx.Int32(_stride_out_head)
|
||||
c_op_ = fx.Int32(_stride_out_part)
|
||||
out_base = seq * c_os + kv_h * c_oh + part * c_op_
|
||||
|
||||
# Gate all global stores by mfma_row < QG. For QG=16 the predicate is
|
||||
# tautological (mfma_row ∈ [0,15]); for QG=8 lanes 8..15 (whose MFMA
|
||||
# outputs are computational waste) skip the stores entirely.
|
||||
valid_row_pred = arith.cmpi(
|
||||
arith.CmpIPredicate.ult,
|
||||
mfma_row.ir_value() if hasattr(mfma_row, "ir_value") else mfma_row,
|
||||
arith.constant(QG, type=T.i32),
|
||||
)
|
||||
_if = _scf.IfOp(valid_row_pred)
|
||||
with ir.InsertionPoint(_if.then_block):
|
||||
for h in range_constexpr(PV_N_CHUNKS):
|
||||
pv_norm = _vsplat_mul(acc_pv[h], rcp)
|
||||
pv_bf16 = arith.trunc_f(T.vec(4, T.bf16), pv_norm)
|
||||
pv_i32x2 = vector.bitcast(T.vec(2, T.i32), pv_bf16)
|
||||
head_dim_start = fx.Int32(h * 16) + mfma_col_grp * fx.Int32(4)
|
||||
out_off_elem = (
|
||||
out_base + mfma_row * fx.Int32(HEAD_SIZE) + head_dim_start
|
||||
)
|
||||
buffer_ops.buffer_store(
|
||||
pv_i32x2,
|
||||
out_rsrc,
|
||||
out_off_elem * fx.Int32(2),
|
||||
offset_is_bytes=True,
|
||||
)
|
||||
|
||||
c_npq = fx.Int32(num_partitions * QG)
|
||||
ml_off = (
|
||||
seq * fx.Int32(_stride_ml_seq) + kv_h * c_npq + part * c_qg + mfma_row
|
||||
)
|
||||
es_off = (
|
||||
seq * fx.Int32(_stride_es_seq) + kv_h * c_npq + part * c_qg + mfma_row
|
||||
)
|
||||
buffer_ops.buffer_store(running_max, ml_rsrc, ml_off)
|
||||
buffer_ops.buffer_store(running_sum, es_rsrc, es_off)
|
||||
_scf.YieldOp([])
|
||||
|
||||
return tq_decode_gqa6_kernel
|
||||
@@ -0,0 +1,874 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""FlyDSL TurboQuant decode launcher (vLLM-side).
|
||||
|
||||
Drop-in replacement for ``triton_turboquant_decode_attention_soa`` for the
|
||||
TQ decode profile HEAD_SIZE=128, MSE_BITS=4 K, VQB=4 V, N_CENTROIDS=16,
|
||||
BLOCK_SIZE in {16, 32}.
|
||||
|
||||
Per-GQA kernel dispatch:
|
||||
* GQA group ∈ {8, 16} → canonical kernels.tq_decode (Qwen-class)
|
||||
* GQA group == 6 → kernels.tq_decode_gqa6 sibling (MiniMax-M2.5)
|
||||
|
||||
Auto-selected on gfx950 when FlyDSL is importable; falls back to the SoA Triton
|
||||
decode otherwise (wrong arch, missing build tree). The GQA-6 sibling is imported
|
||||
best-effort: missing it does not affect Qwen GQA-{8,16} paths.
|
||||
|
||||
Architecture:
|
||||
1. Q rotation: ``q_rot = (query.float() @ PiT).bfloat16()`` — the same
|
||||
launcher-side rocBLAS GEMM the SoA Triton decode uses.
|
||||
2. FlyDSL kernel writes per-partition outputs into
|
||||
``[N, Hk, P, QG, D]`` bf16 + ``[N, Hk, P, QG]`` fp32 max/sum buffers.
|
||||
3. A small Triton reducer combines partitions in the kernel's native layout
|
||||
(no permute/cast), writing the final ``[N, Hq, D]`` output.
|
||||
|
||||
Kernel module is built once per ``(num_kv_heads, num_partitions,
|
||||
max_blocks_per_seq, scale)`` shape and cached.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# -- FlyDSL import bootstrap --------------------------------------------------
|
||||
# FlyDSL is imported directly and must be importable (pip-installed, or on
|
||||
# PYTHONPATH). Import failures are caught below and fall back to the SoA Triton
|
||||
# decode, so vLLM startup is unaffected on hosts without FlyDSL.
|
||||
|
||||
|
||||
_FLYDSL_AVAILABLE: bool | None = None
|
||||
_TQ_MOD: Any = None # kernels.tq_decode module (Qwen GQA-{8,16})
|
||||
_TQ_MOD_GQA6: Any = None # kernels.tq_decode_gqa6 module (MiniMax GQA-6, optional)
|
||||
_FLYC: Any = None # flydsl.compiler
|
||||
_FX: Any = None # flydsl.expr
|
||||
_TYPING_T: Any = None
|
||||
_CC: Any = None # CompilationContext
|
||||
_IR: Any = None
|
||||
|
||||
|
||||
def is_flydsl_available() -> bool:
|
||||
"""Return True iff on gfx950 and FlyDSL imports + kernel module load.
|
||||
|
||||
The GQA-6 sibling kernel (``kernels.tq_decode_gqa6``) is imported
|
||||
best-effort: if it's missing (older FlyDSL checkout that pre-dates
|
||||
MiniMax support) the canonical Qwen path stays fully functional and
|
||||
only GQA-6 dispatches will fail with a clear error at launch time.
|
||||
"""
|
||||
global _FLYDSL_AVAILABLE, _TQ_MOD, _TQ_MOD_GQA6
|
||||
global _FLYC, _FX, _TYPING_T, _CC, _IR
|
||||
if _FLYDSL_AVAILABLE is not None:
|
||||
return _FLYDSL_AVAILABLE
|
||||
try:
|
||||
# gfx950-only kernel (CDNA4 intrinsics); gate before importing FlyDSL.
|
||||
from vllm.platforms.rocm import on_gfx950
|
||||
|
||||
if not on_gfx950():
|
||||
_FLYDSL_AVAILABLE = False
|
||||
return _FLYDSL_AVAILABLE
|
||||
import flydsl.compiler as flyc # noqa: F401
|
||||
import flydsl.expr as fx # noqa: F401
|
||||
from flydsl._mlir import ir
|
||||
from flydsl.compiler.kernel_function import CompilationContext
|
||||
from flydsl.expr.typing import T # noqa: F401
|
||||
|
||||
# Kernel is vendored in-tree (ships with vLLM); FlyDSL provides only
|
||||
# the compiler/runtime framework (imported above).
|
||||
from vllm.v1.attention.ops.flydsl_kernels import tq_decode as tq_mod
|
||||
|
||||
_FLYC = flyc
|
||||
_FX = fx
|
||||
_TYPING_T = T
|
||||
_CC = CompilationContext
|
||||
_IR = ir
|
||||
_TQ_MOD = tq_mod
|
||||
_FLYDSL_AVAILABLE = True
|
||||
logger.info_once("FlyDSL TQ decode launcher: available")
|
||||
except Exception as ex: # noqa: BLE001
|
||||
_FLYDSL_AVAILABLE = False
|
||||
logger.warning_once(
|
||||
"FlyDSL TQ decode launcher: unavailable (%s). "
|
||||
"Falling back to SoA Triton decode.",
|
||||
ex,
|
||||
)
|
||||
return _FLYDSL_AVAILABLE
|
||||
# Best-effort GQA-6 sibling import (does NOT gate Qwen availability).
|
||||
try:
|
||||
from vllm.v1.attention.ops.flydsl_kernels import (
|
||||
tq_decode_gqa6 as tq_mod_gqa6,
|
||||
)
|
||||
|
||||
_TQ_MOD_GQA6 = tq_mod_gqa6
|
||||
logger.info_once("FlyDSL TQ decode GQA-6 sibling: available (MiniMax-class)")
|
||||
except Exception as ex: # noqa: BLE001
|
||||
_TQ_MOD_GQA6 = None
|
||||
logger.info_once(
|
||||
"FlyDSL TQ decode GQA-6 sibling: not available (%s); "
|
||||
"GQA-6 models will fall back to SoA Triton decode.",
|
||||
ex,
|
||||
)
|
||||
return _FLYDSL_AVAILABLE
|
||||
|
||||
|
||||
def is_flydsl_gqa6_available() -> bool:
|
||||
"""True iff the optional GQA-6 sibling kernel module loaded.
|
||||
|
||||
Used by the eligibility gate in turboquant_attn.py to decide whether
|
||||
a layer with num_kv_groups==6 can run on FlyDSL or must fall back
|
||||
to the SoA Triton decode.
|
||||
"""
|
||||
if _FLYDSL_AVAILABLE is None:
|
||||
is_flydsl_available()
|
||||
return _TQ_MOD_GQA6 is not None
|
||||
|
||||
|
||||
# -- Kernel module cache -------------------------------------------------------
|
||||
# Compiling a FlyDSL kernel is expensive; cache by the constexprs that
|
||||
# parameterize stride math: num_kv_heads, num_partitions, max_blocks_per_seq,
|
||||
# scale. ``num_seqs`` is irrelevant to the kernel body so we omit it.
|
||||
_KERN_CACHE: dict[tuple, Any] = {}
|
||||
|
||||
|
||||
# -- One-shot log guards ------------------------------------------------------
|
||||
# Replaces vllm `logger.{info,warning}_once` (which hashes the format string
|
||||
# every call to dedup) with a true zero-overhead bool guard.
|
||||
_LOG_INVOKED_ONCE: bool = False
|
||||
_LOG_SINKS_WARNED: bool = False
|
||||
_LOG_NORM_WARNED: bool = False
|
||||
|
||||
|
||||
def _detect_max_capture_B() -> int:
|
||||
"""Probe vLLM compilation config for the largest cudagraph capture size.
|
||||
|
||||
Returns max(cudagraph_capture_sizes) if available, else falls back to
|
||||
512 (vLLM's default cap). The segm-pool auto-grows if a larger batch is
|
||||
seen (see ``_SegmBufPool.get``).
|
||||
|
||||
Used to size the segm-pool bucket once, so that all distinct B's
|
||||
(whether captured in a graph, or hit eagerly in mixed batches) share
|
||||
a single allocation per shape — eliminating the per-B growth that
|
||||
previously dominated dense-capture VRAM cost.
|
||||
"""
|
||||
try:
|
||||
from vllm.config import get_current_vllm_config
|
||||
|
||||
cfg = get_current_vllm_config()
|
||||
sizes = cfg.compilation_config.cudagraph_capture_sizes
|
||||
if sizes:
|
||||
return int(max(sizes))
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return 512
|
||||
|
||||
|
||||
class _SegmBufPool:
|
||||
"""Single-bucket buffer pool for segm_out/segm_max/segm_sum/output.
|
||||
|
||||
Each unique shape signature (Hk, Hq, num_partitions, QG, D, device,
|
||||
dtype) is backed by ONE allocation sized at ``max_B`` (= max captured
|
||||
cudagraph batch size, or env override). Per-call ``get(B, ...)``
|
||||
returns ``[:B]`` views — same data_ptr base, narrower N dim. The
|
||||
kernel grid is ``(B, ...)`` so it only ever writes the first B rows
|
||||
of segm_out / segm_max / segm_sum; downstream readers (Triton
|
||||
reducer + caller) only consume the first B rows of output.
|
||||
|
||||
Why a single bucket: cudagraph records launch pointers at capture
|
||||
time. With per-B allocations, every captured size triggered a fresh
|
||||
allocation, so total VRAM scaled with the number of captured sizes.
|
||||
With single-bucket, the pool tops out at one max_B-sized buffer per
|
||||
shape, shared across ALL captured sizes and ALL eager mixed-batch
|
||||
calls. The data_ptr is stable from first allocation onward, so
|
||||
cudagraph capture is happy across all captured B's.
|
||||
|
||||
Eliminates per-decode-step ``cudaMalloc`` (× 4) plus the device-side
|
||||
memset kernels behind ``torch.full(-inf)`` and ``torch.zeros``.
|
||||
The kernel always writes the FULL ``[B, Hk, P, QG, D]`` slice and
|
||||
every (n, kv_h, p, qg) position of segm_max / segm_sum (it stores
|
||||
``-inf`` and ``0`` for empty partitions itself), so uninitialized
|
||||
bytes from the pool are safe to reuse for any B ≤ B_bucket.
|
||||
"""
|
||||
|
||||
__slots__ = ("_bufs", "_max_B")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._bufs: dict[tuple, dict[str, torch.Tensor]] = {}
|
||||
self._max_B: int | None = None # lazy-init on first get()
|
||||
|
||||
def get(
|
||||
self,
|
||||
B: int,
|
||||
Hk: int,
|
||||
Hq: int,
|
||||
num_partitions: int,
|
||||
QG: int,
|
||||
D: int,
|
||||
device: torch.device,
|
||||
q_dtype: torch.dtype,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
if self._max_B is None:
|
||||
self._max_B = _detect_max_capture_B()
|
||||
# B may exceed the detected max if user set max-num-seqs higher
|
||||
# than the largest captured size — still safe, we'll grow once.
|
||||
# Note: growing AFTER cudagraph capture would invalidate captured
|
||||
# pointers, so this should only happen during warmup or in
|
||||
# always-eager configs.
|
||||
B_bucket = max(self._max_B, int(B))
|
||||
key = (
|
||||
int(Hk),
|
||||
int(Hq),
|
||||
int(num_partitions),
|
||||
int(QG),
|
||||
int(D),
|
||||
str(device),
|
||||
q_dtype,
|
||||
)
|
||||
bufs = self._bufs.get(key)
|
||||
if bufs is None or bufs["segm_out"].shape[0] < B_bucket:
|
||||
if bufs is not None and bufs["segm_out"].shape[0] < B_bucket:
|
||||
# First-time grow: warn so user knows cudagraphs may be
|
||||
# invalidated. This should only happen during warmup or in
|
||||
# always-eager configs (a batch larger than any captured
|
||||
# cudagraph size).
|
||||
logger.warning_once(
|
||||
"FlyDSL _SegmBufPool: growing bucket from %d to %d "
|
||||
"(B=%d). If you see this AFTER cudagraph warmup, the "
|
||||
"previously-captured graphs hold stale pointers and "
|
||||
"will GPU-fault (max cudagraph capture size < B).",
|
||||
bufs["segm_out"].shape[0],
|
||||
B_bucket,
|
||||
B,
|
||||
)
|
||||
bufs = {
|
||||
"segm_out": torch.empty(
|
||||
(B_bucket, Hk, num_partitions, QG, D),
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
),
|
||||
"segm_max": torch.empty(
|
||||
(B_bucket, Hk, num_partitions, QG),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
"segm_sum": torch.empty(
|
||||
(B_bucket, Hk, num_partitions, QG),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
"output": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=q_dtype,
|
||||
device=device,
|
||||
),
|
||||
# Stable pre-allocated buffer for the rotated query
|
||||
# tensor. On ROCm, fresh torch.empty / matmul-result allocations
|
||||
# after HIP graph capture can land in the graph memory pool and
|
||||
# cause GPU memory faults when passed as kernel arguments in the
|
||||
# eager mixed-batch path. By pooling q_rot here (same pattern as
|
||||
# segm_out/output above), the data_ptr is stable from first
|
||||
# allocation onward — same for all captured B's and eager calls.
|
||||
"q_rot": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=q_dtype,
|
||||
device=device,
|
||||
),
|
||||
# q_float: fp32 copy of query (bf16→fp32 for mm input).
|
||||
# Pooled to avoid fresh allocations post-capture.
|
||||
"q_float": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
# q_rot_fp32: fp32 output of the rotation mm (before bf16 cast).
|
||||
# Using torch.mm(..., out=this) eliminates the fresh mm-result
|
||||
# allocation that would otherwise land in the HIP graph pool.
|
||||
"q_rot_fp32": torch.empty(
|
||||
(B_bucket, Hq, D),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
}
|
||||
self._bufs[key] = bufs
|
||||
self._max_B = B_bucket
|
||||
logger.info_once(
|
||||
"FlyDSL _SegmBufPool: allocated single-bucket "
|
||||
"shape=(Hk=%d, Hq=%d, P=%d, QG=%d, D=%d, dtype=%s) "
|
||||
"B_bucket=%d (covers all captured + eager B's). "
|
||||
"VRAM = %.1f MiB / shape.",
|
||||
Hk,
|
||||
Hq,
|
||||
num_partitions,
|
||||
QG,
|
||||
D,
|
||||
q_dtype,
|
||||
B_bucket,
|
||||
sum(t.numel() * t.element_size() for t in bufs.values()) / (1 << 20),
|
||||
)
|
||||
# Return [:B] views — same data_ptr base, narrower N dim.
|
||||
return {
|
||||
"segm_out": bufs["segm_out"][:B],
|
||||
"segm_max": bufs["segm_max"][:B],
|
||||
"segm_sum": bufs["segm_sum"][:B],
|
||||
"output": bufs["output"][:B],
|
||||
"q_rot": bufs["q_rot"][:B],
|
||||
"q_float": bufs["q_float"][:B],
|
||||
"q_rot_fp32": bufs["q_rot_fp32"][:B],
|
||||
}
|
||||
|
||||
def stats(self) -> dict[str, int]:
|
||||
return {
|
||||
"shapes": len(self._bufs),
|
||||
"max_B": int(self._max_B or 0),
|
||||
"bytes": sum(
|
||||
sum(t.numel() * t.element_size() for t in d.values())
|
||||
for d in self._bufs.values()
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
_SEGM_POOL = _SegmBufPool()
|
||||
|
||||
|
||||
_HW_TR_CACHED: bool | None = None
|
||||
|
||||
|
||||
def _hw_tr_enabled() -> bool:
|
||||
"""Resolve the HW V-transpose build flag.
|
||||
|
||||
ON for gfx950+ (ds_read_tr16_b64 is bit-exact vs baseline and faster
|
||||
on Qwen-class shapes), off elsewhere. Resolved from the GPU arch.
|
||||
"""
|
||||
global _HW_TR_CACHED
|
||||
if _HW_TR_CACHED is not None:
|
||||
return _HW_TR_CACHED
|
||||
try:
|
||||
from flydsl.runtime.device import get_rocm_arch as _arch
|
||||
|
||||
a = str(_arch() or "")
|
||||
_HW_TR_CACHED = a.startswith("gfx950")
|
||||
except Exception: # noqa: BLE001
|
||||
_HW_TR_CACHED = False
|
||||
if _HW_TR_CACHED:
|
||||
logger.info_once("FlyDSL TQ: HW V transpose ON (default for gfx950+)")
|
||||
else:
|
||||
logger.info_once("FlyDSL TQ: HW V transpose OFF")
|
||||
return _HW_TR_CACHED
|
||||
|
||||
|
||||
_GET_KERNEL_STATS = {"hits": 0, "misses": 0, "build_total_s": 0.0}
|
||||
|
||||
|
||||
def _get_kernel(
|
||||
num_kv_heads: int,
|
||||
num_partitions: int,
|
||||
max_blocks_per_seq: int,
|
||||
scale: float,
|
||||
query_group_size: int,
|
||||
kv_block_size: int,
|
||||
use_hw_v_transpose: bool = False,
|
||||
num_seqs_hint: int = 1,
|
||||
tile_groups_per_partition: int = 1,
|
||||
):
|
||||
# ``num_seqs_hint`` (= runtime B) is forwarded to the build for shape
|
||||
# awareness; it does NOT participate in the cache key because the
|
||||
# kernel body uses gpu.block_idx.x (= seq index at runtime) and is
|
||||
# GQA-symmetric across B at build time.
|
||||
#
|
||||
# ``tile_groups_per_partition`` IS in the cache key — each
|
||||
# value compiles a different unrolled K-tile loop body.
|
||||
key = (
|
||||
num_kv_heads,
|
||||
int(num_partitions),
|
||||
int(max_blocks_per_seq),
|
||||
round(float(scale), 8),
|
||||
int(query_group_size),
|
||||
int(kv_block_size),
|
||||
bool(use_hw_v_transpose),
|
||||
int(tile_groups_per_partition),
|
||||
)
|
||||
cached = _KERN_CACHE.get(key)
|
||||
if cached is not None:
|
||||
_GET_KERNEL_STATS["hits"] += 1
|
||||
return cached
|
||||
assert is_flydsl_available()
|
||||
# Time the FlyDSL build path so we can quantify cudagraph capture cost.
|
||||
import time as _t
|
||||
|
||||
_build_t0 = _t.perf_counter()
|
||||
|
||||
# Per-GQA dispatch: GQA-6 (MiniMax-M2.5) lives in the sibling module
|
||||
# tq_decode_gqa6 to keep the Qwen kernel's invariants untouched.
|
||||
# GQA-{8,16} (Qwen) keep using the canonical tq_decode kernel.
|
||||
qg = int(query_group_size)
|
||||
if qg == 6:
|
||||
if _TQ_MOD_GQA6 is None:
|
||||
raise RuntimeError(
|
||||
"FlyDSL TQ GQA-6 sibling kernel "
|
||||
"(vllm.v1.attention.ops.flydsl_kernels.tq_decode_gqa6) "
|
||||
"failed to import; check that FlyDSL is installed/importable."
|
||||
)
|
||||
kmod = _TQ_MOD_GQA6
|
||||
kfn = kmod.build_tq_decode_gqa6_module(
|
||||
num_seqs=int(num_seqs_hint),
|
||||
num_kv_heads=num_kv_heads,
|
||||
num_partitions=num_partitions,
|
||||
max_blocks_per_seq=max_blocks_per_seq,
|
||||
softmax_scale=float(scale),
|
||||
query_group_size=qg,
|
||||
kv_block_size=int(kv_block_size),
|
||||
use_hw_v_transpose=bool(use_hw_v_transpose),
|
||||
tile_groups_per_partition=int(tile_groups_per_partition),
|
||||
)
|
||||
else:
|
||||
kmod = _TQ_MOD
|
||||
kfn = kmod.build_tq_decode_module(
|
||||
num_seqs=int(num_seqs_hint),
|
||||
num_kv_heads=num_kv_heads,
|
||||
num_partitions=num_partitions,
|
||||
max_blocks_per_seq=max_blocks_per_seq,
|
||||
softmax_scale=float(scale),
|
||||
query_group_size=qg,
|
||||
kv_block_size=int(kv_block_size),
|
||||
use_hw_v_transpose=bool(use_hw_v_transpose),
|
||||
tile_groups_per_partition=int(tile_groups_per_partition),
|
||||
)
|
||||
al = kmod.allocator
|
||||
block_threads = kmod.BLOCK_THREADS
|
||||
|
||||
flyc = _FLYC
|
||||
fx = _FX
|
||||
T = _TYPING_T
|
||||
CompilationContext = _CC
|
||||
ir_mod = _IR
|
||||
|
||||
@flyc.jit
|
||||
def _launch(
|
||||
out,
|
||||
es,
|
||||
ml,
|
||||
q,
|
||||
kvc,
|
||||
cents,
|
||||
bt,
|
||||
sl,
|
||||
gx: fx.Int32, # type: ignore[name-defined]
|
||||
gy: fx.Int32, # type: ignore[name-defined]
|
||||
gz: fx.Int32, # type: ignore[name-defined]
|
||||
stream: fx.Stream, # type: ignore[name-defined]
|
||||
):
|
||||
# Re-finalize the LDS allocator for this launch (idempotent per build).
|
||||
al.finalized = False
|
||||
ctx = CompilationContext.get_current()
|
||||
with ir_mod.InsertionPoint(ctx.gpu_module_body):
|
||||
al.finalize()
|
||||
from flydsl.expr import arith
|
||||
|
||||
grid_x = arith.index_cast(T.index, gx.ir_value())
|
||||
grid_y = arith.index_cast(T.index, gy.ir_value())
|
||||
grid_z = arith.index_cast(T.index, gz.ir_value())
|
||||
kfn(out, es, ml, q, kvc, cents, bt, sl).launch(
|
||||
grid=(grid_x, grid_y, grid_z),
|
||||
block=(block_threads, 1, 1),
|
||||
stream=stream,
|
||||
)
|
||||
|
||||
_KERN_CACHE[key] = _launch
|
||||
_build_dt = _t.perf_counter() - _build_t0
|
||||
_GET_KERNEL_STATS["misses"] += 1
|
||||
_GET_KERNEL_STATS["build_total_s"] += _build_dt
|
||||
logger.info(
|
||||
"FlyDSL _get_kernel BUILD #%d dt=%.2fs (cumulative=%.1fs) key=%s",
|
||||
_GET_KERNEL_STATS["misses"],
|
||||
_build_dt,
|
||||
_GET_KERNEL_STATS["build_total_s"],
|
||||
key,
|
||||
)
|
||||
return _launch
|
||||
|
||||
|
||||
# -- Partition reducer (Triton, native kernel layout) -------------------------
|
||||
# Reduces the FA2 split-KV partials produced by the FlyDSL kernel.
|
||||
# segm_out [N, Hk, P, QG, D] bf16
|
||||
# segm_max [N, Hk, P, QG] fp32
|
||||
# segm_sum [N, Hk, P, QG] fp32
|
||||
# → output [N, Hq=Hk*QG, D] of any (bf16/fp16/fp32) dtype.
|
||||
@triton.jit
|
||||
def _reduce_partitions(
|
||||
output_ptr, # [N, Hq, D] in OUT_DTYPE
|
||||
segm_out_ptr, # [N, Hk, P, QG, D] bf16
|
||||
segm_max_ptr, # [N, Hk, P, QG] fp32
|
||||
segm_sum_ptr, # [N, Hk, P, QG] fp32
|
||||
out_stride_n: tl.int64, # stride on N axis (in elems of OUT_DTYPE)
|
||||
out_stride_h: tl.int64, # stride on Hq axis
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
QG: tl.constexpr,
|
||||
NUM_PARTS: tl.constexpr,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
):
|
||||
# grid = (N, Hq)
|
||||
n = tl.program_id(0)
|
||||
hq = tl.program_id(1)
|
||||
kv_h = hq // QG
|
||||
qg = hq % QG
|
||||
|
||||
# Per-partition pointers for this (n, kv_h, qg) row.
|
||||
# Strides in elems of source dtype.
|
||||
msum_base = n * (NUM_KV_HEADS * NUM_PARTS * QG) + kv_h * (NUM_PARTS * QG) + qg
|
||||
so_base = (
|
||||
n * (NUM_KV_HEADS * NUM_PARTS * QG * HEAD_SIZE)
|
||||
+ kv_h * (NUM_PARTS * QG * HEAD_SIZE)
|
||||
+ qg * HEAD_SIZE
|
||||
)
|
||||
|
||||
p_off = tl.arange(0, NUM_PARTS)
|
||||
d_off = tl.arange(0, HEAD_SIZE)
|
||||
|
||||
# Load per-partition max/sum.
|
||||
m_idx = msum_base + p_off * QG
|
||||
seg_max = tl.load(segm_max_ptr + m_idx)
|
||||
seg_sum = tl.load(segm_sum_ptr + m_idx)
|
||||
|
||||
# Mask out empty partitions (max == -inf).
|
||||
valid = seg_max > float("-inf")
|
||||
overall_max = tl.max(tl.where(valid, seg_max, float("-inf")))
|
||||
|
||||
# Rescale exp sums.
|
||||
rescale = tl.where(valid, tl.exp(seg_max - overall_max), 0.0)
|
||||
seg_sum_rescaled = seg_sum * rescale
|
||||
overall_sum = tl.sum(seg_sum_rescaled)
|
||||
|
||||
# Load segment outputs and combine.
|
||||
so_idx = so_base + p_off[:, None] * (QG * HEAD_SIZE) + d_off[None, :]
|
||||
seg_out = tl.load(segm_out_ptr + so_idx).to(tl.float32)
|
||||
# acc = sum_p( seg_out_p * (seg_sum_p * rescale_p) ) / overall_sum
|
||||
weighted = seg_out * seg_sum_rescaled[:, None]
|
||||
acc_sum = tl.sum(weighted, axis=0)
|
||||
acc = tl.where(overall_sum > 0.0, acc_sum / overall_sum, 0.0)
|
||||
|
||||
# Store.
|
||||
out_off = n * out_stride_n + hq * out_stride_h + d_off
|
||||
tl.store(output_ptr + out_off, acc.to(output_ptr.dtype.element_ty))
|
||||
|
||||
|
||||
# -- Public launcher ----------------------------------------------------------
|
||||
def flydsl_turboquant_decode_attention(
|
||||
query: torch.Tensor, # [B, Hq, D] bf16/fp16
|
||||
kv_cache: torch.Tensor, # [num_blocks, BS, Hk, slot_size_aligned]
|
||||
block_table: torch.Tensor, # [B, max_blocks_per_seq] int32
|
||||
seq_lens: torch.Tensor, # [B] int32
|
||||
Pi: torch.Tensor, # [D, D] fp32
|
||||
centroids: torch.Tensor, # [N_CENTROIDS] fp32
|
||||
scale: float,
|
||||
mse_bits: int,
|
||||
key_packed_size: int,
|
||||
value_quant_bits: int,
|
||||
value_packed_size: int,
|
||||
key_fp8: bool = False,
|
||||
norm_correction: bool = False,
|
||||
PiT: torch.Tensor | None = None,
|
||||
max_seq_len: int = 0,
|
||||
mid_o_buf: Any = None,
|
||||
output_buf: torch.Tensor | None = None,
|
||||
lse_buf: Any = None,
|
||||
buf_holder: Any = None,
|
||||
max_num_kv_splits: int = 32,
|
||||
sinks: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""SoA-decode-compatible launcher backed by the FlyDSL decode kernel.
|
||||
|
||||
Constraints:
|
||||
* key_fp8 == False
|
||||
* mse_bits == 4
|
||||
* value_quant_bits == 4
|
||||
* centroids.numel() == 16
|
||||
* D == 128
|
||||
* block_size in {16, 32}
|
||||
* Hq // Hk in {6, 8, 16}
|
||||
- 8/16 → canonical tq_decode kernel (Qwen2.5-72B / Qwen3-32B)
|
||||
- 6 → tq_decode_gqa6 sibling kernel (MiniMax-M2.5)
|
||||
|
||||
Sinks are NYI and silently ignored if set. norm_correction is honored
|
||||
implicitly via the pre-folded stored K-norm (see footer comment).
|
||||
"""
|
||||
del mid_o_buf, lse_buf, key_packed_size, value_packed_size
|
||||
if not is_flydsl_available():
|
||||
raise RuntimeError(
|
||||
"FlyDSL decode requested but FlyDSL is not available (needs gfx950 "
|
||||
"+ importable FlyDSL); the SoA Triton decode is the fallback."
|
||||
)
|
||||
assert not key_fp8, "FlyDSL supports MSE-key path only"
|
||||
assert mse_bits == 4, f"FlyDSL expects mse_bits=4, got {mse_bits}"
|
||||
assert value_quant_bits == 4, (
|
||||
f"FlyDSL expects value_quant_bits=4, got {value_quant_bits}"
|
||||
)
|
||||
|
||||
B, Hq, D = query.shape
|
||||
Hk = kv_cache.shape[2]
|
||||
block_size = kv_cache.shape[1]
|
||||
QG = Hq // Hk
|
||||
assert D == _TQ_MOD.HEAD_SIZE
|
||||
assert block_size in (16, 32), (
|
||||
f"FlyDSL supports kv_block_size 16 or 32, got {block_size}"
|
||||
)
|
||||
assert QG in (6, 8, 16), f"FlyDSL supports GQA factor 6, 8 or 16, got {QG}"
|
||||
if QG == 6 and _TQ_MOD_GQA6 is None:
|
||||
raise RuntimeError(
|
||||
"FlyDSL launcher: GQA-6 requested (MiniMax-class) but the "
|
||||
"tq_decode_gqa6 sibling module is not available. Update your "
|
||||
"FlyDSL checkout (must include kernels/tq_decode_gqa6.py); "
|
||||
"otherwise the SoA Triton decode is the fallback."
|
||||
)
|
||||
assert centroids.numel() == _TQ_MOD.N_CENTROIDS, (
|
||||
f"centroids.numel={centroids.numel()} != {_TQ_MOD.N_CENTROIDS}"
|
||||
)
|
||||
|
||||
# ---- Per-layer cache for PiT_f32 + contiguous centroids ---
|
||||
# PiT and centroids are model constants (set once at layer warmup).
|
||||
# Avoid the per-decode-step transpose+cast and `.contiguous()` no-op
|
||||
# check by stashing the pre-cooked tensors on ``buf_holder`` (= layer).
|
||||
PiT_f32: torch.Tensor
|
||||
centroids_c: torch.Tensor
|
||||
if buf_holder is not None:
|
||||
PiT_f32 = getattr(buf_holder, "_tq_PiT_f32", None)
|
||||
if PiT_f32 is None:
|
||||
_PiT_src = PiT if PiT is not None else Pi.T.contiguous()
|
||||
PiT_f32 = (
|
||||
_PiT_src
|
||||
if _PiT_src.dtype == torch.float32
|
||||
else _PiT_src.to(torch.float32)
|
||||
)
|
||||
buf_holder._tq_PiT_f32 = PiT_f32
|
||||
centroids_c = getattr(buf_holder, "_tq_centroids_c", None)
|
||||
if centroids_c is None:
|
||||
centroids_c = centroids.contiguous()
|
||||
buf_holder._tq_centroids_c = centroids_c
|
||||
else:
|
||||
# Defensive fallback (unit tests may pass ``buf_holder=None``).
|
||||
_PiT_src = PiT if PiT is not None else Pi.T.contiguous()
|
||||
PiT_f32 = (
|
||||
_PiT_src if _PiT_src.dtype == torch.float32 else _PiT_src.to(torch.float32)
|
||||
)
|
||||
centroids_c = centroids.contiguous()
|
||||
|
||||
# ---- Pooled q_rot (stable data_ptr post-capture) ------
|
||||
# On ROCm, fresh tensor allocations made AFTER HIP graph capture can land
|
||||
# in the graph memory pool and cause GPU memory faults when passed as kernel
|
||||
# pointer arguments in the eager mixed-batch path. We pool q_rot and the
|
||||
# fp32 intermediate the same way segm_out/output are pooled: one max-B
|
||||
# allocation per shape, reused across all B's.
|
||||
#
|
||||
# Note: pool_bufs is computed BELOW after num_partitions is determined;
|
||||
# q_rot needs pool_bufs["q_float"] and pool_bufs["q_rot"]. We compute the
|
||||
# rotation inline after the pool is fetched (see the pooled-buffers
|
||||
# section below).
|
||||
|
||||
# ---- Partition count (FA2 split-KV) ----------------------------------
|
||||
#
|
||||
# FlyDSL runs INSIDE FULL cudagraph (TurboQuantMetadataBuilder._cudagraph_
|
||||
# support = UNIFORM_BATCH). That means the gridDim baked at capture
|
||||
# time MUST equal the gridDim at replay — the kernel launch parameters
|
||||
# are recorded into the captured graph. So `num_partitions` (which
|
||||
# becomes gridDim.z) MUST be derived from a stable source that produces
|
||||
# the same value at capture and at runtime.
|
||||
#
|
||||
# We use the worst-case context length implied by the block table's
|
||||
# allocation (block_table.shape[1] * block_size) as that source. It's
|
||||
# bounded by `max_model_len` (vLLM allocates the block table for the
|
||||
# configured max), and it's identical at capture and at runtime
|
||||
# because `block_table.shape[1]` is fixed once the model is loaded.
|
||||
# Per-tile OOB redirects + the masked FA-2 reducer ensure that for
|
||||
# short sequences the extra partitions contribute zero to the output.
|
||||
#
|
||||
# We always size from the worst case (block_table.shape[1] * block_size),
|
||||
# never from the per-step actual max_seq_len: a per-step size would make
|
||||
# the captured gridDim mismatch runtime and GPU-fault under cudagraph.
|
||||
kv_compute_block = _TQ_MOD.KV_COMPUTE_BLOCK
|
||||
worst_case_max_seq_len = int(block_table.shape[1]) * int(block_size)
|
||||
if max_seq_len <= 0:
|
||||
max_seq_len = worst_case_max_seq_len
|
||||
sizing_max_seq_len = worst_case_max_seq_len
|
||||
|
||||
# ── Bounded num_partitions + internal tile-group looping ──
|
||||
# The kernel previously hardcoded one partition = 256 tokens (=
|
||||
# KV_COMPUTE_BLOCK), forcing num_partitions to scale linearly with
|
||||
# max_model_len, which made the cudagraph capture cost grow
|
||||
# prohibitively at long context. The new kernel takes
|
||||
# ``tile_groups_per_partition`` (TGPP); each CTA processes
|
||||
# ``TGPP * 16`` K-tiles = ``TGPP * KV_COMPUTE_BLOCK`` tokens with the
|
||||
# FA-2 online-softmax state accumulating across all of them. This
|
||||
# mirrors what the SoA Triton decode / HIP SoA-fusion already do.
|
||||
#
|
||||
# Strategy: cap num_partitions at ``MAX_PARTITIONS`` (default 32),
|
||||
# then derive TGPP so that ``num_partitions * TGPP * KV_COMPUTE_BLOCK
|
||||
# >= sizing_max_seq_len``.
|
||||
#
|
||||
# Examples (block_size=32 → worst_case max_bps*32):
|
||||
# max_model_len=8K: required=32, parts=32, TGPP=1 (no waste)
|
||||
# max_model_len=16K: required=64, parts=32, TGPP=2 (no waste)
|
||||
# max_model_len=32K: required=128, parts=32, TGPP=4 (no waste)
|
||||
# max_model_len=64K: required=256, parts=32, TGPP=8 (no waste)
|
||||
# max_model_len=128K: required=512, parts=32, TGPP=16
|
||||
MAX_PARTITIONS = 32
|
||||
required_num_partitions = (
|
||||
sizing_max_seq_len + kv_compute_block - 1
|
||||
) // kv_compute_block
|
||||
# max_num_kv_splits acts as a parallelism floor (ensure at least this
|
||||
# many CTAs along grid.z), but is itself capped by MAX_PARTITIONS.
|
||||
parallelism_floor = min(MAX_PARTITIONS, max(1, max_num_kv_splits))
|
||||
# Cap num_partitions at MAX_PARTITIONS, but never go below the floor.
|
||||
num_partitions_actual = max(
|
||||
parallelism_floor,
|
||||
min(MAX_PARTITIONS, required_num_partitions),
|
||||
)
|
||||
# Round up to next power of 2 for the Triton reducer's tl.arange(0, N)
|
||||
# constraint (Triton requires power-of-2 ≥ 2).
|
||||
num_partitions = max(2, triton.next_power_of_2(num_partitions_actual))
|
||||
# Now derive tile_groups_per_partition (TGPP) so total coverage
|
||||
# ``num_partitions * TGPP * KV_COMPUTE_BLOCK`` is >= sizing_max_seq_len.
|
||||
# This guarantees no work is dropped at runtime regardless of seq_len.
|
||||
# Round TGPP up to the next power-of-2 so the JIT cache key is bounded
|
||||
# to a small set of values (e.g. {1, 2, 4, 8, 16}) — limits the number
|
||||
# of distinct kernel binaries that need to be compiled across requests.
|
||||
_tgpp_required = max(
|
||||
1,
|
||||
(required_num_partitions + num_partitions - 1) // num_partitions,
|
||||
)
|
||||
tile_groups_per_partition = int(triton.next_power_of_2(_tgpp_required))
|
||||
|
||||
# ---- Pooled buffers (no per-call cudaMalloc / memset) ----------
|
||||
# The FlyDSL kernel writes the FULL [B, Hk, P, QG, D] segm_out and the
|
||||
# FULL [B, Hk, P, QG] segm_max / segm_sum (running_max=-inf, sum=0 for
|
||||
# empty partitions are stored by the kernel itself), so uninitialized
|
||||
# buffers from the pool are safe to reuse.
|
||||
device = query.device
|
||||
pool_bufs = _SEGM_POOL.get(
|
||||
B,
|
||||
Hk,
|
||||
Hq,
|
||||
num_partitions,
|
||||
QG,
|
||||
D,
|
||||
device,
|
||||
query.dtype,
|
||||
)
|
||||
segm_out = pool_bufs["segm_out"]
|
||||
segm_max = pool_bufs["segm_max"]
|
||||
segm_sum = pool_bufs["segm_sum"]
|
||||
if output_buf is None:
|
||||
output = pool_bufs["output"]
|
||||
else:
|
||||
output = output_buf[:B] if output_buf.shape[0] != B else output_buf
|
||||
|
||||
# ---- q rotation: GEMM q @ PiT via stable pooled buffers -------------
|
||||
_q_float = pool_bufs["q_float"] # [B, Hq, D] fp32, stable
|
||||
_q_rot_f32 = pool_bufs["q_rot_fp32"] # [B, Hq, D] fp32, stable mm output
|
||||
_q_rot_out = pool_bufs["q_rot"] # [B, Hq, D] query.dtype, stable
|
||||
_q_float.copy_(query) # bf16 → fp32 in-place, no alloc
|
||||
# mm into stable fp32 buffer via out= to avoid fresh allocations.
|
||||
# PiT_f32 is cached on buf_holder (stable ptr since first layer warmup).
|
||||
torch.mm(
|
||||
_q_float.view(B * Hq, D),
|
||||
PiT_f32,
|
||||
out=_q_rot_f32.view(B * Hq, D), # in-place into pool buffer
|
||||
)
|
||||
_q_rot_out.copy_(_q_rot_f32) # fp32 → bf16, into stable buf
|
||||
q_for_kernel = _q_rot_out # stable ptr for kernel launch
|
||||
|
||||
# ---- FlyDSL kernel launch -------------------------------------------
|
||||
max_bps = int(block_table.shape[1])
|
||||
use_hw_tr = _hw_tr_enabled()
|
||||
launch = _get_kernel(
|
||||
Hk,
|
||||
num_partitions,
|
||||
max_bps,
|
||||
scale,
|
||||
QG,
|
||||
block_size,
|
||||
use_hw_v_transpose=use_hw_tr,
|
||||
num_seqs_hint=int(B),
|
||||
tile_groups_per_partition=int(tile_groups_per_partition),
|
||||
)
|
||||
# Zero-overhead one-shot info log (replaces logger.info_once which
|
||||
# hashes its format string on every call to dedup).
|
||||
global _LOG_INVOKED_ONCE, _LOG_SINKS_WARNED, _LOG_NORM_WARNED
|
||||
if not _LOG_INVOKED_ONCE:
|
||||
_LOG_INVOKED_ONCE = True
|
||||
logger.info(
|
||||
"FlyDSL launcher invoked (UNIFORM_BATCH cudagraph): "
|
||||
"B=%d Hk=%d Hq=%d D=%d QG=%d num_partitions=%d (actual=%d, "
|
||||
"cap=%d) TGPP=%d max_bps=%d block_size=%d max_seq_len=%d "
|
||||
"hw_v_transpose=%s (coverage=%d tokens, worst_case=%d tokens)",
|
||||
B,
|
||||
Hk,
|
||||
Hq,
|
||||
D,
|
||||
QG,
|
||||
num_partitions,
|
||||
num_partitions_actual,
|
||||
MAX_PARTITIONS,
|
||||
tile_groups_per_partition,
|
||||
max_bps,
|
||||
int(block_size),
|
||||
int(max_seq_len),
|
||||
use_hw_tr,
|
||||
num_partitions * tile_groups_per_partition * kv_compute_block,
|
||||
worst_case_max_seq_len,
|
||||
)
|
||||
launch(
|
||||
segm_out,
|
||||
segm_sum,
|
||||
segm_max,
|
||||
q_for_kernel,
|
||||
kv_cache,
|
||||
centroids_c,
|
||||
block_table,
|
||||
seq_lens,
|
||||
B,
|
||||
Hk,
|
||||
num_partitions,
|
||||
torch.cuda.current_stream(),
|
||||
)
|
||||
|
||||
# ---- Reduce partitions -> [B, Hq, D] --------------------------------
|
||||
_reduce_partitions[(B, Hq)](
|
||||
output_ptr=output,
|
||||
segm_out_ptr=segm_out,
|
||||
segm_max_ptr=segm_max,
|
||||
segm_sum_ptr=segm_sum,
|
||||
out_stride_n=output.stride(0),
|
||||
out_stride_h=output.stride(1),
|
||||
NUM_KV_HEADS=Hk,
|
||||
QG=QG,
|
||||
NUM_PARTS=num_partitions,
|
||||
HEAD_SIZE=D,
|
||||
)
|
||||
if sinks is not None and not _LOG_SINKS_WARNED:
|
||||
_LOG_SINKS_WARNED = True
|
||||
logger.warning(
|
||||
"FlyDSL launcher: sinks ignored (NYI). Disable sinks if they "
|
||||
"are required for correctness."
|
||||
)
|
||||
# ── norm_correction is honored IMPLICITLY ───────────────────────────
|
||||
# When the model was stored with norm_correction=True (the *_nc presets
|
||||
# turboquant_4bit_nc / k3v4_nc / 3bit_nc), the per-token K-norm scalar
|
||||
# was pre-folded to ||k_t|| / ||c_t|| at store time by the TurboQuant
|
||||
# store path. The decode kernel just multiplies `c_vals * stored_knorm`,
|
||||
# which then equals `(c_vals / ||c_t||) * ||k_t||` — exactly the
|
||||
# unit-norm-renormalized centroid times the original key norm. The
|
||||
# Triton decode does the identical multiply, so both decoders honor
|
||||
# norm_correction equivalently and there is nothing to "do" at decode
|
||||
# time. The launcher's `norm_correction` arg is only kept for API
|
||||
# parity; we emit a one-shot INFO log to make the contract explicit.
|
||||
if norm_correction and not _LOG_NORM_WARNED:
|
||||
_LOG_NORM_WARNED = True
|
||||
logger.info(
|
||||
"FlyDSL launcher: norm_correction honored implicitly via "
|
||||
"pre-folded stored K-norm (cf. triton_turboquant_store step 3); "
|
||||
"no decode-time work required, identical to the SoA decode behavior."
|
||||
)
|
||||
return output
|
||||
@@ -0,0 +1,3 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# HIP-free SoA Triton subset for the FlyDSL TurboQuant decode path.
|
||||
@@ -0,0 +1,643 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Triton fused TurboQuant decode attention.
|
||||
|
||||
Decode path: Triton stage1 (split-KV tiled attention scoring + value
|
||||
accumulation) + stage2 (log-sum-exp reduction across splits).
|
||||
|
||||
Supports FP8 (E4M3) keys, 3-bit and 4-bit uniform quantized values.
|
||||
"""
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.ops.triton_decode_attention import (
|
||||
_fwd_kernel_stage2,
|
||||
)
|
||||
|
||||
_FP8_E4B15: dict[int, int] = {}
|
||||
|
||||
|
||||
def _use_fp8_e4b15(device: int = 0) -> int:
|
||||
"""Return 1 if device needs fp8e4b15 (Ampere/Ada, SM < 8.9), else 0.
|
||||
On non-CUDA platforms (e.g. XPU), always returns 0 (use e4nv format).
|
||||
"""
|
||||
if device not in _FP8_E4B15:
|
||||
if current_platform.is_cuda_alike():
|
||||
cap = torch.cuda.get_device_capability(device)
|
||||
_FP8_E4B15[device] = 1 if cap < (8, 9) else 0
|
||||
else:
|
||||
_FP8_E4B15[device] = 0
|
||||
return _FP8_E4B15[device]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 1: Fused TQ score + value accumulation (BLOCK_KV tiled)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_decode_stage1(
|
||||
# Precomputed query projection
|
||||
Q_rot_ptr, # [B, Hq, D] float32
|
||||
# Compressed KV cache (combined K+V)
|
||||
KV_cache_ptr, # [num_blocks, block_size, Hk, padded_slot] uint8
|
||||
KV_cache_u16_ptr, # uint16 view of same storage — Opt#3 SoA loads
|
||||
# Block table and sequence info
|
||||
Block_table_ptr, # [B, max_num_blocks] int32
|
||||
Seq_lens_ptr, # [B] int32
|
||||
# TQ parameters
|
||||
Centroids_ptr, # [n_centroids] float32
|
||||
# Output (intermediate for stage2)
|
||||
Mid_o_ptr, # [B, Hq, NUM_KV_SPLITS, D+1] float32
|
||||
# Sink support
|
||||
Sink_ptr, # [Hq] float32, pre-computed sink attention logits per head
|
||||
# Strides
|
||||
stride_qb,
|
||||
stride_qh, # Q strides: [B, Hq, D]
|
||||
stride_cache_block: tl.int64, # bytes per block (bs*H*slot_aligned)
|
||||
stride_bt_b: tl.int64, # block_table stride per batch
|
||||
stride_mid_b,
|
||||
stride_mid_h,
|
||||
stride_mid_s, # mid_o strides
|
||||
# Constexpr dims
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr, # KV cache block_size (pages)
|
||||
NUM_KV_SPLITS: tl.constexpr,
|
||||
KV_GROUP_SIZE: tl.constexpr, # Hq // Hk
|
||||
# TQ layout constants
|
||||
MSE_BITS: tl.constexpr, # 3 or 4
|
||||
MSE_BYTES: tl.constexpr, # ceil(D * mse_bits / 8)
|
||||
VQB: tl.constexpr, # value_quant_bits (4 or 8=FP8)
|
||||
VAL_DATA_BYTES: tl.constexpr, # ceil(D * vqb / 8) or D for FP8
|
||||
# Opt#3 SoA layout
|
||||
KEY_DATA_BYTES: tl.constexpr, # MSE_BYTES (MSE) or HEAD_DIM (FP8)
|
||||
META_REGION_OFFSET: tl.constexpr, # bytes from block start to SoA region
|
||||
NUM_SOA_FIELDS: tl.constexpr, # 3 (MSE) or 2 (FP8)
|
||||
SOA_K_NORM: tl.constexpr, # 0 (MSE); unused for FP8
|
||||
SOA_V_SCALE: tl.constexpr, # 1 (MSE) / 0 (FP8)
|
||||
SOA_V_ZERO: tl.constexpr, # 2 (MSE) / 1 (FP8)
|
||||
# Score constants
|
||||
ATTN_SCALE: tl.constexpr, # 1/sqrt(D)
|
||||
# Block tile sizes
|
||||
BLOCK_D: tl.constexpr, # next_power_of_2(HEAD_DIM)
|
||||
BLOCK_KV: tl.constexpr, # tokens per tile (16)
|
||||
KEY_FP8: tl.constexpr, # 1 if K is stored as FP8
|
||||
NORM_CORRECTION: tl.constexpr = 0, # 1 = re-normalize centroids
|
||||
FP8_E4B15: tl.constexpr = 0, # 1 = use e4b15 (Ampere/Ada), 0 = e4nv (Hopper+)
|
||||
USE_SINKS: tl.constexpr = 0, # 1 = use sink tokens for attention anchoring
|
||||
):
|
||||
bid = tl.program_id(0) # batch index
|
||||
hid = tl.program_id(1) # q_head index
|
||||
sid = tl.program_id(2) # kv_split index
|
||||
|
||||
kv_head = hid // KV_GROUP_SIZE
|
||||
|
||||
# Sequence length for this batch
|
||||
seq_len = tl.load(Seq_lens_ptr + bid)
|
||||
|
||||
# KV split range
|
||||
split_len = tl.cdiv(seq_len, NUM_KV_SPLITS)
|
||||
split_start = split_len * sid
|
||||
split_end = tl.minimum(split_start + split_len, seq_len)
|
||||
|
||||
if split_start >= split_end:
|
||||
return
|
||||
|
||||
# Dimension offsets
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < HEAD_DIM
|
||||
kv_range = tl.arange(0, BLOCK_KV)
|
||||
|
||||
# Load query vector: q_rot — [BLOCK_D] float32
|
||||
q_base = bid * stride_qb + hid * stride_qh
|
||||
q_rot = tl.load(Q_rot_ptr + q_base + d_offs, mask=d_mask, other=0.0).to(tl.float32)
|
||||
|
||||
# Precompute byte/bit index vectors for MSE gather loads
|
||||
if not KEY_FP8:
|
||||
mse_bit_off = d_offs * MSE_BITS
|
||||
mse_byte_idx = mse_bit_off // 8
|
||||
mse_bit_shift = mse_bit_off % 8
|
||||
mse_mask = (1 << MSE_BITS) - 1
|
||||
|
||||
# Precompute value bit/byte index vectors (loop-invariant)
|
||||
if VQB == 3:
|
||||
val_bit_off = d_offs * 3
|
||||
val_byte_idx = val_bit_off // 8
|
||||
val_bit_shift = val_bit_off % 8
|
||||
|
||||
# Online softmax accumulators
|
||||
# Sink tokens provide a pre-computed attention bias that should be
|
||||
# included in the softmax normalization. For the first split (sid==0),
|
||||
# we initialize m_prev with the sink logit. L is always 1.0 for all
|
||||
# splits to match unified attention semantics.
|
||||
if USE_SINKS:
|
||||
if sid == 0:
|
||||
m_prev = tl.load(Sink_ptr + hid).to(tl.float32)
|
||||
l_prev = 1.0
|
||||
else:
|
||||
m_prev = -float("inf")
|
||||
l_prev = 0.0
|
||||
else:
|
||||
m_prev = -float("inf")
|
||||
l_prev = 0.0
|
||||
acc = tl.zeros([BLOCK_D], dtype=tl.float32)
|
||||
|
||||
bt_base = bid * stride_bt_b
|
||||
|
||||
# ================================================================
|
||||
# TILED LOOP: process BLOCK_KV tokens per iteration
|
||||
# ================================================================
|
||||
for start_n in range(split_start, split_end, BLOCK_KV):
|
||||
kv_offs = start_n + kv_range
|
||||
kv_mask = kv_offs < split_end
|
||||
|
||||
page_idx = kv_offs // BLOCK_SIZE
|
||||
page_off = kv_offs % BLOCK_SIZE
|
||||
block_nums = tl.load(
|
||||
Block_table_ptr + bt_base + page_idx,
|
||||
mask=kv_mask,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
|
||||
# Opt#3 SoA addressing: data region then metadata region per block.
|
||||
slot_within_block = page_off.to(tl.int64)
|
||||
block_base = block_nums * stride_cache_block
|
||||
DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES
|
||||
data_bases = (
|
||||
block_base
|
||||
+ slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT)
|
||||
+ tl.cast(kv_head, tl.int64) * DATA_BYTES_PER_SLOT
|
||||
)
|
||||
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast(
|
||||
kv_head, tl.int64
|
||||
) * (NUM_SOA_FIELDS * BLOCK_SIZE)
|
||||
knorm_u16_addrs = (
|
||||
head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block
|
||||
)
|
||||
vscale_u16_addrs = (
|
||||
head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block
|
||||
)
|
||||
vzero_u16_addrs = (
|
||||
head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# COMPUTE ATTENTION SCORES: [BLOCK_KV]
|
||||
# ============================================================
|
||||
if KEY_FP8:
|
||||
k_addrs = data_bases[:, None] + d_offs[None, :]
|
||||
k_raw = tl.load(
|
||||
KV_cache_ptr + k_addrs,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
)
|
||||
if FP8_E4B15:
|
||||
k_float = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32)
|
||||
else:
|
||||
k_float = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32)
|
||||
scores = (
|
||||
tl.sum(
|
||||
tl.where(d_mask[None, :], q_rot[None, :] * k_float, 0.0),
|
||||
axis=1,
|
||||
)
|
||||
* ATTN_SCALE
|
||||
)
|
||||
scores = tl.where(kv_mask, scores, -float("inf"))
|
||||
else:
|
||||
# MSE unpack + norms
|
||||
mse_addrs0 = data_bases[:, None] + mse_byte_idx[None, :]
|
||||
mse_raw0 = tl.load(
|
||||
KV_cache_ptr + mse_addrs0,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
mse_raw1 = tl.load(
|
||||
KV_cache_ptr + mse_addrs0 + 1,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
raw16 = mse_raw0 | (mse_raw1 << 8)
|
||||
mse_idx = (raw16 >> mse_bit_shift[None, :]) & mse_mask
|
||||
|
||||
# Centroid gather + dot product
|
||||
c_vals = tl.load(
|
||||
Centroids_ptr + mse_idx,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
# Opt#1: norm-correction (1/||c_vec||) is pre-folded into the
|
||||
# stored per-token scalar by the MSE store kernel when
|
||||
# NORM_CORRECTION=1, so we skip the per-tile sum+sqrt+divide.
|
||||
# NORM_CORRECTION is kept in the signature for API parity; it is
|
||||
# implicit in the stored norm value now.
|
||||
|
||||
term1 = tl.sum(
|
||||
tl.where(d_mask[None, :], q_rot[None, :] * c_vals, 0.0),
|
||||
axis=1,
|
||||
)
|
||||
|
||||
# Opt#3: K-norms live in the per-block SoA region. Single u16
|
||||
# load per token; contiguous across tokens within a block.
|
||||
norm_u16 = tl.load(
|
||||
KV_cache_u16_ptr + knorm_u16_addrs, mask=kv_mask, other=0
|
||||
)
|
||||
vec_norms = norm_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
|
||||
scores = vec_norms * term1 * ATTN_SCALE
|
||||
scores = tl.where(kv_mask, scores, -float("inf"))
|
||||
|
||||
# ============================================================
|
||||
# ONLINE SOFTMAX UPDATE (block-level)
|
||||
# ============================================================
|
||||
n_e_max = tl.maximum(tl.max(scores, 0), m_prev)
|
||||
re_scale = tl.exp(m_prev - n_e_max)
|
||||
p = tl.exp(scores - n_e_max)
|
||||
|
||||
# ============================================================
|
||||
# VALUE LOAD + DEQUANTIZE: [BLOCK_KV, BLOCK_D]
|
||||
# Opt#3: V data immediately follows K data in the slot (no
|
||||
# interleaved metadata). Scale/zero come from the SoA metadata
|
||||
# region as single u16 loads per field per token.
|
||||
# ============================================================
|
||||
val_bases = data_bases + KEY_DATA_BYTES
|
||||
|
||||
if VQB == 3:
|
||||
val_addrs0 = val_bases[:, None] + val_byte_idx[None, :]
|
||||
val_raw0 = tl.load(
|
||||
KV_cache_ptr + val_addrs0,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
val_raw1 = tl.load(
|
||||
KV_cache_ptr + val_addrs0 + 1,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
raw16 = val_raw0 | (val_raw1 << 8)
|
||||
v_idx = ((raw16 >> val_bit_shift[None, :]) & 0x7).to(tl.float32)
|
||||
else: # VQB == 4
|
||||
vb_idx = d_offs // 2
|
||||
vb_shift = (d_offs % 2) * 4
|
||||
val_addrs = val_bases[:, None] + vb_idx[None, :]
|
||||
val_raw = tl.load(
|
||||
KV_cache_ptr + val_addrs,
|
||||
mask=kv_mask[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
v_idx = ((val_raw >> vb_shift[None, :]) & 0xF).to(tl.float32)
|
||||
|
||||
scale_u16 = tl.load(KV_cache_u16_ptr + vscale_u16_addrs, mask=kv_mask, other=0)
|
||||
zero_u16 = tl.load(KV_cache_u16_ptr + vzero_u16_addrs, mask=kv_mask, other=0)
|
||||
v_scales = scale_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
v_zeros = zero_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
values = v_idx * v_scales[:, None] + v_zeros[:, None]
|
||||
|
||||
# ============================================================
|
||||
# WEIGHTED VALUE ACCUMULATION
|
||||
# ============================================================
|
||||
acc = acc * re_scale + tl.sum(p[:, None] * values, 0)
|
||||
l_prev = l_prev * re_scale + tl.sum(p, 0)
|
||||
m_prev = n_e_max
|
||||
|
||||
# Store partial result
|
||||
out_base = bid * stride_mid_b + hid * stride_mid_h + sid * stride_mid_s
|
||||
safe_l = tl.where(l_prev > 0.0, l_prev, 1.0)
|
||||
tl.store(Mid_o_ptr + out_base + d_offs, acc / safe_l, mask=d_mask)
|
||||
lse = m_prev + tl.log(safe_l)
|
||||
tl.store(Mid_o_ptr + out_base + HEAD_DIM, lse)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pre-dequant kernel: Bulk dequant K (MSE+norms) and V to fp16
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_full_dequant_kv(
|
||||
KV_cache_ptr,
|
||||
KV_cache_u16_ptr,
|
||||
Block_table_ptr,
|
||||
Centroids_ptr,
|
||||
K_out_ptr, # [B, Hk, max_seq, D] float16
|
||||
V_out_ptr, # [B, Hk, max_seq, D] float16
|
||||
stride_ko_b: tl.int64,
|
||||
stride_ko_h: tl.int64,
|
||||
stride_ko_s: tl.int64,
|
||||
stride_vo_b: tl.int64,
|
||||
stride_vo_h: tl.int64,
|
||||
stride_vo_s: tl.int64,
|
||||
stride_cache_block: tl.int64,
|
||||
stride_bt_b: tl.int64,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
MSE_BYTES: tl.constexpr,
|
||||
VQB: tl.constexpr,
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
MSE_BITS: tl.constexpr,
|
||||
KEY_FP8: tl.constexpr,
|
||||
# Opt#3 SoA layout
|
||||
KEY_DATA_BYTES: tl.constexpr,
|
||||
META_REGION_OFFSET: tl.constexpr,
|
||||
NUM_SOA_FIELDS: tl.constexpr,
|
||||
SOA_K_NORM: tl.constexpr,
|
||||
SOA_V_SCALE: tl.constexpr,
|
||||
SOA_V_ZERO: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
NORM_CORRECTION: tl.constexpr = 0,
|
||||
FP8_E4B15: tl.constexpr = 0, # 1 = use e4b15 (Ampere/Ada), 0 = e4nv (Hopper+)
|
||||
):
|
||||
"""Full dequant: reconstruct K (MSE centroids * norm or FP8) and V to fp16.
|
||||
|
||||
Opt#3 SoA layout: data region [bs, H, KD+VD] + per-block SoA metadata.
|
||||
"""
|
||||
pos = tl.program_id(0)
|
||||
bh = tl.program_id(1)
|
||||
bid = bh // NUM_KV_HEADS
|
||||
hid = bh % NUM_KV_HEADS
|
||||
|
||||
page_idx = pos // BLOCK_SIZE
|
||||
page_off = pos % BLOCK_SIZE
|
||||
block_num = tl.load(Block_table_ptr + bid * stride_bt_b + page_idx).to(tl.int64)
|
||||
|
||||
block_base = block_num * stride_cache_block
|
||||
slot_within_block = tl.cast(page_off, tl.int64)
|
||||
DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES
|
||||
data_base = (
|
||||
block_base
|
||||
+ slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT)
|
||||
+ tl.cast(hid, tl.int64) * DATA_BYTES_PER_SLOT
|
||||
)
|
||||
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast(
|
||||
hid, tl.int64
|
||||
) * (NUM_SOA_FIELDS * BLOCK_SIZE)
|
||||
knorm_u16_addr = head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block
|
||||
vscale_u16_addr = head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block
|
||||
vzero_u16_addr = head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block
|
||||
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < HEAD_DIM
|
||||
|
||||
# === K dequant ===
|
||||
ko_base = bid * stride_ko_b + hid * stride_ko_h + pos * stride_ko_s
|
||||
if KEY_FP8:
|
||||
k_raw = tl.load(KV_cache_ptr + data_base + d_offs, mask=d_mask, other=0)
|
||||
if FP8_E4B15:
|
||||
k_recon = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32)
|
||||
else:
|
||||
k_recon = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32)
|
||||
tl.store(K_out_ptr + ko_base + d_offs, k_recon.to(tl.float16), mask=d_mask)
|
||||
else:
|
||||
mse_bit_off = d_offs * MSE_BITS
|
||||
mse_byte_idx = mse_bit_off // 8
|
||||
mse_bit_shift = mse_bit_off % 8
|
||||
mse_umask = (1 << MSE_BITS) - 1
|
||||
|
||||
mse_raw0 = tl.load(
|
||||
KV_cache_ptr + data_base + mse_byte_idx, mask=d_mask, other=0
|
||||
).to(tl.int32)
|
||||
mse_raw1 = tl.load(
|
||||
KV_cache_ptr + data_base + mse_byte_idx + 1, mask=d_mask, other=0
|
||||
).to(tl.int32)
|
||||
raw16_key = mse_raw0 | (mse_raw1 << 8)
|
||||
mse_idx = (raw16_key >> mse_bit_shift) & mse_umask
|
||||
|
||||
k_mse = tl.load(Centroids_ptr + mse_idx, mask=d_mask, other=0.0)
|
||||
|
||||
# Opt#1: norm-correction is pre-folded into the stored scalar.
|
||||
# Opt#3: K-norm lives in the SoA region — single u16 load.
|
||||
norm_u16 = tl.load(KV_cache_u16_ptr + knorm_u16_addr)
|
||||
vec_norm = norm_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
|
||||
k_recon = vec_norm * k_mse
|
||||
tl.store(K_out_ptr + ko_base + d_offs, k_recon.to(tl.float16), mask=d_mask)
|
||||
|
||||
# === V dequant ===
|
||||
val_base = data_base + KEY_DATA_BYTES
|
||||
if VQB == 4:
|
||||
vb_idx = d_offs // 2
|
||||
vb_shift = (d_offs % 2) * 4
|
||||
val_raw = tl.load(KV_cache_ptr + val_base + vb_idx, mask=d_mask, other=0).to(
|
||||
tl.int32
|
||||
)
|
||||
v_idx = ((val_raw >> vb_shift) & 0xF).to(tl.float32)
|
||||
elif VQB == 3:
|
||||
val_bit_off = d_offs * 3
|
||||
val_byte_idx = val_bit_off // 8
|
||||
val_bit_shift = val_bit_off % 8
|
||||
val_raw0 = tl.load(
|
||||
KV_cache_ptr + val_base + val_byte_idx, mask=d_mask, other=0
|
||||
).to(tl.int32)
|
||||
val_raw1 = tl.load(
|
||||
KV_cache_ptr + val_base + val_byte_idx + 1, mask=d_mask, other=0
|
||||
).to(tl.int32)
|
||||
raw16_val = val_raw0 | (val_raw1 << 8)
|
||||
v_idx = ((raw16_val >> val_bit_shift) & 0x7).to(tl.float32)
|
||||
else:
|
||||
v_idx = tl.zeros([BLOCK_D], dtype=tl.float32)
|
||||
|
||||
# Opt#3: V scale/zero single-u16 loads from SoA region.
|
||||
sc_u16 = tl.load(KV_cache_u16_ptr + vscale_u16_addr)
|
||||
zr_u16 = tl.load(KV_cache_u16_ptr + vzero_u16_addr)
|
||||
v_scale = sc_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
v_zero = zr_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
v_vals = v_idx * v_scale + v_zero
|
||||
|
||||
vo_base = bid * stride_vo_b + hid * stride_vo_h + pos * stride_vo_s
|
||||
tl.store(V_out_ptr + vo_base + d_offs, v_vals.to(tl.float16), mask=d_mask)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 2: Reuse from triton_decode_attention.py
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Launcher — cached constants + fused GEMM
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_layout_cache: dict = {}
|
||||
|
||||
|
||||
def _get_layout(D, mse_bits, value_quant_bits, key_packed_size):
|
||||
"""Get cached layout constants."""
|
||||
key = (D, mse_bits, value_quant_bits, key_packed_size)
|
||||
cfg = _layout_cache.get(key)
|
||||
if cfg is None:
|
||||
val_data_bytes = math.ceil(D * value_quant_bits / 8)
|
||||
cfg = {
|
||||
"mse_bytes": math.ceil(D * mse_bits / 8),
|
||||
"val_data_bytes": val_data_bytes,
|
||||
"mse_bits": mse_bits,
|
||||
"n_centroids": 2**mse_bits,
|
||||
"BLOCK_D": triton.next_power_of_2(D),
|
||||
}
|
||||
_layout_cache[key] = cfg
|
||||
return cfg
|
||||
|
||||
|
||||
def triton_turboquant_decode_attention(
|
||||
query: torch.Tensor, # [B, Hq, D] — original query
|
||||
kv_cache: torch.Tensor, # [num_blocks, block_size, Hk, padded_slot] uint8
|
||||
block_table: torch.Tensor, # [B, max_num_blocks] int32
|
||||
seq_lens: torch.Tensor, # [B] int32
|
||||
Pi: torch.Tensor, # [D, D] float32
|
||||
centroids: torch.Tensor, # [n_centroids] float32
|
||||
scale: float,
|
||||
mse_bits: int,
|
||||
key_packed_size: int,
|
||||
value_quant_bits: int,
|
||||
key_fp8: bool = False,
|
||||
norm_correction: bool = False,
|
||||
PiT: torch.Tensor | None = None, # [D, D] pre-computed Pi.T contiguous
|
||||
# Pre-allocated buffers (optional, avoids per-call allocation)
|
||||
mid_o_buf: torch.Tensor | None = None,
|
||||
output_buf: torch.Tensor | None = None,
|
||||
lse_buf: torch.Tensor | None = None,
|
||||
buf_holder: Any = None,
|
||||
max_num_kv_splits: int = 32, # fixed split count (must be constant for cudagraph)
|
||||
sinks: torch.Tensor | None = None, # [Hq] float32, pre-computed sink logits
|
||||
) -> torch.Tensor:
|
||||
"""Launch fused TQ decode attention (Triton stage1 + stage2).
|
||||
|
||||
Returns: output tensor [B, Hq, D] in query's dtype.
|
||||
"""
|
||||
B, Hq, D = query.shape
|
||||
Hk = kv_cache.shape[2]
|
||||
block_size = kv_cache.shape[1]
|
||||
kv_group_size = Hq // Hk
|
||||
device = query.device
|
||||
|
||||
cfg = _get_layout(D, mse_bits, value_quant_bits, key_packed_size)
|
||||
|
||||
# Opt#3 SoA layout constants (derived locally; match store-side values).
|
||||
key_data_bytes = D if key_fp8 else cfg["mse_bytes"]
|
||||
data_bytes_per_slot = key_data_bytes + cfg["val_data_bytes"]
|
||||
meta_region_offset = block_size * Hk * data_bytes_per_slot
|
||||
num_soa_fields = 2 if key_fp8 else 3
|
||||
soa_k_norm = 0
|
||||
soa_v_scale = 0 if key_fp8 else 1
|
||||
soa_v_zero = 1 if key_fp8 else 2
|
||||
kv_cache_u16 = kv_cache.view(torch.uint16)
|
||||
|
||||
# Compute q_rot = q @ Pi.T (rotated query for MSE key scoring)
|
||||
# FP8 path: pass query directly (float16); kernel casts inline.
|
||||
# MSE path: still needs external GEMM (cuBLAS), so q_rot is float32.
|
||||
if key_fp8:
|
||||
q_rot = query.contiguous()
|
||||
else:
|
||||
q_float = query.float()
|
||||
if PiT is None:
|
||||
PiT = Pi.T.contiguous()
|
||||
q_rot = (q_float @ PiT).contiguous()
|
||||
|
||||
NUM_KV_SPLITS = max_num_kv_splits
|
||||
|
||||
if (
|
||||
mid_o_buf is not None
|
||||
and mid_o_buf.shape[0] >= B
|
||||
and mid_o_buf.shape[2] >= NUM_KV_SPLITS
|
||||
):
|
||||
mid_o = mid_o_buf[:B, :Hq, :NUM_KV_SPLITS, :]
|
||||
else:
|
||||
mid_o = torch.empty(
|
||||
B,
|
||||
Hq,
|
||||
NUM_KV_SPLITS,
|
||||
D + 1,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
if buf_holder is not None:
|
||||
buf_holder._tq_mid_o_buf = mid_o
|
||||
|
||||
# Stage 1: split-KV tiled attention scoring + value accumulation
|
||||
fp8_e4b15 = _use_fp8_e4b15(device.index or 0)
|
||||
BLOCK_KV = 4
|
||||
grid = (B, Hq, NUM_KV_SPLITS)
|
||||
|
||||
# Prepare sink pointer (convert to float32 on device if provided)
|
||||
use_sinks = sinks is not None
|
||||
|
||||
_tq_decode_stage1[grid](
|
||||
q_rot,
|
||||
kv_cache,
|
||||
kv_cache_u16,
|
||||
block_table,
|
||||
seq_lens,
|
||||
centroids,
|
||||
mid_o,
|
||||
sinks,
|
||||
q_rot.stride(0),
|
||||
q_rot.stride(1),
|
||||
kv_cache.stride(0),
|
||||
block_table.stride(0),
|
||||
mid_o.stride(0),
|
||||
mid_o.stride(1),
|
||||
mid_o.stride(2),
|
||||
NUM_KV_HEADS=Hk,
|
||||
HEAD_DIM=D,
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_KV_SPLITS=NUM_KV_SPLITS,
|
||||
KV_GROUP_SIZE=kv_group_size,
|
||||
MSE_BITS=mse_bits,
|
||||
MSE_BYTES=cfg["mse_bytes"],
|
||||
VQB=value_quant_bits,
|
||||
VAL_DATA_BYTES=cfg["val_data_bytes"],
|
||||
KEY_DATA_BYTES=key_data_bytes,
|
||||
META_REGION_OFFSET=meta_region_offset,
|
||||
NUM_SOA_FIELDS=num_soa_fields,
|
||||
SOA_K_NORM=soa_k_norm,
|
||||
SOA_V_SCALE=soa_v_scale,
|
||||
SOA_V_ZERO=soa_v_zero,
|
||||
ATTN_SCALE=scale,
|
||||
BLOCK_D=cfg["BLOCK_D"],
|
||||
BLOCK_KV=BLOCK_KV,
|
||||
KEY_FP8=1 if key_fp8 else 0,
|
||||
NORM_CORRECTION=1 if norm_correction else 0,
|
||||
FP8_E4B15=fp8_e4b15,
|
||||
USE_SINKS=1 if use_sinks else 0,
|
||||
num_warps=1,
|
||||
num_stages=1,
|
||||
)
|
||||
|
||||
# Stage 2: Reduce across KV splits
|
||||
if output_buf is not None and output_buf.shape[0] >= B:
|
||||
output = output_buf[:B, :Hq, :D]
|
||||
else:
|
||||
output = torch.empty(B, Hq, D, dtype=torch.float32, device=device)
|
||||
if buf_holder is not None:
|
||||
buf_holder._tq_output_buf = output
|
||||
if lse_buf is not None and lse_buf.shape[0] >= B:
|
||||
lse = lse_buf[:B, :Hq]
|
||||
else:
|
||||
lse = torch.empty(B, Hq, dtype=torch.float32, device=device)
|
||||
if buf_holder is not None:
|
||||
buf_holder._tq_lse_buf = lse
|
||||
|
||||
grid2 = (B, Hq)
|
||||
_fwd_kernel_stage2[grid2](
|
||||
mid_o,
|
||||
output,
|
||||
lse,
|
||||
seq_lens,
|
||||
mid_o.stride(0),
|
||||
mid_o.stride(1),
|
||||
mid_o.stride(2),
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
lse.stride(0),
|
||||
NUM_KV_SPLITS=NUM_KV_SPLITS,
|
||||
BLOCK_DV=cfg["BLOCK_D"],
|
||||
Lv=D,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
|
||||
return output.to(query.dtype)
|
||||
@@ -0,0 +1,669 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Optimized Triton TurboQuant decode attention (v2).
|
||||
|
||||
FLUTE-paper optimizations applied:
|
||||
1. Grouped Q heads: grid over (B, Hk, splits) instead of (B, Hq, splits).
|
||||
Each program loads BLOCK_M Q heads sharing a KV head into a 2D tile,
|
||||
enabling tl.dot on tensor cores (MFMA/WMMA) for both Q·K and P·V.
|
||||
2. Vectorized pair LUT: precompute pair_table[i][j] = (T[i], T[j])
|
||||
offline. At runtime, extract adjacent index pairs and fetch two
|
||||
dequantized centroids with a single gather, halving LUT lookups.
|
||||
3. exp2 instead of exp: scores pre-scaled by log2(e) so the hardware-
|
||||
native exp2 instruction replaces the more expensive exp.
|
||||
4. Wider index extraction: for 4-bit MSE, two adjacent 4-bit indices share
|
||||
a byte. One byte load yields both, eliminating redundant loads.
|
||||
5. Centroids pre-warmed in L1 at kernel start.
|
||||
6. BLOCK_KV = TILE_SIZE raised to 16-32 (from 4), reducing loop iterations
|
||||
and softmax rescaling overhead.
|
||||
|
||||
Stage 2 is reused unchanged from triton_decode_attention.py.
|
||||
"""
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.ops.triton_decode_attention import _fwd_kernel_stage2
|
||||
|
||||
from .triton_turboquant_decode import _use_fp8_e4b15
|
||||
|
||||
# ROCm prefers num_stages=1 in attention-like kernels to reduce shared-memory
|
||||
# pressure (mirrors the pattern used in triton_decode_attention.py and
|
||||
# triton_turboquant_decode.py).
|
||||
_is_hip = current_platform.is_rocm()
|
||||
|
||||
# On ROCm, bf16 has the same MFMA throughput as fp16 but wider dynamic range
|
||||
# (8-bit exponent vs 5-bit), which is safer for attention scores. On CUDA,
|
||||
# fp16 tensor cores may be faster than bf16 for some shapes.
|
||||
_DOT_DTYPE = tl.bfloat16 if _is_hip else tl.float16
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pair LUT construction (FLUTE) — called once at launcher time
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_pair_lut(centroids: torch.Tensor) -> torch.Tensor:
|
||||
"""Build vectorized pair lookup table.
|
||||
|
||||
For N centroids, returns a [N, N, 2] float32 tensor where
|
||||
pair_lut[i, j] = (centroids[i], centroids[j]).
|
||||
Flattened to [N*N, 2] for kernel indexing: pair_lut[i*N + j].
|
||||
|
||||
For 4-bit MSE (N=16): 16*16*2*4 = 2048 bytes — fits in L1/smem.
|
||||
For 3-bit MSE (N=8): 8*8*2*4 = 512 bytes.
|
||||
"""
|
||||
N = centroids.shape[0]
|
||||
# pair_lut[i,j,0] = centroids[i], pair_lut[i,j,1] = centroids[j]
|
||||
c = centroids.float()
|
||||
lut = torch.empty(N, N, 2, dtype=torch.float32, device=centroids.device)
|
||||
lut[:, :, 0] = c[:, None]
|
||||
lut[:, :, 1] = c[None, :]
|
||||
return lut.reshape(N * N, 2).contiguous()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 1 v2: Grouped-Q + pair LUT + tl.dot + exp2
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_decode_stage1_v2(
|
||||
Q_rot_ptr,
|
||||
KV_cache_ptr,
|
||||
KV_cache_u16_ptr, # uint16 view — Opt#3 SoA loads
|
||||
Block_table_ptr,
|
||||
Seq_lens_ptr,
|
||||
Centroids_ptr,
|
||||
Pair_lut_ptr,
|
||||
Mid_o_ptr,
|
||||
stride_qb,
|
||||
stride_qh,
|
||||
stride_cache_block, # bytes per block (bs*H*slot_aligned)
|
||||
stride_bt_b,
|
||||
stride_mid_b,
|
||||
stride_mid_h,
|
||||
stride_mid_s,
|
||||
NUM_Q_HEADS: tl.constexpr,
|
||||
NUM_KV_HEADS: tl.constexpr,
|
||||
HEAD_DIM: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
PADDED_SLOT: tl.constexpr,
|
||||
MAX_NUM_BLOCKS: tl.constexpr,
|
||||
NUM_KV_SPLITS: tl.constexpr,
|
||||
KV_GROUP_SIZE: tl.constexpr,
|
||||
MSE_BITS: tl.constexpr,
|
||||
MSE_BYTES: tl.constexpr,
|
||||
VQB: tl.constexpr,
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
# Opt#3 SoA layout
|
||||
KEY_DATA_BYTES: tl.constexpr,
|
||||
META_REGION_OFFSET: tl.constexpr,
|
||||
NUM_SOA_FIELDS: tl.constexpr,
|
||||
SOA_K_NORM: tl.constexpr,
|
||||
SOA_V_SCALE: tl.constexpr,
|
||||
SOA_V_ZERO: tl.constexpr,
|
||||
N_CENTROIDS: tl.constexpr,
|
||||
ATTN_SCALE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
TILE_SIZE: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
KEY_FP8: tl.constexpr,
|
||||
NORM_CORRECTION: tl.constexpr = 0,
|
||||
FP8_E4B15: tl.constexpr = 0,
|
||||
USE_PAIR_LUT: tl.constexpr = 0,
|
||||
USE_BF16_DOT: tl.constexpr = 0,
|
||||
):
|
||||
# Grid: (B, Hk, NUM_KV_SPLITS) — one program per KV head group
|
||||
bid = tl.program_id(0)
|
||||
kv_hid = tl.program_id(1)
|
||||
sid = tl.program_id(2)
|
||||
|
||||
seq_len = tl.load(Seq_lens_ptr + bid)
|
||||
split_len = tl.cdiv(seq_len, NUM_KV_SPLITS)
|
||||
split_start = split_len * sid
|
||||
split_end = tl.minimum(split_start + split_len, seq_len)
|
||||
|
||||
if split_start >= split_end:
|
||||
return
|
||||
|
||||
# Dimension and tile offsets
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < HEAD_DIM
|
||||
m_offs = tl.arange(0, BLOCK_M)
|
||||
t_offs = tl.arange(0, TILE_SIZE)
|
||||
|
||||
# Q head indices for this KV group: [kv_hid * GS, kv_hid * GS + GS)
|
||||
q_head_offs = kv_hid * KV_GROUP_SIZE + m_offs
|
||||
q_mask = m_offs < KV_GROUP_SIZE
|
||||
|
||||
# Load Q tile: [BLOCK_M, BLOCK_D] float32
|
||||
q_addrs = bid * stride_qb + q_head_offs[:, None] * stride_qh + d_offs[None, :]
|
||||
Q = tl.load(
|
||||
Q_rot_ptr + q_addrs,
|
||||
mask=q_mask[:, None] & d_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
# Pre-warm centroids in L1
|
||||
if not KEY_FP8:
|
||||
_c_warm = tl.load(Centroids_ptr + tl.arange(0, N_CENTROIDS)) # noqa: F841
|
||||
|
||||
# Loop-invariant bit-extraction constants for MSE keys
|
||||
if not KEY_FP8:
|
||||
if MSE_BITS == 4:
|
||||
pass # 4-bit byte extraction is computed inline in the tile loop
|
||||
else:
|
||||
# Generic bit extraction
|
||||
mse_bit_off = d_offs * MSE_BITS
|
||||
mse_byte_idx = mse_bit_off // 8
|
||||
mse_bit_shift = mse_bit_off % 8
|
||||
mse_mask_val = (1 << MSE_BITS) - 1
|
||||
|
||||
# Loop-invariant bit-extraction for 3-bit values
|
||||
if VQB == 3:
|
||||
val_bit_off = d_offs * 3
|
||||
val_byte_idx = val_bit_off // 8
|
||||
val_bit_shift = val_bit_off % 8
|
||||
|
||||
# exp2 pre-scaling
|
||||
RCP_LN2: tl.constexpr = 1.4426950408889634
|
||||
LN2: tl.constexpr = 0.6931471805599453
|
||||
QK_SCALE = ATTN_SCALE * RCP_LN2
|
||||
|
||||
# Online softmax accumulators: [BLOCK_M] for max, [BLOCK_M] for L
|
||||
M = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32)
|
||||
L = tl.zeros([BLOCK_M], dtype=tl.float32)
|
||||
|
||||
acc = tl.zeros([BLOCK_M, BLOCK_D], dtype=tl.float32)
|
||||
|
||||
bt_base = bid * stride_bt_b
|
||||
|
||||
# ================================================================
|
||||
# TILED LOOP: TILE_SIZE KV tokens per iteration
|
||||
# ================================================================
|
||||
for start_n in range(split_start, split_end, TILE_SIZE):
|
||||
kv_offs = start_n + t_offs
|
||||
kv_mask_1d = kv_offs < split_end
|
||||
|
||||
page_idx = kv_offs // BLOCK_SIZE
|
||||
page_off = kv_offs % BLOCK_SIZE
|
||||
block_nums = tl.load(
|
||||
Block_table_ptr + bt_base + page_idx,
|
||||
mask=kv_mask_1d,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
|
||||
# Opt#3 SoA addressing: data region then per-block SoA metadata.
|
||||
slot_within_block = page_off.to(tl.int64)
|
||||
block_base = block_nums * stride_cache_block
|
||||
DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES
|
||||
data_bases = (
|
||||
block_base
|
||||
+ slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT)
|
||||
+ tl.cast(kv_hid, tl.int64) * DATA_BYTES_PER_SLOT
|
||||
)
|
||||
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast(
|
||||
kv_hid, tl.int64
|
||||
) * (NUM_SOA_FIELDS * BLOCK_SIZE)
|
||||
knorm_u16_addrs = (
|
||||
head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block
|
||||
)
|
||||
vscale_u16_addrs = (
|
||||
head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block
|
||||
)
|
||||
vzero_u16_addrs = (
|
||||
head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# KEY DEQUANT → K_T: [BLOCK_D, TILE_SIZE] for tl.dot(Q, K_T)
|
||||
# ============================================================
|
||||
if KEY_FP8:
|
||||
k_addrs = data_bases[:, None] + d_offs[None, :]
|
||||
k_raw = tl.load(
|
||||
KV_cache_ptr + k_addrs,
|
||||
mask=kv_mask_1d[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
)
|
||||
if FP8_E4B15:
|
||||
K_f = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32)
|
||||
else:
|
||||
K_f = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32)
|
||||
# K_f: [TILE_SIZE, BLOCK_D] → transpose for tl.dot
|
||||
K_T = tl.trans(K_f)
|
||||
# S: [BLOCK_M, TILE_SIZE] (bf16 dot on ROCm for better dynamic range)
|
||||
if USE_BF16_DOT:
|
||||
S = QK_SCALE * tl.dot(Q.to(tl.bfloat16), K_T.to(tl.bfloat16))
|
||||
else:
|
||||
S = QK_SCALE * tl.dot(Q.to(tl.float16), K_T.to(tl.float16))
|
||||
else:
|
||||
# --- MSE dequantization ---
|
||||
if MSE_BITS == 4 and USE_PAIR_LUT:
|
||||
# OPT#1 (load-halving): FLUTE §3.2 pair LUT, but with each
|
||||
# packed byte loaded EXACTLY ONCE instead of twice (the
|
||||
# previous code built byte_addrs from d_offs // 2 which has
|
||||
# the pattern [0,0,1,1,2,2,...], duplicating every byte
|
||||
# load). We load the packed bytes as a [TILE_SIZE, HALF_D]
|
||||
# tile, decode both nibbles, then issue a single 3-D gather
|
||||
# against pair_lut which returns (T[lo], T[hi]) in slots
|
||||
# 0/1. Reshape [TILE_SIZE, HALF_D, 2] -> [TILE_SIZE, BLOCK_D]
|
||||
# interleaves the pairs into the expected dim order
|
||||
# (d=2k -> T[lo_k], d=2k+1 -> T[hi_k]).
|
||||
HALF_D: tl.constexpr = BLOCK_D // 2
|
||||
half_offs = tl.arange(0, HALF_D)
|
||||
# Each byte k covers dims (2k, 2k+1); it is in-range iff
|
||||
# 2k < HEAD_DIM. HEAD_DIM is always even in our layouts,
|
||||
# so this is equivalent to k < HEAD_DIM // 2.
|
||||
byte_mask = (half_offs * 2) < HEAD_DIM
|
||||
|
||||
byte_addrs = data_bases[:, None] + half_offs[None, :]
|
||||
byte_raw = tl.load(
|
||||
KV_cache_ptr + byte_addrs,
|
||||
mask=kv_mask_1d[:, None] & byte_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
|
||||
lo_idx = byte_raw & 0xF # [TILE_SIZE, HALF_D]
|
||||
hi_idx = (byte_raw >> 4) & 0xF # [TILE_SIZE, HALF_D]
|
||||
pair_key = lo_idx * N_CENTROIDS + hi_idx # [TILE_SIZE, HALF_D]
|
||||
|
||||
# Single 3-D gather: pair_lut[pair_key*2 + {0,1}]
|
||||
# -> [TILE_SIZE, HALF_D, 2] = (T[lo_k], T[hi_k]) per element.
|
||||
pair_slot = tl.arange(0, 2)
|
||||
c_pair = tl.load(
|
||||
Pair_lut_ptr + pair_key[:, :, None] * 2 + pair_slot[None, None, :],
|
||||
mask=(kv_mask_1d[:, None, None] & byte_mask[None, :, None]),
|
||||
other=0.0,
|
||||
)
|
||||
# Row-major reshape interleaves pairs:
|
||||
# out[t, 2k] = c_pair[t, k, 0] = T[lo_k]
|
||||
# out[t, 2k+1] = c_pair[t, k, 1] = T[hi_k]
|
||||
c_vals = tl.reshape(c_pair, [TILE_SIZE, BLOCK_D])
|
||||
elif MSE_BITS == 4:
|
||||
# 4-bit without pair LUT: exploit byte alignment
|
||||
half_idx = d_offs // 2
|
||||
nibble_shift = (d_offs % 2) * 4
|
||||
mse_addrs = data_bases[:, None] + half_idx[None, :]
|
||||
mse_raw = tl.load(
|
||||
KV_cache_ptr + mse_addrs,
|
||||
mask=kv_mask_1d[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
mse_idx = (mse_raw >> nibble_shift[None, :]) & 0xF
|
||||
c_vals = tl.load(
|
||||
Centroids_ptr + mse_idx,
|
||||
mask=kv_mask_1d[:, None] & d_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
else:
|
||||
# Generic 3-bit path: two byte loads → 16-bit → shift/mask
|
||||
mse_addrs0 = data_bases[:, None] + mse_byte_idx[None, :]
|
||||
mse_raw0 = tl.load(
|
||||
KV_cache_ptr + mse_addrs0,
|
||||
mask=kv_mask_1d[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
mse_raw1 = tl.load(
|
||||
KV_cache_ptr + mse_addrs0 + 1,
|
||||
mask=kv_mask_1d[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
raw16 = mse_raw0 | (mse_raw1 << 8)
|
||||
mse_idx = (raw16 >> mse_bit_shift[None, :]) & mse_mask_val
|
||||
c_vals = tl.load(
|
||||
Centroids_ptr + mse_idx,
|
||||
mask=kv_mask_1d[:, None] & d_mask[None, :],
|
||||
other=0.0,
|
||||
)
|
||||
|
||||
# Opt#1: norm-correction is pre-folded into the stored per-token
|
||||
# scalar at store time, so no per-tile sum+sqrt+divide here. The
|
||||
# stored vec_norm carries ||k||/||c_vec||; multiply by c_vals
|
||||
# below gives the original c_vec/||c_vec|| * ||k||.
|
||||
|
||||
# Opt#3: K-norms from per-block SoA region. Single u16 load per
|
||||
# token; contiguous across tokens within a block.
|
||||
norm_u16 = tl.load(
|
||||
KV_cache_u16_ptr + knorm_u16_addrs, mask=kv_mask_1d, other=0
|
||||
)
|
||||
vec_norms = norm_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
|
||||
# Reconstruct K: K_recon[t, d] = norm[t] * centroid[t, d]
|
||||
K_recon = c_vals * vec_norms[:, None]
|
||||
# K_T: [BLOCK_D, TILE_SIZE]
|
||||
K_T = tl.trans(K_recon)
|
||||
|
||||
# S: [BLOCK_M, TILE_SIZE] via tensor core MMA
|
||||
if USE_BF16_DOT:
|
||||
S = QK_SCALE * tl.dot(Q.to(tl.bfloat16), K_T.to(tl.bfloat16))
|
||||
else:
|
||||
S = QK_SCALE * tl.dot(Q.to(tl.float16), K_T.to(tl.float16))
|
||||
|
||||
# Mask out-of-range positions
|
||||
S = tl.where(kv_mask_1d[None, :], S, float("-inf"))
|
||||
|
||||
# ============================================================
|
||||
# ONLINE SOFTMAX with exp2
|
||||
# ============================================================
|
||||
m_j = tl.maximum(M, tl.max(S, axis=1))
|
||||
m_j = tl.where(m_j > float("-inf"), m_j, 0.0)
|
||||
P = tl.math.exp2(S - m_j[:, None])
|
||||
l_j = tl.sum(P, axis=1)
|
||||
alpha = tl.math.exp2(M - m_j)
|
||||
|
||||
acc = acc * alpha[:, None]
|
||||
L = L * alpha + l_j
|
||||
M = m_j
|
||||
|
||||
# ============================================================
|
||||
# VALUE DEQUANT → V: [TILE_SIZE, BLOCK_D] for tl.dot(P, V)
|
||||
# Opt#3: V data lives at data_base + KEY_DATA_BYTES; scale/zero
|
||||
# move to the SoA metadata region (one u16 load each per token).
|
||||
# ============================================================
|
||||
val_bases = data_bases + KEY_DATA_BYTES
|
||||
|
||||
if VQB == 3:
|
||||
val_addrs0 = val_bases[:, None] + val_byte_idx[None, :]
|
||||
val_raw0 = tl.load(
|
||||
KV_cache_ptr + val_addrs0,
|
||||
mask=kv_mask_1d[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
val_raw1 = tl.load(
|
||||
KV_cache_ptr + val_addrs0 + 1,
|
||||
mask=kv_mask_1d[:, None] & d_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
raw16 = val_raw0 | (val_raw1 << 8)
|
||||
v_idx = ((raw16 >> val_bit_shift[None, :]) & 0x7).to(tl.float32)
|
||||
else: # VQB == 4
|
||||
# OPT#3 (load-halving, value path): same pattern as OPT#1 for
|
||||
# keys -- load each byte exactly once as [TILE_SIZE, HALF_D],
|
||||
# decode both nibbles, then tl.interleave to [TILE_SIZE, BLOCK_D]
|
||||
# with pattern [v_lo0, v_hi0, v_lo1, v_hi1, ...].
|
||||
V_HALF_D: tl.constexpr = BLOCK_D // 2
|
||||
v_half_offs = tl.arange(0, V_HALF_D)
|
||||
v_byte_mask = (v_half_offs * 2) < HEAD_DIM
|
||||
|
||||
val_addrs = val_bases[:, None] + v_half_offs[None, :]
|
||||
val_byte = tl.load(
|
||||
KV_cache_ptr + val_addrs,
|
||||
mask=kv_mask_1d[:, None] & v_byte_mask[None, :],
|
||||
other=0,
|
||||
).to(tl.int32)
|
||||
v_lo = (val_byte & 0xF).to(tl.float32) # [T, HALF_D]
|
||||
v_hi = ((val_byte >> 4) & 0xF).to(tl.float32) # [T, HALF_D]
|
||||
v_idx = tl.interleave(v_lo, v_hi) # [T, BLOCK_D]
|
||||
|
||||
scale_u16 = tl.load(
|
||||
KV_cache_u16_ptr + vscale_u16_addrs, mask=kv_mask_1d, other=0
|
||||
)
|
||||
zero_u16 = tl.load(KV_cache_u16_ptr + vzero_u16_addrs, mask=kv_mask_1d, other=0)
|
||||
v_scales = scale_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
v_zeros = zero_u16.to(tl.float16, bitcast=True).to(tl.float32)
|
||||
V = v_idx * v_scales[:, None] + v_zeros[:, None]
|
||||
|
||||
# P·V accumulation via tensor core MMA
|
||||
# P: [BLOCK_M, TILE_SIZE], V: [TILE_SIZE, BLOCK_D]
|
||||
if USE_BF16_DOT:
|
||||
acc += tl.dot(P.to(tl.bfloat16), V.to(tl.bfloat16))
|
||||
else:
|
||||
acc += tl.dot(P.to(tl.float16), V.to(tl.float16))
|
||||
|
||||
# ================================================================
|
||||
# EPILOGUE: Store per-Q-head partial results for stage2
|
||||
# ================================================================
|
||||
# acc: [BLOCK_M, BLOCK_D], L: [BLOCK_M], M: [BLOCK_M]
|
||||
# Stage2 expects per-Q-head: mid_o[B, Hq, split, D+1]
|
||||
safe_L = tl.where(L > 0.0, L, 1.0)
|
||||
one_over_L = 1.0 / safe_L
|
||||
acc = acc * one_over_L[:, None]
|
||||
|
||||
# Convert M from log2 space back to natural log for stage2 compatibility
|
||||
lse = M * LN2 + tl.log(safe_L)
|
||||
|
||||
# Store each Q head's result using 2D scatter
|
||||
# q_head_offs: [BLOCK_M] = kv_hid * KV_GROUP_SIZE + m_offs
|
||||
out_addrs = (
|
||||
bid * stride_mid_b
|
||||
+ q_head_offs[:, None] * stride_mid_h
|
||||
+ sid * stride_mid_s
|
||||
+ d_offs[None, :]
|
||||
)
|
||||
tl.store(
|
||||
Mid_o_ptr + out_addrs,
|
||||
acc,
|
||||
mask=q_mask[:, None] & d_mask[None, :],
|
||||
)
|
||||
# Store LSE per Q head
|
||||
lse_addrs = (
|
||||
bid * stride_mid_b + q_head_offs * stride_mid_h + sid * stride_mid_s + HEAD_DIM
|
||||
)
|
||||
tl.store(Mid_o_ptr + lse_addrs, lse, mask=q_mask)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Launcher v2
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_layout_cache: dict = {}
|
||||
|
||||
|
||||
def _get_layout(D, mse_bits, value_quant_bits, key_packed_size):
|
||||
key = (D, mse_bits, value_quant_bits, key_packed_size)
|
||||
cfg = _layout_cache.get(key)
|
||||
if cfg is None:
|
||||
val_data_bytes = math.ceil(D * value_quant_bits / 8)
|
||||
cfg = {
|
||||
"mse_bytes": math.ceil(D * mse_bits / 8),
|
||||
"val_data_bytes": val_data_bytes,
|
||||
"mse_bits": mse_bits,
|
||||
"n_centroids": 2**mse_bits,
|
||||
"BLOCK_D": triton.next_power_of_2(D),
|
||||
}
|
||||
_layout_cache[key] = cfg
|
||||
return cfg
|
||||
|
||||
|
||||
def _get_pair_lut(centroids: torch.Tensor) -> torch.Tensor:
|
||||
"""Return a fresh pair-LUT for ``centroids`` on each call.
|
||||
|
||||
The LUT is tiny (N*N*2 fp32, e.g. 2KB for 4-bit MSE) so the build cost
|
||||
is negligible compared to attention. We avoid caching by data_ptr()
|
||||
because CUDA allocator memory reuse across different centroid tensors
|
||||
can silently return a stale LUT (subtle correctness bug). If this ever
|
||||
shows up on a profile, cache by a hash-of-values fingerprint instead.
|
||||
"""
|
||||
return build_pair_lut(centroids)
|
||||
|
||||
|
||||
def triton_turboquant_decode_attention_v2(
|
||||
query: torch.Tensor,
|
||||
kv_cache: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
seq_lens: torch.Tensor,
|
||||
Pi: torch.Tensor,
|
||||
centroids: torch.Tensor,
|
||||
scale: float,
|
||||
mse_bits: int,
|
||||
key_packed_size: int,
|
||||
value_quant_bits: int,
|
||||
value_packed_size: int,
|
||||
max_seq_len: int = 0, # unused; kept for backward compatibility
|
||||
key_fp8: bool = False,
|
||||
norm_correction: bool = False,
|
||||
PiT: torch.Tensor | None = None,
|
||||
# Pre-allocated buffers (optional, required for CUDA-graph stability).
|
||||
mid_o_buf: torch.Tensor | None = None,
|
||||
output_buf: torch.Tensor | None = None,
|
||||
lse_buf: torch.Tensor | None = None,
|
||||
buf_holder: Any = None,
|
||||
# Fixed split count — MUST be a compile-time constant across iterations
|
||||
# for CUDA-graph capture/replay to work. Mirrors the v1 launcher contract.
|
||||
max_num_kv_splits: int = 32,
|
||||
) -> torch.Tensor:
|
||||
"""Launch optimized TQ decode attention (v2 stage1 + shared stage2).
|
||||
|
||||
Follows the same buffer-reuse + fixed-grid contract as the v1 launcher
|
||||
(triton_turboquant_decode_attention) so the backend can capture a CUDA
|
||||
graph across both versions.
|
||||
"""
|
||||
B, Hq, D = query.shape
|
||||
Hk = kv_cache.shape[2]
|
||||
block_size = kv_cache.shape[1]
|
||||
padded_slot = kv_cache.shape[3]
|
||||
max_num_blocks = block_table.shape[1]
|
||||
n_centroids = centroids.shape[0]
|
||||
kv_group_size = Hq // Hk
|
||||
device = query.device
|
||||
del max_seq_len # no longer used: splits is fixed via max_num_kv_splits
|
||||
|
||||
cfg = _get_layout(D, mse_bits, value_quant_bits, key_packed_size)
|
||||
|
||||
# Opt#3 SoA layout constants (match store-side computation).
|
||||
key_data_bytes = D if key_fp8 else cfg["mse_bytes"]
|
||||
data_bytes_per_slot = key_data_bytes + cfg["val_data_bytes"]
|
||||
meta_region_offset = block_size * Hk * data_bytes_per_slot
|
||||
num_soa_fields = 2 if key_fp8 else 3
|
||||
soa_k_norm = 0
|
||||
soa_v_scale = 0 if key_fp8 else 1
|
||||
soa_v_zero = 1 if key_fp8 else 2
|
||||
kv_cache_u16 = kv_cache.view(torch.uint16)
|
||||
|
||||
# Compute q_rot = q @ Pi.T
|
||||
if key_fp8:
|
||||
q_rot = query.float().contiguous()
|
||||
else:
|
||||
q_float = query.float()
|
||||
if PiT is None:
|
||||
PiT = Pi.T.contiguous()
|
||||
q_rot = (q_float @ PiT).contiguous()
|
||||
|
||||
# BLOCK_M: pad KV_GROUP_SIZE to power of 2, minimum 16 for tensor cores
|
||||
BLOCK_M = max(16, triton.next_power_of_2(kv_group_size))
|
||||
|
||||
# TILE_SIZE (BLOCK_KV): tokens per inner-loop iteration
|
||||
TILE_SIZE = 16
|
||||
|
||||
# Fixed split count — must be constant across calls for cudagraph replay.
|
||||
NUM_KV_SPLITS = max_num_kv_splits
|
||||
|
||||
# --- mid_o buffer reuse (same pattern as v1 launcher) ---
|
||||
if (
|
||||
mid_o_buf is not None
|
||||
and mid_o_buf.shape[0] >= B
|
||||
and mid_o_buf.shape[2] >= NUM_KV_SPLITS
|
||||
):
|
||||
mid_o = mid_o_buf[:B, :Hq, :NUM_KV_SPLITS, :]
|
||||
else:
|
||||
mid_o = torch.empty(
|
||||
B,
|
||||
Hq,
|
||||
NUM_KV_SPLITS,
|
||||
D + 1,
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
)
|
||||
if buf_holder is not None:
|
||||
buf_holder._tq_mid_o_buf = mid_o
|
||||
|
||||
fp8_e4b15 = _use_fp8_e4b15(device.index or 0)
|
||||
|
||||
# Build pair LUT for 4-bit MSE (FLUTE §3.2)
|
||||
use_pair_lut = mse_bits == 4 and not key_fp8
|
||||
pair_lut = _get_pair_lut(centroids) if use_pair_lut else centroids
|
||||
|
||||
# Platform-dependent pipelining depth: ROCm prefers num_stages=1.
|
||||
stage1_num_stages = 1 if _is_hip else 2
|
||||
|
||||
# --- Stage 1: v2 kernel ---
|
||||
# Grid over KV heads (not Q heads) — each program handles all Q heads
|
||||
grid = (B, Hk, NUM_KV_SPLITS)
|
||||
_tq_decode_stage1_v2[grid](
|
||||
q_rot,
|
||||
kv_cache,
|
||||
kv_cache_u16,
|
||||
block_table,
|
||||
seq_lens,
|
||||
centroids,
|
||||
pair_lut,
|
||||
mid_o,
|
||||
q_rot.stride(0),
|
||||
q_rot.stride(1),
|
||||
kv_cache.stride(0),
|
||||
block_table.stride(0),
|
||||
mid_o.stride(0),
|
||||
mid_o.stride(1),
|
||||
mid_o.stride(2),
|
||||
NUM_Q_HEADS=Hq,
|
||||
NUM_KV_HEADS=Hk,
|
||||
HEAD_DIM=D,
|
||||
BLOCK_SIZE=block_size,
|
||||
PADDED_SLOT=padded_slot,
|
||||
MAX_NUM_BLOCKS=max_num_blocks,
|
||||
NUM_KV_SPLITS=NUM_KV_SPLITS,
|
||||
KV_GROUP_SIZE=kv_group_size,
|
||||
MSE_BITS=mse_bits,
|
||||
MSE_BYTES=cfg["mse_bytes"],
|
||||
VQB=value_quant_bits,
|
||||
VAL_DATA_BYTES=cfg["val_data_bytes"],
|
||||
KEY_DATA_BYTES=key_data_bytes,
|
||||
META_REGION_OFFSET=meta_region_offset,
|
||||
NUM_SOA_FIELDS=num_soa_fields,
|
||||
SOA_K_NORM=soa_k_norm,
|
||||
SOA_V_SCALE=soa_v_scale,
|
||||
SOA_V_ZERO=soa_v_zero,
|
||||
N_CENTROIDS=n_centroids,
|
||||
ATTN_SCALE=scale,
|
||||
BLOCK_D=cfg["BLOCK_D"],
|
||||
TILE_SIZE=TILE_SIZE,
|
||||
BLOCK_M=BLOCK_M,
|
||||
KEY_FP8=1 if key_fp8 else 0,
|
||||
NORM_CORRECTION=1 if norm_correction else 0,
|
||||
FP8_E4B15=fp8_e4b15,
|
||||
USE_PAIR_LUT=1 if use_pair_lut else 0,
|
||||
USE_BF16_DOT=1 if _is_hip else 0,
|
||||
num_warps=4,
|
||||
num_stages=stage1_num_stages,
|
||||
)
|
||||
|
||||
# --- output / lse buffer reuse (same pattern as v1 launcher) ---
|
||||
if output_buf is not None and output_buf.shape[0] >= B:
|
||||
output = output_buf[:B, :Hq, :D]
|
||||
else:
|
||||
output = torch.empty(B, Hq, D, dtype=torch.float32, device=device)
|
||||
if buf_holder is not None:
|
||||
buf_holder._tq_output_buf = output
|
||||
if lse_buf is not None and lse_buf.shape[0] >= B:
|
||||
lse = lse_buf[:B, :Hq]
|
||||
else:
|
||||
lse = torch.empty(B, Hq, dtype=torch.float32, device=device)
|
||||
if buf_holder is not None:
|
||||
buf_holder._tq_lse_buf = lse
|
||||
|
||||
# --- Stage 2: reduce across KV splits (unchanged) ---
|
||||
grid2 = (B, Hq)
|
||||
_fwd_kernel_stage2[grid2](
|
||||
mid_o,
|
||||
output,
|
||||
lse,
|
||||
seq_lens,
|
||||
mid_o.stride(0),
|
||||
mid_o.stride(1),
|
||||
mid_o.stride(2),
|
||||
output.stride(0),
|
||||
output.stride(1),
|
||||
lse.stride(0),
|
||||
NUM_KV_SPLITS=NUM_KV_SPLITS,
|
||||
BLOCK_DV=cfg["BLOCK_D"],
|
||||
Lv=D,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
|
||||
return output.to(query.dtype)
|
||||
@@ -0,0 +1,524 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fused Triton kernels for TurboQuant KV store.
|
||||
|
||||
Two kernels:
|
||||
1. _tq_fused_store_fp8: FP8 key scatter + value uniform quantization.
|
||||
2. _tq_fused_store_mse: Fused binary-search bucketize + MSE index
|
||||
packing + value quantization.
|
||||
|
||||
The launcher `triton_turboquant_store` selects the appropriate kernel.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
from .triton_turboquant_decode import _use_fp8_e4b15
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Shared: value uniform quantization + pack + SoA scale/zero store
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_quantized_value(
|
||||
Value_ptr,
|
||||
KV_cache_ptr,
|
||||
KV_cache_u16_ptr, # Opt#3: u16 view for SoA scale/zero stores
|
||||
base, # pid * D offset into Value_ptr
|
||||
data_base, # byte offset into KV_cache_ptr for this slot+head's DATA region
|
||||
vscale_u16_addr, # u16 element index for V-scale in the SoA region
|
||||
vzero_u16_addr, # u16 element index for V-zero in the SoA region
|
||||
d_offs, # tl.arange(0, BLOCK_D)
|
||||
d_mask, # d_offs < D
|
||||
D: tl.constexpr,
|
||||
# Offset (bytes) from data_base to V-data region within the slot's data.
|
||||
# For MSE keys this equals MSE_BYTES; for FP8 keys it equals D.
|
||||
V_DATA_OFFSET: tl.constexpr,
|
||||
VQB: tl.constexpr,
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
BLOCK_VAL: tl.constexpr,
|
||||
BLOCK_GRP: tl.constexpr,
|
||||
):
|
||||
"""Uniform quantization of values to VQB bits, pack, and store with
|
||||
scale/zero in the per-block SoA metadata region (Opt#3 layout)."""
|
||||
if VQB == 3:
|
||||
val_vec = tl.load(Value_ptr + base + d_offs, mask=d_mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
val_min = tl.min(tl.where(d_mask, val_vec, float("inf")), axis=0)
|
||||
val_max = tl.max(tl.where(d_mask, val_vec, -float("inf")), axis=0)
|
||||
v_scale = (val_max - val_min) / 7.0
|
||||
v_scale = tl.where(v_scale > 1e-8, v_scale, 1e-8)
|
||||
|
||||
q_vals = tl.minimum(
|
||||
tl.maximum(((val_vec - val_min) / v_scale + 0.5).to(tl.int32), 0), 7
|
||||
)
|
||||
|
||||
grp_offs = tl.arange(0, BLOCK_GRP)
|
||||
grp_mask = grp_offs < (D // 8)
|
||||
q_grp = tl.reshape(q_vals, [BLOCK_GRP, 8])
|
||||
shifts_3bit = tl.arange(0, 8) * 3
|
||||
packed_24 = tl.sum(q_grp << shifts_3bit[None, :], axis=1)
|
||||
b0 = (packed_24 & 0xFF).to(tl.uint8)
|
||||
b1 = ((packed_24 >> 8) & 0xFF).to(tl.uint8)
|
||||
b2 = ((packed_24 >> 16) & 0xFF).to(tl.uint8)
|
||||
tl.store(
|
||||
KV_cache_ptr + data_base + V_DATA_OFFSET + grp_offs * 3,
|
||||
b0,
|
||||
mask=grp_mask,
|
||||
)
|
||||
tl.store(
|
||||
KV_cache_ptr + data_base + V_DATA_OFFSET + grp_offs * 3 + 1,
|
||||
b1,
|
||||
mask=grp_mask,
|
||||
)
|
||||
tl.store(
|
||||
KV_cache_ptr + data_base + V_DATA_OFFSET + grp_offs * 3 + 2,
|
||||
b2,
|
||||
mask=grp_mask,
|
||||
)
|
||||
else: # VQB == 4
|
||||
val_vec = tl.load(Value_ptr + base + d_offs, mask=d_mask, other=0.0).to(
|
||||
tl.float32
|
||||
)
|
||||
val_min = tl.min(tl.where(d_mask, val_vec, float("inf")), axis=0)
|
||||
val_max = tl.max(tl.where(d_mask, val_vec, -float("inf")), axis=0)
|
||||
v_scale = (val_max - val_min) / 15.0
|
||||
v_scale = tl.where(v_scale > 1e-8, v_scale, 1e-8)
|
||||
|
||||
# Quantize all D elements from register (no re-load)
|
||||
q_all = tl.minimum(
|
||||
tl.maximum(((val_vec - val_min) / v_scale + 0.5).to(tl.int32), 0), 15
|
||||
)
|
||||
q_pairs = tl.reshape(q_all, [BLOCK_D // 2, 2])
|
||||
shifts_4 = tl.arange(0, 2) * 4
|
||||
packed_val = tl.sum((q_pairs & 0xF) << shifts_4[None, :], axis=1).to(tl.uint8)
|
||||
val_offs = tl.arange(0, BLOCK_D // 2)
|
||||
val_mask = val_offs < VAL_DATA_BYTES
|
||||
tl.store(
|
||||
KV_cache_ptr + data_base + V_DATA_OFFSET + val_offs,
|
||||
packed_val,
|
||||
mask=val_mask,
|
||||
)
|
||||
|
||||
# ── SoA scale/zero store (Opt#3) ──────────────────────────────────
|
||||
# Scale and zero each get one fp16 stored at their token's slot index
|
||||
# within the per-head SoA region. Single u16 store per field vs the
|
||||
# previous 2×u8 writes per field.
|
||||
sc_u16 = v_scale.to(tl.float16).to(tl.uint16, bitcast=True)
|
||||
zr_u16 = val_min.to(tl.float16).to(tl.uint16, bitcast=True)
|
||||
tl.store(KV_cache_u16_ptr + vscale_u16_addr, sc_u16)
|
||||
tl.store(KV_cache_u16_ptr + vzero_u16_addr, zr_u16)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# FP8 key store + value uniform quantization
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_fused_store_fp8(
|
||||
Key_ptr, # [NH, D] float16/bfloat16 — raw keys
|
||||
Value_ptr, # [NH, D] float16/bfloat16 — raw values
|
||||
KV_cache_ptr, # [total_bytes] uint8 (flattened view)
|
||||
KV_cache_u16_ptr, # uint16-aliased view of the same storage (Opt#3)
|
||||
Slot_mapping_ptr, # [N] int32 — per-token slot indices
|
||||
# Cache strides (stride_cache_block = bytes per block = bs*H*slot_aligned)
|
||||
stride_cache_block: tl.constexpr,
|
||||
# Dimensions
|
||||
D: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
# SoA layout (Opt#3): data region is [bs, H, KEY_DATA_BYTES+VAL_DATA_BYTES],
|
||||
# followed by metadata region [H, NUM_SOA_FIELDS, bs] of fp16.
|
||||
KEY_DATA_BYTES: tl.constexpr, # FP8: = D
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
META_REGION_OFFSET: tl.constexpr, # bytes from block start to SoA region
|
||||
NUM_SOA_FIELDS: tl.constexpr, # FP8: 2 (V-scale, V-zero)
|
||||
SOA_V_SCALE: tl.constexpr, # FP8: 0
|
||||
SOA_V_ZERO: tl.constexpr, # FP8: 1
|
||||
# Value quantization
|
||||
VQB: tl.constexpr,
|
||||
# Packing block sizes
|
||||
BLOCK_VAL: tl.constexpr,
|
||||
BLOCK_GRP: tl.constexpr = 16,
|
||||
FP8_E4B15: tl.constexpr = 0, # 1 = e4b15 (Ampere/Ada), 0 = e4nv (Hopper+)
|
||||
):
|
||||
"""FP8 key cast+scatter + value uniform quantization (SoA layout)."""
|
||||
pid = tl.program_id(0)
|
||||
token_idx = pid // H
|
||||
head_idx = pid % H
|
||||
|
||||
slot = tl.load(Slot_mapping_ptr + token_idx)
|
||||
if slot < 0:
|
||||
return
|
||||
blk = (slot // BLOCK_SIZE).to(tl.int64)
|
||||
off = (slot % BLOCK_SIZE).to(tl.int64)
|
||||
head_idx_i64 = tl.cast(head_idx, tl.int64)
|
||||
|
||||
# Block start byte address — stride_cache_block is unchanged (same total
|
||||
# bytes per block as the AoS layout).
|
||||
block_base = blk * stride_cache_block
|
||||
|
||||
# Data region: per-slot stride = H * DATA_BYTES_PER_SLOT; per-head stride
|
||||
# within a slot = DATA_BYTES_PER_SLOT.
|
||||
DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES
|
||||
data_base = (
|
||||
block_base
|
||||
+ off * (H * DATA_BYTES_PER_SLOT)
|
||||
+ head_idx_i64 * DATA_BYTES_PER_SLOT
|
||||
)
|
||||
|
||||
# SoA metadata region (u16 element indexing): head strip = NUM_SOA_FIELDS
|
||||
# consecutive bs-long fp16 arrays.
|
||||
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + head_idx_i64 * (
|
||||
NUM_SOA_FIELDS * BLOCK_SIZE
|
||||
)
|
||||
vscale_u16_addr = head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + off
|
||||
vzero_u16_addr = head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + off
|
||||
|
||||
base = pid * D
|
||||
|
||||
# ── FP8 KEY: cast to FP8 in-kernel and store at data_base ────────
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < D
|
||||
k_vals = tl.load(Key_ptr + base + d_offs, mask=d_mask, other=0.0)
|
||||
k_fp8 = k_vals.to(tl.float8e4b15) if FP8_E4B15 else k_vals.to(tl.float8e4nv)
|
||||
k_bytes = k_fp8.to(tl.uint8, bitcast=True)
|
||||
tl.store(KV_cache_ptr + data_base + d_offs, k_bytes, mask=d_mask)
|
||||
|
||||
# ── VALUE QUANTIZE + PACK + SoA scale/zero store ─────────────────
|
||||
_store_quantized_value(
|
||||
Value_ptr,
|
||||
KV_cache_ptr,
|
||||
KV_cache_u16_ptr,
|
||||
base,
|
||||
data_base,
|
||||
vscale_u16_addr,
|
||||
vzero_u16_addr,
|
||||
d_offs,
|
||||
d_mask,
|
||||
D=D,
|
||||
V_DATA_OFFSET=KEY_DATA_BYTES, # V follows K in the data region
|
||||
VQB=VQB,
|
||||
VAL_DATA_BYTES=VAL_DATA_BYTES,
|
||||
BLOCK_D=BLOCK_D,
|
||||
BLOCK_VAL=BLOCK_VAL,
|
||||
BLOCK_GRP=BLOCK_GRP,
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Fused MSE store: bucketize + MSE index pack + norm store + value pack
|
||||
# (eliminates 4 PyTorch kernel launches per layer vs pack-only kernel)
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _tq_fused_store_mse(
|
||||
# Post-rotation inputs
|
||||
Y_ptr, # [NH, D] float32 — rotated normalized keys (x_hat @ PiT)
|
||||
Norms_ptr, # [NH] float32 — key vector norms (||k||)
|
||||
Value_ptr, # [NH, D] float32 — raw values
|
||||
# Quantization tables
|
||||
Midpoints_ptr, # [n_centroids-1] float32
|
||||
Centroids_ptr, # [n_centroids] float32 — used only when NORM_CORRECTION=1
|
||||
# Cache and indexing
|
||||
KV_cache_ptr, # [total_bytes] uint8 (flattened view)
|
||||
KV_cache_u16_ptr, # uint16-aliased view of the same storage (Opt#3)
|
||||
Slot_mapping_ptr, # [N] int32 — per-token slot indices
|
||||
# Cache strides (stride_cache_block = bs*H*slot_aligned, unchanged)
|
||||
stride_cache_block: tl.constexpr,
|
||||
# Dimensions
|
||||
D: tl.constexpr,
|
||||
H: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr,
|
||||
BLOCK_D: tl.constexpr,
|
||||
# SoA layout (Opt#3): data region [bs, H, MSE_BYTES+VAL_DATA_BYTES],
|
||||
# then metadata region [H, NUM_SOA_FIELDS, bs] of fp16.
|
||||
MSE_BYTES: tl.constexpr,
|
||||
VAL_DATA_BYTES: tl.constexpr,
|
||||
META_REGION_OFFSET: tl.constexpr, # bytes
|
||||
NUM_SOA_FIELDS: tl.constexpr, # MSE: 3 (k_norm, v_scale, v_zero)
|
||||
SOA_K_NORM: tl.constexpr, # MSE: 0
|
||||
SOA_V_SCALE: tl.constexpr, # MSE: 1
|
||||
SOA_V_ZERO: tl.constexpr, # MSE: 2
|
||||
# Value quantization
|
||||
VQB: tl.constexpr,
|
||||
# Packing block sizes
|
||||
BLOCK_VAL: tl.constexpr,
|
||||
# MSE params
|
||||
MSE_BITS: tl.constexpr,
|
||||
N_CENTROIDS: tl.constexpr,
|
||||
# When 1, fold the per-token centroid-vector norm ||c_t|| into the
|
||||
# stored scalar: store ||k_t|| / ||c_t|| instead of ||k_t||. This
|
||||
# removes the per-tile sum+sqrt+divide cost from the attention kernel's
|
||||
# K-load path. Load-side math becomes a plain multiply K = c * stored.
|
||||
NORM_CORRECTION: tl.constexpr = 0,
|
||||
BLOCK_GRP: tl.constexpr = 16,
|
||||
):
|
||||
"""Fused MSE quantize + pack + SoA-metadata store (Opt#3 layout).
|
||||
|
||||
Binary-search bucketize → MSE-index pack → K-norm store (SoA) → value
|
||||
uniform quantize + pack → V scale/zero store (SoA), in one kernel.
|
||||
"""
|
||||
pid = tl.program_id(0)
|
||||
token_idx = pid // H
|
||||
head_idx = pid % H
|
||||
|
||||
slot = tl.load(Slot_mapping_ptr + token_idx)
|
||||
if slot < 0:
|
||||
return
|
||||
blk = (slot // BLOCK_SIZE).to(tl.int64)
|
||||
off = (slot % BLOCK_SIZE).to(tl.int64)
|
||||
head_idx_i64 = tl.cast(head_idx, tl.int64)
|
||||
|
||||
# Block start byte address — stride unchanged from AoS (same total
|
||||
# bytes per block); layout inside the block differs.
|
||||
block_base = blk * stride_cache_block
|
||||
|
||||
# Data region offset (packed K indices + packed V data).
|
||||
DATA_BYTES_PER_SLOT: tl.constexpr = MSE_BYTES + VAL_DATA_BYTES
|
||||
data_base = (
|
||||
block_base
|
||||
+ off * (H * DATA_BYTES_PER_SLOT)
|
||||
+ head_idx_i64 * DATA_BYTES_PER_SLOT
|
||||
)
|
||||
|
||||
# SoA metadata addresses (u16 element indexing).
|
||||
head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + head_idx_i64 * (
|
||||
NUM_SOA_FIELDS * BLOCK_SIZE
|
||||
)
|
||||
knorm_u16_addr = head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + off
|
||||
vscale_u16_addr = head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + off
|
||||
vzero_u16_addr = head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + off
|
||||
|
||||
base = pid * D
|
||||
d_offs = tl.arange(0, BLOCK_D)
|
||||
d_mask = d_offs < D
|
||||
|
||||
# ── 1. BINARY SEARCH BUCKETIZE ────────────────────────────────────
|
||||
y_vec = tl.load(Y_ptr + base + d_offs, mask=d_mask, other=0.0)
|
||||
lo = tl.zeros([BLOCK_D], dtype=tl.int32)
|
||||
hi = tl.full([BLOCK_D], N_CENTROIDS - 1, dtype=tl.int32)
|
||||
for _ in range(MSE_BITS):
|
||||
mid = (lo + hi) >> 1
|
||||
safe_mid = tl.minimum(mid, N_CENTROIDS - 2)
|
||||
mid_val = tl.load(Midpoints_ptr + safe_mid, mask=d_mask, other=0.0)
|
||||
lo = tl.where(y_vec >= mid_val, mid + 1, lo)
|
||||
hi = tl.where(y_vec >= mid_val, hi, mid)
|
||||
idx = tl.minimum(lo, N_CENTROIDS - 1)
|
||||
|
||||
# ── 2. PACK MSE INDICES into data region at data_base[0:MSE_BYTES] ──
|
||||
if MSE_BITS == 4:
|
||||
idx_pairs = tl.reshape(idx, [BLOCK_D // 2, 2])
|
||||
shifts_4 = tl.arange(0, 2) * 4
|
||||
packed = tl.sum((idx_pairs & 0xF) << shifts_4[None, :], axis=1).to(tl.uint8)
|
||||
mse_offs = tl.arange(0, BLOCK_D // 2)
|
||||
mse_mask = mse_offs < MSE_BYTES
|
||||
tl.store(KV_cache_ptr + data_base + mse_offs, packed, mask=mse_mask)
|
||||
|
||||
elif MSE_BITS == 3:
|
||||
grp_offs = tl.arange(0, BLOCK_GRP)
|
||||
grp_mask = grp_offs < (D // 8)
|
||||
idx_grp = tl.reshape(idx, [BLOCK_GRP, 8])
|
||||
shifts_3 = tl.arange(0, 8) * 3
|
||||
packed_24 = tl.sum((idx_grp & 0x7) << shifts_3[None, :], axis=1)
|
||||
b0 = (packed_24 & 0xFF).to(tl.uint8)
|
||||
b1 = ((packed_24 >> 8) & 0xFF).to(tl.uint8)
|
||||
b2 = ((packed_24 >> 16) & 0xFF).to(tl.uint8)
|
||||
tl.store(KV_cache_ptr + data_base + grp_offs * 3, b0, mask=grp_mask)
|
||||
tl.store(KV_cache_ptr + data_base + grp_offs * 3 + 1, b1, mask=grp_mask)
|
||||
tl.store(KV_cache_ptr + data_base + grp_offs * 3 + 2, b2, mask=grp_mask)
|
||||
|
||||
# ── 3. STORE K-norm in SoA region (single u16 store) ─────────────
|
||||
# Norm-correction (Opt#1) still folds 1/||c_vec|| into the stored value;
|
||||
# only the destination moved to the SoA K-norm array.
|
||||
vn_f32 = tl.load(Norms_ptr + pid) # ||k_t||, fp32
|
||||
|
||||
if NORM_CORRECTION:
|
||||
safe_idx = tl.minimum(tl.maximum(idx, 0), N_CENTROIDS - 1)
|
||||
c_vals = tl.load(Centroids_ptr + safe_idx, mask=d_mask, other=0.0)
|
||||
c_norm_sq = tl.sum(tl.where(d_mask, c_vals * c_vals, 0.0), axis=0)
|
||||
c_inv_norm = 1.0 / tl.sqrt(c_norm_sq + 1e-16)
|
||||
vn_f32 = vn_f32 * c_inv_norm
|
||||
|
||||
vn_u16 = vn_f32.to(tl.float16).to(tl.uint16, bitcast=True)
|
||||
tl.store(KV_cache_u16_ptr + knorm_u16_addr, vn_u16)
|
||||
|
||||
# ── 4. VALUE QUANTIZE + PACK + SoA scale/zero store ──────────────
|
||||
_store_quantized_value(
|
||||
Value_ptr,
|
||||
KV_cache_ptr,
|
||||
KV_cache_u16_ptr,
|
||||
base,
|
||||
data_base,
|
||||
vscale_u16_addr,
|
||||
vzero_u16_addr,
|
||||
d_offs,
|
||||
d_mask,
|
||||
D=D,
|
||||
V_DATA_OFFSET=MSE_BYTES, # V follows K indices in the data region
|
||||
VQB=VQB,
|
||||
VAL_DATA_BYTES=VAL_DATA_BYTES,
|
||||
BLOCK_D=BLOCK_D,
|
||||
BLOCK_VAL=BLOCK_VAL,
|
||||
BLOCK_GRP=BLOCK_GRP,
|
||||
)
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
# Launcher
|
||||
# ═══════════════════════════════════════════════════════════════════════
|
||||
|
||||
|
||||
def triton_turboquant_store(
|
||||
key: torch.Tensor, # [N, H, D] — raw keys (post-RoPE)
|
||||
value: torch.Tensor, # [N, H, D] — raw values
|
||||
kv_cache: torch.Tensor, # [num_blocks, block_size, Hk, padded_slot] uint8
|
||||
slot_mapping: torch.Tensor, # [N] int32
|
||||
PiT: torch.Tensor, # [D, D] float32
|
||||
midpoints: torch.Tensor, # [n_centroids-1] float32
|
||||
mse_bits: int,
|
||||
key_packed_size: int,
|
||||
value_quant_bits: int,
|
||||
key_fp8: bool = False,
|
||||
centroids: torch.Tensor
|
||||
| None = None, # [n_centroids] float32 — required when norm_correction=True
|
||||
norm_correction: bool = False,
|
||||
):
|
||||
"""Launch TQ store kernel (FP8 or MSE path).
|
||||
|
||||
When ``norm_correction=True`` (default off; set by MSE norm-correction
|
||||
presets), the centroid-vector norm ||c_t|| is folded into the stored
|
||||
per-token scalar at store time (stored value = ||k_t|| / ||c_t||). The
|
||||
load-side attention kernel no longer computes per-tile sum+sqrt+divide,
|
||||
saving ~26% of TQ overhead on gpt-oss decode (ctx=4096). ``centroids``
|
||||
must be provided in this mode.
|
||||
"""
|
||||
if norm_correction and not key_fp8:
|
||||
assert centroids is not None, (
|
||||
"norm_correction=True requires centroids tensor; got None. "
|
||||
"Pass the same [n_centroids] fp32 tensor used to derive midpoints."
|
||||
)
|
||||
N, H, D = key.shape
|
||||
NH = N * H
|
||||
block_size = kv_cache.shape[1]
|
||||
BLOCK_D = triton.next_power_of_2(D)
|
||||
mse_bytes = math.ceil(D * mse_bits / 8)
|
||||
n_centroids = 2**mse_bits
|
||||
|
||||
val_data_bytes = math.ceil(D * value_quant_bits / 8)
|
||||
|
||||
BLOCK_VAL = triton.next_power_of_2(val_data_bytes)
|
||||
|
||||
# Cache strides (element_size=1 for uint8, so stride in bytes = stride()).
|
||||
# Only stride_cache_block (== bs*H*slot_aligned) is used under Opt#3;
|
||||
# per-slot and per-head strides are computed from DATA_BYTES_PER_SLOT.
|
||||
stride_block = kv_cache.stride(0)
|
||||
|
||||
block_grp = triton.next_power_of_2(D // 8) if D >= 8 else 1
|
||||
|
||||
# ── Opt#3 SoA layout constants (derived locally so the launcher API
|
||||
# stays unchanged). Invariant: key_packed_size + value_packed_size
|
||||
# == slot_size_aligned == (key_data + val_data) + (k_meta + v_meta).
|
||||
key_data_bytes = D if key_fp8 else mse_bytes
|
||||
data_bytes_per_slot = key_data_bytes + val_data_bytes
|
||||
# Data region occupies [0, block_size * H * data_bytes_per_slot) bytes
|
||||
# at the start of each block; metadata region starts at that offset.
|
||||
meta_region_offset = block_size * H * data_bytes_per_slot
|
||||
num_soa_fields = 2 if key_fp8 else 3
|
||||
soa_v_scale = 0 if key_fp8 else 1
|
||||
soa_v_zero = 1 if key_fp8 else 2
|
||||
|
||||
# uint16-aliased view of the same storage — enables single-instruction
|
||||
# u16 writes for the SoA K-norm / V-scale / V-zero stores. The cache
|
||||
# bytes are always contiguous and 2-byte aligned under the Opt#3 layout
|
||||
# (MSE_BYTES, VAL_DATA_BYTES, meta_region_offset are all even).
|
||||
kv_cache_u16 = kv_cache.view(torch.uint16)
|
||||
|
||||
# ── FP8 PATH: in-kernel FP8 cast + scatter via fp8 kernel ──
|
||||
if key_fp8:
|
||||
k_flat = key.reshape(NH, D).contiguous()
|
||||
v_flat = value.reshape(NH, D).contiguous()
|
||||
|
||||
fp8_e4b15 = _use_fp8_e4b15(key.device.index or 0)
|
||||
|
||||
grid = (NH,)
|
||||
_tq_fused_store_fp8[grid](
|
||||
k_flat,
|
||||
v_flat,
|
||||
kv_cache.view(-1),
|
||||
kv_cache_u16.view(-1),
|
||||
slot_mapping,
|
||||
stride_cache_block=stride_block,
|
||||
D=D,
|
||||
H=H,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_D=BLOCK_D,
|
||||
KEY_DATA_BYTES=key_data_bytes,
|
||||
VAL_DATA_BYTES=val_data_bytes,
|
||||
META_REGION_OFFSET=meta_region_offset,
|
||||
NUM_SOA_FIELDS=num_soa_fields,
|
||||
SOA_V_SCALE=soa_v_scale,
|
||||
SOA_V_ZERO=soa_v_zero,
|
||||
VQB=value_quant_bits,
|
||||
BLOCK_VAL=BLOCK_VAL,
|
||||
BLOCK_GRP=block_grp,
|
||||
FP8_E4B15=fp8_e4b15,
|
||||
num_warps=4,
|
||||
num_stages=1,
|
||||
)
|
||||
return
|
||||
|
||||
# ── MSE PATH: external GEMM + fused bucketize/pack kernel ──
|
||||
# Normalize + rotation GEMM externally (cuBLAS is faster than in-kernel)
|
||||
k_flat = key.float().reshape(NH, D)
|
||||
norms = k_flat.norm(dim=1, keepdim=True)
|
||||
x_hat = k_flat / (norms + 1e-8)
|
||||
y = x_hat @ PiT
|
||||
|
||||
v_flat = value.float().reshape(NH, D)
|
||||
|
||||
# When norm_correction=True, the kernel folds 1/||c_t|| into the
|
||||
# stored per-token K-norm so the load kernel can skip per-tile norm
|
||||
# math (Opt#1). centroids is only dereferenced inside the kernel when
|
||||
# NORM_CORRECTION=1; pass midpoints as a harmless placeholder otherwise.
|
||||
centroids_ptr = centroids if centroids is not None else midpoints
|
||||
grid = (NH,)
|
||||
_tq_fused_store_mse[grid](
|
||||
y,
|
||||
norms.squeeze(1),
|
||||
v_flat,
|
||||
midpoints,
|
||||
centroids_ptr,
|
||||
kv_cache.view(-1),
|
||||
kv_cache_u16.view(-1),
|
||||
slot_mapping,
|
||||
stride_cache_block=stride_block,
|
||||
D=D,
|
||||
H=H,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_D=BLOCK_D,
|
||||
MSE_BYTES=mse_bytes,
|
||||
VAL_DATA_BYTES=val_data_bytes,
|
||||
META_REGION_OFFSET=meta_region_offset,
|
||||
NUM_SOA_FIELDS=num_soa_fields,
|
||||
SOA_K_NORM=0,
|
||||
SOA_V_SCALE=soa_v_scale,
|
||||
SOA_V_ZERO=soa_v_zero,
|
||||
VQB=value_quant_bits,
|
||||
BLOCK_VAL=BLOCK_VAL,
|
||||
MSE_BITS=mse_bits,
|
||||
N_CENTROIDS=n_centroids,
|
||||
NORM_CORRECTION=1 if norm_correction else 0,
|
||||
BLOCK_GRP=block_grp,
|
||||
num_warps=4,
|
||||
num_stages=1,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user