From 5426311d9115c597c561967ec3f277ee6e970420 Mon Sep 17 00:00:00 2001 From: aditi-amd Date: Tue, 11 Aug 2026 08:45:39 -0700 Subject: [PATCH] [Kernel][ROCm][Perf] FlyDSL decode-attention kernel for 4-bit TurboQuant KV cache (#47896) Signed-off-by: root Co-authored-by: root Co-authored-by: Andreas Karatzas Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/kernels/turboquant/__init__.py | 0 .../test_flydsl_turboquant_decode.py | 261 ++++ vllm/platforms/rocm.py | 31 +- vllm/v1/attention/backends/turboquant_attn.py | 394 ++++- .../attention/ops/flydsl_kernels/__init__.py | 2 + .../attention/ops/flydsl_kernels/tq_decode.py | 823 ++++++++++ .../ops/flydsl_kernels/tq_decode_gqa6.py | 875 +++++++++++ .../attention/ops/flydsl_turboquant_decode.py | 874 +++++++++++ .../attention/ops/turboquant_soa/__init__.py | 3 + .../triton_turboquant_decode.py | 643 ++++++++ .../triton_turboquant_decode_v2.py | 669 ++++++++ .../turboquant_soa/triton_turboquant_store.py | 524 +++++++ .../triton_turboquant_unified_attention.py | 1350 +++++++++++++++++ 13 files changed, 6396 insertions(+), 53 deletions(-) create mode 100644 tests/kernels/turboquant/__init__.py create mode 100644 tests/kernels/turboquant/test_flydsl_turboquant_decode.py create mode 100644 vllm/v1/attention/ops/flydsl_kernels/__init__.py create mode 100644 vllm/v1/attention/ops/flydsl_kernels/tq_decode.py create mode 100644 vllm/v1/attention/ops/flydsl_kernels/tq_decode_gqa6.py create mode 100644 vllm/v1/attention/ops/flydsl_turboquant_decode.py create mode 100644 vllm/v1/attention/ops/turboquant_soa/__init__.py create mode 100644 vllm/v1/attention/ops/turboquant_soa/triton_turboquant_decode.py create mode 100644 vllm/v1/attention/ops/turboquant_soa/triton_turboquant_decode_v2.py create mode 100644 vllm/v1/attention/ops/turboquant_soa/triton_turboquant_store.py create mode 100644 vllm/v1/attention/ops/turboquant_soa/triton_turboquant_unified_attention.py diff --git a/tests/kernels/turboquant/__init__.py b/tests/kernels/turboquant/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/kernels/turboquant/test_flydsl_turboquant_decode.py b/tests/kernels/turboquant/test_flydsl_turboquant_decode.py new file mode 100644 index 0000000000..d5c1403385 --- /dev/null +++ b/tests/kernels/turboquant/test_flydsl_turboquant_decode.py @@ -0,0 +1,261 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# Copyright (c) 2025 FlyDSL Project Contributors +"""Correctness test for the FlyDSL TurboQuant 4-bit KV decode kernel. + +The kernel is validated against a pure-PyTorch full-precision attention +oracle rather than another kernel: the synthetic KV cache is built from +known centroid indices, K-norms and V-scales/zeros, so we can dequantize +the exact stored values and run fp32 softmax attention as ground truth. +Both sides therefore see identical quantized values; the only difference +is bf16-vs-fp32 accumulation, which is bounded by ``ATOL`` below. + +Skipped unless running on ROCm/gfx950 with FlyDSL importable. +""" + +import pytest +import torch + +from vllm.platforms import current_platform + +# Guard the ROCm-only import: `vllm.platforms.rocm` pulls in amdsmi and other +# ROCm-specific modules that are not importable on non-ROCm platforms, so it +# must not be imported at module load time on CUDA/CPU CI. +if not current_platform.is_rocm(): + pytest.skip( + "TurboQuant FlyDSL decode only runs on ROCm gfx950.", + allow_module_level=True, + ) + +from vllm.platforms.rocm import on_gfx950 # noqa: E402 + +if not on_gfx950(): + pytest.skip( + "TurboQuant FlyDSL decode only runs on ROCm gfx950.", + allow_module_level=True, + ) + +from vllm.v1.attention.ops.flydsl_turboquant_decode import ( # noqa: E402 + flydsl_turboquant_decode_attention, + is_flydsl_available, + is_flydsl_gqa6_available, +) + +if not is_flydsl_available(): + pytest.skip("FlyDSL runtime is not available.", allow_module_level=True) + +HEAD_SIZE = 128 +N_CENTROIDS = 16 +KEY_DATA_BYTES = HEAD_SIZE // 2 # 4-bit packed key nibbles +DATA_BYTES_PER_SLOT = HEAD_SIZE # packed K (64) + packed V (64) +NUM_SOA_FIELDS = 3 +SOA_K_NORM, SOA_V_SCALE, SOA_V_ZERO = 0, 1, 2 + +# bf16-vs-fp32 accumulation tolerance (FlyDSL test_pa.py default). +ATOL = 5e-3 + + +def _build_cache( + num_seqs, num_kv_heads, seq_len, qg, kv_block_size, seed=0xC0FFEE, device="cuda" +): + """Vectorized builder for a SoA-layout 4-bit KV cache + fp32 ground truth. + + Returns ``(centroids, q_bf16, kv_cache_4d, block_table, seq_lens, + K_ref, V_ref)`` where ``K_ref``/``V_ref`` are the dequantized + full-precision keys/values used by the reference attention. + """ + g = torch.Generator(device=device).manual_seed(seed) + hk = num_kv_heads + hq = hk * qg + + centroids = ( + torch.randn(N_CENTROIDS, generator=g, dtype=torch.float32, device=device) * 0.5 + ) + q = ( + torch.randn( + num_seqs, hq, HEAD_SIZE, generator=g, dtype=torch.float32, device=device + ) + * 0.1 + ) + q_bf16 = q.to(torch.bfloat16) + + blocks_per_seq = (seq_len + kv_block_size - 1) // kv_block_size + max_bps = blocks_per_seq + 4 + block_table = torch.zeros(num_seqs, max_bps, dtype=torch.int32, device=device) + block_table[:, :blocks_per_seq] = ( + torch.arange( + num_seqs * blocks_per_seq, dtype=torch.int32, device=device + ).reshape(num_seqs, blocks_per_seq) + + 1 + ) + total_blocks = num_seqs * blocks_per_seq + 8 + + slot_size_aligned = (KEY_DATA_BYTES + 2) + (KEY_DATA_BYTES + 4) # 134 + if slot_size_aligned % 2: + slot_size_aligned += 1 + bytes_per_block = kv_block_size * hk * slot_size_aligned + + k_idx = torch.randint( + 0, + N_CENTROIDS, + (num_seqs, hk, seq_len, HEAD_SIZE), + generator=g, + dtype=torch.uint8, + device=device, + ) + v_idx = torch.randint( + 0, + N_CENTROIDS, + (num_seqs, hk, seq_len, HEAD_SIZE), + generator=g, + dtype=torch.uint8, + device=device, + ) + knorm = ( + torch.rand(num_seqs, hk, seq_len, generator=g, device=device) * 0.5 + 0.5 + ).to(torch.float16) + vscale = ( + torch.rand(num_seqs, hk, seq_len, generator=g, device=device) * 0.05 + 0.01 + ).to(torch.float16) + vzero = ( + (torch.rand(num_seqs, hk, seq_len, generator=g, device=device) - 0.5) * 0.1 + ).to(torch.float16) + + k_ref = centroids[k_idx.long()].float() * knorm.float().unsqueeze(-1) + v_ref = v_idx.float() * vscale.float().unsqueeze(-1) + vzero.float().unsqueeze(-1) + + k_packed = k_idx[..., 0::2] | (k_idx[..., 1::2] << 4) + v_packed = v_idx[..., 0::2] | (v_idx[..., 1::2] << 4) + + kv_cache = torch.zeros( + total_blocks, bytes_per_block, dtype=torch.uint8, device=device + ) + + seq_ax = ( + torch.arange(num_seqs, device=device) + .view(num_seqs, 1) + .expand(num_seqs, seq_len) + ) + tok_ax = ( + torch.arange(seq_len, device=device).view(1, seq_len).expand(num_seqs, seq_len) + ) + blk_for_tok = block_table[seq_ax, tok_ax // kv_block_size] + slot_for_tok = tok_ax % kv_block_size + + h_ax = torch.arange(hk, device=device) + base_data = ( + slot_for_tok.unsqueeze(1) * hk * DATA_BYTES_PER_SLOT + + h_ax.view(1, hk, 1) * DATA_BYTES_PER_SLOT + ) + blk_b = blk_for_tok.unsqueeze(1).expand(num_seqs, hk, seq_len) + dst_base = blk_b * bytes_per_block + base_data + rng_k = torch.arange(KEY_DATA_BYTES, device=device) + rng_v = torch.arange(KEY_DATA_BYTES, device=device) + KEY_DATA_BYTES + k_dst = (dst_base.unsqueeze(-1) + rng_k.view(1, 1, 1, -1)).reshape(-1) + v_dst = (dst_base.unsqueeze(-1) + rng_v.view(1, 1, 1, -1)).reshape(-1) + flat = kv_cache.view(-1) + flat[k_dst] = k_packed.reshape(-1) + flat[v_dst] = v_packed.reshape(-1) + + meta_off = kv_block_size * hk * DATA_BYTES_PER_SLOT + meta_view = kv_cache.view(torch.float16).view(total_blocks, -1) + meta_off_hw = meta_off // 2 + common = h_ax.view( + 1, hk, 1 + ) * NUM_SOA_FIELDS * kv_block_size + slot_for_tok.unsqueeze(1) + blk_flat = blk_for_tok.unsqueeze(1).expand(num_seqs, hk, seq_len) + knorm_off = meta_off_hw + common + SOA_K_NORM * kv_block_size + vscale_off = meta_off_hw + common + SOA_V_SCALE * kv_block_size + vzero_off = meta_off_hw + common + SOA_V_ZERO * kv_block_size + meta_view[blk_flat.reshape(-1).long(), knorm_off.reshape(-1).long()] = ( + knorm.reshape(-1) + ) + meta_view[blk_flat.reshape(-1).long(), vscale_off.reshape(-1).long()] = ( + vscale.reshape(-1) + ) + meta_view[blk_flat.reshape(-1).long(), vzero_off.reshape(-1).long()] = ( + vzero.reshape(-1) + ) + + seq_lens = torch.full((num_seqs,), seq_len, dtype=torch.int32, device=device) + kv_cache_4d = kv_cache.view(total_blocks, kv_block_size, hk, slot_size_aligned) + return ( + centroids, + q_bf16, + kv_cache_4d, + block_table, + seq_lens, + k_ref.cpu(), + v_ref.cpu(), + ) + + +def _reference_attention(q_bf16, k_ref, v_ref, seq_lens, scale): + """fp32 softmax attention on dequantized keys/values (ground truth).""" + num_seqs, hq, d = q_bf16.shape + hk = k_ref.shape[1] + qg = hq // hk + q = q_bf16.float().reshape(num_seqs, hk, qg, d) + out = torch.zeros(num_seqs, hq, d, dtype=torch.float32) + for s in range(num_seqs): + for h in range(hk): + ql = int(seq_lens[s].item()) + k = k_ref[s, h, :ql] + v = v_ref[s, h, :ql] + scores = (q[s, h] @ k.T) * scale + m = scores.max(dim=-1, keepdim=True).values + e = torch.exp(scores - m) + p = e / e.sum(dim=-1, keepdim=True) + out[s, h * qg : (h + 1) * qg] = p @ v + return out + + +@pytest.mark.parametrize("num_seqs", [1, 4, 16]) +@pytest.mark.parametrize("seq_len", [256, 1024, 4096]) +@pytest.mark.parametrize( + "num_kv_heads,qg", + [ + (8, 8), # Qwen2.5-72B class + (8, 16), # Qwen3-32B class + pytest.param( + 4, + 6, # MiniMax-M2.5 class (GQA-6 sibling kernel) + marks=pytest.mark.skipif( + not is_flydsl_gqa6_available(), + reason="FlyDSL GQA-6 sibling kernel not available", + ), + ), + ], +) +@pytest.mark.parametrize("kv_block_size", [16, 32]) +def test_flydsl_matches_reference(num_seqs, seq_len, num_kv_heads, qg, kv_block_size): + """FlyDSL decode must match fp32 attention on dequantized KV.""" + centroids, q_bf16, kv_cache, block_table, seq_lens, k_ref, v_ref = _build_cache( + num_seqs, num_kv_heads, seq_len, qg, kv_block_size + ) + scale = 1.0 / (HEAD_SIZE**0.5) + identity = torch.eye(HEAD_SIZE, dtype=torch.float32, device="cuda") + + out = flydsl_turboquant_decode_attention( + query=q_bf16, + kv_cache=kv_cache, + block_table=block_table, + seq_lens=seq_lens, + Pi=identity, + centroids=centroids, + scale=scale, + mse_bits=4, + key_packed_size=KEY_DATA_BYTES + 2, + value_quant_bits=4, + value_packed_size=KEY_DATA_BYTES + 4, + key_fp8=False, + norm_correction=False, + PiT=identity.T.contiguous(), + max_seq_len=seq_len, + max_num_kv_splits=32, + sinks=None, + ) + + ref = _reference_attention(q_bf16.cpu(), k_ref, v_ref, seq_lens.cpu(), scale) + + torch.testing.assert_close(out.cpu().float(), ref, atol=ATOL, rtol=0.0) diff --git a/vllm/platforms/rocm.py b/vllm/platforms/rocm.py index 83c95edbf6..651c4cadbe 100644 --- a/vllm/platforms/rocm.py +++ b/vllm/platforms/rocm.py @@ -623,23 +623,38 @@ class RocmPlatform(Platform): if selected_backend is not None: try: backend_class = selected_backend.get_class() - invalid_reasons = backend_class.validate_configuration( + sel_invalid_reasons = backend_class.validate_configuration( device_capability=device_capability, **attn_selector_config._asdict(), ) except ImportError: - invalid_reasons = ["ImportError"] - if invalid_reasons: - raise ValueError( - f"Selected backend {selected_backend} is not valid for " - f"this configuration. Reason: {invalid_reasons}" - ) - else: + sel_invalid_reasons = ["ImportError"] + if not sel_invalid_reasons: logger.info_once( "Using %s backend (selected via --attention-backend).", selected_backend.name, ) return selected_backend.get_path() + # Only tolerate the mismatch for turboquant_* KV-cache layers: + # boundary layers keep the native dtype (served by the selected + # backend) while turboquant_* layers need TURBOQUANT, so no single + # --attention-backend can serve every layer. For any other dtype + # the explicit selection is genuinely invalid -> fail loud. + kv_dtype = attn_selector_config.kv_cache_dtype + if not (kv_dtype is not None and str(kv_dtype).startswith("turboquant")): + raise ValueError( + f"Selected backend {selected_backend} is not valid for " + f"this configuration. Reason: {sel_invalid_reasons}" + ) + # NOTE: pass a str (not the list) -- info_once hashes its args. + logger.info_once( + "Selected backend %s is incompatible with this turboquant " + "layer (%s); using the auto-selected per-layer backend. " + "Reason: %s", + selected_backend.name, + attn_selector_config.attn_type, + str(sel_invalid_reasons), + ) # No selected backend or the selected backend is invalid, # so we try finding a valid backend. diff --git a/vllm/v1/attention/backends/turboquant_attn.py b/vllm/v1/attention/backends/turboquant_attn.py index 981aa9b32a..ab9e375b0a 100644 --- a/vllm/v1/attention/backends/turboquant_attn.py +++ b/vllm/v1/attention/backends/turboquant_attn.py @@ -16,6 +16,7 @@ Per-head per-position slot layout: For turboquant_k3v4_nc head_dim=256: [100 bytes key | 512 bytes value] = 612 """ +import contextlib import functools import math from dataclasses import dataclass @@ -26,6 +27,7 @@ import torch.nn.functional as F from vllm.config import get_current_vllm_config from vllm.config.cache import CacheDType +from vllm.logger import init_logger from vllm.model_executor.layers.quantization.turboquant.centroids import ( get_centroids, ) @@ -47,6 +49,15 @@ from vllm.v1.attention.backends.fa_utils import ( is_flash_attn_varlen_func_available, ) from vllm.v1.attention.backends.utils import split_decodes_and_prefills + +# FlyDSL TurboQuant decode (AMD gfx950). Auto-selected when FlyDSL is available +# for eligible layers (SoA store + FlyDSL decode + SoA-aware continuation); +# non-gfx950 or ineligible layers use the SoA Triton decode. +from vllm.v1.attention.ops.flydsl_turboquant_decode import ( + flydsl_turboquant_decode_attention, + is_flydsl_available, + is_flydsl_gqa6_available, +) from vllm.v1.attention.ops.triton_turboquant_decode import ( _tq_full_dequant_kv, _use_fp8_e4b15, @@ -59,6 +70,8 @@ from vllm.v1.worker.workspace import ( is_workspace_manager_initialized, ) +logger = init_logger(__name__) + _HAS_FLASH_ATTN = is_flash_attn_varlen_func_available() if _HAS_FLASH_ATTN: from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func @@ -71,6 +84,24 @@ if _HAS_FLASH_ATTN: _CONTINUATION_DECODE_THRESHOLD = 128 +def _soa_imports(): + """Lazy import of the HIP-free SoA Triton subset (store / dequant / decode). + + Kept lazy so the default (FlyDSL-off) path never imports these modules. + """ + from vllm.v1.attention.ops.turboquant_soa.triton_turboquant_decode import ( + _tq_full_dequant_kv as soa_dequant, + ) + from vllm.v1.attention.ops.turboquant_soa.triton_turboquant_store import ( + triton_turboquant_store as soa_store, + ) + from vllm.v1.attention.ops.turboquant_soa.triton_turboquant_unified_attention import ( # noqa: E501 + triton_turboquant_decode_attention_soa as soa_decode, + ) + + return soa_store, soa_dequant, soa_decode + + def _build_hadamard(d: int, device_str: str) -> torch.Tensor: """Orthonormal Hadamard matrix (Sylvester construction), cached per (d, device). @@ -290,6 +321,10 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): supports_quant_query_input: bool = False + # Lazily populated before cudagraph capture (FlyDSL decode path only). + _arange_cache: torch.Tensor + _cu_2: torch.Tensor + def __init__( self, num_heads: int, @@ -337,6 +372,17 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): vllm_config.attention_config.tq_max_kv_splits_for_cuda_graph ) + # FlyDSL decode state. Auto-enabled on gfx950 when FlyDSL is available. + self.sliding_window = sliding_window + self.sinks = kwargs.get("sinks") + # Cache max_model_len now (config is available at __init__ but NOT + # during CUDA-graph capture when _ensure_on_device is re-entered). + self._max_model_len = vllm_config.model_config.max_model_len + # SoA store is required by the FlyDSL decode/continuation path, so it + # tracks FlyDSL availability (single switch for the whole pipeline). + self._use_flydsl = is_flydsl_available() + self._soa_store = self._use_flydsl + def _flash_attn_varlen( self, q: torch.Tensor, @@ -382,6 +428,44 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): quantizer is symmetric around zero (sign-flipping a coordinate maps it to the mirror centroid with identical distortion). """ + if self._soa_store: + # CUDA-graph capture safety for the FlyDSL decode path on ROCm. + # (1) Pre-allocate _arange_cache / _cu_2 BEFORE any capture; lazy + # allocation during graph replay lands in the HIP graph memory + # pool and yields stale addresses (GPU fault / garbage). + # (2) Pre-warm the WorkspaceManager to its max size before capture + # so mid-capture growth cannot invalidate pointers baked into + # already-captured batch sizes. + _max_len = self._max_model_len + _already_ok = ( + hasattr(self, "_arange_cache") + and self._arange_cache.device.type == str(device).split(":")[0] + and self._arange_cache.shape[0] >= _max_len + 2 + ) + if not _already_ok: + self._arange_cache = torch.arange( + 0, _max_len + 2, device=device, dtype=torch.int32 + ) + if not hasattr(self, "_cu_2") or self._cu_2.device != torch.device(device): + self._cu_2 = torch.zeros(2, device=device, dtype=torch.int32) + if ( + is_workspace_manager_initialized() + and not current_workspace_manager().is_locked() + ): + B_max = self._max_capture_batch_size() + D = self.head_size + Hq = self.num_heads + S = self.max_num_kv_splits + _pre_warm_bytes = ( + B_max * Hq * (S * (D + 1) + D) * 4 # fp32 mid_o + fp32 lse + + B_max * Hq * D * 2 # query-dtype output (bf16 = 2 B) + + 512 # alignment padding + ) + with contextlib.suppress(AssertionError): + current_workspace_manager().get_simultaneous( + ((_pre_warm_bytes,), torch.uint8) + ) + if not hasattr(layer, "_tq_cached"): D = self.head_size @@ -402,6 +486,29 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): layer._tq_midpoints = (c_sorted[:-1] + c_sorted[1:]) / 2 layer._tq_cached = True + def _max_capture_batch_size(self) -> int: + """Largest decode batch we might see at runtime (for workspace pre-warm). + + Take max(cudagraph_capture_sizes, scheduler.max_num_seqs): a forward + pass exceeding the largest captured graph size falls back to eager, + but the workspace is locked and that eager path can still hit batch + sizes up to max_num_seqs. Falls back to 1024 if config is unavailable. + """ + try: + cfg = get_current_vllm_config() + candidates: list[int] = [] + sizes = cfg.compilation_config.cudagraph_capture_sizes + if sizes: + candidates.append(int(max(sizes))) + sched = getattr(cfg, "scheduler_config", None) + if sched is not None and getattr(sched, "max_num_seqs", None): + candidates.append(int(sched.max_num_seqs)) + if candidates: + return max(candidates) + except Exception: # noqa: BLE001 + pass + return 1024 + def do_kv_cache_update( self, layer: torch.nn.Module, @@ -590,6 +697,27 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): layer: Any, ): """Quantize + store via fused Triton kernel.""" + if self._soa_store: + # SoA layout (data region + metadata region separated per block), + # required by the FlyDSL decode kernel. Pure-Triton store; the + # cache tensor shape is identical to the default AoS store, only + # the within-block byte convention differs. + soa_store, _, _ = _soa_imports() + soa_store( + key=key, + value=value, + kv_cache=kv_cache, + slot_mapping=slot_mapping, + PiT=layer._tq_PiT, + midpoints=layer._tq_midpoints, + mse_bits=self.tq_config.key_mse_bits, + key_packed_size=self.tq_config.key_packed_size, + value_quant_bits=self.tq_config.effective_value_quant_bits, + key_fp8=self.tq_config.key_fp8, + centroids=layer._tq_centroids, + norm_correction=self.tq_config.norm_correction, + ) + return triton_turboquant_store( key, value, @@ -722,21 +850,55 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): # Slice from pre-built arange (no kernel launch) synth_seq_lens = _arange_cache[cached_len + 1 : seq_len + 1] synth_bt = attn_metadata.block_table[i : i + 1].expand(q_len, -1) - out = triton_turboquant_decode_attention( - query=q_seq, - kv_cache=kv_cache, - block_table=synth_bt, - seq_lens=synth_seq_lens, - Pi=Pi, - centroids=centroids, - scale=self.scale, - mse_bits=self.tq_config.key_mse_bits, - key_packed_size=self.tq_config.key_packed_size, - value_quant_bits=(self.tq_config.effective_value_quant_bits), - key_fp8=self.tq_config.key_fp8, - norm_correction=self.tq_config.norm_correction, - PiT=PiT, - ) + if self._soa_store: + # The cache was written in SoA layout (always, for FlyDSL), + # so it MUST be read with the SoA-aware decode. The + # default AoS decode reads k_norm/v_scale/v_zero from + # the wrong offsets in a SoA cache -> garbage + # cached-prefix output -> + # accuracy collapse on every multi-turn / prefix-cached + # (APC) request. Continuation stays on the Triton SoA + # path even when FlyDSL is the main decode kernel + # (FlyDSL is decode-batch only). + out = self._dispatch_decode_soa( + query=q_seq, + kv_cache=kv_cache, + block_table=synth_bt, + seq_lens=synth_seq_lens, + Pi=Pi, + centroids=centroids, + scale=self.scale, + mse_bits=self.tq_config.key_mse_bits, + key_packed_size=self.tq_config.key_packed_size, + value_quant_bits=( + self.tq_config.effective_value_quant_bits + ), + value_packed_size=self.tq_config.value_packed_size, + max_seq_len=int(seq_len), + key_fp8=self.tq_config.key_fp8, + norm_correction=self.tq_config.norm_correction, + PiT=PiT, + sinks=self.sinks, + sliding_window=self.sliding_window, + ) + else: + out = triton_turboquant_decode_attention( + query=q_seq, + kv_cache=kv_cache, + block_table=synth_bt, + seq_lens=synth_seq_lens, + Pi=Pi, + centroids=centroids, + scale=self.scale, + mse_bits=self.tq_config.key_mse_bits, + key_packed_size=self.tq_config.key_packed_size, + value_quant_bits=( + self.tq_config.effective_value_quant_bits + ), + key_fp8=self.tq_config.key_fp8, + norm_correction=self.tq_config.norm_correction, + PiT=PiT, + ) else: # Large continuation: dequant cached K/V and use # flash_attn for better throughput. @@ -801,36 +963,84 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): v_cached = v_buf[:, :, :alloc_len, :] grid = (alloc_len, 1 * Hk) - _tq_full_dequant_kv[grid]( - kv_cache, - block_table, - centroids, - k_cached, - v_cached, - k_cached.stride(0), - k_cached.stride(1), - k_cached.stride(2), - v_cached.stride(0), - v_cached.stride(1), - v_cached.stride(2), - kv_cache.stride(0), - kv_cache.stride(1), - kv_cache.stride(2), - block_table.stride(0), - HEAD_DIM=D, - BLOCK_SIZE=block_size, - NUM_KV_HEADS=Hk, - MSE_BYTES=mse_bytes, - KPS=self.tq_config.key_packed_size, - VQB=self.tq_config.effective_value_quant_bits, - VAL_DATA_BYTES=val_data_bytes, - MSE_BITS=self.tq_config.key_mse_bits, - KEY_FP8=1 if self.tq_config.key_fp8 else 0, - BLOCK_D=BLOCK_D, - NORM_CORRECTION=1 if self.tq_config.norm_correction else 0, - FP8_E4B15=_use_fp8_e4b15(device.index or 0), - num_warps=4, - ) + if self._soa_store: + # SoA-aware dequant: read the data/metadata-separated SoA cache + # written by the SoA store. Constants must match the store side. + _, soa_dequant, _ = _soa_imports() + key_fp8 = self.tq_config.key_fp8 + key_data_bytes = D if key_fp8 else mse_bytes + data_bytes_per_slot = key_data_bytes + val_data_bytes + meta_region_offset = block_size * Hk * data_bytes_per_slot + num_soa_fields = 2 if key_fp8 else 3 + soa_k_norm = 0 + soa_v_scale = 0 if key_fp8 else 1 + soa_v_zero = 1 if key_fp8 else 2 + kv_cache_u16 = kv_cache.view(torch.uint16) + soa_dequant[grid]( + kv_cache, + kv_cache_u16, + block_table, + centroids, + k_cached, + v_cached, + k_cached.stride(0), + k_cached.stride(1), + k_cached.stride(2), + v_cached.stride(0), + v_cached.stride(1), + v_cached.stride(2), + kv_cache.stride(0), + block_table.stride(0), + HEAD_DIM=D, + BLOCK_SIZE=block_size, + NUM_KV_HEADS=Hk, + MSE_BYTES=mse_bytes, + VQB=self.tq_config.effective_value_quant_bits, + VAL_DATA_BYTES=val_data_bytes, + MSE_BITS=self.tq_config.key_mse_bits, + KEY_FP8=1 if key_fp8 else 0, + KEY_DATA_BYTES=key_data_bytes, + META_REGION_OFFSET=meta_region_offset, + NUM_SOA_FIELDS=num_soa_fields, + SOA_K_NORM=soa_k_norm, + SOA_V_SCALE=soa_v_scale, + SOA_V_ZERO=soa_v_zero, + BLOCK_D=BLOCK_D, + NORM_CORRECTION=1 if self.tq_config.norm_correction else 0, + FP8_E4B15=_use_fp8_e4b15(device.index or 0), + num_warps=4, + ) + else: + _tq_full_dequant_kv[grid]( + kv_cache, + block_table, + centroids, + k_cached, + v_cached, + k_cached.stride(0), + k_cached.stride(1), + k_cached.stride(2), + v_cached.stride(0), + v_cached.stride(1), + v_cached.stride(2), + kv_cache.stride(0), + kv_cache.stride(1), + kv_cache.stride(2), + block_table.stride(0), + HEAD_DIM=D, + BLOCK_SIZE=block_size, + NUM_KV_HEADS=Hk, + MSE_BYTES=mse_bytes, + KPS=self.tq_config.key_packed_size, + VQB=self.tq_config.effective_value_quant_bits, + VAL_DATA_BYTES=val_data_bytes, + MSE_BITS=self.tq_config.key_mse_bits, + KEY_FP8=1 if self.tq_config.key_fp8 else 0, + BLOCK_D=BLOCK_D, + NORM_CORRECTION=1 if self.tq_config.norm_correction else 0, + FP8_E4B15=_use_fp8_e4b15(device.index or 0), + num_warps=4, + ) # Inverse-rotate MSE keys back to original space if not self.tq_config.key_fp8: @@ -930,6 +1140,82 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): ) ) + if self._use_flydsl: + # FlyDSL decode (gfx950, MSE-key, HEAD_SIZE=128, GQA in {6, 8, 16}). + # GQA-6 routes to the MiniMax sibling kernel. Ineligible layers fall + # back to SoA Triton decode. + _gqa = self.num_kv_groups + flydsl_gqa_ok = (_gqa in (8, 16)) or ( + _gqa == 6 and is_flydsl_gqa6_available() + ) + flydsl_eligible = ( + not self.tq_config.key_fp8 + and self.tq_config.key_mse_bits == 4 + and self.tq_config.effective_value_quant_bits == 4 + and self.head_size == 128 + and flydsl_gqa_ok + and self.sinks is None + and not (self.sliding_window and self.sliding_window > 0) + ) + if flydsl_eligible: + return flydsl_turboquant_decode_attention( + query=query, + kv_cache=kv_cache, + block_table=attn_metadata.block_table, + seq_lens=attn_metadata.seq_lens, + Pi=Pi, + centroids=centroids, + scale=self.scale, + mse_bits=self.tq_config.key_mse_bits, + key_packed_size=self.tq_config.key_packed_size, + value_quant_bits=self.tq_config.effective_value_quant_bits, + value_packed_size=self.tq_config.value_packed_size, + max_seq_len=attn_metadata.max_seq_len, + key_fp8=self.tq_config.key_fp8, + norm_correction=self.tq_config.norm_correction, + PiT=PiT, + mid_o_buf=mid_o_buf, + output_buf=output_buf, + lse_buf=lse_buf, + buf_holder=layer, + max_num_kv_splits=self.max_num_kv_splits, + sinks=self.sinks, + ) + logger.warning_once( + "TurboQuant FlyDSL ineligible (key_fp8=%s mse_bits=%s vqb=%s " + "head_size=%s num_kv_groups=%s sinks=%s) -> SoA Triton decode", + self.tq_config.key_fp8, + self.tq_config.key_mse_bits, + self.tq_config.effective_value_quant_bits, + self.head_size, + self.num_kv_groups, + self.sinks is not None, + ) + return self._dispatch_decode_soa( + query=query, + kv_cache=kv_cache, + block_table=attn_metadata.block_table, + seq_lens=attn_metadata.seq_lens, + Pi=Pi, + centroids=centroids, + scale=self.scale, + mse_bits=self.tq_config.key_mse_bits, + key_packed_size=self.tq_config.key_packed_size, + value_quant_bits=self.tq_config.effective_value_quant_bits, + value_packed_size=self.tq_config.value_packed_size, + max_seq_len=attn_metadata.max_seq_len, + key_fp8=self.tq_config.key_fp8, + norm_correction=self.tq_config.norm_correction, + PiT=PiT, + mid_o_buf=mid_o_buf, + output_buf=output_buf, + lse_buf=lse_buf, + buf_holder=layer, + max_num_kv_splits=self.max_num_kv_splits, + sinks=self.sinks, + sliding_window=self.sliding_window, + ) + result = triton_turboquant_decode_attention( query=query, kv_cache=kv_cache, @@ -951,3 +1237,21 @@ class TurboQuantAttentionImpl(AttentionImpl["TurboQuantMetadata"]): max_num_kv_splits=self.max_num_kv_splits, ) return result + + def _dispatch_decode_soa(self, **kwargs): + """SoA-aware Triton decode — fallback for FlyDSL-ineligible layers. + + The SoA decode launcher accepts a subset of the FlyDSL kwargs; filter to + the params it accepts and raise if a *meaningful* (non-None) kwarg would + be silently dropped (so we never mask a real feature gap). + """ + import inspect + + _, _, soa_decode = _soa_imports() + accepted = set(inspect.signature(soa_decode).parameters) + dropped = [k for k, v in kwargs.items() if k not in accepted and v is not None] + if dropped: + raise NotImplementedError( + f"SoA decode does not support kwargs {sorted(dropped)}" + ) + return soa_decode(**{k: v for k, v in kwargs.items() if k in accepted}) diff --git a/vllm/v1/attention/ops/flydsl_kernels/__init__.py b/vllm/v1/attention/ops/flydsl_kernels/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/v1/attention/ops/flydsl_kernels/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/v1/attention/ops/flydsl_kernels/tq_decode.py b/vllm/v1/attention/ops/flydsl_kernels/tq_decode.py new file mode 100644 index 0000000000..d20559e70a --- /dev/null +++ b/vllm/v1/attention/ops/flydsl_kernels/tq_decode.py @@ -0,0 +1,823 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# TurboQuant decode (FlyDSL) — MI355X / gfx950 / CDNA4 +# +# Decode-only paged attention over a TurboQuant SoA KV cache. +# Targets Qwen-class models: HEAD_SIZE=128, GQA group=8, MSE_BITS=4 K, VQB=4 V, +# N_CENTROIDS=16, BLOCK_SIZE=16, partitioned mode. +# +# === Design === +# * 1 wave (64 lanes) per CTA — simpler than 4-warp; trades parallelism for +# correctness/debug-ability. A later revision will fan out to 4 warps. +# * Flash-Attention-2 online softmax: rescale running PV/sum per K-tile, +# no QK score storage across tiles, no P-LDS round-trip (qk_acc layout +# matches PV B-operand directly). +# * Centroids loaded once into LDS at kernel start (LDS-resident LUT). +# * K dequant → LDS [token, head_dim] (natural layout) +# * V dequant → LDS [head_dim, token] (TRANSPOSED — required so PV MFMA's +# A=V operand can be loaded as 4-bf16 contiguous via 1 ds_read_b64). +# +# MFMA layouts used: +# QK: mfma(A=K, B=Q, C=qk_acc) → C[m=token, n=query], 4 fp32/lane +# = 4 tokens for SAME query → softmax via +# xor_shuffle(16),(32) reduces per-row +# PV: mfma(A=V_T, B=P, C=acc_pv) → C[m=head_dim, n=query], 4 fp32/lane +# = 4 contiguous head_dim cols for SAME +# query → 1 buffer_store of 8 bytes +# per chunk per lane + +from __future__ import annotations + +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl._mlir import ir +from flydsl._mlir.dialects import scf as _scf +from flydsl.expr import ( + arith, + buffer_ops, + const_expr, # noqa: F401 kept for kernel-DSL API parity + gpu, + range_constexpr, + rocdl, + vector, +) +from flydsl.expr.typing import Int32, T # noqa: F401 Int32 via fx.Int32 +from flydsl.runtime.device import get_rocm_arch as get_hip_arch +from flydsl.utils.smem_allocator import SmemAllocator, SmemPtr + +# === Constants (Qwen-class TQ decode profile) =============================== +HEAD_SIZE = 128 +# default; overridable via build_tq_decode_module(kv_block_size=...) +KV_BLOCK_SIZE = 16 +# MFMA tile = 16 tokens (do not change without re-deriving MFMA shapes) +TILE_SIZE = 16 +N_CENTROIDS = 16 +# default for tests; overridable via build_tq_decode_module(query_group_size=...) +QUERY_GROUP_SIZE = 16 +WARP_SIZE = 64 +NUM_WARPS = 1 +BLOCK_THREADS = NUM_WARPS * WARP_SIZE +KV_COMPUTE_BLOCK = 256 # 16 K-tiles × 16 tokens + +# TQ SoA layout +NUM_SOA_FIELDS = 3 +SOA_K_NORM = 0 +SOA_V_SCALE = 1 +SOA_V_ZERO = 2 +KEY_DATA_BYTES = HEAD_SIZE // 2 +VAL_DATA_BYTES = HEAD_SIZE // 2 +DATA_BYTES_PER_SLOT = KEY_DATA_BYTES + VAL_DATA_BYTES + +# MFMA +MFMA_M = MFMA_N = 16 +MFMA_K_BF16_QK = 32 # CDNA4 wide-K (mfma_f32_16x16x32_bf16) +MFMA_K_BF16_PV = 16 # PV's K = token tile = 16 +QK_K_CHUNKS = HEAD_SIZE // MFMA_K_BF16_QK # 4 +PV_N_CHUNKS = HEAD_SIZE // MFMA_N # 8 + +# LDS regions +CENTROID_LDS_BYTES = N_CENTROIDS * 4 # 64 +Q_LDS_BYTES = QUERY_GROUP_SIZE * HEAD_SIZE * 2 # 4096 +KV_TILE_LDS_BYTES = TILE_SIZE * HEAD_SIZE * 2 # 4096 + +LOG2E = 1.4426950408889634 +NEG_INF_VAL = float("-inf") + + +def _vsplat_mul(vec, scalar): + s = scalar.ir_value() if hasattr(scalar, "ir_value") else scalar + return vec * vector.broadcast(T.f32x4, s) + + +allocator = None + + +def build_tq_decode_module( + num_seqs: int, + num_kv_heads: int, + num_partitions: int, + max_blocks_per_seq: int = 512, + softmax_scale: float | None = None, + query_group_size: int = QUERY_GROUP_SIZE, + kv_block_size: int = KV_BLOCK_SIZE, + use_hw_v_transpose: bool = False, + tile_groups_per_partition: int = 1, +): + """Build a TQ decode kernel module. + + ``query_group_size`` (= num_query_heads // num_kv_heads) selects the + GQA factor. Supported values: 8 and 16. For 8 the MFMA's 16-row + capacity is half-used; lanes 8..15 compute garbage and are gated out + of all global-memory writes via an OOB-offset trick on the buffer + resource. Q load iterates ``QG // 4`` chunks (= 2 for QG=8, 4 for + QG=16) so we never read past the real Q rows in HBM. + + ``kv_block_size`` is the number of tokens per vLLM cache block. Must + be a multiple of TILE_SIZE (=16). Supported values: 16 and 32. With + block_size=32, two MFMA K-tiles fit in each block; the kernel walks + block-table entries every ``kv_block_size // TILE_SIZE`` tiles and + re-uses the entry for sub-tiles within the same block. SoA metadata + region grows linearly with block_size. + + ``use_hw_v_transpose`` (CDNA4 / gfx950 only) replaces the 32-element + scattered ``ds_write_b16`` V-dequant pattern with a row-major + ``V[token][head_dim]`` LDS layout written as 4 wide ``ds_write_b128`` + per lane, then read back into the MFMA A operand via the hardware + ``ds_read_tr16_b64`` transpose. This eliminates ~28 ds_write + instructions per lane per K-tile (32 -> 4) and 8 strided ds_read_b64 + per PV chunk (replaces them with 1 hw-transpose read each). + + ``tile_groups_per_partition`` (default 1) controls the FA-2 split-K + granularity. With value G each partition processes ``G * 16`` K-tiles + = ``G * KV_COMPUTE_BLOCK`` tokens, with the existing 16-tile loop + body iterated G times. The launcher uses this to bound + ``num_partitions`` (e.g. cap at 32) for long context: at 32K / + block_size=32 / num_partitions=32, G=5 covers 32*5*256 = 40 960 + tokens of worst-case context with grid.z=32 instead of 256. + Behavior at G=1 is bit-identical to the single-partition kernel. + """ + assert query_group_size in (8, 16), ( + f"query_group_size must be 8 or 16; got {query_group_size}" + ) + assert kv_block_size in (16, 32), ( + f"kv_block_size must be 16 or 32; got {kv_block_size}" + ) + assert kv_block_size % TILE_SIZE == 0 + assert int(tile_groups_per_partition) >= 1, ( + f"tile_groups_per_partition must be >= 1; got {tile_groups_per_partition}" + ) + USE_HW_TR = bool(use_hw_v_transpose) + TGPP = int(tile_groups_per_partition) + PARTITION_EXTENT_TOKENS = TGPP * KV_COMPUTE_BLOCK + + QG = int(query_group_size) + QG_LOAD_ITERS = QG // 4 # 2 for QG=8, 4 for QG=16 + OOB_OFFSET = 0x7FFFFFF0 # noqa: F841 ~2GB byte offset; > any plausible buffer + + _BS = int(kv_block_size) + _TILES_PER_BLOCK = _BS // TILE_SIZE # 1 for BS=16, 2 for BS=32 + + global allocator + arch = get_hip_arch() + + if softmax_scale is None: + softmax_scale = 1.0 / (HEAD_SIZE**0.5) + _qk_scale = float(softmax_scale) + + # --- Strides --- + _Hq = num_kv_heads * QG + _stride_q_seq = _Hq * HEAD_SIZE + _stride_q_head = HEAD_SIZE + _stride_bt_seq = max_blocks_per_seq + + _data_region_bytes = _BS * num_kv_heads * DATA_BYTES_PER_SLOT + _meta_region_bytes = num_kv_heads * NUM_SOA_FIELDS * _BS * 2 + _stride_cache_block = _data_region_bytes + _meta_region_bytes + _meta_region_offset = _data_region_bytes + + _stride_out_part = QG * HEAD_SIZE + _stride_out_head = num_partitions * QG * HEAD_SIZE + _stride_out_seq = num_kv_heads * num_partitions * QG * HEAD_SIZE + _stride_es_seq = num_kv_heads * num_partitions * QG + _stride_ml_seq = _stride_es_seq + + # --- LDS layout --- + allocator = SmemAllocator(None, arch=arch, global_sym_name="tq_smem") + centroid_off = 0 + allocator.ptr = CENTROID_LDS_BYTES + q_off = allocator.ptr + allocator.ptr += Q_LDS_BYTES + kv_off = allocator.ptr + allocator.ptr += KV_TILE_LDS_BYTES + + @flyc.kernel + def tq_decode_kernel( + out_ptr: fx.Tensor, + exp_sums_ptr: fx.Tensor, + max_logits_ptr: fx.Tensor, + query_ptr: fx.Tensor, + kv_cache_ptr: fx.Tensor, + centroids_ptr: fx.Tensor, + block_tables_ptr: fx.Tensor, + seq_lens_ptr: fx.Tensor, + ): + # ---- IDs --------------------------------------------------------- + tid = gpu.thread_idx.x + seq = gpu.block_idx.x + kv_h = gpu.block_idx.y + part = gpu.block_idx.z + lane = tid # 0..63 + mfma_row = lane & fx.Int32(15) # = query_row (B's N) / + # token (A's M for QK) + mfma_col_grp = lane >> fx.Int32(4) # 0..3, K-group dim + + # ---- Buffer resources ------------------------------------------- + q_rsrc = buffer_ops.create_buffer_resource(query_ptr, max_size=True) + kv_rsrc = buffer_ops.create_buffer_resource(kv_cache_ptr, max_size=True) + bt_rsrc = buffer_ops.create_buffer_resource(block_tables_ptr, max_size=True) + sl_rsrc = buffer_ops.create_buffer_resource(seq_lens_ptr, max_size=True) + cent_rsrc = buffer_ops.create_buffer_resource(centroids_ptr, max_size=True) + out_rsrc = buffer_ops.create_buffer_resource(out_ptr, max_size=True) + es_rsrc = buffer_ops.create_buffer_resource(exp_sums_ptr, max_size=True) + ml_rsrc = buffer_ops.create_buffer_resource(max_logits_ptr, max_size=True) + + # ---- LDS pointers ----------------------------------------------- + base = allocator.get_base() + cent_lds = SmemPtr(base, centroid_off, T.f32, shape=(N_CENTROIDS,)) + q_lds_i32 = SmemPtr(base, q_off, T.i32, shape=(Q_LDS_BYTES // 4,)).get() + q_lds_i64 = SmemPtr(base, q_off, T.i64, shape=(Q_LDS_BYTES // 8,)).get() + kv_lds_i32 = SmemPtr(base, kv_off, T.i32, shape=(KV_TILE_LDS_BYTES // 4,)).get() # noqa: F841 + kv_lds_i64 = SmemPtr(base, kv_off, T.i64, shape=(KV_TILE_LDS_BYTES // 8,)).get() + kv_lds_i16 = SmemPtr(base, kv_off, T.i16, shape=(KV_TILE_LDS_BYTES // 2,)).get() + + # ---- Constants --------------------------------------------------- + c_sq = fx.Int32(_stride_q_seq) + c_qh = fx.Int32(_stride_q_head) + c_qg = fx.Int32(QG) + c_bt = fx.Int32(_stride_bt_seq) + c_block = fx.Int32(_stride_cache_block) + c_meta_off = fx.Int32(_meta_region_offset) + c_data_per_slot = fx.Int32(DATA_BYTES_PER_SLOT) + c_kv_heads = fx.Int32(num_kv_heads) + c_keydata = fx.Int32(KEY_DATA_BYTES) + c_w = fx.Int32(WARP_SIZE) + + NEG_INF = arith.constant(NEG_INF_VAL, type=T.f32) + ZERO_F = fx.Float32(0.0) + ONE_F = fx.Float32(1.0) + LOG2E_C = arith.constant(LOG2E, type=T.f32) + QK_SCALE = arith.constant(_qk_scale, type=T.f32) + + # Helper: unwrap fx wrapper → raw ir.Value (used in STEP F) + def _ival(v): + return v.ir_value() if hasattr(v, "ir_value") else v + + # ===== STEP A: Load centroids → LDS (cooperative, race-safe) ===== + c_idx_safe = lane & fx.Int32(N_CENTROIDS - 1) + c_val = buffer_ops.buffer_load(cent_rsrc, c_idx_safe, vec_width=1, dtype=T.f32) + cent_lds.store(c_val, [arith.index_cast(T.index, c_idx_safe)]) + gpu.barrier() + + # ===== STEP B: Load pre-rotated q_rot → row-major LDS ============ + # q_rsrc points to the externally-rotated q_rot tensor. Load 8 bf16 + # per lane per iter (vec_width=4 i32) in QG_LOAD_ITERS iterations. + for c in range_constexpr(QG_LOAD_ITERS): + row_chunk = lane + fx.Int32(c * WARP_SIZE) # 0..QG*16-1 + row = row_chunk >> fx.Int32(4) # 0..QG-1 + col_b = row_chunk & fx.Int32(15) # 0..15 + col_elem = col_b * fx.Int32(8) # 0..120 in bf16 + q_off_elem = seq * c_sq + (kv_h * c_qg + row) * c_qh + col_elem + q_v = buffer_ops.buffer_load( + q_rsrc, + q_off_elem // fx.Int32(2), + vec_width=4, + dtype=T.i32, + ) + q_lds_byte = row * fx.Int32(HEAD_SIZE * 2) + col_elem * fx.Int32(2) + vector.store( + q_v, + q_lds_i32, + [arith.index_cast(T.index, q_lds_byte // fx.Int32(4))], + ) + gpu.barrier() + + # ===== STEP C: Pre-load Q operands for QK_K_CHUNKS = 4 K-chunks === + # Wide-K MFMA: K_per_chunk = 32, K_per_lane = 32/4 = 8 bf16 per chunk. + # Lane t holds 8 bf16 at row=mfma_row, K-cols + # chunk*32 + mfma_col_grp*8 + 0..7 + # Byte addr = mfma_row * HEAD_SIZE*2 + (chunk*32 + col_grp*8)*2 + # = mfma_row * 256 + chunk*64 + col_grp*16 + # i64 idx = mfma_row * 32 + chunk*8 + col_grp*2 (each i64 = 4 bf16) + # Load 16 bytes (= 8 bf16 = 2 i64) per chunk via vec(2, i64). + q_chunks = [] + for chk in range_constexpr(QK_K_CHUNKS): + q_idx_i64 = ( + mfma_row * fx.Int32(HEAD_SIZE * 2 // 8) + + fx.Int32(chk * 8) + + mfma_col_grp * fx.Int32(2) + ) + qv = vector.load_op( + T.vec(2, T.i64), + q_lds_i64, + [arith.index_cast(T.index, q_idx_i64)], + ) + q_chunks.append(vector.bitcast(T.vec(8, T.bf16), qv)) + + # ===== STEP D: Online softmax + PV state ========================= + running_max = NEG_INF + running_sum = ZERO_F + zero_v4 = arith.constant_vector(0.0, T.f32x4) + acc_pv = [zero_v4 for _ in range(PV_N_CHUNKS)] + + # ===== STEP E: Sequence-len + partition base ==================== + seq_len = buffer_ops.buffer_load(sl_rsrc, seq, vec_width=1, dtype=T.i32) + # ── Bounded num_partitions + internal looping ─────────────────── + # With TGPP > 1 this CTA owns a contiguous slice of length + # ``PARTITION_EXTENT_TOKENS`` that is iterated as ``TGPP`` groups + # of 16 K-tiles each. ``partition_start`` is recomputed per + # outer ``tg`` iteration below; the FA-2 online-softmax state + # (running_max, running_sum, acc_pv) accumulates across all + # ``TGPP * 16`` tiles for this CTA — semantically identical to + # processing one large 16*TGPP-tile partition. + partition_base = part * fx.Int32(PARTITION_EXTENT_TOKENS) + + # Per-K-tile dequant lane assignment: + # 16 tokens × 64 packed bytes per token = 1024 bytes + # 64 lanes × 16 bytes/lane = 1024 ✓ + # Lane t: token = t / 4, sub-chunk = t % 4 (0..3) → 16 bytes contiguous + tok_in_tile = lane >> fx.Int32(2) + chunk_in_tok = lane & fx.Int32(3) + # ``slot`` (= absolute slot index within the cache block, range + # 0.._BS-1) is recomputed per tile inside the K-tile loop below + # so it reflects ``tile_in_block * TILE_SIZE + tok_in_tile``. + + c_meta_u16_per_kvh = fx.Int32(NUM_SOA_FIELDS * _BS) + c_knorm_off_u16 = fx.Int32(SOA_K_NORM * _BS) + c_vscale_off_u16 = fx.Int32(SOA_V_SCALE * _BS) + c_vzero_off_u16 = fx.Int32(SOA_V_ZERO * _BS) + + # ===== STEP F: K-tile loop ======================================= + # With kv_block_size > TILE_SIZE there are _TILES_PER_BLOCK tiles + # per cache block. Tiles within the same block share a block-table + # entry but address different slot ranges within the block. + # + # ── Per-tile block-table OOB redirect ──────────────────────────── + # The K-tile loop is unrolled 16 times so every CTA issues 16 + # block-table reads regardless of (partition, seq_len). When a + # partition extends past ``seq_len`` (e.g. num_partitions=2 for + # a 256-token sequence — partition 1 covers tokens 256..511, all + # masked out) the per-tile bt offset can land beyond the bt + # allocation. With ``max_size=True`` the descriptor advertises + # 4 GB so the HW returns whatever pre-existing HBM bytes live at + # that offset; the resulting garbage ``phys_block`` is multiplied + # by ``stride_cache_block`` and the subsequent kv_cache read + # jumps into pages that may be unmapped (→ "Memory access fault + # by GPU node") or may decode as NaN bytes (→ NaN segm_out). + # Allocator-pattern dependent, but reproduces deterministically + # at large B (e.g. B=64 seq=256 in the test sweep produces 8 192 + # NaN entries even on canonical QG=8). + # + # Fix: when the tile starts at or past ``seq_len`` (so its + # qk_acc is going to be killed by the per-token + # ``kv_tok < seq_len`` mask anyway), redirect the bt read to + # ``bt[seq, 0]`` — always in bounds. The redundant phys_block + # decode + kv_cache read is wasted work, but correctness is + # preserved without changing the kernel's iteration count. + # ===== Runtime-adaptive scf.ForOp w/ iter_args ========= + # Runtime-adaptive outer loop: trip count derives from + # actual seq_len, NOT TGPP_max. At cudagraph-capture warmup + # (seq_len=1) only 1 iteration runs; at 32K production + # decode all TGPP iterations run. Kernel binary stays small + # (single body, looped at runtime) — capture time matches + # the legacy 16-tile-only kernel rather than scaling 4x. + # + # FA-2 state (running_max, running_sum, acc_pv[8]) threads + # through scf iter_args; the body reads them at top, runs + # the unchanged 16-tile inner body, and yields the new + # state at bottom. After the loop, results are pulled out + # of for_op.results back into the local Python names so the + # downstream STEP G (output) is unmodified. + c_kcb = fx.Int32(KV_COMPUTE_BLOCK) + c_tgpp = fx.Int32(TGPP) + c_zero_i32 = fx.Int32(0) + c_one_i32 = fx.Int32(1) + # remaining = seq_len - partition_base (signed, may be <=0) + remaining = seq_len - partition_base + in_range = remaining > c_zero_i32 + # trip_raw = ceil(remaining / KV_COMPUTE_BLOCK) + # (when in_range is false, the divisor branch produces + # garbage that the select below overrides with zero, so we + # don't pre-clamp) + trip_raw = (remaining + c_kcb - c_one_i32) // c_kcb + trip_clamped = (trip_raw > c_tgpp).select(c_tgpp, trip_raw) + trip_or_zero = in_range.select(trip_clamped, c_zero_i32) + c_zero_idx = arith.constant(0, index=True) + c_one_idx = arith.constant(1, index=True) + trip_idx = arith.index_cast( + T.index, + trip_or_zero.ir_value() + if hasattr(trip_or_zero, "ir_value") + else trip_or_zero, + ) + # Initial iter_args list: FA-2 accumulator state. + # Order: running_max, running_sum, acc_pv[0..PV_N_CHUNKS-1]. + # NOTE: scf.ForOp requires ir.Value (with .type). The DSL + # constants ZERO_F / ONE_F are fx.Float32 (Numeric) wrappers, + # not raw ir.Value, so we unwrap via .ir_value() before + # passing. _ival() is defined once near the top of the kernel body. + _init_iter = [ + _ival(running_max), + _ival(running_sum), + *[_ival(p) for p in acc_pv], + ] + _for_op = _scf.ForOp( + c_zero_idx, + trip_idx, + c_one_idx, + _init_iter, + ) + _for_ip = ir.InsertionPoint(_for_op.body) + _for_ip.__enter__() + try: + tg_idx = _for_op.induction_variable + tg_i32 = fx.Int32(arith.index_cast(T.i32, tg_idx)) + partition_start = partition_base + tg_i32 * c_kcb + bt_seq_base = seq * c_bt + (partition_start // fx.Int32(_BS)) + running_max = _for_op.inner_iter_args[0] + running_sum = _for_op.inner_iter_args[1] + acc_pv = list(_for_op.inner_iter_args[2:]) + for n_tile in range_constexpr(16): + block_in_part = n_tile // _TILES_PER_BLOCK + tile_in_block = n_tile % _TILES_PER_BLOCK + tile_start_tok = partition_start + fx.Int32(n_tile * TILE_SIZE) + tile_in_seq = tile_start_tok < seq_len + bt_off = bt_seq_base + fx.Int32(block_in_part) + bt_off_safe = tile_in_seq.select(bt_off, seq * c_bt) + phys_block = buffer_ops.buffer_load( + bt_rsrc, + bt_off_safe, + vec_width=1, + dtype=T.i32, + ) + block_base = phys_block * c_block + data_region = block_base + meta_region = block_base + c_meta_off + + # ``slot`` is the absolute slot index within the cache block + # (range 0.._BS-1). For BS=16 it equals tok_in_tile; for BS=32 + # it equals tile_in_block * 16 + tok_in_tile. Used both for + # the K/V data region and SoA metadata region addressing. + slot = fx.Int32(tile_in_block * TILE_SIZE) + tok_in_tile + data_bases_byte = ( + data_region + + slot * (c_kv_heads * c_data_per_slot) + + kv_h * c_data_per_slot + ) + k_byte = data_bases_byte + chunk_in_tok * fx.Int32(16) + k_packed = buffer_ops.buffer_load( + kv_rsrc, + k_byte // fx.Int32(4), + vec_width=4, + dtype=T.i32, + ) + + # ---- HOISTED: issue V data + meta HBM loads early --------- + # These are async; their s_waitcnt is pushed by the compiler + # past the K dequant + QK MFMA + softmax block, hiding most + # of the V HBM latency behind compute. + v_byte = data_bases_byte + c_keydata + chunk_in_tok * fx.Int32(16) + v_packed = buffer_ops.buffer_load( + kv_rsrc, + v_byte // fx.Int32(4), + vec_width=4, + dtype=T.i32, + ) + vscale_u16 = ( + meta_region // fx.Int32(2) + + kv_h * c_meta_u16_per_kvh + + c_vscale_off_u16 + + slot + ) + vzero_u16 = ( + meta_region // fx.Int32(2) + + kv_h * c_meta_u16_per_kvh + + c_vzero_off_u16 + + slot + ) + vscale_raw = buffer_ops.buffer_load( + kv_rsrc, + vscale_u16, + vec_width=1, + dtype=T.i16, + ) + vzero_raw = buffer_ops.buffer_load( + kv_rsrc, + vzero_u16, + vec_width=1, + dtype=T.i16, + ) + + knorm_u16 = ( + meta_region // fx.Int32(2) + + kv_h * c_meta_u16_per_kvh + + c_knorm_off_u16 + + slot + ) + knorm_raw = buffer_ops.buffer_load( + kv_rsrc, + knorm_u16, + vec_width=1, + dtype=T.i16, + ) + knorm_f16 = arith.bitcast(T.f16, knorm_raw) + knorm_f32 = arith.extf(T.f32, knorm_f16) + + # K dequant → LDS [token, head_dim] (natural) + # Lane writes 32 bf16 (= 4×8) for token=tok_in_tile, + # head_dims chunk_in_tok*32..+31 (4 sub-chunks of 8). + tok_kreg = tok_in_tile * fx.Int32(HEAD_SIZE * 2 // 8) + chunk_kreg = tok_kreg + chunk_in_tok * fx.Int32(8) + for w in range_constexpr(4): + word_i32 = vector.extract(k_packed, static_position=[w]) + bf16_elems = [] + for n in range_constexpr(8): + nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF) + nibble_idx = arith.index_cast(T.index, nibble) + cent_f32 = cent_lds.load([nibble_idx]) + elem_bf16 = arith.trunc_f(T.bf16, cent_f32 * knorm_f32) + bf16_elems.append(elem_bf16) + v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems) + v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16) + vector.store( + v_i64, + kv_lds_i64, + [arith.index_cast(T.index, chunk_kreg + fx.Int32(w * 2))], + ) + gpu.barrier() + + # QK MFMA (CDNA4 wide-K): A=K[token, head_dim], B=Q (= Q^T). + # K read: lane t = K[token=mfma_row, head_dim=chunk*32+col_grp*8..+7] + # Same i64×2 indexing as Q. + qk_acc = zero_v4 + for chk in range_constexpr(QK_K_CHUNKS): + k_idx_i64 = ( + mfma_row * fx.Int32(HEAD_SIZE * 2 // 8) + + fx.Int32(chk * 8) + + mfma_col_grp * fx.Int32(2) + ) + kv_load = vector.load_op( + T.vec(2, T.i64), + kv_lds_i64, + [arith.index_cast(T.index, k_idx_i64)], + ) + k_op = vector.bitcast(T.vec(8, T.bf16), kv_load) + qk_acc = rocdl.mfma_f32_16x16x32_bf16( + T.f32x4, [k_op, q_chunks[chk], qk_acc, 0, 0, 0] + ) + + # qk_acc layout: lane t holds C[token=(t/16)*4..+3, query=t%16] + # 4 fp32/lane = 4 different tokens at SAME query_row. + + # Scale + mask out-of-context tokens. + qk_acc = _vsplat_mul(qk_acc, QK_SCALE) + for elem in range_constexpr(4): + kv_tok = ( + partition_start + + fx.Int32(n_tile * TILE_SIZE) + + mfma_col_grp * fx.Int32(4) + + fx.Int32(elem) + ) + in_b = kv_tok < seq_len + v = vector.extract(qk_acc, static_position=[elem]) + qk_acc = vector.insert( + in_b.select(v, NEG_INF), + qk_acc, + static_position=[elem], + dynamic_position=[], + ) + + # FA2 online softmax: per-query-row reduce. + # Per-row max: max over 4 fp32 in lane, then xor-shuffle 16, 32 + # (across the 4 col_grps that share same mfma_row). + local_max = vector.reduction(T.f32, "maxnumf", qk_acc) + r1 = local_max.shuffle_xor(fx.Int32(16), c_w) + local_max = local_max.maximumf(r1) + r2 = local_max.shuffle_xor(fx.Int32(32), c_w) + tile_max = local_max.maximumf(r2) + + new_max = running_max.maximumf(tile_max) + max_diff = running_max - new_max + safe_diff = (running_max > NEG_INF).select(max_diff, ZERO_F) + scale = (safe_diff * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast) + running_sum = running_sum * scale + for h in range_constexpr(PV_N_CHUNKS): + acc_pv[h] = _vsplat_mul(acc_pv[h], scale) + running_max = new_max + + # Compute probs: p = exp((qk - new_max) * LOG2E) + tile_sum = ZERO_F + for elem in range_constexpr(4): + s = vector.extract(qk_acc, static_position=[elem]) + d = s - new_max + d = (new_max > NEG_INF).select(d, NEG_INF) + p = (d * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast) + tile_sum = tile_sum + p + qk_acc = vector.insert( + p, qk_acc, static_position=[elem], dynamic_position=[] + ) + + ts1 = tile_sum.shuffle_xor(fx.Int32(16), c_w) + tile_sum = tile_sum + ts1 + ts2 = tile_sum.shuffle_xor(fx.Int32(32), c_w) + tile_sum = tile_sum + ts2 + running_sum = running_sum + tile_sum + + # ---- V dequant → LDS (V/scale/zero raw loads were hoisted) - + # v_packed, vscale_raw, vzero_raw are already in flight from + # the top of this iteration; only the f32 conversion stays + # here so the actual consume site is still close to the use. + vscale_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vscale_raw)) + vzero_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vzero_raw)) + + if USE_HW_TR: + # V dequant → LDS [token][head_dim] (ROW-MAJOR, no transpose). + # Each lane writes 32 contiguous bf16 (one token, head_dims + # chunk_in_tok*32..+31) as 4× ds_write_b128 = 4× vec(2,i64). + # Replaces 32× ds_write_b16 of the legacy transposed path. + v_lds_elem_base = tok_in_tile * fx.Int32( + HEAD_SIZE + ) + chunk_in_tok * fx.Int32(32) + for w in range_constexpr(4): + word_i32 = vector.extract(v_packed, static_position=[w]) + bf16_elems = [] + for n in range_constexpr(8): + nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF) + nibble_f32 = arith.sitofp(T.f32, nibble) + elem_f32 = nibble_f32 * vscale_f32 + vzero_f32 + elem_bf16 = arith.trunc_f(T.bf16, elem_f32) + bf16_elems.append(elem_bf16) + v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems) + v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16) + v_lds_i64_idx = (v_lds_elem_base + fx.Int32(w * 8)) // fx.Int32( + 4 + ) + vector.store( + v_i64, + kv_lds_i64, + [arith.index_cast(T.index, v_lds_i64_idx)], + ) + # ── HW V transpose: cross-lane LDS sync ───────────────────── + # ds_read_tr16_b64 below introduces a cross-lane LDS read: + # consumer lane t reads bytes written by source lanes + # 0,4,8,12 (etc.). The compiler's automatic waitcnt insertion + # is per-lane and has no awareness of the HW transpose's + # cross-lane forwarding, so it can sink the next iteration's + # ds_read ahead of these ds_write_b128 ops in the schedule. + # + # Two-part fence: + # 1. sched_barrier(0): MachineScheduler reorder barrier + # (mask=0 → no instruction class may cross). Pure hint, + # generates no runtime instruction. Use the simple + # sched_barrier (NOT sched_group_barrier, which is an + # IGLP marker that requires partner barriers and + # corrupts the schedule when used in isolation). + # 2. s_waitcnt lgkmcnt=0: runtime drain of the LDS write + # queue. vmcnt/expcnt left at no-wait so we don't + # stall on speculatively-issued HBM loads (which can + # include OOB-but-masked buffer_load reads from the + # next K-tile's hoisted V prefetch). + # + # The SW path below does NOT need this because each lane + # only reads cells it itself wrote (no cross-lane dep), and + # the per-lane same-address waitcnt the compiler emits is + # correct. + # + # Without this fence the reordered ds_read produces + # wrong results on high-partition-count configs (large + # padded num_partitions); low-partition configs are + # unaffected. + rocdl.sched_barrier(0) + # encode_waitcnt(vmcnt=63, expcnt=7, lgkmcnt=0) + # = 0xF | (7<<4) | (0<<8) | (3<<14) = 0xC07F + rocdl.s_waitcnt(0xC07F) + else: + # Legacy transposed V_LDS path: 32 ds_write_b16 per lane. + for w in range_constexpr(4): + word_i32 = vector.extract(v_packed, static_position=[w]) + for n in range_constexpr(8): + nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF) + nibble_f32 = arith.sitofp(T.f32, nibble) + elem_f32 = nibble_f32 * vscale_f32 + vzero_f32 + elem_bf16 = arith.trunc_f(T.bf16, elem_f32) + elem_i16 = arith.bitcast(T.i16, elem_bf16) + head_dim = chunk_in_tok * fx.Int32(32) + fx.Int32(w * 8 + n) + v_idx_i16 = head_dim * fx.Int32(TILE_SIZE) + tok_in_tile + v_vec = vector.from_elements(T.vec(1, T.i16), [elem_i16]) + vector.store( + v_vec, + kv_lds_i16, + [arith.index_cast(T.index, v_idx_i16)], + ) + gpu.barrier() + + # ---- PV MFMA: A=V[head_dim, token], B=P (=qk_acc bf16) ----- + # P operand B layout matches qk_acc: lane t holds 4 bf16 at + # K=token=(t/16)*4..+3, N=query=t%16. Just trunc_f to bf16. + p_bf16 = arith.trunc_f(T.vec(4, T.bf16), qk_acc) + p_op = vector.bitcast(T.vec(4, T.i16), p_bf16) + + if USE_HW_TR: + # HW-transpose PV path: V_lds is row-major V[token][head_dim]. + # MFMA A operand layout: lane t holds 4 bf16 at + # M=head_dim=mfma_row + h*16, K=token=(t/16)*4..+3 + # ds_read_tr16_b64 (4-element 16-bit transpose, per-16-lane block): + # result[lane=t, elem=e] = Input[source_lane=e*4 + (t%16)//4, + # col=t%4] + # Per-lane address: token_idx = lane // 4 (covers 0..15 across + # all four 16-lane MFMA blocks), hd_sub = (lane % 4)*4 selects + # the 4-element column window inside the h*16 chunk. + # Total LDS byte offset = kv_off + token_idx*HEAD_SIZE*2 + # + (h*16 + hd_sub)*2 + token_idx = lane >> fx.Int32(2) + hd_sub = (lane & fx.Int32(3)) * fx.Int32(4) + v_lane_byte = ( + fx.Int32(kv_off) + + token_idx * fx.Int32(HEAD_SIZE * 2) + + hd_sub * fx.Int32(2) + ) + for h in range_constexpr(PV_N_CHUNKS): + v_byte_off = v_lane_byte + fx.Int32(h * 32) + v_byte_i64 = fx.Int64(v_byte_off) + v_ptr = buffer_ops.create_llvm_ptr( + v_byte_i64, + address_space=3, + ) + v_op_raw = rocdl.ds_read_tr16_b64( + T.vec(4, T.i16), + v_ptr, + ).result + acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k( + T.f32x4, [v_op_raw, p_op, acc_pv[h], 0, 0, 0] + ) + else: + # Legacy transposed-LDS PV: 1 ds_read_b64 per chunk. + # Byte addr = (mfma_row + h*16) * TILE_SIZE * 2 + (col_grp*4) * 2 + # = (mfma_row + h*16) * 32 + col_grp*8 + # i64 idx = (mfma_row + h*16) * 4 + col_grp + for h in range_constexpr(PV_N_CHUNKS): + v_idx_i64 = (mfma_row + fx.Int32(h * 16)) * fx.Int32( + 4 + ) + mfma_col_grp + kv_load = vector.load_op( + T.vec(1, T.i64), + kv_lds_i64, + [arith.index_cast(T.index, v_idx_i64)], + ) + v_op = vector.bitcast(T.vec(4, T.i16), kv_load) + acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k( + T.f32x4, [v_op, p_op, acc_pv[h], 0, 0, 0] + ) + + _scf.YieldOp( + [ + _ival(running_max), + _ival(running_sum), + *[_ival(p) for p in acc_pv], + ] + ) + finally: + _for_ip.__exit__(None, None, None) + # Pull final accumulator values out of the for_op results. + running_max = _for_op.results[0] + running_sum = _for_op.results[1] + acc_pv = list(_for_op.results[2:]) + + # ===== STEP G: Output =========================================== + # acc_pv[h] layout: lane t holds 4 fp32 at M=head_dim=(t/16)*4..+3 + h*16, + # N=query_row=t%16. + # 4 fp32/lane = 4 contiguous head_dim values for ONE query_row. + safe_sum = (running_sum > ZERO_F).select(running_sum, ONE_F) + rcp = ONE_F / safe_sum + + c_os = fx.Int32(_stride_out_seq) + c_oh = fx.Int32(_stride_out_head) + c_op_ = fx.Int32(_stride_out_part) + out_base = seq * c_os + kv_h * c_oh + part * c_op_ + + # Gate all global stores by mfma_row < QG. For QG=16 the predicate is + # tautological (mfma_row ∈ [0,15]); for QG=8 lanes 8..15 (whose MFMA + # outputs are computational waste) skip the stores entirely. + valid_row_pred = arith.cmpi( + arith.CmpIPredicate.ult, + mfma_row.ir_value() if hasattr(mfma_row, "ir_value") else mfma_row, + arith.constant(QG, type=T.i32), + ) + _if = _scf.IfOp(valid_row_pred) + with ir.InsertionPoint(_if.then_block): + for h in range_constexpr(PV_N_CHUNKS): + pv_norm = _vsplat_mul(acc_pv[h], rcp) + pv_bf16 = arith.trunc_f(T.vec(4, T.bf16), pv_norm) + pv_i32x2 = vector.bitcast(T.vec(2, T.i32), pv_bf16) + head_dim_start = fx.Int32(h * 16) + mfma_col_grp * fx.Int32(4) + out_off_elem = ( + out_base + mfma_row * fx.Int32(HEAD_SIZE) + head_dim_start + ) + buffer_ops.buffer_store( + pv_i32x2, + out_rsrc, + out_off_elem * fx.Int32(2), + offset_is_bytes=True, + ) + + c_npq = fx.Int32(num_partitions * QG) + ml_off = ( + seq * fx.Int32(_stride_ml_seq) + kv_h * c_npq + part * c_qg + mfma_row + ) + es_off = ( + seq * fx.Int32(_stride_es_seq) + kv_h * c_npq + part * c_qg + mfma_row + ) + buffer_ops.buffer_store(running_max, ml_rsrc, ml_off) + buffer_ops.buffer_store(running_sum, es_rsrc, es_off) + _scf.YieldOp([]) + + return tq_decode_kernel diff --git a/vllm/v1/attention/ops/flydsl_kernels/tq_decode_gqa6.py b/vllm/v1/attention/ops/flydsl_kernels/tq_decode_gqa6.py new file mode 100644 index 0000000000..e7c4faca5a --- /dev/null +++ b/vllm/v1/attention/ops/flydsl_kernels/tq_decode_gqa6.py @@ -0,0 +1,875 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# TurboQuant decode GQA-6 sibling (FlyDSL) — MI355X / gfx950 / CDNA4 +# +# Decode-only paged attention over a TurboQuant SoA KV cache. +# Targets MiniMax-M2.5 class models: HEAD_SIZE=128, GQA group=6, MSE_BITS=4 K, +# VQB=4 V, N_CENTROIDS=16, BLOCK_SIZE=16, partitioned mode. +# +# === Sibling-file rationale === +# Structurally identical to tq_decode.py except: +# * QUERY_GROUP_SIZE default = 6 (was 8) +# * Build-time assert allows query_group_size = 6 only (Qwen sizes go to +# the canonical tq_decode kernel — NOT this file) +# * QG_LOAD_ITERS = (QG + 3) // 4 (round-up; 2 iters for QG=6 loads +# 8 LDS rows of which only 6 carry real Q; rows 6..7 are zero-padded +# by buffer_load OOB protection — gated out by the same mfma_row < QG +# predicate the canonical kernel already uses for QG=8) +# +# All cross-lane operations (xor_shuffle softmax reductions, ds_read_tr16_b64 +# HW V transpose) are mfma_row-isolated and thus already GQA-agnostic. The +# kernel is functionally a direct copy of tq_decode.py with the QG knob +# turned to 6 — kept as a sibling per FlyDSL convention to preserve the +# canonical kernel's invariants for its production callers (Qwen2.5-72B, +# Qwen3-32B). +# +# === Design === +# * 1 wave (64 lanes) per CTA — simpler than 4-warp; trades parallelism for +# correctness/debug-ability. A later revision will fan out to 4 warps. +# * Flash-Attention-2 online softmax: rescale running PV/sum per K-tile, +# no QK score storage across tiles, no P-LDS round-trip (qk_acc layout +# matches PV B-operand directly). +# * Centroids loaded once into LDS at kernel start (LDS-resident LUT). +# * K dequant → LDS [token, head_dim] (natural layout) +# * V dequant → LDS [head_dim, token] (TRANSPOSED — required so PV MFMA's +# A=V operand can be loaded as 4-bf16 contiguous via 1 ds_read_b64). +# +# MFMA layouts used: +# QK: mfma(A=K, B=Q, C=qk_acc) → C[m=token, n=query], 4 fp32/lane +# = 4 tokens for SAME query → softmax via +# xor_shuffle(16),(32) reduces per-row +# PV: mfma(A=V_T, B=P, C=acc_pv) → C[m=head_dim, n=query], 4 fp32/lane +# = 4 contiguous head_dim cols for SAME +# query → 1 buffer_store of 8 bytes +# per chunk per lane + +from __future__ import annotations + +import flydsl.compiler as flyc +import flydsl.expr as fx +from flydsl._mlir import ir +from flydsl._mlir.dialects import scf as _scf +from flydsl.expr import ( + arith, + buffer_ops, + const_expr, # noqa: F401 kept for kernel-DSL API parity + gpu, + range_constexpr, + rocdl, + vector, +) +from flydsl.expr.typing import Int32, T # noqa: F401 Int32 via fx.Int32 +from flydsl.runtime.device import get_rocm_arch as get_hip_arch +from flydsl.utils.smem_allocator import SmemAllocator, SmemPtr + +# === Constants (MiniMax-M2.5-class TQ decode profile) ======================= +HEAD_SIZE = 128 +# default; overridable via build_tq_decode_gqa6_module(kv_block_size=...) +KV_BLOCK_SIZE = 16 +# MFMA tile = 16 tokens (do not change without re-deriving MFMA shapes) +TILE_SIZE = 16 +N_CENTROIDS = 16 +QUERY_GROUP_SIZE = 6 # GQA-6 default for MiniMax-M2.5 +WARP_SIZE = 64 +NUM_WARPS = 1 +BLOCK_THREADS = NUM_WARPS * WARP_SIZE +KV_COMPUTE_BLOCK = 256 # 16 K-tiles × 16 tokens + +# TQ SoA layout +NUM_SOA_FIELDS = 3 +SOA_K_NORM = 0 +SOA_V_SCALE = 1 +SOA_V_ZERO = 2 +KEY_DATA_BYTES = HEAD_SIZE // 2 +VAL_DATA_BYTES = HEAD_SIZE // 2 +DATA_BYTES_PER_SLOT = KEY_DATA_BYTES + VAL_DATA_BYTES + +# MFMA +MFMA_M = MFMA_N = 16 +MFMA_K_BF16_QK = 32 # CDNA4 wide-K (mfma_f32_16x16x32_bf16) +MFMA_K_BF16_PV = 16 # PV's K = token tile = 16 +QK_K_CHUNKS = HEAD_SIZE // MFMA_K_BF16_QK # 4 +PV_N_CHUNKS = HEAD_SIZE // MFMA_N # 8 + +# LDS regions +# Q region uses the full 16-row footprint (4096 B) rather than a tight +# QG=6 sizing, keeping the LDS layout identical to the canonical kernel. +CENTROID_LDS_BYTES = N_CENTROIDS * 4 # 64 +Q_LDS_BYTES = 16 * HEAD_SIZE * 2 # 4096 +KV_TILE_LDS_BYTES = TILE_SIZE * HEAD_SIZE * 2 # 4096 + +LOG2E = 1.4426950408889634 +NEG_INF_VAL = float("-inf") + + +def _vsplat_mul(vec, scalar): + s = scalar.ir_value() if hasattr(scalar, "ir_value") else scalar + return vec * vector.broadcast(T.f32x4, s) + + +allocator = None + + +def build_tq_decode_gqa6_module( + num_seqs: int, + num_kv_heads: int, + num_partitions: int, + max_blocks_per_seq: int = 512, + softmax_scale: float | None = None, + query_group_size: int = QUERY_GROUP_SIZE, + kv_block_size: int = KV_BLOCK_SIZE, + use_hw_v_transpose: bool = False, + tile_groups_per_partition: int = 1, +): + """Build a TQ decode GQA-6 kernel module (MiniMax-M2.5). + + ``query_group_size`` (= num_query_heads // num_kv_heads) is fixed at 6 + for this sibling. The MFMA's 16-row capacity is 6/16 = 37.5% used; + lanes 6..15 compute garbage and are gated out of all global-memory + writes via the same ``mfma_row < QG`` predicate the canonical kernel + uses for QG=8. Q load iterates ``(QG + 3) // 4`` = 2 chunks (loads + 8 LDS rows of which rows 6..7 are zero-padded by the buffer-resource + OOB protection — they're never consumed because the gating lanes + don't write). + + For GQA-8 or GQA-16 (Qwen models), use the canonical + ``build_tq_decode_module`` from ``tq_decode.py`` instead. + + ``kv_block_size`` is the number of tokens per vLLM cache block. Must + be a multiple of TILE_SIZE (=16). Supported values: 16 and 32. + + ``use_hw_v_transpose`` (CDNA4 / gfx950 only) — see canonical kernel + docstring for details. Same fence pattern applies here. + """ + assert query_group_size == 6, ( + f"build_tq_decode_gqa6_module is GQA-6 only; got " + f"query_group_size={query_group_size}. Use build_tq_decode_module " + f"from tq_decode.py for GQA-8 or GQA-16." + ) + assert kv_block_size in (16, 32), ( + f"kv_block_size must be 16 or 32; got {kv_block_size}" + ) + assert kv_block_size % TILE_SIZE == 0 + assert int(tile_groups_per_partition) >= 1, ( + f"tile_groups_per_partition must be >= 1; got {tile_groups_per_partition}" + ) + USE_HW_TR = bool(use_hw_v_transpose) + TGPP = int(tile_groups_per_partition) + PARTITION_EXTENT_TOKENS = TGPP * KV_COMPUTE_BLOCK + + QG = int(query_group_size) + # Round-up: 2 iters for QG=6 → loads 8 LDS rows (rows 6..7 are + # zero-padded by buffer_load OOB protection; never consumed because + # mfma_row >= QG lanes are gated out of all global stores below). + QG_LOAD_ITERS = (QG + 3) // 4 + OOB_OFFSET = 0x7FFFFFF0 # noqa: F841 ~2GB byte offset; > any plausible buffer + + _BS = int(kv_block_size) + _TILES_PER_BLOCK = _BS // TILE_SIZE # 1 for BS=16, 2 for BS=32 + + global allocator + arch = get_hip_arch() + + if softmax_scale is None: + softmax_scale = 1.0 / (HEAD_SIZE**0.5) + _qk_scale = float(softmax_scale) + + # --- Strides --- + _Hq = num_kv_heads * QG + _stride_q_seq = _Hq * HEAD_SIZE + _stride_q_head = HEAD_SIZE + _stride_bt_seq = max_blocks_per_seq + + _data_region_bytes = _BS * num_kv_heads * DATA_BYTES_PER_SLOT + _meta_region_bytes = num_kv_heads * NUM_SOA_FIELDS * _BS * 2 + _stride_cache_block = _data_region_bytes + _meta_region_bytes + _meta_region_offset = _data_region_bytes + + _stride_out_part = QG * HEAD_SIZE + _stride_out_head = num_partitions * QG * HEAD_SIZE + _stride_out_seq = num_kv_heads * num_partitions * QG * HEAD_SIZE + _stride_es_seq = num_kv_heads * num_partitions * QG + _stride_ml_seq = _stride_es_seq + + # --- LDS layout --- + allocator = SmemAllocator(None, arch=arch, global_sym_name="tq_gqa6_smem") + centroid_off = 0 + allocator.ptr = CENTROID_LDS_BYTES + q_off = allocator.ptr + allocator.ptr += Q_LDS_BYTES + kv_off = allocator.ptr + allocator.ptr += KV_TILE_LDS_BYTES + + @flyc.kernel + def tq_decode_gqa6_kernel( + out_ptr: fx.Tensor, + exp_sums_ptr: fx.Tensor, + max_logits_ptr: fx.Tensor, + query_ptr: fx.Tensor, + kv_cache_ptr: fx.Tensor, + centroids_ptr: fx.Tensor, + block_tables_ptr: fx.Tensor, + seq_lens_ptr: fx.Tensor, + ): + # ---- IDs --------------------------------------------------------- + tid = gpu.thread_idx.x + seq = gpu.block_idx.x + kv_h = gpu.block_idx.y + part = gpu.block_idx.z + lane = tid # 0..63 + mfma_row = lane & fx.Int32(15) # = query_row (B's N) / + # token (A's M for QK) + mfma_col_grp = lane >> fx.Int32(4) # 0..3, K-group dim + + # ---- Buffer resources ------------------------------------------- + q_rsrc = buffer_ops.create_buffer_resource(query_ptr, max_size=True) + kv_rsrc = buffer_ops.create_buffer_resource(kv_cache_ptr, max_size=True) + bt_rsrc = buffer_ops.create_buffer_resource(block_tables_ptr, max_size=True) + sl_rsrc = buffer_ops.create_buffer_resource(seq_lens_ptr, max_size=True) + cent_rsrc = buffer_ops.create_buffer_resource(centroids_ptr, max_size=True) + out_rsrc = buffer_ops.create_buffer_resource(out_ptr, max_size=True) + es_rsrc = buffer_ops.create_buffer_resource(exp_sums_ptr, max_size=True) + ml_rsrc = buffer_ops.create_buffer_resource(max_logits_ptr, max_size=True) + + # ---- LDS pointers ----------------------------------------------- + base = allocator.get_base() + cent_lds = SmemPtr(base, centroid_off, T.f32, shape=(N_CENTROIDS,)) + q_lds_i32 = SmemPtr(base, q_off, T.i32, shape=(Q_LDS_BYTES // 4,)).get() + q_lds_i64 = SmemPtr(base, q_off, T.i64, shape=(Q_LDS_BYTES // 8,)).get() + kv_lds_i32 = SmemPtr(base, kv_off, T.i32, shape=(KV_TILE_LDS_BYTES // 4,)).get() # noqa: F841 + kv_lds_i64 = SmemPtr(base, kv_off, T.i64, shape=(KV_TILE_LDS_BYTES // 8,)).get() + kv_lds_i16 = SmemPtr(base, kv_off, T.i16, shape=(KV_TILE_LDS_BYTES // 2,)).get() + + # ---- Constants --------------------------------------------------- + c_sq = fx.Int32(_stride_q_seq) + c_qh = fx.Int32(_stride_q_head) + c_qg = fx.Int32(QG) + c_bt = fx.Int32(_stride_bt_seq) + c_block = fx.Int32(_stride_cache_block) + c_meta_off = fx.Int32(_meta_region_offset) + c_data_per_slot = fx.Int32(DATA_BYTES_PER_SLOT) + c_kv_heads = fx.Int32(num_kv_heads) + c_keydata = fx.Int32(KEY_DATA_BYTES) + c_w = fx.Int32(WARP_SIZE) + + NEG_INF = arith.constant(NEG_INF_VAL, type=T.f32) + ZERO_F = fx.Float32(0.0) + ONE_F = fx.Float32(1.0) + LOG2E_C = arith.constant(LOG2E, type=T.f32) + QK_SCALE = arith.constant(_qk_scale, type=T.f32) + + # ===== STEP A: Load centroids → LDS (cooperative, race-safe) ===== + c_idx_safe = lane & fx.Int32(N_CENTROIDS - 1) + c_val = buffer_ops.buffer_load(cent_rsrc, c_idx_safe, vec_width=1, dtype=T.f32) + cent_lds.store(c_val, [arith.index_cast(T.index, c_idx_safe)]) + gpu.barrier() + + # ===== STEP B: Load Q → row-major LDS [query_row, head_dim] ====== + # Q layout in HBM: [N, Hq, D] bf16, slice = QG rows × D=128 cols × 2B. + # QG=6 → 1536 B real, 96 chunks; QG_LOAD_ITERS=2 issues 128 chunks + # (8 rows × 16 col_b) so lanes whose row ∈ {6, 7} address heads that + # do NOT exist in the Q tensor (Hq = num_kv_heads * 6). + # + # ── Per-lane Q-load OOB redirect ───────────────────────────────── + # Hq = num_kv_heads * QG, so head index = kv_h * QG + row. For QG=6 + # the upper rows alias to (kv_h+1)'s real heads (still in-bounds + # for most kv_h), but for the *last* (seq, kv_h) the read tips + # past the Q tensor entirely. With ``max_size=True`` the buffer + # descriptor advertises 4 GB so the HW does no OOB clamping; the + # padding read then touches unmapped pages and the kernel dies + # with "Memory access fault by GPU node" the moment the + # surrounding allocator state leaves the next page unmapped + # (allocator-pattern-dependent → nondeterministic fault). + # + # Fix: redirect OOB lanes (``row >= QG``) to load Q[0,0,col_elem] + # — a byte offset that is always inside the actual Q tensor. + # The redundant load is wasted but harmless; the resulting LDS + # write to row ∈ {6, 7} is overwritten/ignored downstream because + # every global store is gated by the output-time + # ``mfma_row < QG`` predicate (the same gate canonical QG=8 uses + # for its garbage rows 8..15). + # + # NB: A ``mask=`` parameter on ``buffer_load`` substitutes the + # masked-out offset with 0x7FFFFFFF *bytes*; with ``max_size=True`` + # this is still inside the 4 GB descriptor and so the HW happily + # dereferences ``q_base + 2 GB`` → another unmapped page → fault. + # The address-redirect below is the only safe option short of + # threading the true Q size through as a runtime kernel arg. + # + # The canonical Qwen kernel does not need this redirect because + # for QG ∈ {8, 16} the (kv_h, row) iteration covers exactly the + # real heads; QG=6 is the only case where + # ``QG_LOAD_ITERS * 4 != QG``. + # STEP B: load pre-rotated q_rot → Q_LDS. QG=6 over-reads + # (QG_LOAD_ITERS * 4 != QG), so rows >= QG redirect to a safe + # in-bounds offset to avoid faulting past the real Q rows. + for c in range_constexpr(QG_LOAD_ITERS): + row_chunk = lane + fx.Int32(c * WARP_SIZE) # 0..127 + row = row_chunk >> fx.Int32(4) # 0..7 + col_b = row_chunk & fx.Int32(15) # 0..15 + col_elem = col_b * fx.Int32(8) # 0..120 in bf16 + q_off_real = seq * c_sq + (kv_h * c_qg + row) * c_qh + col_elem + row_in_qg = row < fx.Int32(QG) + q_off_elem = row_in_qg.select(q_off_real, col_elem) + q_v = buffer_ops.buffer_load( + q_rsrc, + q_off_elem // fx.Int32(2), + vec_width=4, + dtype=T.i32, + ) + q_lds_byte = row * fx.Int32(HEAD_SIZE * 2) + col_elem * fx.Int32(2) + vector.store( + q_v, + q_lds_i32, + [arith.index_cast(T.index, q_lds_byte // fx.Int32(4))], + ) + gpu.barrier() + + # ===== STEP C: Pre-load Q operands for QK_K_CHUNKS = 4 K-chunks === + # Wide-K MFMA: K_per_chunk = 32, K_per_lane = 32/4 = 8 bf16 per chunk. + # Lane t holds 8 bf16 at row=mfma_row, K-cols + # chunk*32 + mfma_col_grp*8 + 0..7 + # Byte addr = mfma_row * HEAD_SIZE*2 + (chunk*32 + col_grp*8)*2 + # = mfma_row * 256 + chunk*64 + col_grp*16 + # i64 idx = mfma_row * 32 + chunk*8 + col_grp*2 (each i64 = 4 bf16) + # Load 16 bytes (= 8 bf16 = 2 i64) per chunk via vec(2, i64). + q_chunks = [] + for chk in range_constexpr(QK_K_CHUNKS): + q_idx_i64 = ( + mfma_row * fx.Int32(HEAD_SIZE * 2 // 8) + + fx.Int32(chk * 8) + + mfma_col_grp * fx.Int32(2) + ) + qv = vector.load_op( + T.vec(2, T.i64), + q_lds_i64, + [arith.index_cast(T.index, q_idx_i64)], + ) + q_chunks.append(vector.bitcast(T.vec(8, T.bf16), qv)) + + # ===== STEP D: Online softmax + PV state ========================= + running_max = NEG_INF + running_sum = ZERO_F + zero_v4 = arith.constant_vector(0.0, T.f32x4) + acc_pv = [zero_v4 for _ in range(PV_N_CHUNKS)] + + # ===== STEP E: Sequence-len + partition base ==================== + seq_len = buffer_ops.buffer_load(sl_rsrc, seq, vec_width=1, dtype=T.i32) + # ── Bounded num_partitions + internal looping ─────────────────── + # Mirrors the canonical tq_decode kernel: each CTA owns + # ``PARTITION_EXTENT_TOKENS = TGPP * KV_COMPUTE_BLOCK`` contiguous + # tokens of the seq, processed as TGPP groups of 16 K-tiles each. + # FA-2 online-softmax state accumulates across all TGPP * 16 tiles. + # With TGPP=1 this is bit-identical to the legacy 16-tile body. + partition_base = part * fx.Int32(PARTITION_EXTENT_TOKENS) + + # Per-K-tile dequant lane assignment: + # 16 tokens × 64 packed bytes per token = 1024 bytes + # 64 lanes × 16 bytes/lane = 1024 ✓ + # Lane t: token = t / 4, sub-chunk = t % 4 (0..3) → 16 bytes contiguous + tok_in_tile = lane >> fx.Int32(2) + chunk_in_tok = lane & fx.Int32(3) + # ``slot`` (= absolute slot index within the cache block, range + # 0.._BS-1) is recomputed per tile inside the K-tile loop below + # so it reflects ``tile_in_block * TILE_SIZE + tok_in_tile``. + + c_meta_u16_per_kvh = fx.Int32(NUM_SOA_FIELDS * _BS) + c_knorm_off_u16 = fx.Int32(SOA_K_NORM * _BS) + c_vscale_off_u16 = fx.Int32(SOA_V_SCALE * _BS) + c_vzero_off_u16 = fx.Int32(SOA_V_ZERO * _BS) + + # ===== STEP F: K-tile loop ======================================= + # With kv_block_size > TILE_SIZE there are _TILES_PER_BLOCK tiles + # per cache block. Tiles within the same block share a block-table + # entry but address different slot ranges within the block. + # + # ── Per-tile block-table OOB redirect ──────────────────────────── + # The K-tile loop is unrolled 16 times so every CTA issues 16 + # block-table reads regardless of (partition, seq_len). When a + # partition extends past ``seq_len`` (e.g. num_partitions=2 for a + # 256-token sequence — partition 1 covers tokens 256..511, all + # masked out) the per-tile bt offset can land beyond the bt + # allocation. With ``max_size=True`` the descriptor is 4 GB so + # the HW returns whatever pre-existing HBM bytes live at that + # offset; the resulting garbage ``phys_block`` is multiplied by + # ``stride_cache_block`` and the subsequent kv_cache read jumps + # into unmapped pages, killing the whole launch with a + # "Memory access fault by GPU node" exactly as the OOB Q load + # used to (same allocator-pattern-dependent fault; B=4 makes + # it deterministic by adding more colliding CTAs). + # + # Fix: when the tile starts at or past ``seq_len`` (so its + # qk_acc is going to be killed by the per-token + # ``kv_tok < seq_len`` mask anyway), redirect the bt read to + # ``bt[seq, 0]`` — always in bounds. The redundant phys_block + # decode + kv_cache read is wasted work, but correctness is + # preserved without changing the kernel's iteration count. + # + # The canonical Qwen kernel reads the same tiles and is also + # exposed to this issue in principle; in practice it surfaces on + # GQA-6 first because the surrounding allocator state differs + # (extra Q-tail allocation) and shifts the page maps so the bt + # overflow lands on unmapped pages. Applying the same redirect to + # the canonical kernel is a possible follow-up. + # ===== Runtime-adaptive scf.ForOp w/ iter_args ========= + # Runtime-adaptive outer loop: trip count derives from + # actual seq_len, NOT TGPP_max. At cudagraph-capture warmup + # (seq_len=1) only 1 iteration runs; at 32K production + # decode all TGPP iterations run. Kernel binary stays small + # (single body, looped at runtime) — capture time matches + # the legacy 16-tile-only kernel rather than scaling 4x. + # + # FA-2 state (running_max, running_sum, acc_pv[8]) threads + # through scf iter_args; the body reads them at top, runs + # the unchanged 16-tile inner body, and yields the new + # state at bottom. After the loop, results are pulled out + # of for_op.results back into the local Python names so the + # downstream STEP G (output) is unmodified. + c_kcb = fx.Int32(KV_COMPUTE_BLOCK) + c_tgpp = fx.Int32(TGPP) + c_zero_i32 = fx.Int32(0) + c_one_i32 = fx.Int32(1) + # remaining = seq_len - partition_base (signed, may be <=0) + remaining = seq_len - partition_base + in_range = remaining > c_zero_i32 + # trip_raw = ceil(remaining / KV_COMPUTE_BLOCK) + # (when in_range is false, the divisor branch produces + # garbage that the select below overrides with zero, so we + # don't pre-clamp) + trip_raw = (remaining + c_kcb - c_one_i32) // c_kcb + trip_clamped = (trip_raw > c_tgpp).select(c_tgpp, trip_raw) + trip_or_zero = in_range.select(trip_clamped, c_zero_i32) + c_zero_idx = arith.constant(0, index=True) + c_one_idx = arith.constant(1, index=True) + trip_idx = arith.index_cast( + T.index, + trip_or_zero.ir_value() + if hasattr(trip_or_zero, "ir_value") + else trip_or_zero, + ) + + # Initial iter_args list: FA-2 accumulator state. + # Order: running_max, running_sum, acc_pv[0..PV_N_CHUNKS-1]. + # NOTE: scf.ForOp requires ir.Value (with .type). The DSL + # constants ZERO_F / ONE_F are fx.Float32 (Numeric) wrappers, + # not raw ir.Value, so we unwrap via .ir_value() before + # passing. + def _ival(v): + return v.ir_value() if hasattr(v, "ir_value") else v + + _init_iter = [ + _ival(running_max), + _ival(running_sum), + *[_ival(p) for p in acc_pv], + ] + _for_op = _scf.ForOp( + c_zero_idx, + trip_idx, + c_one_idx, + _init_iter, + ) + _for_ip = ir.InsertionPoint(_for_op.body) + _for_ip.__enter__() + try: + tg_idx = _for_op.induction_variable + tg_i32 = fx.Int32(arith.index_cast(T.i32, tg_idx)) + partition_start = partition_base + tg_i32 * c_kcb + bt_seq_base = seq * c_bt + (partition_start // fx.Int32(_BS)) + running_max = _for_op.inner_iter_args[0] + running_sum = _for_op.inner_iter_args[1] + acc_pv = list(_for_op.inner_iter_args[2:]) + for n_tile in range_constexpr(16): + block_in_part = n_tile // _TILES_PER_BLOCK + tile_in_block = n_tile % _TILES_PER_BLOCK + tile_start_tok = partition_start + fx.Int32(n_tile * TILE_SIZE) + tile_in_seq = tile_start_tok < seq_len + bt_off = bt_seq_base + fx.Int32(block_in_part) + bt_off_safe = tile_in_seq.select(bt_off, seq * c_bt) + phys_block = buffer_ops.buffer_load( + bt_rsrc, + bt_off_safe, + vec_width=1, + dtype=T.i32, + ) + block_base = phys_block * c_block + data_region = block_base + meta_region = block_base + c_meta_off + + # ``slot`` is the absolute slot index within the cache block + # (range 0.._BS-1). For BS=16 it equals tok_in_tile; for BS=32 + # it equals tile_in_block * 16 + tok_in_tile. Used both for + # the K/V data region and SoA metadata region addressing. + slot = fx.Int32(tile_in_block * TILE_SIZE) + tok_in_tile + data_bases_byte = ( + data_region + + slot * (c_kv_heads * c_data_per_slot) + + kv_h * c_data_per_slot + ) + k_byte = data_bases_byte + chunk_in_tok * fx.Int32(16) + k_packed = buffer_ops.buffer_load( + kv_rsrc, + k_byte // fx.Int32(4), + vec_width=4, + dtype=T.i32, + ) + + # ---- HOISTED: issue V data + meta HBM loads early --------- + # These are async; their s_waitcnt is pushed by the compiler + # past the K dequant + QK MFMA + softmax block, hiding most + # of the V HBM latency behind compute. + v_byte = data_bases_byte + c_keydata + chunk_in_tok * fx.Int32(16) + v_packed = buffer_ops.buffer_load( + kv_rsrc, + v_byte // fx.Int32(4), + vec_width=4, + dtype=T.i32, + ) + vscale_u16 = ( + meta_region // fx.Int32(2) + + kv_h * c_meta_u16_per_kvh + + c_vscale_off_u16 + + slot + ) + vzero_u16 = ( + meta_region // fx.Int32(2) + + kv_h * c_meta_u16_per_kvh + + c_vzero_off_u16 + + slot + ) + vscale_raw = buffer_ops.buffer_load( + kv_rsrc, + vscale_u16, + vec_width=1, + dtype=T.i16, + ) + vzero_raw = buffer_ops.buffer_load( + kv_rsrc, + vzero_u16, + vec_width=1, + dtype=T.i16, + ) + + knorm_u16 = ( + meta_region // fx.Int32(2) + + kv_h * c_meta_u16_per_kvh + + c_knorm_off_u16 + + slot + ) + knorm_raw = buffer_ops.buffer_load( + kv_rsrc, + knorm_u16, + vec_width=1, + dtype=T.i16, + ) + knorm_f16 = arith.bitcast(T.f16, knorm_raw) + knorm_f32 = arith.extf(T.f32, knorm_f16) + + # K dequant → LDS [token, head_dim] (natural) + # Lane writes 32 bf16 (= 4×8) for token=tok_in_tile, + # head_dims chunk_in_tok*32..+31 (4 sub-chunks of 8). + tok_kreg = tok_in_tile * fx.Int32(HEAD_SIZE * 2 // 8) + chunk_kreg = tok_kreg + chunk_in_tok * fx.Int32(8) + for w in range_constexpr(4): + word_i32 = vector.extract(k_packed, static_position=[w]) + bf16_elems = [] + for n in range_constexpr(8): + nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF) + nibble_idx = arith.index_cast(T.index, nibble) + cent_f32 = cent_lds.load([nibble_idx]) + elem_bf16 = arith.trunc_f(T.bf16, cent_f32 * knorm_f32) + bf16_elems.append(elem_bf16) + v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems) + v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16) + vector.store( + v_i64, + kv_lds_i64, + [arith.index_cast(T.index, chunk_kreg + fx.Int32(w * 2))], + ) + gpu.barrier() + + # QK MFMA (CDNA4 wide-K): A=K[token, head_dim], B=Q (= Q^T). + # K read: lane t = K[token=mfma_row, head_dim=chunk*32+col_grp*8..+7] + # Same i64×2 indexing as Q. + qk_acc = zero_v4 + for chk in range_constexpr(QK_K_CHUNKS): + k_idx_i64 = ( + mfma_row * fx.Int32(HEAD_SIZE * 2 // 8) + + fx.Int32(chk * 8) + + mfma_col_grp * fx.Int32(2) + ) + kv_load = vector.load_op( + T.vec(2, T.i64), + kv_lds_i64, + [arith.index_cast(T.index, k_idx_i64)], + ) + k_op = vector.bitcast(T.vec(8, T.bf16), kv_load) + qk_acc = rocdl.mfma_f32_16x16x32_bf16( + T.f32x4, [k_op, q_chunks[chk], qk_acc, 0, 0, 0] + ) + + # qk_acc layout: lane t holds C[token=(t/16)*4..+3, query=t%16] + # 4 fp32/lane = 4 different tokens at SAME query_row. + + # Scale + mask out-of-context tokens. + qk_acc = _vsplat_mul(qk_acc, QK_SCALE) + for elem in range_constexpr(4): + kv_tok = ( + partition_start + + fx.Int32(n_tile * TILE_SIZE) + + mfma_col_grp * fx.Int32(4) + + fx.Int32(elem) + ) + in_b = kv_tok < seq_len + v = vector.extract(qk_acc, static_position=[elem]) + qk_acc = vector.insert( + in_b.select(v, NEG_INF), + qk_acc, + static_position=[elem], + dynamic_position=[], + ) + + # FA2 online softmax: per-query-row reduce. + # Per-row max: max over 4 fp32 in lane, then xor-shuffle 16, 32 + # (across the 4 col_grps that share same mfma_row). + local_max = vector.reduction(T.f32, "maxnumf", qk_acc) + r1 = local_max.shuffle_xor(fx.Int32(16), c_w) + local_max = local_max.maximumf(r1) + r2 = local_max.shuffle_xor(fx.Int32(32), c_w) + tile_max = local_max.maximumf(r2) + + new_max = running_max.maximumf(tile_max) + max_diff = running_max - new_max + safe_diff = (running_max > NEG_INF).select(max_diff, ZERO_F) + scale = (safe_diff * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast) + running_sum = running_sum * scale + for h in range_constexpr(PV_N_CHUNKS): + acc_pv[h] = _vsplat_mul(acc_pv[h], scale) + running_max = new_max + + # Compute probs: p = exp((qk - new_max) * LOG2E) + tile_sum = ZERO_F + for elem in range_constexpr(4): + s = vector.extract(qk_acc, static_position=[elem]) + d = s - new_max + d = (new_max > NEG_INF).select(d, NEG_INF) + p = (d * LOG2E_C).exp2(fastmath=arith.FastMathFlags.fast) + tile_sum = tile_sum + p + qk_acc = vector.insert( + p, qk_acc, static_position=[elem], dynamic_position=[] + ) + + ts1 = tile_sum.shuffle_xor(fx.Int32(16), c_w) + tile_sum = tile_sum + ts1 + ts2 = tile_sum.shuffle_xor(fx.Int32(32), c_w) + tile_sum = tile_sum + ts2 + running_sum = running_sum + tile_sum + + # ---- V dequant → LDS (V/scale/zero raw loads were hoisted) - + # v_packed, vscale_raw, vzero_raw are already in flight from + # the top of this iteration; only the f32 conversion stays + # here so the actual consume site is still close to the use. + vscale_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vscale_raw)) + vzero_f32 = arith.extf(T.f32, arith.bitcast(T.f16, vzero_raw)) + + if USE_HW_TR: + # V dequant → LDS [token][head_dim] (ROW-MAJOR, no transpose). + # Each lane writes 32 contiguous bf16 (one token, head_dims + # chunk_in_tok*32..+31) as 4× ds_write_b128 = 4× vec(2,i64). + # Replaces 32× ds_write_b16 of the legacy transposed path. + v_lds_elem_base = tok_in_tile * fx.Int32( + HEAD_SIZE + ) + chunk_in_tok * fx.Int32(32) + for w in range_constexpr(4): + word_i32 = vector.extract(v_packed, static_position=[w]) + bf16_elems = [] + for n in range_constexpr(8): + nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF) + nibble_f32 = arith.sitofp(T.f32, nibble) + elem_f32 = nibble_f32 * vscale_f32 + vzero_f32 + elem_bf16 = arith.trunc_f(T.bf16, elem_f32) + bf16_elems.append(elem_bf16) + v_bf16 = vector.from_elements(T.vec(8, T.bf16), bf16_elems) + v_i64 = vector.bitcast(T.vec(2, T.i64), v_bf16) + v_lds_i64_idx = (v_lds_elem_base + fx.Int32(w * 8)) // fx.Int32( + 4 + ) + vector.store( + v_i64, + kv_lds_i64, + [arith.index_cast(T.index, v_lds_i64_idx)], + ) + # ── HW V transpose: cross-lane LDS sync ───────────────────── + # ds_read_tr16_b64 below introduces a cross-lane LDS read: + # consumer lane t reads bytes written by source lanes + # 0,4,8,12 (etc.). The compiler's automatic waitcnt insertion + # is per-lane and has no awareness of the HW transpose's + # cross-lane forwarding, so it can sink the next iteration's + # ds_read ahead of these ds_write_b128 ops in the schedule. + # + # Two-part fence: + # 1. sched_barrier(0): MachineScheduler reorder barrier + # (mask=0 → no instruction class may cross). Pure hint, + # generates no runtime instruction. Use the simple + # sched_barrier (NOT sched_group_barrier, which is an + # IGLP marker that requires partner barriers and + # corrupts the schedule when used in isolation). + # 2. s_waitcnt lgkmcnt=0: runtime drain of the LDS write + # queue. vmcnt/expcnt left at no-wait so we don't + # stall on speculatively-issued HBM loads (which can + # include OOB-but-masked buffer_load reads from the + # next K-tile's hoisted V prefetch). + # + # The SW path below does NOT need this because each lane + # only reads cells it itself wrote (no cross-lane dep), and + # the per-lane same-address waitcnt the compiler emits is + # correct. + # + # Without this fence the reordered ds_read produces + # wrong results on high-partition-count configs (large + # padded num_partitions); low-partition configs are + # unaffected. + rocdl.sched_barrier(0) + # encode_waitcnt(vmcnt=63, expcnt=7, lgkmcnt=0) + # = 0xF | (7<<4) | (0<<8) | (3<<14) = 0xC07F + rocdl.s_waitcnt(0xC07F) + else: + # Legacy transposed V_LDS path: 32 ds_write_b16 per lane. + for w in range_constexpr(4): + word_i32 = vector.extract(v_packed, static_position=[w]) + for n in range_constexpr(8): + nibble = (word_i32 >> fx.Int32(n * 4)) & fx.Int32(0xF) + nibble_f32 = arith.sitofp(T.f32, nibble) + elem_f32 = nibble_f32 * vscale_f32 + vzero_f32 + elem_bf16 = arith.trunc_f(T.bf16, elem_f32) + elem_i16 = arith.bitcast(T.i16, elem_bf16) + head_dim = chunk_in_tok * fx.Int32(32) + fx.Int32(w * 8 + n) + v_idx_i16 = head_dim * fx.Int32(TILE_SIZE) + tok_in_tile + v_vec = vector.from_elements(T.vec(1, T.i16), [elem_i16]) + vector.store( + v_vec, + kv_lds_i16, + [arith.index_cast(T.index, v_idx_i16)], + ) + gpu.barrier() + + # ---- PV MFMA: A=V[head_dim, token], B=P (=qk_acc bf16) ----- + # P operand B layout matches qk_acc: lane t holds 4 bf16 at + # K=token=(t/16)*4..+3, N=query=t%16. Just trunc_f to bf16. + p_bf16 = arith.trunc_f(T.vec(4, T.bf16), qk_acc) + p_op = vector.bitcast(T.vec(4, T.i16), p_bf16) + + if USE_HW_TR: + # HW-transpose PV path: V_lds is row-major V[token][head_dim]. + # MFMA A operand layout: lane t holds 4 bf16 at + # M=head_dim=mfma_row + h*16, K=token=(t/16)*4..+3 + # ds_read_tr16_b64 (4-element 16-bit transpose, per-16-lane block): + # result[lane=t, elem=e] = Input[source_lane=e*4 + (t%16)//4, + # col=t%4] + # Per-lane address: token_idx = lane // 4 (covers 0..15 across + # all four 16-lane MFMA blocks), hd_sub = (lane % 4)*4 selects + # the 4-element column window inside the h*16 chunk. + # Total LDS byte offset = kv_off + token_idx*HEAD_SIZE*2 + # + (h*16 + hd_sub)*2 + token_idx = lane >> fx.Int32(2) + hd_sub = (lane & fx.Int32(3)) * fx.Int32(4) + v_lane_byte = ( + fx.Int32(kv_off) + + token_idx * fx.Int32(HEAD_SIZE * 2) + + hd_sub * fx.Int32(2) + ) + for h in range_constexpr(PV_N_CHUNKS): + v_byte_off = v_lane_byte + fx.Int32(h * 32) + v_byte_i64 = fx.Int64(v_byte_off) + v_ptr = buffer_ops.create_llvm_ptr( + v_byte_i64, + address_space=3, + ) + v_op_raw = rocdl.ds_read_tr16_b64( + T.vec(4, T.i16), + v_ptr, + ).result + acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k( + T.f32x4, [v_op_raw, p_op, acc_pv[h], 0, 0, 0] + ) + else: + # Legacy transposed-LDS PV: 1 ds_read_b64 per chunk. + # Byte addr = (mfma_row + h*16) * TILE_SIZE * 2 + (col_grp*4) * 2 + # = (mfma_row + h*16) * 32 + col_grp*8 + # i64 idx = (mfma_row + h*16) * 4 + col_grp + for h in range_constexpr(PV_N_CHUNKS): + v_idx_i64 = (mfma_row + fx.Int32(h * 16)) * fx.Int32( + 4 + ) + mfma_col_grp + kv_load = vector.load_op( + T.vec(1, T.i64), + kv_lds_i64, + [arith.index_cast(T.index, v_idx_i64)], + ) + v_op = vector.bitcast(T.vec(4, T.i16), kv_load) + acc_pv[h] = rocdl.mfma_f32_16x16x16bf16_1k( + T.f32x4, [v_op, p_op, acc_pv[h], 0, 0, 0] + ) + + _scf.YieldOp( + [ + _ival(running_max), + _ival(running_sum), + *[_ival(p) for p in acc_pv], + ] + ) + finally: + _for_ip.__exit__(None, None, None) + # Pull final accumulator values out of the for_op results. + running_max = _for_op.results[0] + running_sum = _for_op.results[1] + acc_pv = list(_for_op.results[2:]) + + # ===== STEP G: Output =========================================== + # acc_pv[h] layout: lane t holds 4 fp32 at M=head_dim=(t/16)*4..+3 + h*16, + # N=query_row=t%16. + # 4 fp32/lane = 4 contiguous head_dim values for ONE query_row. + safe_sum = (running_sum > ZERO_F).select(running_sum, ONE_F) + rcp = ONE_F / safe_sum + + c_os = fx.Int32(_stride_out_seq) + c_oh = fx.Int32(_stride_out_head) + c_op_ = fx.Int32(_stride_out_part) + out_base = seq * c_os + kv_h * c_oh + part * c_op_ + + # Gate all global stores by mfma_row < QG. For QG=16 the predicate is + # tautological (mfma_row ∈ [0,15]); for QG=8 lanes 8..15 (whose MFMA + # outputs are computational waste) skip the stores entirely. + valid_row_pred = arith.cmpi( + arith.CmpIPredicate.ult, + mfma_row.ir_value() if hasattr(mfma_row, "ir_value") else mfma_row, + arith.constant(QG, type=T.i32), + ) + _if = _scf.IfOp(valid_row_pred) + with ir.InsertionPoint(_if.then_block): + for h in range_constexpr(PV_N_CHUNKS): + pv_norm = _vsplat_mul(acc_pv[h], rcp) + pv_bf16 = arith.trunc_f(T.vec(4, T.bf16), pv_norm) + pv_i32x2 = vector.bitcast(T.vec(2, T.i32), pv_bf16) + head_dim_start = fx.Int32(h * 16) + mfma_col_grp * fx.Int32(4) + out_off_elem = ( + out_base + mfma_row * fx.Int32(HEAD_SIZE) + head_dim_start + ) + buffer_ops.buffer_store( + pv_i32x2, + out_rsrc, + out_off_elem * fx.Int32(2), + offset_is_bytes=True, + ) + + c_npq = fx.Int32(num_partitions * QG) + ml_off = ( + seq * fx.Int32(_stride_ml_seq) + kv_h * c_npq + part * c_qg + mfma_row + ) + es_off = ( + seq * fx.Int32(_stride_es_seq) + kv_h * c_npq + part * c_qg + mfma_row + ) + buffer_ops.buffer_store(running_max, ml_rsrc, ml_off) + buffer_ops.buffer_store(running_sum, es_rsrc, es_off) + _scf.YieldOp([]) + + return tq_decode_gqa6_kernel diff --git a/vllm/v1/attention/ops/flydsl_turboquant_decode.py b/vllm/v1/attention/ops/flydsl_turboquant_decode.py new file mode 100644 index 0000000000..747ce10f81 --- /dev/null +++ b/vllm/v1/attention/ops/flydsl_turboquant_decode.py @@ -0,0 +1,874 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""FlyDSL TurboQuant decode launcher (vLLM-side). + +Drop-in replacement for ``triton_turboquant_decode_attention_soa`` for the +TQ decode profile HEAD_SIZE=128, MSE_BITS=4 K, VQB=4 V, N_CENTROIDS=16, +BLOCK_SIZE in {16, 32}. + +Per-GQA kernel dispatch: + * GQA group ∈ {8, 16} → canonical kernels.tq_decode (Qwen-class) + * GQA group == 6 → kernels.tq_decode_gqa6 sibling (MiniMax-M2.5) + +Auto-selected on gfx950 when FlyDSL is importable; falls back to the SoA Triton +decode otherwise (wrong arch, missing build tree). The GQA-6 sibling is imported +best-effort: missing it does not affect Qwen GQA-{8,16} paths. + +Architecture: + 1. Q rotation: ``q_rot = (query.float() @ PiT).bfloat16()`` — the same + launcher-side rocBLAS GEMM the SoA Triton decode uses. + 2. FlyDSL kernel writes per-partition outputs into + ``[N, Hk, P, QG, D]`` bf16 + ``[N, Hk, P, QG]`` fp32 max/sum buffers. + 3. A small Triton reducer combines partitions in the kernel's native layout + (no permute/cast), writing the final ``[N, Hq, D]`` output. + +Kernel module is built once per ``(num_kv_heads, num_partitions, +max_blocks_per_seq, scale)`` shape and cached. +""" + +from __future__ import annotations + +from typing import Any + +import torch + +from vllm.logger import init_logger +from vllm.triton_utils import tl, triton + +logger = init_logger(__name__) + +# -- FlyDSL import bootstrap -------------------------------------------------- +# FlyDSL is imported directly and must be importable (pip-installed, or on +# PYTHONPATH). Import failures are caught below and fall back to the SoA Triton +# decode, so vLLM startup is unaffected on hosts without FlyDSL. + + +_FLYDSL_AVAILABLE: bool | None = None +_TQ_MOD: Any = None # kernels.tq_decode module (Qwen GQA-{8,16}) +_TQ_MOD_GQA6: Any = None # kernels.tq_decode_gqa6 module (MiniMax GQA-6, optional) +_FLYC: Any = None # flydsl.compiler +_FX: Any = None # flydsl.expr +_TYPING_T: Any = None +_CC: Any = None # CompilationContext +_IR: Any = None + + +def is_flydsl_available() -> bool: + """Return True iff on gfx950 and FlyDSL imports + kernel module load. + + The GQA-6 sibling kernel (``kernels.tq_decode_gqa6``) is imported + best-effort: if it's missing (older FlyDSL checkout that pre-dates + MiniMax support) the canonical Qwen path stays fully functional and + only GQA-6 dispatches will fail with a clear error at launch time. + """ + global _FLYDSL_AVAILABLE, _TQ_MOD, _TQ_MOD_GQA6 + global _FLYC, _FX, _TYPING_T, _CC, _IR + if _FLYDSL_AVAILABLE is not None: + return _FLYDSL_AVAILABLE + try: + # gfx950-only kernel (CDNA4 intrinsics); gate before importing FlyDSL. + from vllm.platforms.rocm import on_gfx950 + + if not on_gfx950(): + _FLYDSL_AVAILABLE = False + return _FLYDSL_AVAILABLE + import flydsl.compiler as flyc # noqa: F401 + import flydsl.expr as fx # noqa: F401 + from flydsl._mlir import ir + from flydsl.compiler.kernel_function import CompilationContext + from flydsl.expr.typing import T # noqa: F401 + + # Kernel is vendored in-tree (ships with vLLM); FlyDSL provides only + # the compiler/runtime framework (imported above). + from vllm.v1.attention.ops.flydsl_kernels import tq_decode as tq_mod + + _FLYC = flyc + _FX = fx + _TYPING_T = T + _CC = CompilationContext + _IR = ir + _TQ_MOD = tq_mod + _FLYDSL_AVAILABLE = True + logger.info_once("FlyDSL TQ decode launcher: available") + except Exception as ex: # noqa: BLE001 + _FLYDSL_AVAILABLE = False + logger.warning_once( + "FlyDSL TQ decode launcher: unavailable (%s). " + "Falling back to SoA Triton decode.", + ex, + ) + return _FLYDSL_AVAILABLE + # Best-effort GQA-6 sibling import (does NOT gate Qwen availability). + try: + from vllm.v1.attention.ops.flydsl_kernels import ( + tq_decode_gqa6 as tq_mod_gqa6, + ) + + _TQ_MOD_GQA6 = tq_mod_gqa6 + logger.info_once("FlyDSL TQ decode GQA-6 sibling: available (MiniMax-class)") + except Exception as ex: # noqa: BLE001 + _TQ_MOD_GQA6 = None + logger.info_once( + "FlyDSL TQ decode GQA-6 sibling: not available (%s); " + "GQA-6 models will fall back to SoA Triton decode.", + ex, + ) + return _FLYDSL_AVAILABLE + + +def is_flydsl_gqa6_available() -> bool: + """True iff the optional GQA-6 sibling kernel module loaded. + + Used by the eligibility gate in turboquant_attn.py to decide whether + a layer with num_kv_groups==6 can run on FlyDSL or must fall back + to the SoA Triton decode. + """ + if _FLYDSL_AVAILABLE is None: + is_flydsl_available() + return _TQ_MOD_GQA6 is not None + + +# -- Kernel module cache ------------------------------------------------------- +# Compiling a FlyDSL kernel is expensive; cache by the constexprs that +# parameterize stride math: num_kv_heads, num_partitions, max_blocks_per_seq, +# scale. ``num_seqs`` is irrelevant to the kernel body so we omit it. +_KERN_CACHE: dict[tuple, Any] = {} + + +# -- One-shot log guards ------------------------------------------------------ +# Replaces vllm `logger.{info,warning}_once` (which hashes the format string +# every call to dedup) with a true zero-overhead bool guard. +_LOG_INVOKED_ONCE: bool = False +_LOG_SINKS_WARNED: bool = False +_LOG_NORM_WARNED: bool = False + + +def _detect_max_capture_B() -> int: + """Probe vLLM compilation config for the largest cudagraph capture size. + + Returns max(cudagraph_capture_sizes) if available, else falls back to + 512 (vLLM's default cap). The segm-pool auto-grows if a larger batch is + seen (see ``_SegmBufPool.get``). + + Used to size the segm-pool bucket once, so that all distinct B's + (whether captured in a graph, or hit eagerly in mixed batches) share + a single allocation per shape — eliminating the per-B growth that + previously dominated dense-capture VRAM cost. + """ + try: + from vllm.config import get_current_vllm_config + + cfg = get_current_vllm_config() + sizes = cfg.compilation_config.cudagraph_capture_sizes + if sizes: + return int(max(sizes)) + except Exception: # noqa: BLE001 + pass + return 512 + + +class _SegmBufPool: + """Single-bucket buffer pool for segm_out/segm_max/segm_sum/output. + + Each unique shape signature (Hk, Hq, num_partitions, QG, D, device, + dtype) is backed by ONE allocation sized at ``max_B`` (= max captured + cudagraph batch size, or env override). Per-call ``get(B, ...)`` + returns ``[:B]`` views — same data_ptr base, narrower N dim. The + kernel grid is ``(B, ...)`` so it only ever writes the first B rows + of segm_out / segm_max / segm_sum; downstream readers (Triton + reducer + caller) only consume the first B rows of output. + + Why a single bucket: cudagraph records launch pointers at capture + time. With per-B allocations, every captured size triggered a fresh + allocation, so total VRAM scaled with the number of captured sizes. + With single-bucket, the pool tops out at one max_B-sized buffer per + shape, shared across ALL captured sizes and ALL eager mixed-batch + calls. The data_ptr is stable from first allocation onward, so + cudagraph capture is happy across all captured B's. + + Eliminates per-decode-step ``cudaMalloc`` (× 4) plus the device-side + memset kernels behind ``torch.full(-inf)`` and ``torch.zeros``. + The kernel always writes the FULL ``[B, Hk, P, QG, D]`` slice and + every (n, kv_h, p, qg) position of segm_max / segm_sum (it stores + ``-inf`` and ``0`` for empty partitions itself), so uninitialized + bytes from the pool are safe to reuse for any B ≤ B_bucket. + """ + + __slots__ = ("_bufs", "_max_B") + + def __init__(self) -> None: + self._bufs: dict[tuple, dict[str, torch.Tensor]] = {} + self._max_B: int | None = None # lazy-init on first get() + + def get( + self, + B: int, + Hk: int, + Hq: int, + num_partitions: int, + QG: int, + D: int, + device: torch.device, + q_dtype: torch.dtype, + ) -> dict[str, torch.Tensor]: + if self._max_B is None: + self._max_B = _detect_max_capture_B() + # B may exceed the detected max if user set max-num-seqs higher + # than the largest captured size — still safe, we'll grow once. + # Note: growing AFTER cudagraph capture would invalidate captured + # pointers, so this should only happen during warmup or in + # always-eager configs. + B_bucket = max(self._max_B, int(B)) + key = ( + int(Hk), + int(Hq), + int(num_partitions), + int(QG), + int(D), + str(device), + q_dtype, + ) + bufs = self._bufs.get(key) + if bufs is None or bufs["segm_out"].shape[0] < B_bucket: + if bufs is not None and bufs["segm_out"].shape[0] < B_bucket: + # First-time grow: warn so user knows cudagraphs may be + # invalidated. This should only happen during warmup or in + # always-eager configs (a batch larger than any captured + # cudagraph size). + logger.warning_once( + "FlyDSL _SegmBufPool: growing bucket from %d to %d " + "(B=%d). If you see this AFTER cudagraph warmup, the " + "previously-captured graphs hold stale pointers and " + "will GPU-fault (max cudagraph capture size < B).", + bufs["segm_out"].shape[0], + B_bucket, + B, + ) + bufs = { + "segm_out": torch.empty( + (B_bucket, Hk, num_partitions, QG, D), + dtype=torch.bfloat16, + device=device, + ), + "segm_max": torch.empty( + (B_bucket, Hk, num_partitions, QG), + dtype=torch.float32, + device=device, + ), + "segm_sum": torch.empty( + (B_bucket, Hk, num_partitions, QG), + dtype=torch.float32, + device=device, + ), + "output": torch.empty( + (B_bucket, Hq, D), + dtype=q_dtype, + device=device, + ), + # Stable pre-allocated buffer for the rotated query + # tensor. On ROCm, fresh torch.empty / matmul-result allocations + # after HIP graph capture can land in the graph memory pool and + # cause GPU memory faults when passed as kernel arguments in the + # eager mixed-batch path. By pooling q_rot here (same pattern as + # segm_out/output above), the data_ptr is stable from first + # allocation onward — same for all captured B's and eager calls. + "q_rot": torch.empty( + (B_bucket, Hq, D), + dtype=q_dtype, + device=device, + ), + # q_float: fp32 copy of query (bf16→fp32 for mm input). + # Pooled to avoid fresh allocations post-capture. + "q_float": torch.empty( + (B_bucket, Hq, D), + dtype=torch.float32, + device=device, + ), + # q_rot_fp32: fp32 output of the rotation mm (before bf16 cast). + # Using torch.mm(..., out=this) eliminates the fresh mm-result + # allocation that would otherwise land in the HIP graph pool. + "q_rot_fp32": torch.empty( + (B_bucket, Hq, D), + dtype=torch.float32, + device=device, + ), + } + self._bufs[key] = bufs + self._max_B = B_bucket + logger.info_once( + "FlyDSL _SegmBufPool: allocated single-bucket " + "shape=(Hk=%d, Hq=%d, P=%d, QG=%d, D=%d, dtype=%s) " + "B_bucket=%d (covers all captured + eager B's). " + "VRAM = %.1f MiB / shape.", + Hk, + Hq, + num_partitions, + QG, + D, + q_dtype, + B_bucket, + sum(t.numel() * t.element_size() for t in bufs.values()) / (1 << 20), + ) + # Return [:B] views — same data_ptr base, narrower N dim. + return { + "segm_out": bufs["segm_out"][:B], + "segm_max": bufs["segm_max"][:B], + "segm_sum": bufs["segm_sum"][:B], + "output": bufs["output"][:B], + "q_rot": bufs["q_rot"][:B], + "q_float": bufs["q_float"][:B], + "q_rot_fp32": bufs["q_rot_fp32"][:B], + } + + def stats(self) -> dict[str, int]: + return { + "shapes": len(self._bufs), + "max_B": int(self._max_B or 0), + "bytes": sum( + sum(t.numel() * t.element_size() for t in d.values()) + for d in self._bufs.values() + ), + } + + +_SEGM_POOL = _SegmBufPool() + + +_HW_TR_CACHED: bool | None = None + + +def _hw_tr_enabled() -> bool: + """Resolve the HW V-transpose build flag. + + ON for gfx950+ (ds_read_tr16_b64 is bit-exact vs baseline and faster + on Qwen-class shapes), off elsewhere. Resolved from the GPU arch. + """ + global _HW_TR_CACHED + if _HW_TR_CACHED is not None: + return _HW_TR_CACHED + try: + from flydsl.runtime.device import get_rocm_arch as _arch + + a = str(_arch() or "") + _HW_TR_CACHED = a.startswith("gfx950") + except Exception: # noqa: BLE001 + _HW_TR_CACHED = False + if _HW_TR_CACHED: + logger.info_once("FlyDSL TQ: HW V transpose ON (default for gfx950+)") + else: + logger.info_once("FlyDSL TQ: HW V transpose OFF") + return _HW_TR_CACHED + + +_GET_KERNEL_STATS = {"hits": 0, "misses": 0, "build_total_s": 0.0} + + +def _get_kernel( + num_kv_heads: int, + num_partitions: int, + max_blocks_per_seq: int, + scale: float, + query_group_size: int, + kv_block_size: int, + use_hw_v_transpose: bool = False, + num_seqs_hint: int = 1, + tile_groups_per_partition: int = 1, +): + # ``num_seqs_hint`` (= runtime B) is forwarded to the build for shape + # awareness; it does NOT participate in the cache key because the + # kernel body uses gpu.block_idx.x (= seq index at runtime) and is + # GQA-symmetric across B at build time. + # + # ``tile_groups_per_partition`` IS in the cache key — each + # value compiles a different unrolled K-tile loop body. + key = ( + num_kv_heads, + int(num_partitions), + int(max_blocks_per_seq), + round(float(scale), 8), + int(query_group_size), + int(kv_block_size), + bool(use_hw_v_transpose), + int(tile_groups_per_partition), + ) + cached = _KERN_CACHE.get(key) + if cached is not None: + _GET_KERNEL_STATS["hits"] += 1 + return cached + assert is_flydsl_available() + # Time the FlyDSL build path so we can quantify cudagraph capture cost. + import time as _t + + _build_t0 = _t.perf_counter() + + # Per-GQA dispatch: GQA-6 (MiniMax-M2.5) lives in the sibling module + # tq_decode_gqa6 to keep the Qwen kernel's invariants untouched. + # GQA-{8,16} (Qwen) keep using the canonical tq_decode kernel. + qg = int(query_group_size) + if qg == 6: + if _TQ_MOD_GQA6 is None: + raise RuntimeError( + "FlyDSL TQ GQA-6 sibling kernel " + "(vllm.v1.attention.ops.flydsl_kernels.tq_decode_gqa6) " + "failed to import; check that FlyDSL is installed/importable." + ) + kmod = _TQ_MOD_GQA6 + kfn = kmod.build_tq_decode_gqa6_module( + num_seqs=int(num_seqs_hint), + num_kv_heads=num_kv_heads, + num_partitions=num_partitions, + max_blocks_per_seq=max_blocks_per_seq, + softmax_scale=float(scale), + query_group_size=qg, + kv_block_size=int(kv_block_size), + use_hw_v_transpose=bool(use_hw_v_transpose), + tile_groups_per_partition=int(tile_groups_per_partition), + ) + else: + kmod = _TQ_MOD + kfn = kmod.build_tq_decode_module( + num_seqs=int(num_seqs_hint), + num_kv_heads=num_kv_heads, + num_partitions=num_partitions, + max_blocks_per_seq=max_blocks_per_seq, + softmax_scale=float(scale), + query_group_size=qg, + kv_block_size=int(kv_block_size), + use_hw_v_transpose=bool(use_hw_v_transpose), + tile_groups_per_partition=int(tile_groups_per_partition), + ) + al = kmod.allocator + block_threads = kmod.BLOCK_THREADS + + flyc = _FLYC + fx = _FX + T = _TYPING_T + CompilationContext = _CC + ir_mod = _IR + + @flyc.jit + def _launch( + out, + es, + ml, + q, + kvc, + cents, + bt, + sl, + gx: fx.Int32, # type: ignore[name-defined] + gy: fx.Int32, # type: ignore[name-defined] + gz: fx.Int32, # type: ignore[name-defined] + stream: fx.Stream, # type: ignore[name-defined] + ): + # Re-finalize the LDS allocator for this launch (idempotent per build). + al.finalized = False + ctx = CompilationContext.get_current() + with ir_mod.InsertionPoint(ctx.gpu_module_body): + al.finalize() + from flydsl.expr import arith + + grid_x = arith.index_cast(T.index, gx.ir_value()) + grid_y = arith.index_cast(T.index, gy.ir_value()) + grid_z = arith.index_cast(T.index, gz.ir_value()) + kfn(out, es, ml, q, kvc, cents, bt, sl).launch( + grid=(grid_x, grid_y, grid_z), + block=(block_threads, 1, 1), + stream=stream, + ) + + _KERN_CACHE[key] = _launch + _build_dt = _t.perf_counter() - _build_t0 + _GET_KERNEL_STATS["misses"] += 1 + _GET_KERNEL_STATS["build_total_s"] += _build_dt + logger.info( + "FlyDSL _get_kernel BUILD #%d dt=%.2fs (cumulative=%.1fs) key=%s", + _GET_KERNEL_STATS["misses"], + _build_dt, + _GET_KERNEL_STATS["build_total_s"], + key, + ) + return _launch + + +# -- Partition reducer (Triton, native kernel layout) ------------------------- +# Reduces the FA2 split-KV partials produced by the FlyDSL kernel. +# segm_out [N, Hk, P, QG, D] bf16 +# segm_max [N, Hk, P, QG] fp32 +# segm_sum [N, Hk, P, QG] fp32 +# → output [N, Hq=Hk*QG, D] of any (bf16/fp16/fp32) dtype. +@triton.jit +def _reduce_partitions( + output_ptr, # [N, Hq, D] in OUT_DTYPE + segm_out_ptr, # [N, Hk, P, QG, D] bf16 + segm_max_ptr, # [N, Hk, P, QG] fp32 + segm_sum_ptr, # [N, Hk, P, QG] fp32 + out_stride_n: tl.int64, # stride on N axis (in elems of OUT_DTYPE) + out_stride_h: tl.int64, # stride on Hq axis + NUM_KV_HEADS: tl.constexpr, + QG: tl.constexpr, + NUM_PARTS: tl.constexpr, + HEAD_SIZE: tl.constexpr, +): + # grid = (N, Hq) + n = tl.program_id(0) + hq = tl.program_id(1) + kv_h = hq // QG + qg = hq % QG + + # Per-partition pointers for this (n, kv_h, qg) row. + # Strides in elems of source dtype. + msum_base = n * (NUM_KV_HEADS * NUM_PARTS * QG) + kv_h * (NUM_PARTS * QG) + qg + so_base = ( + n * (NUM_KV_HEADS * NUM_PARTS * QG * HEAD_SIZE) + + kv_h * (NUM_PARTS * QG * HEAD_SIZE) + + qg * HEAD_SIZE + ) + + p_off = tl.arange(0, NUM_PARTS) + d_off = tl.arange(0, HEAD_SIZE) + + # Load per-partition max/sum. + m_idx = msum_base + p_off * QG + seg_max = tl.load(segm_max_ptr + m_idx) + seg_sum = tl.load(segm_sum_ptr + m_idx) + + # Mask out empty partitions (max == -inf). + valid = seg_max > float("-inf") + overall_max = tl.max(tl.where(valid, seg_max, float("-inf"))) + + # Rescale exp sums. + rescale = tl.where(valid, tl.exp(seg_max - overall_max), 0.0) + seg_sum_rescaled = seg_sum * rescale + overall_sum = tl.sum(seg_sum_rescaled) + + # Load segment outputs and combine. + so_idx = so_base + p_off[:, None] * (QG * HEAD_SIZE) + d_off[None, :] + seg_out = tl.load(segm_out_ptr + so_idx).to(tl.float32) + # acc = sum_p( seg_out_p * (seg_sum_p * rescale_p) ) / overall_sum + weighted = seg_out * seg_sum_rescaled[:, None] + acc_sum = tl.sum(weighted, axis=0) + acc = tl.where(overall_sum > 0.0, acc_sum / overall_sum, 0.0) + + # Store. + out_off = n * out_stride_n + hq * out_stride_h + d_off + tl.store(output_ptr + out_off, acc.to(output_ptr.dtype.element_ty)) + + +# -- Public launcher ---------------------------------------------------------- +def flydsl_turboquant_decode_attention( + query: torch.Tensor, # [B, Hq, D] bf16/fp16 + kv_cache: torch.Tensor, # [num_blocks, BS, Hk, slot_size_aligned] + block_table: torch.Tensor, # [B, max_blocks_per_seq] int32 + seq_lens: torch.Tensor, # [B] int32 + Pi: torch.Tensor, # [D, D] fp32 + centroids: torch.Tensor, # [N_CENTROIDS] fp32 + scale: float, + mse_bits: int, + key_packed_size: int, + value_quant_bits: int, + value_packed_size: int, + key_fp8: bool = False, + norm_correction: bool = False, + PiT: torch.Tensor | None = None, + max_seq_len: int = 0, + mid_o_buf: Any = None, + output_buf: torch.Tensor | None = None, + lse_buf: Any = None, + buf_holder: Any = None, + max_num_kv_splits: int = 32, + sinks: torch.Tensor | None = None, +) -> torch.Tensor: + """SoA-decode-compatible launcher backed by the FlyDSL decode kernel. + + Constraints: + * key_fp8 == False + * mse_bits == 4 + * value_quant_bits == 4 + * centroids.numel() == 16 + * D == 128 + * block_size in {16, 32} + * Hq // Hk in {6, 8, 16} + - 8/16 → canonical tq_decode kernel (Qwen2.5-72B / Qwen3-32B) + - 6 → tq_decode_gqa6 sibling kernel (MiniMax-M2.5) + + Sinks are NYI and silently ignored if set. norm_correction is honored + implicitly via the pre-folded stored K-norm (see footer comment). + """ + del mid_o_buf, lse_buf, key_packed_size, value_packed_size + if not is_flydsl_available(): + raise RuntimeError( + "FlyDSL decode requested but FlyDSL is not available (needs gfx950 " + "+ importable FlyDSL); the SoA Triton decode is the fallback." + ) + assert not key_fp8, "FlyDSL supports MSE-key path only" + assert mse_bits == 4, f"FlyDSL expects mse_bits=4, got {mse_bits}" + assert value_quant_bits == 4, ( + f"FlyDSL expects value_quant_bits=4, got {value_quant_bits}" + ) + + B, Hq, D = query.shape + Hk = kv_cache.shape[2] + block_size = kv_cache.shape[1] + QG = Hq // Hk + assert D == _TQ_MOD.HEAD_SIZE + assert block_size in (16, 32), ( + f"FlyDSL supports kv_block_size 16 or 32, got {block_size}" + ) + assert QG in (6, 8, 16), f"FlyDSL supports GQA factor 6, 8 or 16, got {QG}" + if QG == 6 and _TQ_MOD_GQA6 is None: + raise RuntimeError( + "FlyDSL launcher: GQA-6 requested (MiniMax-class) but the " + "tq_decode_gqa6 sibling module is not available. Update your " + "FlyDSL checkout (must include kernels/tq_decode_gqa6.py); " + "otherwise the SoA Triton decode is the fallback." + ) + assert centroids.numel() == _TQ_MOD.N_CENTROIDS, ( + f"centroids.numel={centroids.numel()} != {_TQ_MOD.N_CENTROIDS}" + ) + + # ---- Per-layer cache for PiT_f32 + contiguous centroids --- + # PiT and centroids are model constants (set once at layer warmup). + # Avoid the per-decode-step transpose+cast and `.contiguous()` no-op + # check by stashing the pre-cooked tensors on ``buf_holder`` (= layer). + PiT_f32: torch.Tensor + centroids_c: torch.Tensor + if buf_holder is not None: + PiT_f32 = getattr(buf_holder, "_tq_PiT_f32", None) + if PiT_f32 is None: + _PiT_src = PiT if PiT is not None else Pi.T.contiguous() + PiT_f32 = ( + _PiT_src + if _PiT_src.dtype == torch.float32 + else _PiT_src.to(torch.float32) + ) + buf_holder._tq_PiT_f32 = PiT_f32 + centroids_c = getattr(buf_holder, "_tq_centroids_c", None) + if centroids_c is None: + centroids_c = centroids.contiguous() + buf_holder._tq_centroids_c = centroids_c + else: + # Defensive fallback (unit tests may pass ``buf_holder=None``). + _PiT_src = PiT if PiT is not None else Pi.T.contiguous() + PiT_f32 = ( + _PiT_src if _PiT_src.dtype == torch.float32 else _PiT_src.to(torch.float32) + ) + centroids_c = centroids.contiguous() + + # ---- Pooled q_rot (stable data_ptr post-capture) ------ + # On ROCm, fresh tensor allocations made AFTER HIP graph capture can land + # in the graph memory pool and cause GPU memory faults when passed as kernel + # pointer arguments in the eager mixed-batch path. We pool q_rot and the + # fp32 intermediate the same way segm_out/output are pooled: one max-B + # allocation per shape, reused across all B's. + # + # Note: pool_bufs is computed BELOW after num_partitions is determined; + # q_rot needs pool_bufs["q_float"] and pool_bufs["q_rot"]. We compute the + # rotation inline after the pool is fetched (see the pooled-buffers + # section below). + + # ---- Partition count (FA2 split-KV) ---------------------------------- + # + # FlyDSL runs INSIDE FULL cudagraph (TurboQuantMetadataBuilder._cudagraph_ + # support = UNIFORM_BATCH). That means the gridDim baked at capture + # time MUST equal the gridDim at replay — the kernel launch parameters + # are recorded into the captured graph. So `num_partitions` (which + # becomes gridDim.z) MUST be derived from a stable source that produces + # the same value at capture and at runtime. + # + # We use the worst-case context length implied by the block table's + # allocation (block_table.shape[1] * block_size) as that source. It's + # bounded by `max_model_len` (vLLM allocates the block table for the + # configured max), and it's identical at capture and at runtime + # because `block_table.shape[1]` is fixed once the model is loaded. + # Per-tile OOB redirects + the masked FA-2 reducer ensure that for + # short sequences the extra partitions contribute zero to the output. + # + # We always size from the worst case (block_table.shape[1] * block_size), + # never from the per-step actual max_seq_len: a per-step size would make + # the captured gridDim mismatch runtime and GPU-fault under cudagraph. + kv_compute_block = _TQ_MOD.KV_COMPUTE_BLOCK + worst_case_max_seq_len = int(block_table.shape[1]) * int(block_size) + if max_seq_len <= 0: + max_seq_len = worst_case_max_seq_len + sizing_max_seq_len = worst_case_max_seq_len + + # ── Bounded num_partitions + internal tile-group looping ── + # The kernel previously hardcoded one partition = 256 tokens (= + # KV_COMPUTE_BLOCK), forcing num_partitions to scale linearly with + # max_model_len, which made the cudagraph capture cost grow + # prohibitively at long context. The new kernel takes + # ``tile_groups_per_partition`` (TGPP); each CTA processes + # ``TGPP * 16`` K-tiles = ``TGPP * KV_COMPUTE_BLOCK`` tokens with the + # FA-2 online-softmax state accumulating across all of them. This + # mirrors what the SoA Triton decode / HIP SoA-fusion already do. + # + # Strategy: cap num_partitions at ``MAX_PARTITIONS`` (default 32), + # then derive TGPP so that ``num_partitions * TGPP * KV_COMPUTE_BLOCK + # >= sizing_max_seq_len``. + # + # Examples (block_size=32 → worst_case max_bps*32): + # max_model_len=8K: required=32, parts=32, TGPP=1 (no waste) + # max_model_len=16K: required=64, parts=32, TGPP=2 (no waste) + # max_model_len=32K: required=128, parts=32, TGPP=4 (no waste) + # max_model_len=64K: required=256, parts=32, TGPP=8 (no waste) + # max_model_len=128K: required=512, parts=32, TGPP=16 + MAX_PARTITIONS = 32 + required_num_partitions = ( + sizing_max_seq_len + kv_compute_block - 1 + ) // kv_compute_block + # max_num_kv_splits acts as a parallelism floor (ensure at least this + # many CTAs along grid.z), but is itself capped by MAX_PARTITIONS. + parallelism_floor = min(MAX_PARTITIONS, max(1, max_num_kv_splits)) + # Cap num_partitions at MAX_PARTITIONS, but never go below the floor. + num_partitions_actual = max( + parallelism_floor, + min(MAX_PARTITIONS, required_num_partitions), + ) + # Round up to next power of 2 for the Triton reducer's tl.arange(0, N) + # constraint (Triton requires power-of-2 ≥ 2). + num_partitions = max(2, triton.next_power_of_2(num_partitions_actual)) + # Now derive tile_groups_per_partition (TGPP) so total coverage + # ``num_partitions * TGPP * KV_COMPUTE_BLOCK`` is >= sizing_max_seq_len. + # This guarantees no work is dropped at runtime regardless of seq_len. + # Round TGPP up to the next power-of-2 so the JIT cache key is bounded + # to a small set of values (e.g. {1, 2, 4, 8, 16}) — limits the number + # of distinct kernel binaries that need to be compiled across requests. + _tgpp_required = max( + 1, + (required_num_partitions + num_partitions - 1) // num_partitions, + ) + tile_groups_per_partition = int(triton.next_power_of_2(_tgpp_required)) + + # ---- Pooled buffers (no per-call cudaMalloc / memset) ---------- + # The FlyDSL kernel writes the FULL [B, Hk, P, QG, D] segm_out and the + # FULL [B, Hk, P, QG] segm_max / segm_sum (running_max=-inf, sum=0 for + # empty partitions are stored by the kernel itself), so uninitialized + # buffers from the pool are safe to reuse. + device = query.device + pool_bufs = _SEGM_POOL.get( + B, + Hk, + Hq, + num_partitions, + QG, + D, + device, + query.dtype, + ) + segm_out = pool_bufs["segm_out"] + segm_max = pool_bufs["segm_max"] + segm_sum = pool_bufs["segm_sum"] + if output_buf is None: + output = pool_bufs["output"] + else: + output = output_buf[:B] if output_buf.shape[0] != B else output_buf + + # ---- q rotation: GEMM q @ PiT via stable pooled buffers ------------- + _q_float = pool_bufs["q_float"] # [B, Hq, D] fp32, stable + _q_rot_f32 = pool_bufs["q_rot_fp32"] # [B, Hq, D] fp32, stable mm output + _q_rot_out = pool_bufs["q_rot"] # [B, Hq, D] query.dtype, stable + _q_float.copy_(query) # bf16 → fp32 in-place, no alloc + # mm into stable fp32 buffer via out= to avoid fresh allocations. + # PiT_f32 is cached on buf_holder (stable ptr since first layer warmup). + torch.mm( + _q_float.view(B * Hq, D), + PiT_f32, + out=_q_rot_f32.view(B * Hq, D), # in-place into pool buffer + ) + _q_rot_out.copy_(_q_rot_f32) # fp32 → bf16, into stable buf + q_for_kernel = _q_rot_out # stable ptr for kernel launch + + # ---- FlyDSL kernel launch ------------------------------------------- + max_bps = int(block_table.shape[1]) + use_hw_tr = _hw_tr_enabled() + launch = _get_kernel( + Hk, + num_partitions, + max_bps, + scale, + QG, + block_size, + use_hw_v_transpose=use_hw_tr, + num_seqs_hint=int(B), + tile_groups_per_partition=int(tile_groups_per_partition), + ) + # Zero-overhead one-shot info log (replaces logger.info_once which + # hashes its format string on every call to dedup). + global _LOG_INVOKED_ONCE, _LOG_SINKS_WARNED, _LOG_NORM_WARNED + if not _LOG_INVOKED_ONCE: + _LOG_INVOKED_ONCE = True + logger.info( + "FlyDSL launcher invoked (UNIFORM_BATCH cudagraph): " + "B=%d Hk=%d Hq=%d D=%d QG=%d num_partitions=%d (actual=%d, " + "cap=%d) TGPP=%d max_bps=%d block_size=%d max_seq_len=%d " + "hw_v_transpose=%s (coverage=%d tokens, worst_case=%d tokens)", + B, + Hk, + Hq, + D, + QG, + num_partitions, + num_partitions_actual, + MAX_PARTITIONS, + tile_groups_per_partition, + max_bps, + int(block_size), + int(max_seq_len), + use_hw_tr, + num_partitions * tile_groups_per_partition * kv_compute_block, + worst_case_max_seq_len, + ) + launch( + segm_out, + segm_sum, + segm_max, + q_for_kernel, + kv_cache, + centroids_c, + block_table, + seq_lens, + B, + Hk, + num_partitions, + torch.cuda.current_stream(), + ) + + # ---- Reduce partitions -> [B, Hq, D] -------------------------------- + _reduce_partitions[(B, Hq)]( + output_ptr=output, + segm_out_ptr=segm_out, + segm_max_ptr=segm_max, + segm_sum_ptr=segm_sum, + out_stride_n=output.stride(0), + out_stride_h=output.stride(1), + NUM_KV_HEADS=Hk, + QG=QG, + NUM_PARTS=num_partitions, + HEAD_SIZE=D, + ) + if sinks is not None and not _LOG_SINKS_WARNED: + _LOG_SINKS_WARNED = True + logger.warning( + "FlyDSL launcher: sinks ignored (NYI). Disable sinks if they " + "are required for correctness." + ) + # ── norm_correction is honored IMPLICITLY ─────────────────────────── + # When the model was stored with norm_correction=True (the *_nc presets + # turboquant_4bit_nc / k3v4_nc / 3bit_nc), the per-token K-norm scalar + # was pre-folded to ||k_t|| / ||c_t|| at store time by the TurboQuant + # store path. The decode kernel just multiplies `c_vals * stored_knorm`, + # which then equals `(c_vals / ||c_t||) * ||k_t||` — exactly the + # unit-norm-renormalized centroid times the original key norm. The + # Triton decode does the identical multiply, so both decoders honor + # norm_correction equivalently and there is nothing to "do" at decode + # time. The launcher's `norm_correction` arg is only kept for API + # parity; we emit a one-shot INFO log to make the contract explicit. + if norm_correction and not _LOG_NORM_WARNED: + _LOG_NORM_WARNED = True + logger.info( + "FlyDSL launcher: norm_correction honored implicitly via " + "pre-folded stored K-norm (cf. triton_turboquant_store step 3); " + "no decode-time work required, identical to the SoA decode behavior." + ) + return output diff --git a/vllm/v1/attention/ops/turboquant_soa/__init__.py b/vllm/v1/attention/ops/turboquant_soa/__init__.py new file mode 100644 index 0000000000..bbb96ff736 --- /dev/null +++ b/vllm/v1/attention/ops/turboquant_soa/__init__.py @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# HIP-free SoA Triton subset for the FlyDSL TurboQuant decode path. diff --git a/vllm/v1/attention/ops/turboquant_soa/triton_turboquant_decode.py b/vllm/v1/attention/ops/turboquant_soa/triton_turboquant_decode.py new file mode 100644 index 0000000000..3241c64508 --- /dev/null +++ b/vllm/v1/attention/ops/turboquant_soa/triton_turboquant_decode.py @@ -0,0 +1,643 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Triton fused TurboQuant decode attention. + +Decode path: Triton stage1 (split-KV tiled attention scoring + value +accumulation) + stage2 (log-sum-exp reduction across splits). + +Supports FP8 (E4M3) keys, 3-bit and 4-bit uniform quantized values. +""" + +import math +from typing import Any + +import torch + +from vllm.platforms import current_platform +from vllm.triton_utils import tl, triton +from vllm.v1.attention.ops.triton_decode_attention import ( + _fwd_kernel_stage2, +) + +_FP8_E4B15: dict[int, int] = {} + + +def _use_fp8_e4b15(device: int = 0) -> int: + """Return 1 if device needs fp8e4b15 (Ampere/Ada, SM < 8.9), else 0. + On non-CUDA platforms (e.g. XPU), always returns 0 (use e4nv format). + """ + if device not in _FP8_E4B15: + if current_platform.is_cuda_alike(): + cap = torch.cuda.get_device_capability(device) + _FP8_E4B15[device] = 1 if cap < (8, 9) else 0 + else: + _FP8_E4B15[device] = 0 + return _FP8_E4B15[device] + + +# --------------------------------------------------------------------------- +# Stage 1: Fused TQ score + value accumulation (BLOCK_KV tiled) +# --------------------------------------------------------------------------- + + +@triton.jit +def _tq_decode_stage1( + # Precomputed query projection + Q_rot_ptr, # [B, Hq, D] float32 + # Compressed KV cache (combined K+V) + KV_cache_ptr, # [num_blocks, block_size, Hk, padded_slot] uint8 + KV_cache_u16_ptr, # uint16 view of same storage — Opt#3 SoA loads + # Block table and sequence info + Block_table_ptr, # [B, max_num_blocks] int32 + Seq_lens_ptr, # [B] int32 + # TQ parameters + Centroids_ptr, # [n_centroids] float32 + # Output (intermediate for stage2) + Mid_o_ptr, # [B, Hq, NUM_KV_SPLITS, D+1] float32 + # Sink support + Sink_ptr, # [Hq] float32, pre-computed sink attention logits per head + # Strides + stride_qb, + stride_qh, # Q strides: [B, Hq, D] + stride_cache_block: tl.int64, # bytes per block (bs*H*slot_aligned) + stride_bt_b: tl.int64, # block_table stride per batch + stride_mid_b, + stride_mid_h, + stride_mid_s, # mid_o strides + # Constexpr dims + NUM_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_SIZE: tl.constexpr, # KV cache block_size (pages) + NUM_KV_SPLITS: tl.constexpr, + KV_GROUP_SIZE: tl.constexpr, # Hq // Hk + # TQ layout constants + MSE_BITS: tl.constexpr, # 3 or 4 + MSE_BYTES: tl.constexpr, # ceil(D * mse_bits / 8) + VQB: tl.constexpr, # value_quant_bits (4 or 8=FP8) + VAL_DATA_BYTES: tl.constexpr, # ceil(D * vqb / 8) or D for FP8 + # Opt#3 SoA layout + KEY_DATA_BYTES: tl.constexpr, # MSE_BYTES (MSE) or HEAD_DIM (FP8) + META_REGION_OFFSET: tl.constexpr, # bytes from block start to SoA region + NUM_SOA_FIELDS: tl.constexpr, # 3 (MSE) or 2 (FP8) + SOA_K_NORM: tl.constexpr, # 0 (MSE); unused for FP8 + SOA_V_SCALE: tl.constexpr, # 1 (MSE) / 0 (FP8) + SOA_V_ZERO: tl.constexpr, # 2 (MSE) / 1 (FP8) + # Score constants + ATTN_SCALE: tl.constexpr, # 1/sqrt(D) + # Block tile sizes + BLOCK_D: tl.constexpr, # next_power_of_2(HEAD_DIM) + BLOCK_KV: tl.constexpr, # tokens per tile (16) + KEY_FP8: tl.constexpr, # 1 if K is stored as FP8 + NORM_CORRECTION: tl.constexpr = 0, # 1 = re-normalize centroids + FP8_E4B15: tl.constexpr = 0, # 1 = use e4b15 (Ampere/Ada), 0 = e4nv (Hopper+) + USE_SINKS: tl.constexpr = 0, # 1 = use sink tokens for attention anchoring +): + bid = tl.program_id(0) # batch index + hid = tl.program_id(1) # q_head index + sid = tl.program_id(2) # kv_split index + + kv_head = hid // KV_GROUP_SIZE + + # Sequence length for this batch + seq_len = tl.load(Seq_lens_ptr + bid) + + # KV split range + split_len = tl.cdiv(seq_len, NUM_KV_SPLITS) + split_start = split_len * sid + split_end = tl.minimum(split_start + split_len, seq_len) + + if split_start >= split_end: + return + + # Dimension offsets + d_offs = tl.arange(0, BLOCK_D) + d_mask = d_offs < HEAD_DIM + kv_range = tl.arange(0, BLOCK_KV) + + # Load query vector: q_rot — [BLOCK_D] float32 + q_base = bid * stride_qb + hid * stride_qh + q_rot = tl.load(Q_rot_ptr + q_base + d_offs, mask=d_mask, other=0.0).to(tl.float32) + + # Precompute byte/bit index vectors for MSE gather loads + if not KEY_FP8: + mse_bit_off = d_offs * MSE_BITS + mse_byte_idx = mse_bit_off // 8 + mse_bit_shift = mse_bit_off % 8 + mse_mask = (1 << MSE_BITS) - 1 + + # Precompute value bit/byte index vectors (loop-invariant) + if VQB == 3: + val_bit_off = d_offs * 3 + val_byte_idx = val_bit_off // 8 + val_bit_shift = val_bit_off % 8 + + # Online softmax accumulators + # Sink tokens provide a pre-computed attention bias that should be + # included in the softmax normalization. For the first split (sid==0), + # we initialize m_prev with the sink logit. L is always 1.0 for all + # splits to match unified attention semantics. + if USE_SINKS: + if sid == 0: + m_prev = tl.load(Sink_ptr + hid).to(tl.float32) + l_prev = 1.0 + else: + m_prev = -float("inf") + l_prev = 0.0 + else: + m_prev = -float("inf") + l_prev = 0.0 + acc = tl.zeros([BLOCK_D], dtype=tl.float32) + + bt_base = bid * stride_bt_b + + # ================================================================ + # TILED LOOP: process BLOCK_KV tokens per iteration + # ================================================================ + for start_n in range(split_start, split_end, BLOCK_KV): + kv_offs = start_n + kv_range + kv_mask = kv_offs < split_end + + page_idx = kv_offs // BLOCK_SIZE + page_off = kv_offs % BLOCK_SIZE + block_nums = tl.load( + Block_table_ptr + bt_base + page_idx, + mask=kv_mask, + other=0, + ).to(tl.int64) + + # Opt#3 SoA addressing: data region then metadata region per block. + slot_within_block = page_off.to(tl.int64) + block_base = block_nums * stride_cache_block + DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES + data_bases = ( + block_base + + slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT) + + tl.cast(kv_head, tl.int64) * DATA_BYTES_PER_SLOT + ) + head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast( + kv_head, tl.int64 + ) * (NUM_SOA_FIELDS * BLOCK_SIZE) + knorm_u16_addrs = ( + head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block + ) + vscale_u16_addrs = ( + head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block + ) + vzero_u16_addrs = ( + head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block + ) + + # ============================================================ + # COMPUTE ATTENTION SCORES: [BLOCK_KV] + # ============================================================ + if KEY_FP8: + k_addrs = data_bases[:, None] + d_offs[None, :] + k_raw = tl.load( + KV_cache_ptr + k_addrs, + mask=kv_mask[:, None] & d_mask[None, :], + other=0, + ) + if FP8_E4B15: + k_float = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32) + else: + k_float = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32) + scores = ( + tl.sum( + tl.where(d_mask[None, :], q_rot[None, :] * k_float, 0.0), + axis=1, + ) + * ATTN_SCALE + ) + scores = tl.where(kv_mask, scores, -float("inf")) + else: + # MSE unpack + norms + mse_addrs0 = data_bases[:, None] + mse_byte_idx[None, :] + mse_raw0 = tl.load( + KV_cache_ptr + mse_addrs0, + mask=kv_mask[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + mse_raw1 = tl.load( + KV_cache_ptr + mse_addrs0 + 1, + mask=kv_mask[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + raw16 = mse_raw0 | (mse_raw1 << 8) + mse_idx = (raw16 >> mse_bit_shift[None, :]) & mse_mask + + # Centroid gather + dot product + c_vals = tl.load( + Centroids_ptr + mse_idx, + mask=kv_mask[:, None] & d_mask[None, :], + other=0.0, + ) + + # Opt#1: norm-correction (1/||c_vec||) is pre-folded into the + # stored per-token scalar by the MSE store kernel when + # NORM_CORRECTION=1, so we skip the per-tile sum+sqrt+divide. + # NORM_CORRECTION is kept in the signature for API parity; it is + # implicit in the stored norm value now. + + term1 = tl.sum( + tl.where(d_mask[None, :], q_rot[None, :] * c_vals, 0.0), + axis=1, + ) + + # Opt#3: K-norms live in the per-block SoA region. Single u16 + # load per token; contiguous across tokens within a block. + norm_u16 = tl.load( + KV_cache_u16_ptr + knorm_u16_addrs, mask=kv_mask, other=0 + ) + vec_norms = norm_u16.to(tl.float16, bitcast=True).to(tl.float32) + + scores = vec_norms * term1 * ATTN_SCALE + scores = tl.where(kv_mask, scores, -float("inf")) + + # ============================================================ + # ONLINE SOFTMAX UPDATE (block-level) + # ============================================================ + n_e_max = tl.maximum(tl.max(scores, 0), m_prev) + re_scale = tl.exp(m_prev - n_e_max) + p = tl.exp(scores - n_e_max) + + # ============================================================ + # VALUE LOAD + DEQUANTIZE: [BLOCK_KV, BLOCK_D] + # Opt#3: V data immediately follows K data in the slot (no + # interleaved metadata). Scale/zero come from the SoA metadata + # region as single u16 loads per field per token. + # ============================================================ + val_bases = data_bases + KEY_DATA_BYTES + + if VQB == 3: + val_addrs0 = val_bases[:, None] + val_byte_idx[None, :] + val_raw0 = tl.load( + KV_cache_ptr + val_addrs0, + mask=kv_mask[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + val_raw1 = tl.load( + KV_cache_ptr + val_addrs0 + 1, + mask=kv_mask[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + raw16 = val_raw0 | (val_raw1 << 8) + v_idx = ((raw16 >> val_bit_shift[None, :]) & 0x7).to(tl.float32) + else: # VQB == 4 + vb_idx = d_offs // 2 + vb_shift = (d_offs % 2) * 4 + val_addrs = val_bases[:, None] + vb_idx[None, :] + val_raw = tl.load( + KV_cache_ptr + val_addrs, + mask=kv_mask[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + v_idx = ((val_raw >> vb_shift[None, :]) & 0xF).to(tl.float32) + + scale_u16 = tl.load(KV_cache_u16_ptr + vscale_u16_addrs, mask=kv_mask, other=0) + zero_u16 = tl.load(KV_cache_u16_ptr + vzero_u16_addrs, mask=kv_mask, other=0) + v_scales = scale_u16.to(tl.float16, bitcast=True).to(tl.float32) + v_zeros = zero_u16.to(tl.float16, bitcast=True).to(tl.float32) + values = v_idx * v_scales[:, None] + v_zeros[:, None] + + # ============================================================ + # WEIGHTED VALUE ACCUMULATION + # ============================================================ + acc = acc * re_scale + tl.sum(p[:, None] * values, 0) + l_prev = l_prev * re_scale + tl.sum(p, 0) + m_prev = n_e_max + + # Store partial result + out_base = bid * stride_mid_b + hid * stride_mid_h + sid * stride_mid_s + safe_l = tl.where(l_prev > 0.0, l_prev, 1.0) + tl.store(Mid_o_ptr + out_base + d_offs, acc / safe_l, mask=d_mask) + lse = m_prev + tl.log(safe_l) + tl.store(Mid_o_ptr + out_base + HEAD_DIM, lse) + + +# --------------------------------------------------------------------------- +# Pre-dequant kernel: Bulk dequant K (MSE+norms) and V to fp16 +# --------------------------------------------------------------------------- + + +@triton.jit +def _tq_full_dequant_kv( + KV_cache_ptr, + KV_cache_u16_ptr, + Block_table_ptr, + Centroids_ptr, + K_out_ptr, # [B, Hk, max_seq, D] float16 + V_out_ptr, # [B, Hk, max_seq, D] float16 + stride_ko_b: tl.int64, + stride_ko_h: tl.int64, + stride_ko_s: tl.int64, + stride_vo_b: tl.int64, + stride_vo_h: tl.int64, + stride_vo_s: tl.int64, + stride_cache_block: tl.int64, + stride_bt_b: tl.int64, + HEAD_DIM: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + MSE_BYTES: tl.constexpr, + VQB: tl.constexpr, + VAL_DATA_BYTES: tl.constexpr, + MSE_BITS: tl.constexpr, + KEY_FP8: tl.constexpr, + # Opt#3 SoA layout + KEY_DATA_BYTES: tl.constexpr, + META_REGION_OFFSET: tl.constexpr, + NUM_SOA_FIELDS: tl.constexpr, + SOA_K_NORM: tl.constexpr, + SOA_V_SCALE: tl.constexpr, + SOA_V_ZERO: tl.constexpr, + BLOCK_D: tl.constexpr, + NORM_CORRECTION: tl.constexpr = 0, + FP8_E4B15: tl.constexpr = 0, # 1 = use e4b15 (Ampere/Ada), 0 = e4nv (Hopper+) +): + """Full dequant: reconstruct K (MSE centroids * norm or FP8) and V to fp16. + + Opt#3 SoA layout: data region [bs, H, KD+VD] + per-block SoA metadata. + """ + pos = tl.program_id(0) + bh = tl.program_id(1) + bid = bh // NUM_KV_HEADS + hid = bh % NUM_KV_HEADS + + page_idx = pos // BLOCK_SIZE + page_off = pos % BLOCK_SIZE + block_num = tl.load(Block_table_ptr + bid * stride_bt_b + page_idx).to(tl.int64) + + block_base = block_num * stride_cache_block + slot_within_block = tl.cast(page_off, tl.int64) + DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES + data_base = ( + block_base + + slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT) + + tl.cast(hid, tl.int64) * DATA_BYTES_PER_SLOT + ) + head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast( + hid, tl.int64 + ) * (NUM_SOA_FIELDS * BLOCK_SIZE) + knorm_u16_addr = head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block + vscale_u16_addr = head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block + vzero_u16_addr = head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block + + d_offs = tl.arange(0, BLOCK_D) + d_mask = d_offs < HEAD_DIM + + # === K dequant === + ko_base = bid * stride_ko_b + hid * stride_ko_h + pos * stride_ko_s + if KEY_FP8: + k_raw = tl.load(KV_cache_ptr + data_base + d_offs, mask=d_mask, other=0) + if FP8_E4B15: + k_recon = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32) + else: + k_recon = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32) + tl.store(K_out_ptr + ko_base + d_offs, k_recon.to(tl.float16), mask=d_mask) + else: + mse_bit_off = d_offs * MSE_BITS + mse_byte_idx = mse_bit_off // 8 + mse_bit_shift = mse_bit_off % 8 + mse_umask = (1 << MSE_BITS) - 1 + + mse_raw0 = tl.load( + KV_cache_ptr + data_base + mse_byte_idx, mask=d_mask, other=0 + ).to(tl.int32) + mse_raw1 = tl.load( + KV_cache_ptr + data_base + mse_byte_idx + 1, mask=d_mask, other=0 + ).to(tl.int32) + raw16_key = mse_raw0 | (mse_raw1 << 8) + mse_idx = (raw16_key >> mse_bit_shift) & mse_umask + + k_mse = tl.load(Centroids_ptr + mse_idx, mask=d_mask, other=0.0) + + # Opt#1: norm-correction is pre-folded into the stored scalar. + # Opt#3: K-norm lives in the SoA region — single u16 load. + norm_u16 = tl.load(KV_cache_u16_ptr + knorm_u16_addr) + vec_norm = norm_u16.to(tl.float16, bitcast=True).to(tl.float32) + + k_recon = vec_norm * k_mse + tl.store(K_out_ptr + ko_base + d_offs, k_recon.to(tl.float16), mask=d_mask) + + # === V dequant === + val_base = data_base + KEY_DATA_BYTES + if VQB == 4: + vb_idx = d_offs // 2 + vb_shift = (d_offs % 2) * 4 + val_raw = tl.load(KV_cache_ptr + val_base + vb_idx, mask=d_mask, other=0).to( + tl.int32 + ) + v_idx = ((val_raw >> vb_shift) & 0xF).to(tl.float32) + elif VQB == 3: + val_bit_off = d_offs * 3 + val_byte_idx = val_bit_off // 8 + val_bit_shift = val_bit_off % 8 + val_raw0 = tl.load( + KV_cache_ptr + val_base + val_byte_idx, mask=d_mask, other=0 + ).to(tl.int32) + val_raw1 = tl.load( + KV_cache_ptr + val_base + val_byte_idx + 1, mask=d_mask, other=0 + ).to(tl.int32) + raw16_val = val_raw0 | (val_raw1 << 8) + v_idx = ((raw16_val >> val_bit_shift) & 0x7).to(tl.float32) + else: + v_idx = tl.zeros([BLOCK_D], dtype=tl.float32) + + # Opt#3: V scale/zero single-u16 loads from SoA region. + sc_u16 = tl.load(KV_cache_u16_ptr + vscale_u16_addr) + zr_u16 = tl.load(KV_cache_u16_ptr + vzero_u16_addr) + v_scale = sc_u16.to(tl.float16, bitcast=True).to(tl.float32) + v_zero = zr_u16.to(tl.float16, bitcast=True).to(tl.float32) + v_vals = v_idx * v_scale + v_zero + + vo_base = bid * stride_vo_b + hid * stride_vo_h + pos * stride_vo_s + tl.store(V_out_ptr + vo_base + d_offs, v_vals.to(tl.float16), mask=d_mask) + + +# --------------------------------------------------------------------------- +# Stage 2: Reuse from triton_decode_attention.py +# --------------------------------------------------------------------------- + +# --------------------------------------------------------------------------- +# Launcher — cached constants + fused GEMM +# --------------------------------------------------------------------------- + +_layout_cache: dict = {} + + +def _get_layout(D, mse_bits, value_quant_bits, key_packed_size): + """Get cached layout constants.""" + key = (D, mse_bits, value_quant_bits, key_packed_size) + cfg = _layout_cache.get(key) + if cfg is None: + val_data_bytes = math.ceil(D * value_quant_bits / 8) + cfg = { + "mse_bytes": math.ceil(D * mse_bits / 8), + "val_data_bytes": val_data_bytes, + "mse_bits": mse_bits, + "n_centroids": 2**mse_bits, + "BLOCK_D": triton.next_power_of_2(D), + } + _layout_cache[key] = cfg + return cfg + + +def triton_turboquant_decode_attention( + query: torch.Tensor, # [B, Hq, D] — original query + kv_cache: torch.Tensor, # [num_blocks, block_size, Hk, padded_slot] uint8 + block_table: torch.Tensor, # [B, max_num_blocks] int32 + seq_lens: torch.Tensor, # [B] int32 + Pi: torch.Tensor, # [D, D] float32 + centroids: torch.Tensor, # [n_centroids] float32 + scale: float, + mse_bits: int, + key_packed_size: int, + value_quant_bits: int, + key_fp8: bool = False, + norm_correction: bool = False, + PiT: torch.Tensor | None = None, # [D, D] pre-computed Pi.T contiguous + # Pre-allocated buffers (optional, avoids per-call allocation) + mid_o_buf: torch.Tensor | None = None, + output_buf: torch.Tensor | None = None, + lse_buf: torch.Tensor | None = None, + buf_holder: Any = None, + max_num_kv_splits: int = 32, # fixed split count (must be constant for cudagraph) + sinks: torch.Tensor | None = None, # [Hq] float32, pre-computed sink logits +) -> torch.Tensor: + """Launch fused TQ decode attention (Triton stage1 + stage2). + + Returns: output tensor [B, Hq, D] in query's dtype. + """ + B, Hq, D = query.shape + Hk = kv_cache.shape[2] + block_size = kv_cache.shape[1] + kv_group_size = Hq // Hk + device = query.device + + cfg = _get_layout(D, mse_bits, value_quant_bits, key_packed_size) + + # Opt#3 SoA layout constants (derived locally; match store-side values). + key_data_bytes = D if key_fp8 else cfg["mse_bytes"] + data_bytes_per_slot = key_data_bytes + cfg["val_data_bytes"] + meta_region_offset = block_size * Hk * data_bytes_per_slot + num_soa_fields = 2 if key_fp8 else 3 + soa_k_norm = 0 + soa_v_scale = 0 if key_fp8 else 1 + soa_v_zero = 1 if key_fp8 else 2 + kv_cache_u16 = kv_cache.view(torch.uint16) + + # Compute q_rot = q @ Pi.T (rotated query for MSE key scoring) + # FP8 path: pass query directly (float16); kernel casts inline. + # MSE path: still needs external GEMM (cuBLAS), so q_rot is float32. + if key_fp8: + q_rot = query.contiguous() + else: + q_float = query.float() + if PiT is None: + PiT = Pi.T.contiguous() + q_rot = (q_float @ PiT).contiguous() + + NUM_KV_SPLITS = max_num_kv_splits + + if ( + mid_o_buf is not None + and mid_o_buf.shape[0] >= B + and mid_o_buf.shape[2] >= NUM_KV_SPLITS + ): + mid_o = mid_o_buf[:B, :Hq, :NUM_KV_SPLITS, :] + else: + mid_o = torch.empty( + B, + Hq, + NUM_KV_SPLITS, + D + 1, + dtype=torch.float32, + device=device, + ) + if buf_holder is not None: + buf_holder._tq_mid_o_buf = mid_o + + # Stage 1: split-KV tiled attention scoring + value accumulation + fp8_e4b15 = _use_fp8_e4b15(device.index or 0) + BLOCK_KV = 4 + grid = (B, Hq, NUM_KV_SPLITS) + + # Prepare sink pointer (convert to float32 on device if provided) + use_sinks = sinks is not None + + _tq_decode_stage1[grid]( + q_rot, + kv_cache, + kv_cache_u16, + block_table, + seq_lens, + centroids, + mid_o, + sinks, + q_rot.stride(0), + q_rot.stride(1), + kv_cache.stride(0), + block_table.stride(0), + mid_o.stride(0), + mid_o.stride(1), + mid_o.stride(2), + NUM_KV_HEADS=Hk, + HEAD_DIM=D, + BLOCK_SIZE=block_size, + NUM_KV_SPLITS=NUM_KV_SPLITS, + KV_GROUP_SIZE=kv_group_size, + MSE_BITS=mse_bits, + MSE_BYTES=cfg["mse_bytes"], + VQB=value_quant_bits, + VAL_DATA_BYTES=cfg["val_data_bytes"], + KEY_DATA_BYTES=key_data_bytes, + META_REGION_OFFSET=meta_region_offset, + NUM_SOA_FIELDS=num_soa_fields, + SOA_K_NORM=soa_k_norm, + SOA_V_SCALE=soa_v_scale, + SOA_V_ZERO=soa_v_zero, + ATTN_SCALE=scale, + BLOCK_D=cfg["BLOCK_D"], + BLOCK_KV=BLOCK_KV, + KEY_FP8=1 if key_fp8 else 0, + NORM_CORRECTION=1 if norm_correction else 0, + FP8_E4B15=fp8_e4b15, + USE_SINKS=1 if use_sinks else 0, + num_warps=1, + num_stages=1, + ) + + # Stage 2: Reduce across KV splits + if output_buf is not None and output_buf.shape[0] >= B: + output = output_buf[:B, :Hq, :D] + else: + output = torch.empty(B, Hq, D, dtype=torch.float32, device=device) + if buf_holder is not None: + buf_holder._tq_output_buf = output + if lse_buf is not None and lse_buf.shape[0] >= B: + lse = lse_buf[:B, :Hq] + else: + lse = torch.empty(B, Hq, dtype=torch.float32, device=device) + if buf_holder is not None: + buf_holder._tq_lse_buf = lse + + grid2 = (B, Hq) + _fwd_kernel_stage2[grid2]( + mid_o, + output, + lse, + seq_lens, + mid_o.stride(0), + mid_o.stride(1), + mid_o.stride(2), + output.stride(0), + output.stride(1), + lse.stride(0), + NUM_KV_SPLITS=NUM_KV_SPLITS, + BLOCK_DV=cfg["BLOCK_D"], + Lv=D, + num_warps=4, + num_stages=2, + ) + + return output.to(query.dtype) diff --git a/vllm/v1/attention/ops/turboquant_soa/triton_turboquant_decode_v2.py b/vllm/v1/attention/ops/turboquant_soa/triton_turboquant_decode_v2.py new file mode 100644 index 0000000000..99198f7a23 --- /dev/null +++ b/vllm/v1/attention/ops/turboquant_soa/triton_turboquant_decode_v2.py @@ -0,0 +1,669 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Optimized Triton TurboQuant decode attention (v2). + +FLUTE-paper optimizations applied: + 1. Grouped Q heads: grid over (B, Hk, splits) instead of (B, Hq, splits). + Each program loads BLOCK_M Q heads sharing a KV head into a 2D tile, + enabling tl.dot on tensor cores (MFMA/WMMA) for both Q·K and P·V. + 2. Vectorized pair LUT: precompute pair_table[i][j] = (T[i], T[j]) + offline. At runtime, extract adjacent index pairs and fetch two + dequantized centroids with a single gather, halving LUT lookups. + 3. exp2 instead of exp: scores pre-scaled by log2(e) so the hardware- + native exp2 instruction replaces the more expensive exp. + 4. Wider index extraction: for 4-bit MSE, two adjacent 4-bit indices share + a byte. One byte load yields both, eliminating redundant loads. + 5. Centroids pre-warmed in L1 at kernel start. + 6. BLOCK_KV = TILE_SIZE raised to 16-32 (from 4), reducing loop iterations + and softmax rescaling overhead. + +Stage 2 is reused unchanged from triton_decode_attention.py. +""" + +import math +from typing import Any + +import torch + +from vllm.platforms import current_platform +from vllm.triton_utils import tl, triton +from vllm.v1.attention.ops.triton_decode_attention import _fwd_kernel_stage2 + +from .triton_turboquant_decode import _use_fp8_e4b15 + +# ROCm prefers num_stages=1 in attention-like kernels to reduce shared-memory +# pressure (mirrors the pattern used in triton_decode_attention.py and +# triton_turboquant_decode.py). +_is_hip = current_platform.is_rocm() + +# On ROCm, bf16 has the same MFMA throughput as fp16 but wider dynamic range +# (8-bit exponent vs 5-bit), which is safer for attention scores. On CUDA, +# fp16 tensor cores may be faster than bf16 for some shapes. +_DOT_DTYPE = tl.bfloat16 if _is_hip else tl.float16 + + +# --------------------------------------------------------------------------- +# Pair LUT construction (FLUTE) — called once at launcher time +# --------------------------------------------------------------------------- + + +def build_pair_lut(centroids: torch.Tensor) -> torch.Tensor: + """Build vectorized pair lookup table. + + For N centroids, returns a [N, N, 2] float32 tensor where + pair_lut[i, j] = (centroids[i], centroids[j]). + Flattened to [N*N, 2] for kernel indexing: pair_lut[i*N + j]. + + For 4-bit MSE (N=16): 16*16*2*4 = 2048 bytes — fits in L1/smem. + For 3-bit MSE (N=8): 8*8*2*4 = 512 bytes. + """ + N = centroids.shape[0] + # pair_lut[i,j,0] = centroids[i], pair_lut[i,j,1] = centroids[j] + c = centroids.float() + lut = torch.empty(N, N, 2, dtype=torch.float32, device=centroids.device) + lut[:, :, 0] = c[:, None] + lut[:, :, 1] = c[None, :] + return lut.reshape(N * N, 2).contiguous() + + +# --------------------------------------------------------------------------- +# Stage 1 v2: Grouped-Q + pair LUT + tl.dot + exp2 +# --------------------------------------------------------------------------- + + +@triton.jit +def _tq_decode_stage1_v2( + Q_rot_ptr, + KV_cache_ptr, + KV_cache_u16_ptr, # uint16 view — Opt#3 SoA loads + Block_table_ptr, + Seq_lens_ptr, + Centroids_ptr, + Pair_lut_ptr, + Mid_o_ptr, + stride_qb, + stride_qh, + stride_cache_block, # bytes per block (bs*H*slot_aligned) + stride_bt_b, + stride_mid_b, + stride_mid_h, + stride_mid_s, + NUM_Q_HEADS: tl.constexpr, + NUM_KV_HEADS: tl.constexpr, + HEAD_DIM: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + PADDED_SLOT: tl.constexpr, + MAX_NUM_BLOCKS: tl.constexpr, + NUM_KV_SPLITS: tl.constexpr, + KV_GROUP_SIZE: tl.constexpr, + MSE_BITS: tl.constexpr, + MSE_BYTES: tl.constexpr, + VQB: tl.constexpr, + VAL_DATA_BYTES: tl.constexpr, + # Opt#3 SoA layout + KEY_DATA_BYTES: tl.constexpr, + META_REGION_OFFSET: tl.constexpr, + NUM_SOA_FIELDS: tl.constexpr, + SOA_K_NORM: tl.constexpr, + SOA_V_SCALE: tl.constexpr, + SOA_V_ZERO: tl.constexpr, + N_CENTROIDS: tl.constexpr, + ATTN_SCALE: tl.constexpr, + BLOCK_D: tl.constexpr, + TILE_SIZE: tl.constexpr, + BLOCK_M: tl.constexpr, + KEY_FP8: tl.constexpr, + NORM_CORRECTION: tl.constexpr = 0, + FP8_E4B15: tl.constexpr = 0, + USE_PAIR_LUT: tl.constexpr = 0, + USE_BF16_DOT: tl.constexpr = 0, +): + # Grid: (B, Hk, NUM_KV_SPLITS) — one program per KV head group + bid = tl.program_id(0) + kv_hid = tl.program_id(1) + sid = tl.program_id(2) + + seq_len = tl.load(Seq_lens_ptr + bid) + split_len = tl.cdiv(seq_len, NUM_KV_SPLITS) + split_start = split_len * sid + split_end = tl.minimum(split_start + split_len, seq_len) + + if split_start >= split_end: + return + + # Dimension and tile offsets + d_offs = tl.arange(0, BLOCK_D) + d_mask = d_offs < HEAD_DIM + m_offs = tl.arange(0, BLOCK_M) + t_offs = tl.arange(0, TILE_SIZE) + + # Q head indices for this KV group: [kv_hid * GS, kv_hid * GS + GS) + q_head_offs = kv_hid * KV_GROUP_SIZE + m_offs + q_mask = m_offs < KV_GROUP_SIZE + + # Load Q tile: [BLOCK_M, BLOCK_D] float32 + q_addrs = bid * stride_qb + q_head_offs[:, None] * stride_qh + d_offs[None, :] + Q = tl.load( + Q_rot_ptr + q_addrs, + mask=q_mask[:, None] & d_mask[None, :], + other=0.0, + ) + + # Pre-warm centroids in L1 + if not KEY_FP8: + _c_warm = tl.load(Centroids_ptr + tl.arange(0, N_CENTROIDS)) # noqa: F841 + + # Loop-invariant bit-extraction constants for MSE keys + if not KEY_FP8: + if MSE_BITS == 4: + pass # 4-bit byte extraction is computed inline in the tile loop + else: + # Generic bit extraction + mse_bit_off = d_offs * MSE_BITS + mse_byte_idx = mse_bit_off // 8 + mse_bit_shift = mse_bit_off % 8 + mse_mask_val = (1 << MSE_BITS) - 1 + + # Loop-invariant bit-extraction for 3-bit values + if VQB == 3: + val_bit_off = d_offs * 3 + val_byte_idx = val_bit_off // 8 + val_bit_shift = val_bit_off % 8 + + # exp2 pre-scaling + RCP_LN2: tl.constexpr = 1.4426950408889634 + LN2: tl.constexpr = 0.6931471805599453 + QK_SCALE = ATTN_SCALE * RCP_LN2 + + # Online softmax accumulators: [BLOCK_M] for max, [BLOCK_M] for L + M = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32) + L = tl.zeros([BLOCK_M], dtype=tl.float32) + + acc = tl.zeros([BLOCK_M, BLOCK_D], dtype=tl.float32) + + bt_base = bid * stride_bt_b + + # ================================================================ + # TILED LOOP: TILE_SIZE KV tokens per iteration + # ================================================================ + for start_n in range(split_start, split_end, TILE_SIZE): + kv_offs = start_n + t_offs + kv_mask_1d = kv_offs < split_end + + page_idx = kv_offs // BLOCK_SIZE + page_off = kv_offs % BLOCK_SIZE + block_nums = tl.load( + Block_table_ptr + bt_base + page_idx, + mask=kv_mask_1d, + other=0, + ).to(tl.int64) + + # Opt#3 SoA addressing: data region then per-block SoA metadata. + slot_within_block = page_off.to(tl.int64) + block_base = block_nums * stride_cache_block + DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES + data_bases = ( + block_base + + slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT) + + tl.cast(kv_hid, tl.int64) * DATA_BYTES_PER_SLOT + ) + head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast( + kv_hid, tl.int64 + ) * (NUM_SOA_FIELDS * BLOCK_SIZE) + knorm_u16_addrs = ( + head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block + ) + vscale_u16_addrs = ( + head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block + ) + vzero_u16_addrs = ( + head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block + ) + + # ============================================================ + # KEY DEQUANT → K_T: [BLOCK_D, TILE_SIZE] for tl.dot(Q, K_T) + # ============================================================ + if KEY_FP8: + k_addrs = data_bases[:, None] + d_offs[None, :] + k_raw = tl.load( + KV_cache_ptr + k_addrs, + mask=kv_mask_1d[:, None] & d_mask[None, :], + other=0, + ) + if FP8_E4B15: + K_f = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32) + else: + K_f = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32) + # K_f: [TILE_SIZE, BLOCK_D] → transpose for tl.dot + K_T = tl.trans(K_f) + # S: [BLOCK_M, TILE_SIZE] (bf16 dot on ROCm for better dynamic range) + if USE_BF16_DOT: + S = QK_SCALE * tl.dot(Q.to(tl.bfloat16), K_T.to(tl.bfloat16)) + else: + S = QK_SCALE * tl.dot(Q.to(tl.float16), K_T.to(tl.float16)) + else: + # --- MSE dequantization --- + if MSE_BITS == 4 and USE_PAIR_LUT: + # OPT#1 (load-halving): FLUTE §3.2 pair LUT, but with each + # packed byte loaded EXACTLY ONCE instead of twice (the + # previous code built byte_addrs from d_offs // 2 which has + # the pattern [0,0,1,1,2,2,...], duplicating every byte + # load). We load the packed bytes as a [TILE_SIZE, HALF_D] + # tile, decode both nibbles, then issue a single 3-D gather + # against pair_lut which returns (T[lo], T[hi]) in slots + # 0/1. Reshape [TILE_SIZE, HALF_D, 2] -> [TILE_SIZE, BLOCK_D] + # interleaves the pairs into the expected dim order + # (d=2k -> T[lo_k], d=2k+1 -> T[hi_k]). + HALF_D: tl.constexpr = BLOCK_D // 2 + half_offs = tl.arange(0, HALF_D) + # Each byte k covers dims (2k, 2k+1); it is in-range iff + # 2k < HEAD_DIM. HEAD_DIM is always even in our layouts, + # so this is equivalent to k < HEAD_DIM // 2. + byte_mask = (half_offs * 2) < HEAD_DIM + + byte_addrs = data_bases[:, None] + half_offs[None, :] + byte_raw = tl.load( + KV_cache_ptr + byte_addrs, + mask=kv_mask_1d[:, None] & byte_mask[None, :], + other=0, + ).to(tl.int32) + + lo_idx = byte_raw & 0xF # [TILE_SIZE, HALF_D] + hi_idx = (byte_raw >> 4) & 0xF # [TILE_SIZE, HALF_D] + pair_key = lo_idx * N_CENTROIDS + hi_idx # [TILE_SIZE, HALF_D] + + # Single 3-D gather: pair_lut[pair_key*2 + {0,1}] + # -> [TILE_SIZE, HALF_D, 2] = (T[lo_k], T[hi_k]) per element. + pair_slot = tl.arange(0, 2) + c_pair = tl.load( + Pair_lut_ptr + pair_key[:, :, None] * 2 + pair_slot[None, None, :], + mask=(kv_mask_1d[:, None, None] & byte_mask[None, :, None]), + other=0.0, + ) + # Row-major reshape interleaves pairs: + # out[t, 2k] = c_pair[t, k, 0] = T[lo_k] + # out[t, 2k+1] = c_pair[t, k, 1] = T[hi_k] + c_vals = tl.reshape(c_pair, [TILE_SIZE, BLOCK_D]) + elif MSE_BITS == 4: + # 4-bit without pair LUT: exploit byte alignment + half_idx = d_offs // 2 + nibble_shift = (d_offs % 2) * 4 + mse_addrs = data_bases[:, None] + half_idx[None, :] + mse_raw = tl.load( + KV_cache_ptr + mse_addrs, + mask=kv_mask_1d[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + mse_idx = (mse_raw >> nibble_shift[None, :]) & 0xF + c_vals = tl.load( + Centroids_ptr + mse_idx, + mask=kv_mask_1d[:, None] & d_mask[None, :], + other=0.0, + ) + else: + # Generic 3-bit path: two byte loads → 16-bit → shift/mask + mse_addrs0 = data_bases[:, None] + mse_byte_idx[None, :] + mse_raw0 = tl.load( + KV_cache_ptr + mse_addrs0, + mask=kv_mask_1d[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + mse_raw1 = tl.load( + KV_cache_ptr + mse_addrs0 + 1, + mask=kv_mask_1d[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + raw16 = mse_raw0 | (mse_raw1 << 8) + mse_idx = (raw16 >> mse_bit_shift[None, :]) & mse_mask_val + c_vals = tl.load( + Centroids_ptr + mse_idx, + mask=kv_mask_1d[:, None] & d_mask[None, :], + other=0.0, + ) + + # Opt#1: norm-correction is pre-folded into the stored per-token + # scalar at store time, so no per-tile sum+sqrt+divide here. The + # stored vec_norm carries ||k||/||c_vec||; multiply by c_vals + # below gives the original c_vec/||c_vec|| * ||k||. + + # Opt#3: K-norms from per-block SoA region. Single u16 load per + # token; contiguous across tokens within a block. + norm_u16 = tl.load( + KV_cache_u16_ptr + knorm_u16_addrs, mask=kv_mask_1d, other=0 + ) + vec_norms = norm_u16.to(tl.float16, bitcast=True).to(tl.float32) + + # Reconstruct K: K_recon[t, d] = norm[t] * centroid[t, d] + K_recon = c_vals * vec_norms[:, None] + # K_T: [BLOCK_D, TILE_SIZE] + K_T = tl.trans(K_recon) + + # S: [BLOCK_M, TILE_SIZE] via tensor core MMA + if USE_BF16_DOT: + S = QK_SCALE * tl.dot(Q.to(tl.bfloat16), K_T.to(tl.bfloat16)) + else: + S = QK_SCALE * tl.dot(Q.to(tl.float16), K_T.to(tl.float16)) + + # Mask out-of-range positions + S = tl.where(kv_mask_1d[None, :], S, float("-inf")) + + # ============================================================ + # ONLINE SOFTMAX with exp2 + # ============================================================ + m_j = tl.maximum(M, tl.max(S, axis=1)) + m_j = tl.where(m_j > float("-inf"), m_j, 0.0) + P = tl.math.exp2(S - m_j[:, None]) + l_j = tl.sum(P, axis=1) + alpha = tl.math.exp2(M - m_j) + + acc = acc * alpha[:, None] + L = L * alpha + l_j + M = m_j + + # ============================================================ + # VALUE DEQUANT → V: [TILE_SIZE, BLOCK_D] for tl.dot(P, V) + # Opt#3: V data lives at data_base + KEY_DATA_BYTES; scale/zero + # move to the SoA metadata region (one u16 load each per token). + # ============================================================ + val_bases = data_bases + KEY_DATA_BYTES + + if VQB == 3: + val_addrs0 = val_bases[:, None] + val_byte_idx[None, :] + val_raw0 = tl.load( + KV_cache_ptr + val_addrs0, + mask=kv_mask_1d[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + val_raw1 = tl.load( + KV_cache_ptr + val_addrs0 + 1, + mask=kv_mask_1d[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + raw16 = val_raw0 | (val_raw1 << 8) + v_idx = ((raw16 >> val_bit_shift[None, :]) & 0x7).to(tl.float32) + else: # VQB == 4 + # OPT#3 (load-halving, value path): same pattern as OPT#1 for + # keys -- load each byte exactly once as [TILE_SIZE, HALF_D], + # decode both nibbles, then tl.interleave to [TILE_SIZE, BLOCK_D] + # with pattern [v_lo0, v_hi0, v_lo1, v_hi1, ...]. + V_HALF_D: tl.constexpr = BLOCK_D // 2 + v_half_offs = tl.arange(0, V_HALF_D) + v_byte_mask = (v_half_offs * 2) < HEAD_DIM + + val_addrs = val_bases[:, None] + v_half_offs[None, :] + val_byte = tl.load( + KV_cache_ptr + val_addrs, + mask=kv_mask_1d[:, None] & v_byte_mask[None, :], + other=0, + ).to(tl.int32) + v_lo = (val_byte & 0xF).to(tl.float32) # [T, HALF_D] + v_hi = ((val_byte >> 4) & 0xF).to(tl.float32) # [T, HALF_D] + v_idx = tl.interleave(v_lo, v_hi) # [T, BLOCK_D] + + scale_u16 = tl.load( + KV_cache_u16_ptr + vscale_u16_addrs, mask=kv_mask_1d, other=0 + ) + zero_u16 = tl.load(KV_cache_u16_ptr + vzero_u16_addrs, mask=kv_mask_1d, other=0) + v_scales = scale_u16.to(tl.float16, bitcast=True).to(tl.float32) + v_zeros = zero_u16.to(tl.float16, bitcast=True).to(tl.float32) + V = v_idx * v_scales[:, None] + v_zeros[:, None] + + # P·V accumulation via tensor core MMA + # P: [BLOCK_M, TILE_SIZE], V: [TILE_SIZE, BLOCK_D] + if USE_BF16_DOT: + acc += tl.dot(P.to(tl.bfloat16), V.to(tl.bfloat16)) + else: + acc += tl.dot(P.to(tl.float16), V.to(tl.float16)) + + # ================================================================ + # EPILOGUE: Store per-Q-head partial results for stage2 + # ================================================================ + # acc: [BLOCK_M, BLOCK_D], L: [BLOCK_M], M: [BLOCK_M] + # Stage2 expects per-Q-head: mid_o[B, Hq, split, D+1] + safe_L = tl.where(L > 0.0, L, 1.0) + one_over_L = 1.0 / safe_L + acc = acc * one_over_L[:, None] + + # Convert M from log2 space back to natural log for stage2 compatibility + lse = M * LN2 + tl.log(safe_L) + + # Store each Q head's result using 2D scatter + # q_head_offs: [BLOCK_M] = kv_hid * KV_GROUP_SIZE + m_offs + out_addrs = ( + bid * stride_mid_b + + q_head_offs[:, None] * stride_mid_h + + sid * stride_mid_s + + d_offs[None, :] + ) + tl.store( + Mid_o_ptr + out_addrs, + acc, + mask=q_mask[:, None] & d_mask[None, :], + ) + # Store LSE per Q head + lse_addrs = ( + bid * stride_mid_b + q_head_offs * stride_mid_h + sid * stride_mid_s + HEAD_DIM + ) + tl.store(Mid_o_ptr + lse_addrs, lse, mask=q_mask) + + +# --------------------------------------------------------------------------- +# Launcher v2 +# --------------------------------------------------------------------------- + +_layout_cache: dict = {} + + +def _get_layout(D, mse_bits, value_quant_bits, key_packed_size): + key = (D, mse_bits, value_quant_bits, key_packed_size) + cfg = _layout_cache.get(key) + if cfg is None: + val_data_bytes = math.ceil(D * value_quant_bits / 8) + cfg = { + "mse_bytes": math.ceil(D * mse_bits / 8), + "val_data_bytes": val_data_bytes, + "mse_bits": mse_bits, + "n_centroids": 2**mse_bits, + "BLOCK_D": triton.next_power_of_2(D), + } + _layout_cache[key] = cfg + return cfg + + +def _get_pair_lut(centroids: torch.Tensor) -> torch.Tensor: + """Return a fresh pair-LUT for ``centroids`` on each call. + + The LUT is tiny (N*N*2 fp32, e.g. 2KB for 4-bit MSE) so the build cost + is negligible compared to attention. We avoid caching by data_ptr() + because CUDA allocator memory reuse across different centroid tensors + can silently return a stale LUT (subtle correctness bug). If this ever + shows up on a profile, cache by a hash-of-values fingerprint instead. + """ + return build_pair_lut(centroids) + + +def triton_turboquant_decode_attention_v2( + query: torch.Tensor, + kv_cache: torch.Tensor, + block_table: torch.Tensor, + seq_lens: torch.Tensor, + Pi: torch.Tensor, + centroids: torch.Tensor, + scale: float, + mse_bits: int, + key_packed_size: int, + value_quant_bits: int, + value_packed_size: int, + max_seq_len: int = 0, # unused; kept for backward compatibility + key_fp8: bool = False, + norm_correction: bool = False, + PiT: torch.Tensor | None = None, + # Pre-allocated buffers (optional, required for CUDA-graph stability). + mid_o_buf: torch.Tensor | None = None, + output_buf: torch.Tensor | None = None, + lse_buf: torch.Tensor | None = None, + buf_holder: Any = None, + # Fixed split count — MUST be a compile-time constant across iterations + # for CUDA-graph capture/replay to work. Mirrors the v1 launcher contract. + max_num_kv_splits: int = 32, +) -> torch.Tensor: + """Launch optimized TQ decode attention (v2 stage1 + shared stage2). + + Follows the same buffer-reuse + fixed-grid contract as the v1 launcher + (triton_turboquant_decode_attention) so the backend can capture a CUDA + graph across both versions. + """ + B, Hq, D = query.shape + Hk = kv_cache.shape[2] + block_size = kv_cache.shape[1] + padded_slot = kv_cache.shape[3] + max_num_blocks = block_table.shape[1] + n_centroids = centroids.shape[0] + kv_group_size = Hq // Hk + device = query.device + del max_seq_len # no longer used: splits is fixed via max_num_kv_splits + + cfg = _get_layout(D, mse_bits, value_quant_bits, key_packed_size) + + # Opt#3 SoA layout constants (match store-side computation). + key_data_bytes = D if key_fp8 else cfg["mse_bytes"] + data_bytes_per_slot = key_data_bytes + cfg["val_data_bytes"] + meta_region_offset = block_size * Hk * data_bytes_per_slot + num_soa_fields = 2 if key_fp8 else 3 + soa_k_norm = 0 + soa_v_scale = 0 if key_fp8 else 1 + soa_v_zero = 1 if key_fp8 else 2 + kv_cache_u16 = kv_cache.view(torch.uint16) + + # Compute q_rot = q @ Pi.T + if key_fp8: + q_rot = query.float().contiguous() + else: + q_float = query.float() + if PiT is None: + PiT = Pi.T.contiguous() + q_rot = (q_float @ PiT).contiguous() + + # BLOCK_M: pad KV_GROUP_SIZE to power of 2, minimum 16 for tensor cores + BLOCK_M = max(16, triton.next_power_of_2(kv_group_size)) + + # TILE_SIZE (BLOCK_KV): tokens per inner-loop iteration + TILE_SIZE = 16 + + # Fixed split count — must be constant across calls for cudagraph replay. + NUM_KV_SPLITS = max_num_kv_splits + + # --- mid_o buffer reuse (same pattern as v1 launcher) --- + if ( + mid_o_buf is not None + and mid_o_buf.shape[0] >= B + and mid_o_buf.shape[2] >= NUM_KV_SPLITS + ): + mid_o = mid_o_buf[:B, :Hq, :NUM_KV_SPLITS, :] + else: + mid_o = torch.empty( + B, + Hq, + NUM_KV_SPLITS, + D + 1, + dtype=torch.float32, + device=device, + ) + if buf_holder is not None: + buf_holder._tq_mid_o_buf = mid_o + + fp8_e4b15 = _use_fp8_e4b15(device.index or 0) + + # Build pair LUT for 4-bit MSE (FLUTE §3.2) + use_pair_lut = mse_bits == 4 and not key_fp8 + pair_lut = _get_pair_lut(centroids) if use_pair_lut else centroids + + # Platform-dependent pipelining depth: ROCm prefers num_stages=1. + stage1_num_stages = 1 if _is_hip else 2 + + # --- Stage 1: v2 kernel --- + # Grid over KV heads (not Q heads) — each program handles all Q heads + grid = (B, Hk, NUM_KV_SPLITS) + _tq_decode_stage1_v2[grid]( + q_rot, + kv_cache, + kv_cache_u16, + block_table, + seq_lens, + centroids, + pair_lut, + mid_o, + q_rot.stride(0), + q_rot.stride(1), + kv_cache.stride(0), + block_table.stride(0), + mid_o.stride(0), + mid_o.stride(1), + mid_o.stride(2), + NUM_Q_HEADS=Hq, + NUM_KV_HEADS=Hk, + HEAD_DIM=D, + BLOCK_SIZE=block_size, + PADDED_SLOT=padded_slot, + MAX_NUM_BLOCKS=max_num_blocks, + NUM_KV_SPLITS=NUM_KV_SPLITS, + KV_GROUP_SIZE=kv_group_size, + MSE_BITS=mse_bits, + MSE_BYTES=cfg["mse_bytes"], + VQB=value_quant_bits, + VAL_DATA_BYTES=cfg["val_data_bytes"], + KEY_DATA_BYTES=key_data_bytes, + META_REGION_OFFSET=meta_region_offset, + NUM_SOA_FIELDS=num_soa_fields, + SOA_K_NORM=soa_k_norm, + SOA_V_SCALE=soa_v_scale, + SOA_V_ZERO=soa_v_zero, + N_CENTROIDS=n_centroids, + ATTN_SCALE=scale, + BLOCK_D=cfg["BLOCK_D"], + TILE_SIZE=TILE_SIZE, + BLOCK_M=BLOCK_M, + KEY_FP8=1 if key_fp8 else 0, + NORM_CORRECTION=1 if norm_correction else 0, + FP8_E4B15=fp8_e4b15, + USE_PAIR_LUT=1 if use_pair_lut else 0, + USE_BF16_DOT=1 if _is_hip else 0, + num_warps=4, + num_stages=stage1_num_stages, + ) + + # --- output / lse buffer reuse (same pattern as v1 launcher) --- + if output_buf is not None and output_buf.shape[0] >= B: + output = output_buf[:B, :Hq, :D] + else: + output = torch.empty(B, Hq, D, dtype=torch.float32, device=device) + if buf_holder is not None: + buf_holder._tq_output_buf = output + if lse_buf is not None and lse_buf.shape[0] >= B: + lse = lse_buf[:B, :Hq] + else: + lse = torch.empty(B, Hq, dtype=torch.float32, device=device) + if buf_holder is not None: + buf_holder._tq_lse_buf = lse + + # --- Stage 2: reduce across KV splits (unchanged) --- + grid2 = (B, Hq) + _fwd_kernel_stage2[grid2]( + mid_o, + output, + lse, + seq_lens, + mid_o.stride(0), + mid_o.stride(1), + mid_o.stride(2), + output.stride(0), + output.stride(1), + lse.stride(0), + NUM_KV_SPLITS=NUM_KV_SPLITS, + BLOCK_DV=cfg["BLOCK_D"], + Lv=D, + num_warps=4, + num_stages=2, + ) + + return output.to(query.dtype) diff --git a/vllm/v1/attention/ops/turboquant_soa/triton_turboquant_store.py b/vllm/v1/attention/ops/turboquant_soa/triton_turboquant_store.py new file mode 100644 index 0000000000..0dc4150cbf --- /dev/null +++ b/vllm/v1/attention/ops/turboquant_soa/triton_turboquant_store.py @@ -0,0 +1,524 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Fused Triton kernels for TurboQuant KV store. + +Two kernels: +1. _tq_fused_store_fp8: FP8 key scatter + value uniform quantization. +2. _tq_fused_store_mse: Fused binary-search bucketize + MSE index + packing + value quantization. + +The launcher `triton_turboquant_store` selects the appropriate kernel. +""" + +import math + +import torch + +from vllm.triton_utils import tl, triton + +from .triton_turboquant_decode import _use_fp8_e4b15 + +# ═══════════════════════════════════════════════════════════════════════ +# Shared: value uniform quantization + pack + SoA scale/zero store +# ═══════════════════════════════════════════════════════════════════════ + + +@triton.jit +def _store_quantized_value( + Value_ptr, + KV_cache_ptr, + KV_cache_u16_ptr, # Opt#3: u16 view for SoA scale/zero stores + base, # pid * D offset into Value_ptr + data_base, # byte offset into KV_cache_ptr for this slot+head's DATA region + vscale_u16_addr, # u16 element index for V-scale in the SoA region + vzero_u16_addr, # u16 element index for V-zero in the SoA region + d_offs, # tl.arange(0, BLOCK_D) + d_mask, # d_offs < D + D: tl.constexpr, + # Offset (bytes) from data_base to V-data region within the slot's data. + # For MSE keys this equals MSE_BYTES; for FP8 keys it equals D. + V_DATA_OFFSET: tl.constexpr, + VQB: tl.constexpr, + VAL_DATA_BYTES: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_VAL: tl.constexpr, + BLOCK_GRP: tl.constexpr, +): + """Uniform quantization of values to VQB bits, pack, and store with + scale/zero in the per-block SoA metadata region (Opt#3 layout).""" + if VQB == 3: + val_vec = tl.load(Value_ptr + base + d_offs, mask=d_mask, other=0.0).to( + tl.float32 + ) + val_min = tl.min(tl.where(d_mask, val_vec, float("inf")), axis=0) + val_max = tl.max(tl.where(d_mask, val_vec, -float("inf")), axis=0) + v_scale = (val_max - val_min) / 7.0 + v_scale = tl.where(v_scale > 1e-8, v_scale, 1e-8) + + q_vals = tl.minimum( + tl.maximum(((val_vec - val_min) / v_scale + 0.5).to(tl.int32), 0), 7 + ) + + grp_offs = tl.arange(0, BLOCK_GRP) + grp_mask = grp_offs < (D // 8) + q_grp = tl.reshape(q_vals, [BLOCK_GRP, 8]) + shifts_3bit = tl.arange(0, 8) * 3 + packed_24 = tl.sum(q_grp << shifts_3bit[None, :], axis=1) + b0 = (packed_24 & 0xFF).to(tl.uint8) + b1 = ((packed_24 >> 8) & 0xFF).to(tl.uint8) + b2 = ((packed_24 >> 16) & 0xFF).to(tl.uint8) + tl.store( + KV_cache_ptr + data_base + V_DATA_OFFSET + grp_offs * 3, + b0, + mask=grp_mask, + ) + tl.store( + KV_cache_ptr + data_base + V_DATA_OFFSET + grp_offs * 3 + 1, + b1, + mask=grp_mask, + ) + tl.store( + KV_cache_ptr + data_base + V_DATA_OFFSET + grp_offs * 3 + 2, + b2, + mask=grp_mask, + ) + else: # VQB == 4 + val_vec = tl.load(Value_ptr + base + d_offs, mask=d_mask, other=0.0).to( + tl.float32 + ) + val_min = tl.min(tl.where(d_mask, val_vec, float("inf")), axis=0) + val_max = tl.max(tl.where(d_mask, val_vec, -float("inf")), axis=0) + v_scale = (val_max - val_min) / 15.0 + v_scale = tl.where(v_scale > 1e-8, v_scale, 1e-8) + + # Quantize all D elements from register (no re-load) + q_all = tl.minimum( + tl.maximum(((val_vec - val_min) / v_scale + 0.5).to(tl.int32), 0), 15 + ) + q_pairs = tl.reshape(q_all, [BLOCK_D // 2, 2]) + shifts_4 = tl.arange(0, 2) * 4 + packed_val = tl.sum((q_pairs & 0xF) << shifts_4[None, :], axis=1).to(tl.uint8) + val_offs = tl.arange(0, BLOCK_D // 2) + val_mask = val_offs < VAL_DATA_BYTES + tl.store( + KV_cache_ptr + data_base + V_DATA_OFFSET + val_offs, + packed_val, + mask=val_mask, + ) + + # ── SoA scale/zero store (Opt#3) ────────────────────────────────── + # Scale and zero each get one fp16 stored at their token's slot index + # within the per-head SoA region. Single u16 store per field vs the + # previous 2×u8 writes per field. + sc_u16 = v_scale.to(tl.float16).to(tl.uint16, bitcast=True) + zr_u16 = val_min.to(tl.float16).to(tl.uint16, bitcast=True) + tl.store(KV_cache_u16_ptr + vscale_u16_addr, sc_u16) + tl.store(KV_cache_u16_ptr + vzero_u16_addr, zr_u16) + + +# ═══════════════════════════════════════════════════════════════════════ +# FP8 key store + value uniform quantization +# ═══════════════════════════════════════════════════════════════════════ + + +@triton.jit +def _tq_fused_store_fp8( + Key_ptr, # [NH, D] float16/bfloat16 — raw keys + Value_ptr, # [NH, D] float16/bfloat16 — raw values + KV_cache_ptr, # [total_bytes] uint8 (flattened view) + KV_cache_u16_ptr, # uint16-aliased view of the same storage (Opt#3) + Slot_mapping_ptr, # [N] int32 — per-token slot indices + # Cache strides (stride_cache_block = bytes per block = bs*H*slot_aligned) + stride_cache_block: tl.constexpr, + # Dimensions + D: tl.constexpr, + H: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + BLOCK_D: tl.constexpr, + # SoA layout (Opt#3): data region is [bs, H, KEY_DATA_BYTES+VAL_DATA_BYTES], + # followed by metadata region [H, NUM_SOA_FIELDS, bs] of fp16. + KEY_DATA_BYTES: tl.constexpr, # FP8: = D + VAL_DATA_BYTES: tl.constexpr, + META_REGION_OFFSET: tl.constexpr, # bytes from block start to SoA region + NUM_SOA_FIELDS: tl.constexpr, # FP8: 2 (V-scale, V-zero) + SOA_V_SCALE: tl.constexpr, # FP8: 0 + SOA_V_ZERO: tl.constexpr, # FP8: 1 + # Value quantization + VQB: tl.constexpr, + # Packing block sizes + BLOCK_VAL: tl.constexpr, + BLOCK_GRP: tl.constexpr = 16, + FP8_E4B15: tl.constexpr = 0, # 1 = e4b15 (Ampere/Ada), 0 = e4nv (Hopper+) +): + """FP8 key cast+scatter + value uniform quantization (SoA layout).""" + pid = tl.program_id(0) + token_idx = pid // H + head_idx = pid % H + + slot = tl.load(Slot_mapping_ptr + token_idx) + if slot < 0: + return + blk = (slot // BLOCK_SIZE).to(tl.int64) + off = (slot % BLOCK_SIZE).to(tl.int64) + head_idx_i64 = tl.cast(head_idx, tl.int64) + + # Block start byte address — stride_cache_block is unchanged (same total + # bytes per block as the AoS layout). + block_base = blk * stride_cache_block + + # Data region: per-slot stride = H * DATA_BYTES_PER_SLOT; per-head stride + # within a slot = DATA_BYTES_PER_SLOT. + DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES + data_base = ( + block_base + + off * (H * DATA_BYTES_PER_SLOT) + + head_idx_i64 * DATA_BYTES_PER_SLOT + ) + + # SoA metadata region (u16 element indexing): head strip = NUM_SOA_FIELDS + # consecutive bs-long fp16 arrays. + head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + head_idx_i64 * ( + NUM_SOA_FIELDS * BLOCK_SIZE + ) + vscale_u16_addr = head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + off + vzero_u16_addr = head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + off + + base = pid * D + + # ── FP8 KEY: cast to FP8 in-kernel and store at data_base ──────── + d_offs = tl.arange(0, BLOCK_D) + d_mask = d_offs < D + k_vals = tl.load(Key_ptr + base + d_offs, mask=d_mask, other=0.0) + k_fp8 = k_vals.to(tl.float8e4b15) if FP8_E4B15 else k_vals.to(tl.float8e4nv) + k_bytes = k_fp8.to(tl.uint8, bitcast=True) + tl.store(KV_cache_ptr + data_base + d_offs, k_bytes, mask=d_mask) + + # ── VALUE QUANTIZE + PACK + SoA scale/zero store ───────────────── + _store_quantized_value( + Value_ptr, + KV_cache_ptr, + KV_cache_u16_ptr, + base, + data_base, + vscale_u16_addr, + vzero_u16_addr, + d_offs, + d_mask, + D=D, + V_DATA_OFFSET=KEY_DATA_BYTES, # V follows K in the data region + VQB=VQB, + VAL_DATA_BYTES=VAL_DATA_BYTES, + BLOCK_D=BLOCK_D, + BLOCK_VAL=BLOCK_VAL, + BLOCK_GRP=BLOCK_GRP, + ) + + +# ═══════════════════════════════════════════════════════════════════════ +# Fused MSE store: bucketize + MSE index pack + norm store + value pack +# (eliminates 4 PyTorch kernel launches per layer vs pack-only kernel) +# ═══════════════════════════════════════════════════════════════════════ + + +@triton.jit +def _tq_fused_store_mse( + # Post-rotation inputs + Y_ptr, # [NH, D] float32 — rotated normalized keys (x_hat @ PiT) + Norms_ptr, # [NH] float32 — key vector norms (||k||) + Value_ptr, # [NH, D] float32 — raw values + # Quantization tables + Midpoints_ptr, # [n_centroids-1] float32 + Centroids_ptr, # [n_centroids] float32 — used only when NORM_CORRECTION=1 + # Cache and indexing + KV_cache_ptr, # [total_bytes] uint8 (flattened view) + KV_cache_u16_ptr, # uint16-aliased view of the same storage (Opt#3) + Slot_mapping_ptr, # [N] int32 — per-token slot indices + # Cache strides (stride_cache_block = bs*H*slot_aligned, unchanged) + stride_cache_block: tl.constexpr, + # Dimensions + D: tl.constexpr, + H: tl.constexpr, + BLOCK_SIZE: tl.constexpr, + BLOCK_D: tl.constexpr, + # SoA layout (Opt#3): data region [bs, H, MSE_BYTES+VAL_DATA_BYTES], + # then metadata region [H, NUM_SOA_FIELDS, bs] of fp16. + MSE_BYTES: tl.constexpr, + VAL_DATA_BYTES: tl.constexpr, + META_REGION_OFFSET: tl.constexpr, # bytes + NUM_SOA_FIELDS: tl.constexpr, # MSE: 3 (k_norm, v_scale, v_zero) + SOA_K_NORM: tl.constexpr, # MSE: 0 + SOA_V_SCALE: tl.constexpr, # MSE: 1 + SOA_V_ZERO: tl.constexpr, # MSE: 2 + # Value quantization + VQB: tl.constexpr, + # Packing block sizes + BLOCK_VAL: tl.constexpr, + # MSE params + MSE_BITS: tl.constexpr, + N_CENTROIDS: tl.constexpr, + # When 1, fold the per-token centroid-vector norm ||c_t|| into the + # stored scalar: store ||k_t|| / ||c_t|| instead of ||k_t||. This + # removes the per-tile sum+sqrt+divide cost from the attention kernel's + # K-load path. Load-side math becomes a plain multiply K = c * stored. + NORM_CORRECTION: tl.constexpr = 0, + BLOCK_GRP: tl.constexpr = 16, +): + """Fused MSE quantize + pack + SoA-metadata store (Opt#3 layout). + + Binary-search bucketize → MSE-index pack → K-norm store (SoA) → value + uniform quantize + pack → V scale/zero store (SoA), in one kernel. + """ + pid = tl.program_id(0) + token_idx = pid // H + head_idx = pid % H + + slot = tl.load(Slot_mapping_ptr + token_idx) + if slot < 0: + return + blk = (slot // BLOCK_SIZE).to(tl.int64) + off = (slot % BLOCK_SIZE).to(tl.int64) + head_idx_i64 = tl.cast(head_idx, tl.int64) + + # Block start byte address — stride unchanged from AoS (same total + # bytes per block); layout inside the block differs. + block_base = blk * stride_cache_block + + # Data region offset (packed K indices + packed V data). + DATA_BYTES_PER_SLOT: tl.constexpr = MSE_BYTES + VAL_DATA_BYTES + data_base = ( + block_base + + off * (H * DATA_BYTES_PER_SLOT) + + head_idx_i64 * DATA_BYTES_PER_SLOT + ) + + # SoA metadata addresses (u16 element indexing). + head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + head_idx_i64 * ( + NUM_SOA_FIELDS * BLOCK_SIZE + ) + knorm_u16_addr = head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + off + vscale_u16_addr = head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + off + vzero_u16_addr = head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + off + + base = pid * D + d_offs = tl.arange(0, BLOCK_D) + d_mask = d_offs < D + + # ── 1. BINARY SEARCH BUCKETIZE ──────────────────────────────────── + y_vec = tl.load(Y_ptr + base + d_offs, mask=d_mask, other=0.0) + lo = tl.zeros([BLOCK_D], dtype=tl.int32) + hi = tl.full([BLOCK_D], N_CENTROIDS - 1, dtype=tl.int32) + for _ in range(MSE_BITS): + mid = (lo + hi) >> 1 + safe_mid = tl.minimum(mid, N_CENTROIDS - 2) + mid_val = tl.load(Midpoints_ptr + safe_mid, mask=d_mask, other=0.0) + lo = tl.where(y_vec >= mid_val, mid + 1, lo) + hi = tl.where(y_vec >= mid_val, hi, mid) + idx = tl.minimum(lo, N_CENTROIDS - 1) + + # ── 2. PACK MSE INDICES into data region at data_base[0:MSE_BYTES] ── + if MSE_BITS == 4: + idx_pairs = tl.reshape(idx, [BLOCK_D // 2, 2]) + shifts_4 = tl.arange(0, 2) * 4 + packed = tl.sum((idx_pairs & 0xF) << shifts_4[None, :], axis=1).to(tl.uint8) + mse_offs = tl.arange(0, BLOCK_D // 2) + mse_mask = mse_offs < MSE_BYTES + tl.store(KV_cache_ptr + data_base + mse_offs, packed, mask=mse_mask) + + elif MSE_BITS == 3: + grp_offs = tl.arange(0, BLOCK_GRP) + grp_mask = grp_offs < (D // 8) + idx_grp = tl.reshape(idx, [BLOCK_GRP, 8]) + shifts_3 = tl.arange(0, 8) * 3 + packed_24 = tl.sum((idx_grp & 0x7) << shifts_3[None, :], axis=1) + b0 = (packed_24 & 0xFF).to(tl.uint8) + b1 = ((packed_24 >> 8) & 0xFF).to(tl.uint8) + b2 = ((packed_24 >> 16) & 0xFF).to(tl.uint8) + tl.store(KV_cache_ptr + data_base + grp_offs * 3, b0, mask=grp_mask) + tl.store(KV_cache_ptr + data_base + grp_offs * 3 + 1, b1, mask=grp_mask) + tl.store(KV_cache_ptr + data_base + grp_offs * 3 + 2, b2, mask=grp_mask) + + # ── 3. STORE K-norm in SoA region (single u16 store) ───────────── + # Norm-correction (Opt#1) still folds 1/||c_vec|| into the stored value; + # only the destination moved to the SoA K-norm array. + vn_f32 = tl.load(Norms_ptr + pid) # ||k_t||, fp32 + + if NORM_CORRECTION: + safe_idx = tl.minimum(tl.maximum(idx, 0), N_CENTROIDS - 1) + c_vals = tl.load(Centroids_ptr + safe_idx, mask=d_mask, other=0.0) + c_norm_sq = tl.sum(tl.where(d_mask, c_vals * c_vals, 0.0), axis=0) + c_inv_norm = 1.0 / tl.sqrt(c_norm_sq + 1e-16) + vn_f32 = vn_f32 * c_inv_norm + + vn_u16 = vn_f32.to(tl.float16).to(tl.uint16, bitcast=True) + tl.store(KV_cache_u16_ptr + knorm_u16_addr, vn_u16) + + # ── 4. VALUE QUANTIZE + PACK + SoA scale/zero store ────────────── + _store_quantized_value( + Value_ptr, + KV_cache_ptr, + KV_cache_u16_ptr, + base, + data_base, + vscale_u16_addr, + vzero_u16_addr, + d_offs, + d_mask, + D=D, + V_DATA_OFFSET=MSE_BYTES, # V follows K indices in the data region + VQB=VQB, + VAL_DATA_BYTES=VAL_DATA_BYTES, + BLOCK_D=BLOCK_D, + BLOCK_VAL=BLOCK_VAL, + BLOCK_GRP=BLOCK_GRP, + ) + + +# ═══════════════════════════════════════════════════════════════════════ +# Launcher +# ═══════════════════════════════════════════════════════════════════════ + + +def triton_turboquant_store( + key: torch.Tensor, # [N, H, D] — raw keys (post-RoPE) + value: torch.Tensor, # [N, H, D] — raw values + kv_cache: torch.Tensor, # [num_blocks, block_size, Hk, padded_slot] uint8 + slot_mapping: torch.Tensor, # [N] int32 + PiT: torch.Tensor, # [D, D] float32 + midpoints: torch.Tensor, # [n_centroids-1] float32 + mse_bits: int, + key_packed_size: int, + value_quant_bits: int, + key_fp8: bool = False, + centroids: torch.Tensor + | None = None, # [n_centroids] float32 — required when norm_correction=True + norm_correction: bool = False, +): + """Launch TQ store kernel (FP8 or MSE path). + + When ``norm_correction=True`` (default off; set by MSE norm-correction + presets), the centroid-vector norm ||c_t|| is folded into the stored + per-token scalar at store time (stored value = ||k_t|| / ||c_t||). The + load-side attention kernel no longer computes per-tile sum+sqrt+divide, + saving ~26% of TQ overhead on gpt-oss decode (ctx=4096). ``centroids`` + must be provided in this mode. + """ + if norm_correction and not key_fp8: + assert centroids is not None, ( + "norm_correction=True requires centroids tensor; got None. " + "Pass the same [n_centroids] fp32 tensor used to derive midpoints." + ) + N, H, D = key.shape + NH = N * H + block_size = kv_cache.shape[1] + BLOCK_D = triton.next_power_of_2(D) + mse_bytes = math.ceil(D * mse_bits / 8) + n_centroids = 2**mse_bits + + val_data_bytes = math.ceil(D * value_quant_bits / 8) + + BLOCK_VAL = triton.next_power_of_2(val_data_bytes) + + # Cache strides (element_size=1 for uint8, so stride in bytes = stride()). + # Only stride_cache_block (== bs*H*slot_aligned) is used under Opt#3; + # per-slot and per-head strides are computed from DATA_BYTES_PER_SLOT. + stride_block = kv_cache.stride(0) + + block_grp = triton.next_power_of_2(D // 8) if D >= 8 else 1 + + # ── Opt#3 SoA layout constants (derived locally so the launcher API + # stays unchanged). Invariant: key_packed_size + value_packed_size + # == slot_size_aligned == (key_data + val_data) + (k_meta + v_meta). + key_data_bytes = D if key_fp8 else mse_bytes + data_bytes_per_slot = key_data_bytes + val_data_bytes + # Data region occupies [0, block_size * H * data_bytes_per_slot) bytes + # at the start of each block; metadata region starts at that offset. + meta_region_offset = block_size * H * data_bytes_per_slot + num_soa_fields = 2 if key_fp8 else 3 + soa_v_scale = 0 if key_fp8 else 1 + soa_v_zero = 1 if key_fp8 else 2 + + # uint16-aliased view of the same storage — enables single-instruction + # u16 writes for the SoA K-norm / V-scale / V-zero stores. The cache + # bytes are always contiguous and 2-byte aligned under the Opt#3 layout + # (MSE_BYTES, VAL_DATA_BYTES, meta_region_offset are all even). + kv_cache_u16 = kv_cache.view(torch.uint16) + + # ── FP8 PATH: in-kernel FP8 cast + scatter via fp8 kernel ── + if key_fp8: + k_flat = key.reshape(NH, D).contiguous() + v_flat = value.reshape(NH, D).contiguous() + + fp8_e4b15 = _use_fp8_e4b15(key.device.index or 0) + + grid = (NH,) + _tq_fused_store_fp8[grid]( + k_flat, + v_flat, + kv_cache.view(-1), + kv_cache_u16.view(-1), + slot_mapping, + stride_cache_block=stride_block, + D=D, + H=H, + BLOCK_SIZE=block_size, + BLOCK_D=BLOCK_D, + KEY_DATA_BYTES=key_data_bytes, + VAL_DATA_BYTES=val_data_bytes, + META_REGION_OFFSET=meta_region_offset, + NUM_SOA_FIELDS=num_soa_fields, + SOA_V_SCALE=soa_v_scale, + SOA_V_ZERO=soa_v_zero, + VQB=value_quant_bits, + BLOCK_VAL=BLOCK_VAL, + BLOCK_GRP=block_grp, + FP8_E4B15=fp8_e4b15, + num_warps=4, + num_stages=1, + ) + return + + # ── MSE PATH: external GEMM + fused bucketize/pack kernel ── + # Normalize + rotation GEMM externally (cuBLAS is faster than in-kernel) + k_flat = key.float().reshape(NH, D) + norms = k_flat.norm(dim=1, keepdim=True) + x_hat = k_flat / (norms + 1e-8) + y = x_hat @ PiT + + v_flat = value.float().reshape(NH, D) + + # When norm_correction=True, the kernel folds 1/||c_t|| into the + # stored per-token K-norm so the load kernel can skip per-tile norm + # math (Opt#1). centroids is only dereferenced inside the kernel when + # NORM_CORRECTION=1; pass midpoints as a harmless placeholder otherwise. + centroids_ptr = centroids if centroids is not None else midpoints + grid = (NH,) + _tq_fused_store_mse[grid]( + y, + norms.squeeze(1), + v_flat, + midpoints, + centroids_ptr, + kv_cache.view(-1), + kv_cache_u16.view(-1), + slot_mapping, + stride_cache_block=stride_block, + D=D, + H=H, + BLOCK_SIZE=block_size, + BLOCK_D=BLOCK_D, + MSE_BYTES=mse_bytes, + VAL_DATA_BYTES=val_data_bytes, + META_REGION_OFFSET=meta_region_offset, + NUM_SOA_FIELDS=num_soa_fields, + SOA_K_NORM=0, + SOA_V_SCALE=soa_v_scale, + SOA_V_ZERO=soa_v_zero, + VQB=value_quant_bits, + BLOCK_VAL=BLOCK_VAL, + MSE_BITS=mse_bits, + N_CENTROIDS=n_centroids, + NORM_CORRECTION=1 if norm_correction else 0, + BLOCK_GRP=block_grp, + num_warps=4, + num_stages=1, + ) diff --git a/vllm/v1/attention/ops/turboquant_soa/triton_turboquant_unified_attention.py b/vllm/v1/attention/ops/turboquant_soa/triton_turboquant_unified_attention.py new file mode 100644 index 0000000000..03ef36e5b6 --- /dev/null +++ b/vllm/v1/attention/ops/turboquant_soa/triton_turboquant_unified_attention.py @@ -0,0 +1,1350 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unified (prefill + decode) Triton attention kernel for TurboQuant. + +Structure is ported from ``vllm/v1/attention/ops/triton_unified_attention.py`` +(the AITER unified attention kernel already upstreamed into vLLM). Only the K +and V load sites are replaced: instead of reading raw fp16 keys/values from +two contiguous caches, this kernel reads TurboQuant-packed bytes from a single +combined cache and dequantizes on the fly inside the tile loop. + +Benefits over the v1/v2 decode-only kernels: + +1. GQA heads are stacked into ``BLOCK_M`` and the Q·K and P·V ops are proper + ``tl.dot`` tensor-core operations (MFMA on MI300X). +2. The same kernel handles decode (``BLOCK_Q=1``) and prefill + (``BLOCK_Q>1``) -- no more Python per-request for-loop for continuation + chunks. +3. Only the subset of features exercised by the current TQ paths is kept + (causal, GQA). Sinks / softcap / ALiBi / sliding-window / qq-bias / + mm-prefix are deferred to follow-ups. + +This is an opt-in v3 path behind ``VLLM_TQ_DECODE_V3``. +""" + +from __future__ import annotations + +import math +from typing import Any + +import torch + +from vllm.platforms import current_platform +from vllm.triton_utils import tl, triton + +# reduce_segments is KV-format-agnostic: by the time it runs, K/V have been +# consumed and only (max, expsum, partial_output) triples remain. Reuse the +# baseline's implementation verbatim to avoid code duplication. +from vllm.v1.attention.ops.triton_unified_attention import reduce_segments + +from .triton_turboquant_decode import _use_fp8_e4b15 +from .triton_turboquant_decode_v2 import build_pair_lut + +_is_hip = current_platform.is_rocm() + + +# --------------------------------------------------------------------------- +# Helper: find which sequence in the batch a global q-block index belongs to. +# Identical to the unified kernel's binary search. +# --------------------------------------------------------------------------- + + +@triton.jit +def _find_seq_idx( + query_start_len_ptr, + target_idx, + num_seqs, + BLOCK_Q: tl.constexpr, + use_q_block_mode: tl.constexpr, +): + left: tl.int32 = 0 + right = num_seqs + while left < right: + mid = (left + right) // 2 + val = tl.load(query_start_len_ptr + mid) + mid_val = val // BLOCK_Q + mid if use_q_block_mode else val + if mid_val <= target_idx: + left = mid + 1 + else: + right = mid + return left - 1 + + +# --------------------------------------------------------------------------- +# Fused Q-rotation prologue (Opt: eliminates the launcher-side +# q_rot = (query.float() @ PiT).to(dtype).contiguous() +# chain, which cost ~50-60 us/step at D=64 — all launch-bound, not compute- +# bound. The fused version does one small MFMA inside the attention kernel, +# once per program, before the KV-tile loop. PiT is [D,D] fp32 and fits in +# L1 easily (16KB at D=64, 64KB at D=128). +# +# Algebra is identical to the launcher path: both compute Q_rot = Q @ PiT +# with fp32 accumulate. Differences are rounding-order only (MFMA tiling +# vs rocBLAS tiling), well within the 4-bit KV quant noise floor. +# --------------------------------------------------------------------------- + + +@triton.jit +def _tq_fuse_q_rotation( + Q, # [BLOCK_M, HEAD_SIZE_PADDED] — raw query in Q.dtype + PiT_ptr, + PiT_stride_0: tl.int64, + PiT_stride_1: tl.int64, + dim_mask, # [HEAD_SIZE_PADDED] int1 — valid head dims + HEAD_SIZE_PADDED: tl.constexpr, +): + """Fused Q @ PiT prologue. Called once per program for the MSE-key path + when the launcher has passed the raw (un-rotated) query. Returns the + rotated Q in the original dtype. + """ + d_offs = tl.arange(0, HEAD_SIZE_PADDED) + pit_offsets = d_offs[:, None] * PiT_stride_0 + d_offs[None, :] * PiT_stride_1 + pit_mask = dim_mask[:, None] & dim_mask[None, :] + PiT_tile = tl.load(PiT_ptr + pit_offsets, mask=pit_mask, other=0.0).to(tl.float32) + # input_precision="ieee" pins both inputs to full fp32 MFMA (no TF32 + # truncation). allow_tf32 is intentionally omitted — Triton rejects + # passing both. This matches the launcher's rocBLAS fp32 GEMM in + # algebra; rounding-order differs, diff is <= a few fp16 ulp. + Q_rot = tl.dot( + Q.to(tl.float32), + PiT_tile, + input_precision="ieee", + ) + return Q_rot.to(Q.dtype) + + +# --------------------------------------------------------------------------- +# TQ K-tile dequant: returns K_T : [HEAD_SIZE_PADDED, TILE_SIZE] in Q.dtype +# ready for tl.dot(Q, K_T). +# +# Packed slot layout (per (block, pos, kv_head)) is the same as v1/v2: +# MSE path: [ key_idx_bytes (MSE_BYTES) | key_norm_fp16 (2B) | ...V... ] +# FP8 path: [ fp8_key_bytes (D) | ...V... ] +# --------------------------------------------------------------------------- + + +@triton.jit +def _tq_load_k_tile( + KV_cache_ptr, + KV_cache_u16_ptr, # uint16 view of KV_cache (same storage) + data_bases, # [TILE_SIZE] int64 — byte offset to each token's DATA region + knorm_u16_addrs, # [TILE_SIZE] int64 — u16 element index for each token's K-norm + d_offs, # [HEAD_SIZE_PADDED] + d_mask, # [HEAD_SIZE_PADDED] int1 + tile_mask, # [TILE_SIZE] int1 + Centroids_ptr, + Pair_lut_ptr, + OUT_DTYPE: tl.constexpr, # tl.float16 or tl.bfloat16 + HEAD_DIM: tl.constexpr, + BLOCK_D: tl.constexpr, # HEAD_SIZE_PADDED — needed for pair-LUT reshape + MSE_BITS: tl.constexpr, + N_CENTROIDS: tl.constexpr, + KEY_FP8: tl.constexpr, + USE_PAIR_LUT: tl.constexpr, + NORM_CORRECTION: tl.constexpr, + FP8_E4B15: tl.constexpr, + TILE_SIZE: tl.constexpr, +): + """Load + dequantize a TILE_SIZE × HEAD_SIZE block of keys and return + the transposed tile K_T : [HEAD_SIZE_PADDED, TILE_SIZE]. + + Opt#3 SoA layout: packed K data is at `data_bases[t] + [0, MSE_BYTES)` + for MSE keys (or `[0, D)` for FP8). The per-token K-norm lives in the + per-block SoA metadata region; `knorm_u16_addrs` already encodes its + u16 element index. For decode tiles aligned with blocks, these addresses + are contiguous → one coalesced wide load replaces TILE_SIZE scattered + loads (the whole point of Opt#3). + """ + if KEY_FP8: + k_addrs = data_bases[:, None] + d_offs[None, :] + k_raw = tl.load( + KV_cache_ptr + k_addrs, + mask=tile_mask[:, None] & d_mask[None, :], + other=0, + ) + if FP8_E4B15: + k_f32 = k_raw.to(tl.float8e4b15, bitcast=True).to(tl.float32) + else: + k_f32 = k_raw.to(tl.float8e4nv, bitcast=True).to(tl.float32) + K = k_f32 # [TILE_SIZE, HEAD_SIZE_PADDED] + else: + # MSE path: gather packed key indices + centroid LUT. + if MSE_BITS == 4 and USE_PAIR_LUT: + # FLUTE pair-LUT fast path — load each packed byte once, decode + # both nibbles, single 3-D gather returns (T[lo], T[hi]) per byte. + HALF_D: tl.constexpr = BLOCK_D // 2 + half_offs = tl.arange(0, HALF_D) + byte_mask = (half_offs * 2) < HEAD_DIM + byte_addrs = data_bases[:, None] + half_offs[None, :] + byte_raw = tl.load( + KV_cache_ptr + byte_addrs, + mask=tile_mask[:, None] & byte_mask[None, :], + other=0, + ).to(tl.int32) + lo_idx = byte_raw & 0xF + hi_idx = (byte_raw >> 4) & 0xF + pair_key = lo_idx * N_CENTROIDS + hi_idx + pair_slot = tl.arange(0, 2) + c_pair = tl.load( + Pair_lut_ptr + pair_key[:, :, None] * 2 + pair_slot[None, None, :], + mask=(tile_mask[:, None, None] & byte_mask[None, :, None]), + other=0.0, + ) + c_vals = tl.reshape(c_pair, [TILE_SIZE, BLOCK_D]) + elif MSE_BITS == 4: + half_idx = d_offs // 2 + nibble_shift = (d_offs % 2) * 4 + mse_addrs = data_bases[:, None] + half_idx[None, :] + mse_raw = tl.load( + KV_cache_ptr + mse_addrs, + mask=tile_mask[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + mse_idx = (mse_raw >> nibble_shift[None, :]) & 0xF + c_vals = tl.load( + Centroids_ptr + mse_idx, + mask=tile_mask[:, None] & d_mask[None, :], + other=0.0, + ) + else: + # Generic bit extraction (3-bit, etc.) + mse_bit_off = d_offs * MSE_BITS + mse_byte_idx = mse_bit_off // 8 + mse_bit_shift = mse_bit_off % 8 + mse_mask_val = (1 << MSE_BITS) - 1 + mse_addrs0 = data_bases[:, None] + mse_byte_idx[None, :] + mse_raw0 = tl.load( + KV_cache_ptr + mse_addrs0, + mask=tile_mask[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + mse_raw1 = tl.load( + KV_cache_ptr + mse_addrs0 + 1, + mask=tile_mask[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + raw16 = mse_raw0 | (mse_raw1 << 8) + mse_idx = (raw16 >> mse_bit_shift[None, :]) & mse_mask_val + c_vals = tl.load( + Centroids_ptr + mse_idx, + mask=tile_mask[:, None] & d_mask[None, :], + other=0.0, + ) + + # Opt#1: 1/||c_vec|| is pre-folded into the stored K-norm at store + # time, so the kernel doesn't recompute norm-correction here. + # Opt#3: K-norms for a tile are contiguous in the per-block SoA + # region when the tile lies within one block (always true for + # aligned decode tiles with TILE_SIZE == BLOCK_SIZE) — one coalesced + # u16 load replaces TILE_SIZE scattered 2-byte loads. + norm_u16 = tl.load(KV_cache_u16_ptr + knorm_u16_addrs, mask=tile_mask, other=0) + vec_norms = norm_u16.to(tl.float16, bitcast=True).to(tl.float32) + K = c_vals * vec_norms[:, None] # [TILE_SIZE, HEAD_SIZE_PADDED] + + K_T = tl.trans(K.to(OUT_DTYPE)) # [HEAD_SIZE_PADDED, TILE_SIZE] + _ = HEAD_DIM + return K_T + + +# --------------------------------------------------------------------------- +# TQ V-tile dequant: returns V : [TILE_SIZE, HEAD_SIZE_PADDED] in Q.dtype +# ready for tl.dot(P, V). +# --------------------------------------------------------------------------- + + +@triton.jit +def _tq_load_v_tile( + KV_cache_ptr, + KV_cache_u16_ptr, # uint16 view of KV_cache (same storage) + val_bases, # [TILE_SIZE] int64 — byte offset to each token's V-data + vscale_u16_addrs, # [TILE_SIZE] int64 — u16 element index for V-scale + vzero_u16_addrs, # [TILE_SIZE] int64 — u16 element index for V-zero + d_offs, + d_mask, + tile_mask, + OUT_DTYPE: tl.constexpr, + HEAD_DIM: tl.constexpr, + VQB: tl.constexpr, +): + """Load + dequantize a TILE_SIZE × HEAD_SIZE block of values. + + Opt#3 SoA layout: packed V data is at `val_bases[t] + [0, VAL_DATA_BYTES)` + (V-data immediately follows K-data within the slot's data region, and + `val_bases` is precomputed by the caller as `data_base + KEY_DATA_BYTES`). + V-scale / V-zero live in the per-block SoA metadata region at indices + `vscale_u16_addrs` / `vzero_u16_addrs`. For tiles aligned to block + boundaries, those addresses are contiguous → one coalesced wide load + per field instead of TILE_SIZE scattered 2-byte loads. + """ + if VQB == 4: + vb_idx = d_offs // 2 + vb_shift = (d_offs % 2) * 4 + val_addrs = val_bases[:, None] + vb_idx[None, :] + val_raw = tl.load( + KV_cache_ptr + val_addrs, + mask=tile_mask[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + v_idx = ((val_raw >> vb_shift[None, :]) & 0xF).to(tl.float32) + else: # VQB == 3 + val_bit_off = d_offs * 3 + val_byte_idx = val_bit_off // 8 + val_bit_shift = val_bit_off % 8 + val_addrs0 = val_bases[:, None] + val_byte_idx[None, :] + val_raw0 = tl.load( + KV_cache_ptr + val_addrs0, + mask=tile_mask[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + val_raw1 = tl.load( + KV_cache_ptr + val_addrs0 + 1, + mask=tile_mask[:, None] & d_mask[None, :], + other=0, + ).to(tl.int32) + raw16 = val_raw0 | (val_raw1 << 8) + v_idx = ((raw16 >> val_bit_shift[None, :]) & 0x7).to(tl.float32) + + # SoA scale / zero loads — coalesced on aligned decode tiles. + scale_u16 = tl.load(KV_cache_u16_ptr + vscale_u16_addrs, mask=tile_mask, other=0) + zero_u16 = tl.load(KV_cache_u16_ptr + vzero_u16_addrs, mask=tile_mask, other=0) + v_scales = scale_u16.to(tl.float16, bitcast=True).to(tl.float32) + v_zeros = zero_u16.to(tl.float16, bitcast=True).to(tl.float32) + + V = v_idx * v_scales[:, None] + v_zeros[:, None] # [TILE_SIZE, HEAD_SIZE_PADDED] + _ = HEAD_DIM + return V.to(OUT_DTYPE) + + +# --------------------------------------------------------------------------- +# Unified 2D attention kernel over a TurboQuant KV cache (causal only, GQA). +# Structure follows ``kernel_unified_attention_2d`` from +# triton_unified_attention.py with: +# * Q-head GQA stacking into BLOCK_M +# * tensor-core tl.dot(Q, K) and tl.dot(P, V) +# * causal + query-padding masking +# * online softmax +# and K/V loads replaced by the TQ helpers above. +# --------------------------------------------------------------------------- + + +@triton.jit +def kernel_tq_unified_attention_2d( + output_ptr, # [num_tokens, Hq, D] + query_ptr, # [num_tokens, Hq, D] — raw if FUSE_Q_ROT else rotated + KV_cache_ptr, # [num_blocks, block_size, Hk, padded_slot] uint8 - TQ packed + KV_cache_u16_ptr, # uint16 view of same storage — Opt#2 wide metadata loads + Centroids_ptr, # [n_centroids] fp32 + Pair_lut_ptr, # [N*N, 2] fp32 (pair-LUT for 4-bit MSE) or dummy + PiT_ptr, # [D, D] fp32 — only dereferenced when FUSE_Q_ROT == 1 + block_tables_ptr, # [num_seqs, max_num_blocks_per_seq] + seq_lens_ptr, # [num_seqs] + query_start_len_ptr, # [num_seqs+1] + sinks_ptr, # [Hq] fp32 — per-head sink logits; dereferenced only when USE_SINKS + scale, # float32 + num_query_heads: tl.constexpr, + num_queries_per_kv: tl.constexpr, + block_table_stride: tl.int64, + query_stride_0: tl.int64, + query_stride_1: tl.int64, + output_stride_0: tl.int64, + output_stride_1: tl.int64, + stride_cache_block: tl.int64, + pit_stride_0: tl.int64, + pit_stride_1: tl.int64, + BLOCK_SIZE: tl.constexpr, + TILE_SIZE: tl.constexpr, + HEAD_SIZE: tl.constexpr, + HEAD_SIZE_PADDED: tl.constexpr, + BLOCK_Q: tl.constexpr, + BLOCK_M: tl.constexpr, + num_seqs: tl.int32, + # TQ layout constants + MSE_BITS: tl.constexpr, + MSE_BYTES: tl.constexpr, + VQB: tl.constexpr, + VAL_DATA_BYTES: tl.constexpr, + N_CENTROIDS: tl.constexpr, + KEY_FP8: tl.constexpr, + USE_PAIR_LUT: tl.constexpr, + # Opt#3 SoA layout constants + NUM_KV_HEADS: tl.constexpr, + KEY_DATA_BYTES: tl.constexpr, # MSE_BYTES for MSE, HEAD_SIZE for FP8 + META_REGION_OFFSET: tl.constexpr, # bytes: bs * H * (KD+VD) + NUM_SOA_FIELDS: tl.constexpr, # 3 for MSE, 2 for FP8 + SOA_K_NORM: tl.constexpr, # 0 for MSE, unused for FP8 + SOA_V_SCALE: tl.constexpr, # 1 (MSE) / 0 (FP8) + SOA_V_ZERO: tl.constexpr, # 2 (MSE) / 1 (FP8) + NORM_CORRECTION: tl.constexpr = 0, + FP8_E4B15: tl.constexpr = 0, + FUSE_Q_ROT: tl.constexpr = 0, # Fused Q @ PiT prologue when 1 (MSE path) + USE_SINKS: tl.constexpr = 0, # Per-head sink logit folded into softmax denom +): + q_block_global_idx = tl.program_id(0) + kv_head_idx = tl.program_id(1) + + seq_idx = _find_seq_idx( + query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, True + ) + q_block_start_idx = tl.load(query_start_len_ptr + seq_idx) // BLOCK_Q + seq_idx + q_block_local_idx = q_block_global_idx - q_block_start_idx + + cur_batch_in_all_start_index = tl.load(query_start_len_ptr + seq_idx) + cur_batch_in_all_stop_index = tl.load(query_start_len_ptr + seq_idx + 1) + cur_batch_query_len = cur_batch_in_all_stop_index - cur_batch_in_all_start_index + + if q_block_local_idx * BLOCK_Q >= cur_batch_query_len: + return + + offs_m = tl.arange(0, BLOCK_M) + offs_d = tl.arange(0, HEAD_SIZE_PADDED) + offs_t = tl.arange(0, TILE_SIZE) + query_pos = q_block_local_idx * BLOCK_Q + offs_m // num_queries_per_kv + + query_offset_0 = cur_batch_in_all_start_index + query_pos + query_offset_1 = kv_head_idx * num_queries_per_kv + offs_m % num_queries_per_kv + query_offset = ( + query_offset_0[:, None] * query_stride_0 + + query_offset_1[:, None] * query_stride_1 + + offs_d[None, :] + ) + + dim_mask = tl.where(offs_d < HEAD_SIZE, 1, 0).to(tl.int1) + query_mask_0 = tl.where(query_pos < cur_batch_query_len, 1, 0).to(tl.int1) + query_mask_1 = tl.where(query_offset_1 < num_query_heads, 1, 0).to(tl.int1) + + # Q : [BLOCK_M, HEAD_SIZE_PADDED] + Q = tl.load( + query_ptr + query_offset, + mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], + other=0.0, + ) + + if FUSE_Q_ROT: + Q = _tq_fuse_q_rotation( + Q, + PiT_ptr, + pit_stride_0, + pit_stride_1, + dim_mask, + HEAD_SIZE_PADDED, + ) + + block_table_offset = seq_idx * block_table_stride + + # Online softmax state. + # When USE_SINKS=1 we fold a per-head sink logit s_h into the softmax + # denominator by initializing M = s_h, L = 1.0 (== exp(s_h - s_h)). + # The standard online-softmax update then correctly rescales the sink + # contribution as real tokens update M, and the final acc / L gives + # out_h = sum_i exp(q.k_i)*V_i / (exp(s_h) + sum_i exp(q.k_i)). + # Masked rows (query_mask_1 == 0) get -inf so the sink cannot pollute + # invalid (q_token, q_head) positions outside the real query range. + if USE_SINKS: + M = tl.load( + sinks_ptr + query_offset_1, mask=query_mask_1, other=float("-inf") + ).to(tl.float32) + else: + M = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32) + L = tl.full([BLOCK_M], 1.0, dtype=tl.float32) + acc = tl.zeros([BLOCK_M, HEAD_SIZE_PADDED], dtype=tl.float32) + + seq_len = tl.load(seq_lens_ptr + seq_idx) + context_len = seq_len - cur_batch_query_len + + # Longest key prefix any query row in this q-block can attend to. + max_seq_prefix_len = ( + context_len + + q_block_local_idx * BLOCK_Q + + (BLOCK_M - 1) // num_queries_per_kv + + 1 + ) + max_seq_prefix_len = tl.minimum(max_seq_prefix_len, seq_len) + num_tiles = tl.cdiv(max_seq_prefix_len, TILE_SIZE) + + for j in range(0, num_tiles): + seq_offset = j * TILE_SIZE + offs_t + tile_mask = seq_offset < max_seq_prefix_len + + physical_block_idx = tl.load( + block_tables_ptr + block_table_offset + seq_offset // BLOCK_SIZE + ).to(tl.int64) + + # Opt#3 SoA addressing: compute data + SoA metadata addresses once + # per tile; the load helpers below issue coalesced wide loads from + # these addresses for decode tiles that align with block boundaries. + slot_within_block = (seq_offset % BLOCK_SIZE).to(tl.int64) + block_base = physical_block_idx * stride_cache_block + DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES + data_bases = ( + block_base + + slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT) + + tl.cast(kv_head_idx, tl.int64) * DATA_BYTES_PER_SLOT + ) + val_bases = data_bases + KEY_DATA_BYTES + + head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast( + kv_head_idx, tl.int64 + ) * (NUM_SOA_FIELDS * BLOCK_SIZE) + knorm_u16_addrs = ( + head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block + ) + vscale_u16_addrs = ( + head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block + ) + vzero_u16_addrs = ( + head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block + ) + + # ---- K : [HEAD_SIZE_PADDED, TILE_SIZE] in Q.dtype ---- + K_T = _tq_load_k_tile( + KV_cache_ptr, + KV_cache_u16_ptr, + data_bases, + knorm_u16_addrs, + offs_d, + dim_mask, + tile_mask, + Centroids_ptr, + Pair_lut_ptr, + OUT_DTYPE=Q.dtype, + HEAD_DIM=HEAD_SIZE, + BLOCK_D=HEAD_SIZE_PADDED, + MSE_BITS=MSE_BITS, + N_CENTROIDS=N_CENTROIDS, + KEY_FP8=KEY_FP8, + USE_PAIR_LUT=USE_PAIR_LUT, + NORM_CORRECTION=NORM_CORRECTION, + FP8_E4B15=FP8_E4B15, + TILE_SIZE=TILE_SIZE, + ) + + # ---- V : [TILE_SIZE, HEAD_SIZE_PADDED] in Q.dtype ---- + V = _tq_load_v_tile( + KV_cache_ptr, + KV_cache_u16_ptr, + val_bases, + vscale_u16_addrs, + vzero_u16_addrs, + offs_d, + dim_mask, + tile_mask, + OUT_DTYPE=Q.dtype, + HEAD_DIM=HEAD_SIZE, + VQB=VQB, + ) + + # S : [BLOCK_M, TILE_SIZE] + S = scale * tl.dot(Q, K_T) + + # Causal + query-padding mask + query_abs_pos = context_len + query_pos[:, None] + seq_mask = seq_offset[None, :] <= query_abs_pos + S = tl.where( + query_mask_1[:, None] & query_mask_0[:, None] & seq_mask, + S, + float("-inf"), + ) + + # Online softmax + m_j = tl.maximum(M, tl.max(S, axis=1)) + m_j = tl.where(m_j > float("-inf"), m_j, 0.0) + P = tl.exp(S - m_j[:, None]) + l_j = tl.sum(P, axis=1) + alpha = tl.exp(M - m_j) + acc = acc * alpha[:, None] + L = L * alpha + l_j + M = m_j + + # acc += P · V + acc += tl.dot(P.to(V.dtype), V) + + # Epilogue: normalize and store + acc = acc / L[:, None] + + output_offset = ( + query_offset_0[:, None] * output_stride_0 + + query_offset_1[:, None] * output_stride_1 + + offs_d[None, :] + ) + tl.store( + output_ptr + output_offset, + acc, + mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], + ) + + +# --------------------------------------------------------------------------- +# Unified 3D (split-KV) attention kernel over a TurboQuant KV cache. +# +# Same loop body as the 2D kernel above, but with an extra ``segm_idx`` grid +# axis: each program walks only ``cdiv(seq_len, NUM_SEGMENTS_PER_SEQ * TILE)`` +# tiles of the KV history and writes a *partial* (max, expsum, acc) triple to +# a scratch buffer. A subsequent ``reduce_segments`` kernel merges the +# ``NUM_SEGMENTS_PER_SEQ`` partials per (q_token, q_head) via the standard +# online-softmax merge. +# +# This gives the decode path ``num_seqs × Hk × NUM_SEGMENTS_PER_SEQ`` CTAs +# instead of just ``num_seqs × Hk``, which is what v2 already uses (and what +# lets v2 run flat at ~0.15 ms across KV lengths on MI300X). +# --------------------------------------------------------------------------- + + +@triton.jit +def kernel_tq_unified_attention_3d( + segm_output_ptr, # [num_tokens, Hq, NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED] fp32 + segm_max_ptr, # [num_tokens, Hq, NUM_SEGMENTS_PER_SEQ] fp32 + segm_expsum_ptr, # [num_tokens, Hq, NUM_SEGMENTS_PER_SEQ] fp32 + query_ptr, # [num_tokens, Hq, D] — raw if FUSE_Q_ROT else rotated + KV_cache_ptr, # [num_blocks, block_size, Hk, padded_slot] uint8 + KV_cache_u16_ptr, # uint16 view of same storage — Opt#2 wide metadata loads + Centroids_ptr, + Pair_lut_ptr, + PiT_ptr, # [D, D] fp32 — only dereferenced when FUSE_Q_ROT == 1 + block_tables_ptr, + seq_lens_ptr, + query_start_len_ptr, + sinks_ptr, # [Hq] fp32 — per-head sink logits; dereferenced only when USE_SINKS + scale, + num_query_heads: tl.constexpr, + num_queries_per_kv: tl.constexpr, + block_table_stride: tl.int64, + query_stride_0: tl.int64, + query_stride_1: tl.int64, + stride_cache_block: tl.int64, + pit_stride_0: tl.int64, + pit_stride_1: tl.int64, + BLOCK_SIZE: tl.constexpr, + TILE_SIZE: tl.constexpr, + HEAD_SIZE: tl.constexpr, + HEAD_SIZE_PADDED: tl.constexpr, + BLOCK_Q: tl.constexpr, + BLOCK_M: tl.constexpr, + num_seqs: tl.int32, + NUM_SEGMENTS_PER_SEQ: tl.constexpr, + # TQ layout constants (same as 2D) + MSE_BITS: tl.constexpr, + MSE_BYTES: tl.constexpr, + VQB: tl.constexpr, + VAL_DATA_BYTES: tl.constexpr, + N_CENTROIDS: tl.constexpr, + KEY_FP8: tl.constexpr, + USE_PAIR_LUT: tl.constexpr, + # Opt#3 SoA layout constants + NUM_KV_HEADS: tl.constexpr, + KEY_DATA_BYTES: tl.constexpr, + META_REGION_OFFSET: tl.constexpr, + NUM_SOA_FIELDS: tl.constexpr, + SOA_K_NORM: tl.constexpr, + SOA_V_SCALE: tl.constexpr, + SOA_V_ZERO: tl.constexpr, + NORM_CORRECTION: tl.constexpr = 0, + FP8_E4B15: tl.constexpr = 0, + FUSE_Q_ROT: tl.constexpr = 0, # Fused Q @ PiT prologue when 1 (MSE path) + USE_SINKS: tl.constexpr = 0, # Per-head sink logit folded into softmax denom +): + q_block_global_idx = tl.program_id(0) + kv_head_idx = tl.program_id(1) + segm_idx = tl.program_id(2) + + seq_idx = _find_seq_idx( + query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, True + ) + q_block_start_idx = tl.load(query_start_len_ptr + seq_idx) // BLOCK_Q + seq_idx + q_block_local_idx = q_block_global_idx - q_block_start_idx + + cur_batch_in_all_start_index = tl.load(query_start_len_ptr + seq_idx) + cur_batch_in_all_stop_index = tl.load(query_start_len_ptr + seq_idx + 1) + cur_batch_query_len = cur_batch_in_all_stop_index - cur_batch_in_all_start_index + + if q_block_local_idx * BLOCK_Q >= cur_batch_query_len: + return + + seq_len = tl.load(seq_lens_ptr + seq_idx) + tiles_per_segment = tl.cdiv(seq_len, NUM_SEGMENTS_PER_SEQ * TILE_SIZE) + + # Segment empty for this sequence — skip. + if segm_idx * tiles_per_segment * TILE_SIZE >= seq_len: + return + + offs_m = tl.arange(0, BLOCK_M) + offs_d = tl.arange(0, HEAD_SIZE_PADDED) + offs_t = tl.arange(0, TILE_SIZE) + query_pos = q_block_local_idx * BLOCK_Q + offs_m // num_queries_per_kv + + query_offset_0 = cur_batch_in_all_start_index + query_pos + query_offset_1 = kv_head_idx * num_queries_per_kv + offs_m % num_queries_per_kv + query_offset = ( + query_offset_0[:, None] * query_stride_0 + + query_offset_1[:, None] * query_stride_1 + + offs_d[None, :] + ) + + dim_mask = tl.where(offs_d < HEAD_SIZE, 1, 0).to(tl.int1) + query_mask_0 = tl.where(query_pos < cur_batch_query_len, 1, 0).to(tl.int1) + query_mask_1 = tl.where(query_offset_1 < num_query_heads, 1, 0).to(tl.int1) + + Q = tl.load( + query_ptr + query_offset, + mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], + other=0.0, + ) + + if FUSE_Q_ROT: + Q = _tq_fuse_q_rotation( + Q, + PiT_ptr, + pit_stride_0, + pit_stride_1, + dim_mask, + HEAD_SIZE_PADDED, + ) + + block_table_offset = seq_idx * block_table_stride + + # Online softmax state. For the split-KV (3D) kernel we must fold the + # sink logit into *only* segment 0's (M, L) so that the stage-2 reducer + # — which is a standard online-softmax merge — counts the sink exactly + # once. Segments with segm_idx > 0 keep the plain -inf / 1.0 init and + # therefore carry no sink contribution; stage-2 then combines them. + # This mirrors the v1 sink pattern from PR #40663 (sid==0 init trick). + if USE_SINKS and segm_idx == 0: + M = tl.load( + sinks_ptr + query_offset_1, mask=query_mask_1, other=float("-inf") + ).to(tl.float32) + else: + M = tl.full([BLOCK_M], float("-inf"), dtype=tl.float32) + L = tl.full([BLOCK_M], 1.0, dtype=tl.float32) + acc = tl.zeros([BLOCK_M, HEAD_SIZE_PADDED], dtype=tl.float32) + + context_len = seq_len - cur_batch_query_len + max_seq_prefix_len = ( + context_len + + q_block_local_idx * BLOCK_Q + + (BLOCK_M - 1) // num_queries_per_kv + + 1 + ) + max_seq_prefix_len = tl.minimum(max_seq_prefix_len, seq_len) + num_tiles = tl.cdiv(max_seq_prefix_len, TILE_SIZE) + + # Segment bounds: clip to the causal prefix range. + tile_lo = segm_idx * tiles_per_segment + tile_hi = tl.minimum((segm_idx + 1) * tiles_per_segment, num_tiles) + + for j in range(tile_lo, tile_hi): + seq_offset = j * TILE_SIZE + offs_t + tile_mask = seq_offset < max_seq_prefix_len + + physical_block_idx = tl.load( + block_tables_ptr + block_table_offset + seq_offset // BLOCK_SIZE + ).to(tl.int64) + + # Opt#3 SoA addressing (see 2D kernel for the layout invariants). + slot_within_block = (seq_offset % BLOCK_SIZE).to(tl.int64) + block_base = physical_block_idx * stride_cache_block + DATA_BYTES_PER_SLOT: tl.constexpr = KEY_DATA_BYTES + VAL_DATA_BYTES + data_bases = ( + block_base + + slot_within_block * (NUM_KV_HEADS * DATA_BYTES_PER_SLOT) + + tl.cast(kv_head_idx, tl.int64) * DATA_BYTES_PER_SLOT + ) + val_bases = data_bases + KEY_DATA_BYTES + + head_meta_u16_base = (block_base + META_REGION_OFFSET) // 2 + tl.cast( + kv_head_idx, tl.int64 + ) * (NUM_SOA_FIELDS * BLOCK_SIZE) + knorm_u16_addrs = ( + head_meta_u16_base + SOA_K_NORM * BLOCK_SIZE + slot_within_block + ) + vscale_u16_addrs = ( + head_meta_u16_base + SOA_V_SCALE * BLOCK_SIZE + slot_within_block + ) + vzero_u16_addrs = ( + head_meta_u16_base + SOA_V_ZERO * BLOCK_SIZE + slot_within_block + ) + + K_T = _tq_load_k_tile( + KV_cache_ptr, + KV_cache_u16_ptr, + data_bases, + knorm_u16_addrs, + offs_d, + dim_mask, + tile_mask, + Centroids_ptr, + Pair_lut_ptr, + OUT_DTYPE=Q.dtype, + HEAD_DIM=HEAD_SIZE, + BLOCK_D=HEAD_SIZE_PADDED, + MSE_BITS=MSE_BITS, + N_CENTROIDS=N_CENTROIDS, + KEY_FP8=KEY_FP8, + USE_PAIR_LUT=USE_PAIR_LUT, + NORM_CORRECTION=NORM_CORRECTION, + FP8_E4B15=FP8_E4B15, + TILE_SIZE=TILE_SIZE, + ) + + V = _tq_load_v_tile( + KV_cache_ptr, + KV_cache_u16_ptr, + val_bases, + vscale_u16_addrs, + vzero_u16_addrs, + offs_d, + dim_mask, + tile_mask, + OUT_DTYPE=Q.dtype, + HEAD_DIM=HEAD_SIZE, + VQB=VQB, + ) + + S = scale * tl.dot(Q, K_T) + + query_abs_pos = context_len + query_pos[:, None] + seq_mask = seq_offset[None, :] <= query_abs_pos + S = tl.where( + query_mask_1[:, None] & query_mask_0[:, None] & seq_mask, + S, + float("-inf"), + ) + + m_j = tl.maximum(M, tl.max(S, axis=1)) + m_j = tl.where(m_j > float("-inf"), m_j, 0.0) + P = tl.exp(S - m_j[:, None]) + l_j = tl.sum(P, axis=1) + alpha = tl.exp(M - m_j) + acc = acc * alpha[:, None] + L = L * alpha + l_j + M = m_j + + acc += tl.dot(P.to(V.dtype), V) + + # -------- Write partial (acc, M, L) to scratch; no normalization -------- + # Layout matches baseline reduce_segments so we can reuse it. + segm_output_offset = ( + query_offset_0[:, None].to(tl.int64) + * (num_query_heads * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + + query_offset_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) + + segm_idx * HEAD_SIZE_PADDED + + tl.arange(0, HEAD_SIZE_PADDED)[None, :] + ) + tl.store( + segm_output_ptr + segm_output_offset, + acc, + mask=dim_mask[None, :] & query_mask_0[:, None] & query_mask_1[:, None], + ) + segm_offset = ( + query_offset_0.to(tl.int64) * (num_query_heads * NUM_SEGMENTS_PER_SEQ) + + query_offset_1 * NUM_SEGMENTS_PER_SEQ + + segm_idx + ) + tl.store(segm_max_ptr + segm_offset, M, mask=query_mask_0 & query_mask_1) + tl.store(segm_expsum_ptr + segm_offset, L, mask=query_mask_0 & query_mask_1) + + +# --------------------------------------------------------------------------- +# Launcher +# --------------------------------------------------------------------------- + + +_layout_cache: dict[tuple, dict[str, int]] = {} + + +def _get_layout(D: int, mse_bits: int, value_quant_bits: int) -> dict[str, int]: + key = (D, mse_bits, value_quant_bits) + cfg = _layout_cache.get(key) + if cfg is None: + cfg = { + "mse_bytes": math.ceil(D * mse_bits / 8), + "val_data_bytes": math.ceil(D * value_quant_bits / 8), + "BLOCK_D": triton.next_power_of_2(D), + } + _layout_cache[key] = cfg + return cfg + + +def _get_pair_lut(centroids: torch.Tensor) -> torch.Tensor: + """Return a fresh pair-LUT for ``centroids`` on each call. + + Previously cached by ``(data_ptr, device)`` — that key is unsafe + because the CUDA allocator can reuse freed addresses for different + centroid tensors, yielding a stale (wrong-values) LUT. The LUT is tiny + (N*N*2 fp32, ~2KB at N=16) so unconditional rebuild is essentially + free compared to attention work. If this shows up on a profile, + replace with a hash-of-values fingerprint, not data_ptr. + """ + return build_pair_lut(centroids) + + +# PiT in Q's dtype cached per (PiT-id, dtype) to avoid a host cast on every +# launcher call. Keyed by the fp32 PiT's data_ptr so ownership/lifetime is +# tied to the caller's tensor. +_pit_qdtype_cache: dict = {} + + +def _get_pit_in_query_dtype(PiT: torch.Tensor, qdtype: torch.dtype) -> torch.Tensor: + key = (PiT.data_ptr(), qdtype) + cached = _pit_qdtype_cache.get(key) + if cached is None: + cached = PiT.to(qdtype).contiguous() + _pit_qdtype_cache[key] = cached + return cached + + +def triton_turboquant_unified_attention( + query: torch.Tensor, # [num_tokens, Hq, D] - fp16/bf16 + kv_cache: torch.Tensor, # [num_blocks, block_size, Hk, padded_slot] uint8 + block_table: torch.Tensor, # [num_seqs, max_num_blocks] int32 + seq_lens: torch.Tensor, # [num_seqs] int32 + query_start_loc: torch.Tensor, # [num_seqs+1] int32 + Pi: torch.Tensor, # [D, D] fp32 + centroids: torch.Tensor, # [n_centroids] fp32 + scale: float, + mse_bits: int, + key_packed_size: int, + value_quant_bits: int, + value_packed_size: int, # unused; kept for signature parity with v2 + key_fp8: bool = False, + norm_correction: bool = False, + PiT: torch.Tensor | None = None, + output: torch.Tensor | None = None, + tile_size: int | None = None, + max_query_len: int | None = None, + max_seq_len: int | None = None, + num_kv_splits: int | None = None, + force_2d: bool = False, + fuse_q_rot: bool = True, + sinks: torch.Tensor | None = None, # [Hq] float — per-head sink logits +) -> torch.Tensor: + """Launch unified TQ attention (v3). + + ``query`` carries *raw* query vectors. For the MSE-key path the query + has to be rotated by ``PiT`` before it can be multiplied against the + (already-rotated) stored K. By default (``fuse_q_rot=True``) that + rotation is done inside the attention kernel prologue as a single + small MFMA — no extra dispatch, no HBM round-trip. Setting + ``fuse_q_rot=False`` restores the original launcher path (fp32 rocBLAS + GEMM + casts + .contiguous()), which is kept as an A/B toggle for + bench harnesses; numerical results match the fused path to within a + few ulp of fp32 round-off. The FP8-key path never rotates Q regardless + of this flag. + + ``tile_size`` defaults to ``32`` for prefill (``max_query_len > 1``) and + ``16`` for pure decode (``max_query_len == 1``). Callers may override. + + Dispatch rule: + * Prefill / chunked-prefill (any query block with ``BLOCK_Q > 1``) and + ``num_tokens > num_seqs``: 2D kernel (plenty of CTAs already). + * Pure decode with long KV: 3D split-KV kernel + ``reduce_segments`` + for extra parallelism; this is the same pattern v2 uses. + * ``force_2d=True``: force the 2D path regardless (used by the + apples-to-apples bench to isolate pure TQ dequant overhead). + + ``num_kv_splits`` controls the 3D-split segment count (default ``16``). + + Returns ``output`` of shape ``[num_tokens, Hq, D]`` in ``query.dtype``. + """ + assert query.dim() == 3, f"query must be [N, Hq, D], got {query.shape}" + num_tokens, Hq, D = query.shape + Hk = kv_cache.shape[2] + block_size = kv_cache.shape[1] + kv_group_size = Hq // Hk + num_seqs = int(query_start_loc.shape[0] - 1) + device = query.device + + cfg = _get_layout(D, mse_bits, value_quant_bits) + _ = value_packed_size # unused + + # Q-rotation strategy: + # * FP8-key path: no rotation at all (keys stored as fp8, no codebook). + # * MSE-key path + fuse_q_rot=True (default): pass the raw query to the + # kernel together with PiT and let the kernel's prologue do one + # fp32 tl.dot(Q, PiT). Eliminates the launcher-side + # (cast, rocBLAS GEMM, cast, contiguous) chain (~50-60 us/step at + # D=64, all launch-bound — see bottleneck report §10). + # * MSE-key path + fuse_q_rot=False: legacy launcher rotation. Kept + # as an A/B toggle for bench harnesses. + # + # PiT is always materialized as fp32 D x D contiguous; the kernel loads + # it as fp32 (cheap, 16KB at D=64) and computes Q @ PiT with IEEE fp32 + # accumulate via tl.dot. + if key_fp8: + q_rot = query.contiguous() + apply_fuse_q_rot = False + else: + if PiT is None: + PiT = Pi.T.contiguous() + apply_fuse_q_rot = bool(fuse_q_rot) + if apply_fuse_q_rot: + q_rot = query.contiguous() + else: + q_rot = (query.float() @ PiT).to(query.dtype).contiguous() + + # PiT in fp32, contiguous. For the fused path this is what the kernel + # loads; for the legacy path it's passed through as a harmless tensor + # (never dereferenced under the FUSE_Q_ROT constexpr guard). On FP8 + # path Pi may be unused at the call site, so we fall back to an + # arbitrary non-null tensor (reuse centroids) to satisfy Triton's + # non-null pointer requirement. + if (not key_fp8) and PiT is not None: + PiT_f32 = PiT if PiT.dtype == torch.float32 else PiT.to(torch.float32) + if not PiT_f32.is_contiguous(): + PiT_f32 = PiT_f32.contiguous() + pit_stride_0 = PiT_f32.stride(0) + pit_stride_1 = PiT_f32.stride(1) + else: + PiT_f32 = centroids # harmless dummy; not dereferenced when FUSE_Q_ROT=0 + pit_stride_0 = 0 + pit_stride_1 = 0 + + # Sinks: per-head fp32 logits, contiguous. The kernel only dereferences + # this pointer when USE_SINKS=1, so when the caller passes None we bind + # a harmless non-null tensor (centroids) to satisfy Triton's non-null + # pointer requirement. See the sink design notes in the 2D/3D kernel + # bodies and the v1 precedent from PR #40663 (sid==0 init trick). + if sinks is not None: + sinks_f32 = sinks if sinks.dtype == torch.float32 else sinks.to(torch.float32) + if not sinks_f32.is_contiguous(): + sinks_f32 = sinks_f32.contiguous() + assert sinks_f32.numel() == Hq, ( + f"sinks must have shape [Hq={Hq}], got numel={sinks_f32.numel()}" + ) + use_sinks = True + else: + sinks_f32 = centroids # harmless dummy; not dereferenced when USE_SINKS=0 + use_sinks = False + + if output is None: + output = torch.empty_like(query) + + # BLOCK_M heuristic (TQ-specific; diverges from stock unified_attention). + # + # Stock fp16 picks BLOCK_M=16 for small GQA because fp16 is ALU-saturated + # at any tile size and smaller BLOCK_M = better occupancy. TQ is a + # DIFFERENT story: the dequant chain (packed-byte loads + centroid gather + # + pair-LUT + norm/scale bitcasts) has big per-block fixed overhead, and + # the pair-LUT gather pattern is latency-bound (NOT HBM-bandwidth-bound, + # confirmed via rocprof: MemUnitStalled=0.01%, VALUBusy=56% at BM=16). + # + # For prefill/chunked (max_query_len > 1) we want BLOCK_M as large as + # possible to (a) amortize the fixed dequant overhead over more query + # tokens per CTA and (b) let MFMA saturate with a 128x32 tile shape. + # Empirical sweep on MI300X across gpt-oss (D=64) and llama (D=128): + # prefill B=1 Q=4k : BM=128 is 3.4x faster than BM=16 + # chunked B=64 Q=1k C=8: BM=128 is 5.6x faster than BM=16 + # With BM=128 v3 matches or BEATS the fp16 baseline on prefill/chunked. + # + # For pure decode (max_query_len == 1) BLOCK_M stays small because + # BLOCK_Q > 1 just pads rows — each sequence contributes at most 1 query + # token and larger BLOCK_M wastes lanes on masked-out rows. + if max_query_len is not None: + is_prefill_like = max_query_len > 1 + else: + is_prefill_like = num_tokens > num_seqs + + if is_prefill_like: + # Prefill / chunked: use BLOCK_M=128 (rounded up to a multiple of + # kv_group_size). Each CTA does very large work (BLOCK_M x TILE_SIZE + # x KV_len MMA + dequant), so even small grids (~17 CTAs) saturate + # MI300X just fine. We verified empirically: BM=128 wins across all + # prefill/chunked sweep points, including small (B=1 Q=256 C=8k) and + # large (B=64 Q=1k C=8k; 5.6x speedup vs BM=16). BLOCK_M must be a + # multiple of kv_group_size so BLOCK_Q is an integer. + BLOCK_M = max(128, triton.next_power_of_2(kv_group_size)) + else: + # Decode: BLOCK_Q > 1 in decode just pads rows (at most 1 query + # token per sequence), so there's little to amortize. Inherit + # stock's small-BLOCK_M heuristic. + BLOCK_M = 16 if kv_group_size <= 16 else triton.next_power_of_2(kv_group_size) + BLOCK_Q = BLOCK_M // kv_group_size + + # Grid: at most ceil(N / BLOCK_Q) + num_seqs q-blocks total, like unified. + total_num_q_blocks = num_tokens // BLOCK_Q + num_seqs + + # TILE_SIZE heuristic (matches stock unified_attention's _get_tile_size): + # prefill (max_query_len > 1): 32 + # decode (max_query_len == 1): 16 + if tile_size is None: + tile_size = 32 if is_prefill_like else 16 + + # Pair-LUT fast path for 4-bit MSE keys. Skipped for FP8 and non-4-bit. + # When USE_PAIR_LUT==0 the kernel never dereferences pair_lut, but Triton + # still requires a tensor pointer, so we fall back to reusing `centroids`. + use_pair_lut = (not key_fp8) and (mse_bits == 4) + pair_lut = _get_pair_lut(centroids) if use_pair_lut else centroids + + fp8_e4b15 = _use_fp8_e4b15(device.index or 0) + num_stages = 1 if _is_hip else 2 + + # uint16-aliased view of the cache. Under the Opt#3 SoA layout, K-norm / + # V-scale / V-zero live in a contiguous per-block metadata region and are + # always 2-byte aligned, so single-instruction u16 loads replace the + # original 2× uint8 + OR sequence for every per-token metadata fetch. + kv_cache_u16 = kv_cache.view(torch.uint16) + + # Opt#3 SoA layout constants (derived locally; matches the store-side + # computation so the launcher signature stays unchanged). Invariant: + # data_bytes_per_slot + meta_bytes_per_slot == slot_size_aligned. + mse_bytes = cfg["mse_bytes"] + val_data_bytes = cfg["val_data_bytes"] + key_data_bytes = D if key_fp8 else mse_bytes + data_bytes_per_slot = key_data_bytes + val_data_bytes + meta_region_offset = block_size * Hk * data_bytes_per_slot + num_soa_fields = 2 if key_fp8 else 3 + soa_k_norm = 0 # unused for FP8; harmless constant + soa_v_scale = 0 if key_fp8 else 1 + soa_v_zero = 1 if key_fp8 else 2 + + # ------------------------------------------------------------------ + # Dispatch: 2D for prefill / chunked; 3D split-KV for pure decode with + # long KV. Falls back to 2D if the sequences are short or if the caller + # forces it. ``is_prefill_like`` was already computed above for the + # BLOCK_M / tile_size heuristic; reuse it here. + # ------------------------------------------------------------------ + + # Use 3D only for pure decode where the KV history is long enough that + # splitting meaningfully increases SM occupancy. Threshold is empirically + # tuned on MI300X: + # seq < 1024 -> 2D path (short KV, reduce overhead > parallelism gain) + # seq >= 1024 -> 3D path (v3/v2 ratio 1.04-1.27x across shapes) + # Below 1024 KV tokens the reduce_segments + scratch-alloc overhead + # exceeds the parallelism gain from splitting. Above it, the single-CTA + # 2D path is serial over the KV history and starves MI300X's 304 CUs. + # + # IMPORTANT: max_seq_len is expected to come from the caller (backends + # track it from the block table). We avoid calling seq_lens.max().item() + # here because that forces a GPU->CPU sync on every launcher call, which + # adds ~150us of overhead to pure-decode steps. + if max_seq_len is None: + # Fallback: infer from block_table (host-side, no sync). This is an + # upper bound, sufficient for the dispatch decision. + max_seq_len_hint = int(block_table.shape[1]) * int(block_size) + else: + max_seq_len_hint = int(max_seq_len) + use_3d = (not force_2d) and (not is_prefill_like) and max_seq_len_hint >= 1024 + + if not use_3d: + kernel_tq_unified_attention_2d[(total_num_q_blocks, Hk)]( + output_ptr=output, + query_ptr=q_rot, + KV_cache_ptr=kv_cache, + KV_cache_u16_ptr=kv_cache_u16, + Centroids_ptr=centroids, + Pair_lut_ptr=pair_lut, + PiT_ptr=PiT_f32, + block_tables_ptr=block_table, + seq_lens_ptr=seq_lens, + query_start_len_ptr=query_start_loc, + sinks_ptr=sinks_f32, + scale=scale, + num_query_heads=Hq, + num_queries_per_kv=kv_group_size, + block_table_stride=block_table.stride(0), + query_stride_0=q_rot.stride(0), + query_stride_1=q_rot.stride(1), + output_stride_0=output.stride(0), + output_stride_1=output.stride(1), + stride_cache_block=kv_cache.stride(0), + pit_stride_0=pit_stride_0, + pit_stride_1=pit_stride_1, + BLOCK_SIZE=block_size, + TILE_SIZE=tile_size, + HEAD_SIZE=D, + HEAD_SIZE_PADDED=cfg["BLOCK_D"], + BLOCK_Q=BLOCK_Q, + BLOCK_M=BLOCK_M, + num_seqs=num_seqs, + MSE_BITS=mse_bits, + MSE_BYTES=mse_bytes, + VQB=value_quant_bits, + VAL_DATA_BYTES=val_data_bytes, + N_CENTROIDS=int(centroids.numel()), + KEY_FP8=1 if key_fp8 else 0, + USE_PAIR_LUT=1 if use_pair_lut else 0, + NUM_KV_HEADS=Hk, + KEY_DATA_BYTES=key_data_bytes, + META_REGION_OFFSET=meta_region_offset, + NUM_SOA_FIELDS=num_soa_fields, + SOA_K_NORM=soa_k_norm, + SOA_V_SCALE=soa_v_scale, + SOA_V_ZERO=soa_v_zero, + NORM_CORRECTION=1 if norm_correction else 0, + FP8_E4B15=fp8_e4b15, + FUSE_Q_ROT=1 if apply_fuse_q_rot else 0, + USE_SINKS=1 if use_sinks else 0, + num_warps=4, + num_stages=num_stages, + ) + return output + + # ---------------- 3D split-KV path ---------------- + # Pick segment count. 16 is a good default for MI300X (304 CUs): at + # Hk=8, num_seqs=1, total CTAs = 1 * 8 * 16 = 128, which saturates when + # combined with warp-level parallelism. Capped at ceil(max_seq_len / + # TILE_SIZE) so we don't launch empty segments. + if num_kv_splits is None: + num_kv_splits = 16 + max_possible_splits = max(1, (max_seq_len_hint + tile_size - 1) // tile_size) + num_segments = max(1, min(num_kv_splits, max_possible_splits)) + + HEAD_SIZE_PADDED = cfg["BLOCK_D"] + # Scratch buffers for segment partials. fp32 to match the kernel's + # accumulator dtype. Allocated per-call for now; a production backend + # would hoist these to persistent buffers reused across layers. + segm_output = torch.empty( + (num_tokens, Hq, num_segments, HEAD_SIZE_PADDED), + dtype=torch.float32, + device=device, + ) + segm_max = torch.empty( + (num_tokens, Hq, num_segments), dtype=torch.float32, device=device + ) + segm_expsum = torch.empty( + (num_tokens, Hq, num_segments), dtype=torch.float32, device=device + ) + + kernel_tq_unified_attention_3d[(total_num_q_blocks, Hk, num_segments)]( + segm_output_ptr=segm_output, + segm_max_ptr=segm_max, + segm_expsum_ptr=segm_expsum, + query_ptr=q_rot, + KV_cache_ptr=kv_cache, + KV_cache_u16_ptr=kv_cache_u16, + Centroids_ptr=centroids, + Pair_lut_ptr=pair_lut, + PiT_ptr=PiT_f32, + block_tables_ptr=block_table, + seq_lens_ptr=seq_lens, + query_start_len_ptr=query_start_loc, + sinks_ptr=sinks_f32, + scale=scale, + num_query_heads=Hq, + num_queries_per_kv=kv_group_size, + block_table_stride=block_table.stride(0), + query_stride_0=q_rot.stride(0), + query_stride_1=q_rot.stride(1), + stride_cache_block=kv_cache.stride(0), + pit_stride_0=pit_stride_0, + pit_stride_1=pit_stride_1, + BLOCK_SIZE=block_size, + TILE_SIZE=tile_size, + HEAD_SIZE=D, + HEAD_SIZE_PADDED=HEAD_SIZE_PADDED, + BLOCK_Q=BLOCK_Q, + BLOCK_M=BLOCK_M, + num_seqs=num_seqs, + NUM_SEGMENTS_PER_SEQ=num_segments, + MSE_BITS=mse_bits, + MSE_BYTES=mse_bytes, + VQB=value_quant_bits, + VAL_DATA_BYTES=val_data_bytes, + N_CENTROIDS=int(centroids.numel()), + KEY_FP8=1 if key_fp8 else 0, + USE_PAIR_LUT=1 if use_pair_lut else 0, + NUM_KV_HEADS=Hk, + KEY_DATA_BYTES=key_data_bytes, + META_REGION_OFFSET=meta_region_offset, + NUM_SOA_FIELDS=num_soa_fields, + SOA_K_NORM=soa_k_norm, + SOA_V_SCALE=soa_v_scale, + SOA_V_ZERO=soa_v_zero, + NORM_CORRECTION=1 if norm_correction else 0, + FP8_E4B15=fp8_e4b15, + FUSE_Q_ROT=1 if apply_fuse_q_rot else 0, + USE_SINKS=1 if use_sinks else 0, + num_warps=4, + num_stages=num_stages, + ) + + # Reduce: merge num_segments partials per (q_token, q_head) into the + # final output via online-softmax merge. Reusing the baseline kernel + # (KV-format-agnostic). + reduce_segments[(num_tokens, Hq)]( + output_ptr=output, + segm_output_ptr=segm_output, + segm_max_ptr=segm_max, + segm_expsum_ptr=segm_expsum, + seq_lens_ptr=seq_lens, + num_seqs=num_seqs, + num_query_heads=Hq, + out_scale_inv=1.0, + output_stride_0=output.stride(0), + output_stride_1=output.stride(1), + block_table_stride=block_table.stride(0), + TILE_SIZE=tile_size, + HEAD_SIZE=D, + HEAD_SIZE_PADDED=HEAD_SIZE_PADDED, + query_start_len_ptr=query_start_loc, + BLOCK_Q=BLOCK_Q, + NUM_SEGMENTS_PER_SEQ=num_segments, + USE_FP8=False, + ) + return output + + +# --------------------------------------------------------------------------- +# Thin decode adapter: lets callers who use the AoS decode launcher signature +# switch to the SoA decode without rebuilding query_start_loc themselves. +# --------------------------------------------------------------------------- + + +def triton_turboquant_decode_attention_soa( + query: torch.Tensor, # [B, Hq, D] + kv_cache: torch.Tensor, + block_table: torch.Tensor, + seq_lens: torch.Tensor, + Pi: torch.Tensor, + centroids: torch.Tensor, + scale: float, + mse_bits: int, + key_packed_size: int, + value_quant_bits: int, + value_packed_size: int, + key_fp8: bool = False, + norm_correction: bool = False, + PiT: torch.Tensor | None = None, + # kept for signature parity with v1/v2 decode launchers; unused here. + max_seq_len: int = 0, + mid_o_buf: Any = None, + output_buf: torch.Tensor | None = None, + lse_buf: Any = None, + buf_holder: Any = None, + max_num_kv_splits: int = 32, + sinks: torch.Tensor | None = None, +) -> torch.Tensor: + """Decode-only convenience wrapper around ``triton_turboquant_unified_attention``. + + Treats a rank-3 ``[B, Hq, D]`` query as one token per request (query_len = 1) + and synthesizes a ``query_start_loc`` of ``[0, 1, 2, ..., B]``. + ``max_num_kv_splits`` is forwarded to the unified launcher as the 3D + split-KV segment count (capped per-call against ``max_seq_len``). + ``sinks`` (optional ``[Hq]`` fp32) are forwarded to the kernel which + folds them into the softmax denominator via the init-time trick. + """ + del mid_o_buf, lse_buf, buf_holder + B = query.shape[0] + cu_seqlens_q = torch.arange(B + 1, device=query.device, dtype=seq_lens.dtype) + + out = triton_turboquant_unified_attention( + query=query.contiguous(), + kv_cache=kv_cache, + block_table=block_table, + seq_lens=seq_lens, + query_start_loc=cu_seqlens_q, + Pi=Pi, + centroids=centroids, + scale=scale, + mse_bits=mse_bits, + key_packed_size=key_packed_size, + value_quant_bits=value_quant_bits, + value_packed_size=value_packed_size, + key_fp8=key_fp8, + norm_correction=norm_correction, + PiT=PiT, + output=output_buf[:B] if output_buf is not None else None, + max_query_len=1, + max_seq_len=max_seq_len if max_seq_len > 0 else None, + num_kv_splits=max_num_kv_splits, + sinks=sinks, + ) + return out