5426311d91
Signed-off-by: root <root@smci355-ccs-aus-m02-09.cs-aus.dcgpu> Co-authored-by: root <root@smci355-ccs-aus-m02-09.cs-aus.dcgpu> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
875 lines
34 KiB
Python
875 lines
34 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||
"""FlyDSL TurboQuant decode launcher (vLLM-side).
|
||
|
||
Drop-in replacement for ``triton_turboquant_decode_attention_soa`` for the
|
||
TQ decode profile HEAD_SIZE=128, MSE_BITS=4 K, VQB=4 V, N_CENTROIDS=16,
|
||
BLOCK_SIZE in {16, 32}.
|
||
|
||
Per-GQA kernel dispatch:
|
||
* GQA group ∈ {8, 16} → canonical kernels.tq_decode (Qwen-class)
|
||
* GQA group == 6 → kernels.tq_decode_gqa6 sibling (MiniMax-M2.5)
|
||
|
||
Auto-selected on gfx950 when FlyDSL is importable; falls back to the SoA Triton
|
||
decode otherwise (wrong arch, missing build tree). The GQA-6 sibling is imported
|
||
best-effort: missing it does not affect Qwen GQA-{8,16} paths.
|
||
|
||
Architecture:
|
||
1. Q rotation: ``q_rot = (query.float() @ PiT).bfloat16()`` — the same
|
||
launcher-side rocBLAS GEMM the SoA Triton decode uses.
|
||
2. FlyDSL kernel writes per-partition outputs into
|
||
``[N, Hk, P, QG, D]`` bf16 + ``[N, Hk, P, QG]`` fp32 max/sum buffers.
|
||
3. A small Triton reducer combines partitions in the kernel's native layout
|
||
(no permute/cast), writing the final ``[N, Hq, D]`` output.
|
||
|
||
Kernel module is built once per ``(num_kv_heads, num_partitions,
|
||
max_blocks_per_seq, scale)`` shape and cached.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from typing import Any
|
||
|
||
import torch
|
||
|
||
from vllm.logger import init_logger
|
||
from vllm.triton_utils import tl, triton
|
||
|
||
logger = init_logger(__name__)
|
||
|
||
# -- FlyDSL import bootstrap --------------------------------------------------
|
||
# FlyDSL is imported directly and must be importable (pip-installed, or on
|
||
# PYTHONPATH). Import failures are caught below and fall back to the SoA Triton
|
||
# decode, so vLLM startup is unaffected on hosts without FlyDSL.
|
||
|
||
|
||
_FLYDSL_AVAILABLE: bool | None = None
|
||
_TQ_MOD: Any = None # kernels.tq_decode module (Qwen GQA-{8,16})
|
||
_TQ_MOD_GQA6: Any = None # kernels.tq_decode_gqa6 module (MiniMax GQA-6, optional)
|
||
_FLYC: Any = None # flydsl.compiler
|
||
_FX: Any = None # flydsl.expr
|
||
_TYPING_T: Any = None
|
||
_CC: Any = None # CompilationContext
|
||
_IR: Any = None
|
||
|
||
|
||
def is_flydsl_available() -> bool:
|
||
"""Return True iff on gfx950 and FlyDSL imports + kernel module load.
|
||
|
||
The GQA-6 sibling kernel (``kernels.tq_decode_gqa6``) is imported
|
||
best-effort: if it's missing (older FlyDSL checkout that pre-dates
|
||
MiniMax support) the canonical Qwen path stays fully functional and
|
||
only GQA-6 dispatches will fail with a clear error at launch time.
|
||
"""
|
||
global _FLYDSL_AVAILABLE, _TQ_MOD, _TQ_MOD_GQA6
|
||
global _FLYC, _FX, _TYPING_T, _CC, _IR
|
||
if _FLYDSL_AVAILABLE is not None:
|
||
return _FLYDSL_AVAILABLE
|
||
try:
|
||
# gfx950-only kernel (CDNA4 intrinsics); gate before importing FlyDSL.
|
||
from vllm.platforms.rocm import on_gfx950
|
||
|
||
if not on_gfx950():
|
||
_FLYDSL_AVAILABLE = False
|
||
return _FLYDSL_AVAILABLE
|
||
import flydsl.compiler as flyc # noqa: F401
|
||
import flydsl.expr as fx # noqa: F401
|
||
from flydsl._mlir import ir
|
||
from flydsl.compiler.kernel_function import CompilationContext
|
||
from flydsl.expr.typing import T # noqa: F401
|
||
|
||
# Kernel is vendored in-tree (ships with vLLM); FlyDSL provides only
|
||
# the compiler/runtime framework (imported above).
|
||
from vllm.v1.attention.ops.flydsl_kernels import tq_decode as tq_mod
|
||
|
||
_FLYC = flyc
|
||
_FX = fx
|
||
_TYPING_T = T
|
||
_CC = CompilationContext
|
||
_IR = ir
|
||
_TQ_MOD = tq_mod
|
||
_FLYDSL_AVAILABLE = True
|
||
logger.info_once("FlyDSL TQ decode launcher: available")
|
||
except Exception as ex: # noqa: BLE001
|
||
_FLYDSL_AVAILABLE = False
|
||
logger.warning_once(
|
||
"FlyDSL TQ decode launcher: unavailable (%s). "
|
||
"Falling back to SoA Triton decode.",
|
||
ex,
|
||
)
|
||
return _FLYDSL_AVAILABLE
|
||
# Best-effort GQA-6 sibling import (does NOT gate Qwen availability).
|
||
try:
|
||
from vllm.v1.attention.ops.flydsl_kernels import (
|
||
tq_decode_gqa6 as tq_mod_gqa6,
|
||
)
|
||
|
||
_TQ_MOD_GQA6 = tq_mod_gqa6
|
||
logger.info_once("FlyDSL TQ decode GQA-6 sibling: available (MiniMax-class)")
|
||
except Exception as ex: # noqa: BLE001
|
||
_TQ_MOD_GQA6 = None
|
||
logger.info_once(
|
||
"FlyDSL TQ decode GQA-6 sibling: not available (%s); "
|
||
"GQA-6 models will fall back to SoA Triton decode.",
|
||
ex,
|
||
)
|
||
return _FLYDSL_AVAILABLE
|
||
|
||
|
||
def is_flydsl_gqa6_available() -> bool:
|
||
"""True iff the optional GQA-6 sibling kernel module loaded.
|
||
|
||
Used by the eligibility gate in turboquant_attn.py to decide whether
|
||
a layer with num_kv_groups==6 can run on FlyDSL or must fall back
|
||
to the SoA Triton decode.
|
||
"""
|
||
if _FLYDSL_AVAILABLE is None:
|
||
is_flydsl_available()
|
||
return _TQ_MOD_GQA6 is not None
|
||
|
||
|
||
# -- Kernel module cache -------------------------------------------------------
|
||
# Compiling a FlyDSL kernel is expensive; cache by the constexprs that
|
||
# parameterize stride math: num_kv_heads, num_partitions, max_blocks_per_seq,
|
||
# scale. ``num_seqs`` is irrelevant to the kernel body so we omit it.
|
||
_KERN_CACHE: dict[tuple, Any] = {}
|
||
|
||
|
||
# -- One-shot log guards ------------------------------------------------------
|
||
# Replaces vllm `logger.{info,warning}_once` (which hashes the format string
|
||
# every call to dedup) with a true zero-overhead bool guard.
|
||
_LOG_INVOKED_ONCE: bool = False
|
||
_LOG_SINKS_WARNED: bool = False
|
||
_LOG_NORM_WARNED: bool = False
|
||
|
||
|
||
def _detect_max_capture_B() -> int:
|
||
"""Probe vLLM compilation config for the largest cudagraph capture size.
|
||
|
||
Returns max(cudagraph_capture_sizes) if available, else falls back to
|
||
512 (vLLM's default cap). The segm-pool auto-grows if a larger batch is
|
||
seen (see ``_SegmBufPool.get``).
|
||
|
||
Used to size the segm-pool bucket once, so that all distinct B's
|
||
(whether captured in a graph, or hit eagerly in mixed batches) share
|
||
a single allocation per shape — eliminating the per-B growth that
|
||
previously dominated dense-capture VRAM cost.
|
||
"""
|
||
try:
|
||
from vllm.config import get_current_vllm_config
|
||
|
||
cfg = get_current_vllm_config()
|
||
sizes = cfg.compilation_config.cudagraph_capture_sizes
|
||
if sizes:
|
||
return int(max(sizes))
|
||
except Exception: # noqa: BLE001
|
||
pass
|
||
return 512
|
||
|
||
|
||
class _SegmBufPool:
|
||
"""Single-bucket buffer pool for segm_out/segm_max/segm_sum/output.
|
||
|
||
Each unique shape signature (Hk, Hq, num_partitions, QG, D, device,
|
||
dtype) is backed by ONE allocation sized at ``max_B`` (= max captured
|
||
cudagraph batch size, or env override). Per-call ``get(B, ...)``
|
||
returns ``[:B]`` views — same data_ptr base, narrower N dim. The
|
||
kernel grid is ``(B, ...)`` so it only ever writes the first B rows
|
||
of segm_out / segm_max / segm_sum; downstream readers (Triton
|
||
reducer + caller) only consume the first B rows of output.
|
||
|
||
Why a single bucket: cudagraph records launch pointers at capture
|
||
time. With per-B allocations, every captured size triggered a fresh
|
||
allocation, so total VRAM scaled with the number of captured sizes.
|
||
With single-bucket, the pool tops out at one max_B-sized buffer per
|
||
shape, shared across ALL captured sizes and ALL eager mixed-batch
|
||
calls. The data_ptr is stable from first allocation onward, so
|
||
cudagraph capture is happy across all captured B's.
|
||
|
||
Eliminates per-decode-step ``cudaMalloc`` (× 4) plus the device-side
|
||
memset kernels behind ``torch.full(-inf)`` and ``torch.zeros``.
|
||
The kernel always writes the FULL ``[B, Hk, P, QG, D]`` slice and
|
||
every (n, kv_h, p, qg) position of segm_max / segm_sum (it stores
|
||
``-inf`` and ``0`` for empty partitions itself), so uninitialized
|
||
bytes from the pool are safe to reuse for any B ≤ B_bucket.
|
||
"""
|
||
|
||
__slots__ = ("_bufs", "_max_B")
|
||
|
||
def __init__(self) -> None:
|
||
self._bufs: dict[tuple, dict[str, torch.Tensor]] = {}
|
||
self._max_B: int | None = None # lazy-init on first get()
|
||
|
||
def get(
|
||
self,
|
||
B: int,
|
||
Hk: int,
|
||
Hq: int,
|
||
num_partitions: int,
|
||
QG: int,
|
||
D: int,
|
||
device: torch.device,
|
||
q_dtype: torch.dtype,
|
||
) -> dict[str, torch.Tensor]:
|
||
if self._max_B is None:
|
||
self._max_B = _detect_max_capture_B()
|
||
# B may exceed the detected max if user set max-num-seqs higher
|
||
# than the largest captured size — still safe, we'll grow once.
|
||
# Note: growing AFTER cudagraph capture would invalidate captured
|
||
# pointers, so this should only happen during warmup or in
|
||
# always-eager configs.
|
||
B_bucket = max(self._max_B, int(B))
|
||
key = (
|
||
int(Hk),
|
||
int(Hq),
|
||
int(num_partitions),
|
||
int(QG),
|
||
int(D),
|
||
str(device),
|
||
q_dtype,
|
||
)
|
||
bufs = self._bufs.get(key)
|
||
if bufs is None or bufs["segm_out"].shape[0] < B_bucket:
|
||
if bufs is not None and bufs["segm_out"].shape[0] < B_bucket:
|
||
# First-time grow: warn so user knows cudagraphs may be
|
||
# invalidated. This should only happen during warmup or in
|
||
# always-eager configs (a batch larger than any captured
|
||
# cudagraph size).
|
||
logger.warning_once(
|
||
"FlyDSL _SegmBufPool: growing bucket from %d to %d "
|
||
"(B=%d). If you see this AFTER cudagraph warmup, the "
|
||
"previously-captured graphs hold stale pointers and "
|
||
"will GPU-fault (max cudagraph capture size < B).",
|
||
bufs["segm_out"].shape[0],
|
||
B_bucket,
|
||
B,
|
||
)
|
||
bufs = {
|
||
"segm_out": torch.empty(
|
||
(B_bucket, Hk, num_partitions, QG, D),
|
||
dtype=torch.bfloat16,
|
||
device=device,
|
||
),
|
||
"segm_max": torch.empty(
|
||
(B_bucket, Hk, num_partitions, QG),
|
||
dtype=torch.float32,
|
||
device=device,
|
||
),
|
||
"segm_sum": torch.empty(
|
||
(B_bucket, Hk, num_partitions, QG),
|
||
dtype=torch.float32,
|
||
device=device,
|
||
),
|
||
"output": torch.empty(
|
||
(B_bucket, Hq, D),
|
||
dtype=q_dtype,
|
||
device=device,
|
||
),
|
||
# Stable pre-allocated buffer for the rotated query
|
||
# tensor. On ROCm, fresh torch.empty / matmul-result allocations
|
||
# after HIP graph capture can land in the graph memory pool and
|
||
# cause GPU memory faults when passed as kernel arguments in the
|
||
# eager mixed-batch path. By pooling q_rot here (same pattern as
|
||
# segm_out/output above), the data_ptr is stable from first
|
||
# allocation onward — same for all captured B's and eager calls.
|
||
"q_rot": torch.empty(
|
||
(B_bucket, Hq, D),
|
||
dtype=q_dtype,
|
||
device=device,
|
||
),
|
||
# q_float: fp32 copy of query (bf16→fp32 for mm input).
|
||
# Pooled to avoid fresh allocations post-capture.
|
||
"q_float": torch.empty(
|
||
(B_bucket, Hq, D),
|
||
dtype=torch.float32,
|
||
device=device,
|
||
),
|
||
# q_rot_fp32: fp32 output of the rotation mm (before bf16 cast).
|
||
# Using torch.mm(..., out=this) eliminates the fresh mm-result
|
||
# allocation that would otherwise land in the HIP graph pool.
|
||
"q_rot_fp32": torch.empty(
|
||
(B_bucket, Hq, D),
|
||
dtype=torch.float32,
|
||
device=device,
|
||
),
|
||
}
|
||
self._bufs[key] = bufs
|
||
self._max_B = B_bucket
|
||
logger.info_once(
|
||
"FlyDSL _SegmBufPool: allocated single-bucket "
|
||
"shape=(Hk=%d, Hq=%d, P=%d, QG=%d, D=%d, dtype=%s) "
|
||
"B_bucket=%d (covers all captured + eager B's). "
|
||
"VRAM = %.1f MiB / shape.",
|
||
Hk,
|
||
Hq,
|
||
num_partitions,
|
||
QG,
|
||
D,
|
||
q_dtype,
|
||
B_bucket,
|
||
sum(t.numel() * t.element_size() for t in bufs.values()) / (1 << 20),
|
||
)
|
||
# Return [:B] views — same data_ptr base, narrower N dim.
|
||
return {
|
||
"segm_out": bufs["segm_out"][:B],
|
||
"segm_max": bufs["segm_max"][:B],
|
||
"segm_sum": bufs["segm_sum"][:B],
|
||
"output": bufs["output"][:B],
|
||
"q_rot": bufs["q_rot"][:B],
|
||
"q_float": bufs["q_float"][:B],
|
||
"q_rot_fp32": bufs["q_rot_fp32"][:B],
|
||
}
|
||
|
||
def stats(self) -> dict[str, int]:
|
||
return {
|
||
"shapes": len(self._bufs),
|
||
"max_B": int(self._max_B or 0),
|
||
"bytes": sum(
|
||
sum(t.numel() * t.element_size() for t in d.values())
|
||
for d in self._bufs.values()
|
||
),
|
||
}
|
||
|
||
|
||
_SEGM_POOL = _SegmBufPool()
|
||
|
||
|
||
_HW_TR_CACHED: bool | None = None
|
||
|
||
|
||
def _hw_tr_enabled() -> bool:
|
||
"""Resolve the HW V-transpose build flag.
|
||
|
||
ON for gfx950+ (ds_read_tr16_b64 is bit-exact vs baseline and faster
|
||
on Qwen-class shapes), off elsewhere. Resolved from the GPU arch.
|
||
"""
|
||
global _HW_TR_CACHED
|
||
if _HW_TR_CACHED is not None:
|
||
return _HW_TR_CACHED
|
||
try:
|
||
from flydsl.runtime.device import get_rocm_arch as _arch
|
||
|
||
a = str(_arch() or "")
|
||
_HW_TR_CACHED = a.startswith("gfx950")
|
||
except Exception: # noqa: BLE001
|
||
_HW_TR_CACHED = False
|
||
if _HW_TR_CACHED:
|
||
logger.info_once("FlyDSL TQ: HW V transpose ON (default for gfx950+)")
|
||
else:
|
||
logger.info_once("FlyDSL TQ: HW V transpose OFF")
|
||
return _HW_TR_CACHED
|
||
|
||
|
||
_GET_KERNEL_STATS = {"hits": 0, "misses": 0, "build_total_s": 0.0}
|
||
|
||
|
||
def _get_kernel(
|
||
num_kv_heads: int,
|
||
num_partitions: int,
|
||
max_blocks_per_seq: int,
|
||
scale: float,
|
||
query_group_size: int,
|
||
kv_block_size: int,
|
||
use_hw_v_transpose: bool = False,
|
||
num_seqs_hint: int = 1,
|
||
tile_groups_per_partition: int = 1,
|
||
):
|
||
# ``num_seqs_hint`` (= runtime B) is forwarded to the build for shape
|
||
# awareness; it does NOT participate in the cache key because the
|
||
# kernel body uses gpu.block_idx.x (= seq index at runtime) and is
|
||
# GQA-symmetric across B at build time.
|
||
#
|
||
# ``tile_groups_per_partition`` IS in the cache key — each
|
||
# value compiles a different unrolled K-tile loop body.
|
||
key = (
|
||
num_kv_heads,
|
||
int(num_partitions),
|
||
int(max_blocks_per_seq),
|
||
round(float(scale), 8),
|
||
int(query_group_size),
|
||
int(kv_block_size),
|
||
bool(use_hw_v_transpose),
|
||
int(tile_groups_per_partition),
|
||
)
|
||
cached = _KERN_CACHE.get(key)
|
||
if cached is not None:
|
||
_GET_KERNEL_STATS["hits"] += 1
|
||
return cached
|
||
assert is_flydsl_available()
|
||
# Time the FlyDSL build path so we can quantify cudagraph capture cost.
|
||
import time as _t
|
||
|
||
_build_t0 = _t.perf_counter()
|
||
|
||
# Per-GQA dispatch: GQA-6 (MiniMax-M2.5) lives in the sibling module
|
||
# tq_decode_gqa6 to keep the Qwen kernel's invariants untouched.
|
||
# GQA-{8,16} (Qwen) keep using the canonical tq_decode kernel.
|
||
qg = int(query_group_size)
|
||
if qg == 6:
|
||
if _TQ_MOD_GQA6 is None:
|
||
raise RuntimeError(
|
||
"FlyDSL TQ GQA-6 sibling kernel "
|
||
"(vllm.v1.attention.ops.flydsl_kernels.tq_decode_gqa6) "
|
||
"failed to import; check that FlyDSL is installed/importable."
|
||
)
|
||
kmod = _TQ_MOD_GQA6
|
||
kfn = kmod.build_tq_decode_gqa6_module(
|
||
num_seqs=int(num_seqs_hint),
|
||
num_kv_heads=num_kv_heads,
|
||
num_partitions=num_partitions,
|
||
max_blocks_per_seq=max_blocks_per_seq,
|
||
softmax_scale=float(scale),
|
||
query_group_size=qg,
|
||
kv_block_size=int(kv_block_size),
|
||
use_hw_v_transpose=bool(use_hw_v_transpose),
|
||
tile_groups_per_partition=int(tile_groups_per_partition),
|
||
)
|
||
else:
|
||
kmod = _TQ_MOD
|
||
kfn = kmod.build_tq_decode_module(
|
||
num_seqs=int(num_seqs_hint),
|
||
num_kv_heads=num_kv_heads,
|
||
num_partitions=num_partitions,
|
||
max_blocks_per_seq=max_blocks_per_seq,
|
||
softmax_scale=float(scale),
|
||
query_group_size=qg,
|
||
kv_block_size=int(kv_block_size),
|
||
use_hw_v_transpose=bool(use_hw_v_transpose),
|
||
tile_groups_per_partition=int(tile_groups_per_partition),
|
||
)
|
||
al = kmod.allocator
|
||
block_threads = kmod.BLOCK_THREADS
|
||
|
||
flyc = _FLYC
|
||
fx = _FX
|
||
T = _TYPING_T
|
||
CompilationContext = _CC
|
||
ir_mod = _IR
|
||
|
||
@flyc.jit
|
||
def _launch(
|
||
out,
|
||
es,
|
||
ml,
|
||
q,
|
||
kvc,
|
||
cents,
|
||
bt,
|
||
sl,
|
||
gx: fx.Int32, # type: ignore[name-defined]
|
||
gy: fx.Int32, # type: ignore[name-defined]
|
||
gz: fx.Int32, # type: ignore[name-defined]
|
||
stream: fx.Stream, # type: ignore[name-defined]
|
||
):
|
||
# Re-finalize the LDS allocator for this launch (idempotent per build).
|
||
al.finalized = False
|
||
ctx = CompilationContext.get_current()
|
||
with ir_mod.InsertionPoint(ctx.gpu_module_body):
|
||
al.finalize()
|
||
from flydsl.expr import arith
|
||
|
||
grid_x = arith.index_cast(T.index, gx.ir_value())
|
||
grid_y = arith.index_cast(T.index, gy.ir_value())
|
||
grid_z = arith.index_cast(T.index, gz.ir_value())
|
||
kfn(out, es, ml, q, kvc, cents, bt, sl).launch(
|
||
grid=(grid_x, grid_y, grid_z),
|
||
block=(block_threads, 1, 1),
|
||
stream=stream,
|
||
)
|
||
|
||
_KERN_CACHE[key] = _launch
|
||
_build_dt = _t.perf_counter() - _build_t0
|
||
_GET_KERNEL_STATS["misses"] += 1
|
||
_GET_KERNEL_STATS["build_total_s"] += _build_dt
|
||
logger.info(
|
||
"FlyDSL _get_kernel BUILD #%d dt=%.2fs (cumulative=%.1fs) key=%s",
|
||
_GET_KERNEL_STATS["misses"],
|
||
_build_dt,
|
||
_GET_KERNEL_STATS["build_total_s"],
|
||
key,
|
||
)
|
||
return _launch
|
||
|
||
|
||
# -- Partition reducer (Triton, native kernel layout) -------------------------
|
||
# Reduces the FA2 split-KV partials produced by the FlyDSL kernel.
|
||
# segm_out [N, Hk, P, QG, D] bf16
|
||
# segm_max [N, Hk, P, QG] fp32
|
||
# segm_sum [N, Hk, P, QG] fp32
|
||
# → output [N, Hq=Hk*QG, D] of any (bf16/fp16/fp32) dtype.
|
||
@triton.jit
|
||
def _reduce_partitions(
|
||
output_ptr, # [N, Hq, D] in OUT_DTYPE
|
||
segm_out_ptr, # [N, Hk, P, QG, D] bf16
|
||
segm_max_ptr, # [N, Hk, P, QG] fp32
|
||
segm_sum_ptr, # [N, Hk, P, QG] fp32
|
||
out_stride_n: tl.int64, # stride on N axis (in elems of OUT_DTYPE)
|
||
out_stride_h: tl.int64, # stride on Hq axis
|
||
NUM_KV_HEADS: tl.constexpr,
|
||
QG: tl.constexpr,
|
||
NUM_PARTS: tl.constexpr,
|
||
HEAD_SIZE: tl.constexpr,
|
||
):
|
||
# grid = (N, Hq)
|
||
n = tl.program_id(0)
|
||
hq = tl.program_id(1)
|
||
kv_h = hq // QG
|
||
qg = hq % QG
|
||
|
||
# Per-partition pointers for this (n, kv_h, qg) row.
|
||
# Strides in elems of source dtype.
|
||
msum_base = n * (NUM_KV_HEADS * NUM_PARTS * QG) + kv_h * (NUM_PARTS * QG) + qg
|
||
so_base = (
|
||
n * (NUM_KV_HEADS * NUM_PARTS * QG * HEAD_SIZE)
|
||
+ kv_h * (NUM_PARTS * QG * HEAD_SIZE)
|
||
+ qg * HEAD_SIZE
|
||
)
|
||
|
||
p_off = tl.arange(0, NUM_PARTS)
|
||
d_off = tl.arange(0, HEAD_SIZE)
|
||
|
||
# Load per-partition max/sum.
|
||
m_idx = msum_base + p_off * QG
|
||
seg_max = tl.load(segm_max_ptr + m_idx)
|
||
seg_sum = tl.load(segm_sum_ptr + m_idx)
|
||
|
||
# Mask out empty partitions (max == -inf).
|
||
valid = seg_max > float("-inf")
|
||
overall_max = tl.max(tl.where(valid, seg_max, float("-inf")))
|
||
|
||
# Rescale exp sums.
|
||
rescale = tl.where(valid, tl.exp(seg_max - overall_max), 0.0)
|
||
seg_sum_rescaled = seg_sum * rescale
|
||
overall_sum = tl.sum(seg_sum_rescaled)
|
||
|
||
# Load segment outputs and combine.
|
||
so_idx = so_base + p_off[:, None] * (QG * HEAD_SIZE) + d_off[None, :]
|
||
seg_out = tl.load(segm_out_ptr + so_idx).to(tl.float32)
|
||
# acc = sum_p( seg_out_p * (seg_sum_p * rescale_p) ) / overall_sum
|
||
weighted = seg_out * seg_sum_rescaled[:, None]
|
||
acc_sum = tl.sum(weighted, axis=0)
|
||
acc = tl.where(overall_sum > 0.0, acc_sum / overall_sum, 0.0)
|
||
|
||
# Store.
|
||
out_off = n * out_stride_n + hq * out_stride_h + d_off
|
||
tl.store(output_ptr + out_off, acc.to(output_ptr.dtype.element_ty))
|
||
|
||
|
||
# -- Public launcher ----------------------------------------------------------
|
||
def flydsl_turboquant_decode_attention(
|
||
query: torch.Tensor, # [B, Hq, D] bf16/fp16
|
||
kv_cache: torch.Tensor, # [num_blocks, BS, Hk, slot_size_aligned]
|
||
block_table: torch.Tensor, # [B, max_blocks_per_seq] int32
|
||
seq_lens: torch.Tensor, # [B] int32
|
||
Pi: torch.Tensor, # [D, D] fp32
|
||
centroids: torch.Tensor, # [N_CENTROIDS] fp32
|
||
scale: float,
|
||
mse_bits: int,
|
||
key_packed_size: int,
|
||
value_quant_bits: int,
|
||
value_packed_size: int,
|
||
key_fp8: bool = False,
|
||
norm_correction: bool = False,
|
||
PiT: torch.Tensor | None = None,
|
||
max_seq_len: int = 0,
|
||
mid_o_buf: Any = None,
|
||
output_buf: torch.Tensor | None = None,
|
||
lse_buf: Any = None,
|
||
buf_holder: Any = None,
|
||
max_num_kv_splits: int = 32,
|
||
sinks: torch.Tensor | None = None,
|
||
) -> torch.Tensor:
|
||
"""SoA-decode-compatible launcher backed by the FlyDSL decode kernel.
|
||
|
||
Constraints:
|
||
* key_fp8 == False
|
||
* mse_bits == 4
|
||
* value_quant_bits == 4
|
||
* centroids.numel() == 16
|
||
* D == 128
|
||
* block_size in {16, 32}
|
||
* Hq // Hk in {6, 8, 16}
|
||
- 8/16 → canonical tq_decode kernel (Qwen2.5-72B / Qwen3-32B)
|
||
- 6 → tq_decode_gqa6 sibling kernel (MiniMax-M2.5)
|
||
|
||
Sinks are NYI and silently ignored if set. norm_correction is honored
|
||
implicitly via the pre-folded stored K-norm (see footer comment).
|
||
"""
|
||
del mid_o_buf, lse_buf, key_packed_size, value_packed_size
|
||
if not is_flydsl_available():
|
||
raise RuntimeError(
|
||
"FlyDSL decode requested but FlyDSL is not available (needs gfx950 "
|
||
"+ importable FlyDSL); the SoA Triton decode is the fallback."
|
||
)
|
||
assert not key_fp8, "FlyDSL supports MSE-key path only"
|
||
assert mse_bits == 4, f"FlyDSL expects mse_bits=4, got {mse_bits}"
|
||
assert value_quant_bits == 4, (
|
||
f"FlyDSL expects value_quant_bits=4, got {value_quant_bits}"
|
||
)
|
||
|
||
B, Hq, D = query.shape
|
||
Hk = kv_cache.shape[2]
|
||
block_size = kv_cache.shape[1]
|
||
QG = Hq // Hk
|
||
assert D == _TQ_MOD.HEAD_SIZE
|
||
assert block_size in (16, 32), (
|
||
f"FlyDSL supports kv_block_size 16 or 32, got {block_size}"
|
||
)
|
||
assert QG in (6, 8, 16), f"FlyDSL supports GQA factor 6, 8 or 16, got {QG}"
|
||
if QG == 6 and _TQ_MOD_GQA6 is None:
|
||
raise RuntimeError(
|
||
"FlyDSL launcher: GQA-6 requested (MiniMax-class) but the "
|
||
"tq_decode_gqa6 sibling module is not available. Update your "
|
||
"FlyDSL checkout (must include kernels/tq_decode_gqa6.py); "
|
||
"otherwise the SoA Triton decode is the fallback."
|
||
)
|
||
assert centroids.numel() == _TQ_MOD.N_CENTROIDS, (
|
||
f"centroids.numel={centroids.numel()} != {_TQ_MOD.N_CENTROIDS}"
|
||
)
|
||
|
||
# ---- Per-layer cache for PiT_f32 + contiguous centroids ---
|
||
# PiT and centroids are model constants (set once at layer warmup).
|
||
# Avoid the per-decode-step transpose+cast and `.contiguous()` no-op
|
||
# check by stashing the pre-cooked tensors on ``buf_holder`` (= layer).
|
||
PiT_f32: torch.Tensor
|
||
centroids_c: torch.Tensor
|
||
if buf_holder is not None:
|
||
PiT_f32 = getattr(buf_holder, "_tq_PiT_f32", None)
|
||
if PiT_f32 is None:
|
||
_PiT_src = PiT if PiT is not None else Pi.T.contiguous()
|
||
PiT_f32 = (
|
||
_PiT_src
|
||
if _PiT_src.dtype == torch.float32
|
||
else _PiT_src.to(torch.float32)
|
||
)
|
||
buf_holder._tq_PiT_f32 = PiT_f32
|
||
centroids_c = getattr(buf_holder, "_tq_centroids_c", None)
|
||
if centroids_c is None:
|
||
centroids_c = centroids.contiguous()
|
||
buf_holder._tq_centroids_c = centroids_c
|
||
else:
|
||
# Defensive fallback (unit tests may pass ``buf_holder=None``).
|
||
_PiT_src = PiT if PiT is not None else Pi.T.contiguous()
|
||
PiT_f32 = (
|
||
_PiT_src if _PiT_src.dtype == torch.float32 else _PiT_src.to(torch.float32)
|
||
)
|
||
centroids_c = centroids.contiguous()
|
||
|
||
# ---- Pooled q_rot (stable data_ptr post-capture) ------
|
||
# On ROCm, fresh tensor allocations made AFTER HIP graph capture can land
|
||
# in the graph memory pool and cause GPU memory faults when passed as kernel
|
||
# pointer arguments in the eager mixed-batch path. We pool q_rot and the
|
||
# fp32 intermediate the same way segm_out/output are pooled: one max-B
|
||
# allocation per shape, reused across all B's.
|
||
#
|
||
# Note: pool_bufs is computed BELOW after num_partitions is determined;
|
||
# q_rot needs pool_bufs["q_float"] and pool_bufs["q_rot"]. We compute the
|
||
# rotation inline after the pool is fetched (see the pooled-buffers
|
||
# section below).
|
||
|
||
# ---- Partition count (FA2 split-KV) ----------------------------------
|
||
#
|
||
# FlyDSL runs INSIDE FULL cudagraph (TurboQuantMetadataBuilder._cudagraph_
|
||
# support = UNIFORM_BATCH). That means the gridDim baked at capture
|
||
# time MUST equal the gridDim at replay — the kernel launch parameters
|
||
# are recorded into the captured graph. So `num_partitions` (which
|
||
# becomes gridDim.z) MUST be derived from a stable source that produces
|
||
# the same value at capture and at runtime.
|
||
#
|
||
# We use the worst-case context length implied by the block table's
|
||
# allocation (block_table.shape[1] * block_size) as that source. It's
|
||
# bounded by `max_model_len` (vLLM allocates the block table for the
|
||
# configured max), and it's identical at capture and at runtime
|
||
# because `block_table.shape[1]` is fixed once the model is loaded.
|
||
# Per-tile OOB redirects + the masked FA-2 reducer ensure that for
|
||
# short sequences the extra partitions contribute zero to the output.
|
||
#
|
||
# We always size from the worst case (block_table.shape[1] * block_size),
|
||
# never from the per-step actual max_seq_len: a per-step size would make
|
||
# the captured gridDim mismatch runtime and GPU-fault under cudagraph.
|
||
kv_compute_block = _TQ_MOD.KV_COMPUTE_BLOCK
|
||
worst_case_max_seq_len = int(block_table.shape[1]) * int(block_size)
|
||
if max_seq_len <= 0:
|
||
max_seq_len = worst_case_max_seq_len
|
||
sizing_max_seq_len = worst_case_max_seq_len
|
||
|
||
# ── Bounded num_partitions + internal tile-group looping ──
|
||
# The kernel previously hardcoded one partition = 256 tokens (=
|
||
# KV_COMPUTE_BLOCK), forcing num_partitions to scale linearly with
|
||
# max_model_len, which made the cudagraph capture cost grow
|
||
# prohibitively at long context. The new kernel takes
|
||
# ``tile_groups_per_partition`` (TGPP); each CTA processes
|
||
# ``TGPP * 16`` K-tiles = ``TGPP * KV_COMPUTE_BLOCK`` tokens with the
|
||
# FA-2 online-softmax state accumulating across all of them. This
|
||
# mirrors what the SoA Triton decode / HIP SoA-fusion already do.
|
||
#
|
||
# Strategy: cap num_partitions at ``MAX_PARTITIONS`` (default 32),
|
||
# then derive TGPP so that ``num_partitions * TGPP * KV_COMPUTE_BLOCK
|
||
# >= sizing_max_seq_len``.
|
||
#
|
||
# Examples (block_size=32 → worst_case max_bps*32):
|
||
# max_model_len=8K: required=32, parts=32, TGPP=1 (no waste)
|
||
# max_model_len=16K: required=64, parts=32, TGPP=2 (no waste)
|
||
# max_model_len=32K: required=128, parts=32, TGPP=4 (no waste)
|
||
# max_model_len=64K: required=256, parts=32, TGPP=8 (no waste)
|
||
# max_model_len=128K: required=512, parts=32, TGPP=16
|
||
MAX_PARTITIONS = 32
|
||
required_num_partitions = (
|
||
sizing_max_seq_len + kv_compute_block - 1
|
||
) // kv_compute_block
|
||
# max_num_kv_splits acts as a parallelism floor (ensure at least this
|
||
# many CTAs along grid.z), but is itself capped by MAX_PARTITIONS.
|
||
parallelism_floor = min(MAX_PARTITIONS, max(1, max_num_kv_splits))
|
||
# Cap num_partitions at MAX_PARTITIONS, but never go below the floor.
|
||
num_partitions_actual = max(
|
||
parallelism_floor,
|
||
min(MAX_PARTITIONS, required_num_partitions),
|
||
)
|
||
# Round up to next power of 2 for the Triton reducer's tl.arange(0, N)
|
||
# constraint (Triton requires power-of-2 ≥ 2).
|
||
num_partitions = max(2, triton.next_power_of_2(num_partitions_actual))
|
||
# Now derive tile_groups_per_partition (TGPP) so total coverage
|
||
# ``num_partitions * TGPP * KV_COMPUTE_BLOCK`` is >= sizing_max_seq_len.
|
||
# This guarantees no work is dropped at runtime regardless of seq_len.
|
||
# Round TGPP up to the next power-of-2 so the JIT cache key is bounded
|
||
# to a small set of values (e.g. {1, 2, 4, 8, 16}) — limits the number
|
||
# of distinct kernel binaries that need to be compiled across requests.
|
||
_tgpp_required = max(
|
||
1,
|
||
(required_num_partitions + num_partitions - 1) // num_partitions,
|
||
)
|
||
tile_groups_per_partition = int(triton.next_power_of_2(_tgpp_required))
|
||
|
||
# ---- Pooled buffers (no per-call cudaMalloc / memset) ----------
|
||
# The FlyDSL kernel writes the FULL [B, Hk, P, QG, D] segm_out and the
|
||
# FULL [B, Hk, P, QG] segm_max / segm_sum (running_max=-inf, sum=0 for
|
||
# empty partitions are stored by the kernel itself), so uninitialized
|
||
# buffers from the pool are safe to reuse.
|
||
device = query.device
|
||
pool_bufs = _SEGM_POOL.get(
|
||
B,
|
||
Hk,
|
||
Hq,
|
||
num_partitions,
|
||
QG,
|
||
D,
|
||
device,
|
||
query.dtype,
|
||
)
|
||
segm_out = pool_bufs["segm_out"]
|
||
segm_max = pool_bufs["segm_max"]
|
||
segm_sum = pool_bufs["segm_sum"]
|
||
if output_buf is None:
|
||
output = pool_bufs["output"]
|
||
else:
|
||
output = output_buf[:B] if output_buf.shape[0] != B else output_buf
|
||
|
||
# ---- q rotation: GEMM q @ PiT via stable pooled buffers -------------
|
||
_q_float = pool_bufs["q_float"] # [B, Hq, D] fp32, stable
|
||
_q_rot_f32 = pool_bufs["q_rot_fp32"] # [B, Hq, D] fp32, stable mm output
|
||
_q_rot_out = pool_bufs["q_rot"] # [B, Hq, D] query.dtype, stable
|
||
_q_float.copy_(query) # bf16 → fp32 in-place, no alloc
|
||
# mm into stable fp32 buffer via out= to avoid fresh allocations.
|
||
# PiT_f32 is cached on buf_holder (stable ptr since first layer warmup).
|
||
torch.mm(
|
||
_q_float.view(B * Hq, D),
|
||
PiT_f32,
|
||
out=_q_rot_f32.view(B * Hq, D), # in-place into pool buffer
|
||
)
|
||
_q_rot_out.copy_(_q_rot_f32) # fp32 → bf16, into stable buf
|
||
q_for_kernel = _q_rot_out # stable ptr for kernel launch
|
||
|
||
# ---- FlyDSL kernel launch -------------------------------------------
|
||
max_bps = int(block_table.shape[1])
|
||
use_hw_tr = _hw_tr_enabled()
|
||
launch = _get_kernel(
|
||
Hk,
|
||
num_partitions,
|
||
max_bps,
|
||
scale,
|
||
QG,
|
||
block_size,
|
||
use_hw_v_transpose=use_hw_tr,
|
||
num_seqs_hint=int(B),
|
||
tile_groups_per_partition=int(tile_groups_per_partition),
|
||
)
|
||
# Zero-overhead one-shot info log (replaces logger.info_once which
|
||
# hashes its format string on every call to dedup).
|
||
global _LOG_INVOKED_ONCE, _LOG_SINKS_WARNED, _LOG_NORM_WARNED
|
||
if not _LOG_INVOKED_ONCE:
|
||
_LOG_INVOKED_ONCE = True
|
||
logger.info(
|
||
"FlyDSL launcher invoked (UNIFORM_BATCH cudagraph): "
|
||
"B=%d Hk=%d Hq=%d D=%d QG=%d num_partitions=%d (actual=%d, "
|
||
"cap=%d) TGPP=%d max_bps=%d block_size=%d max_seq_len=%d "
|
||
"hw_v_transpose=%s (coverage=%d tokens, worst_case=%d tokens)",
|
||
B,
|
||
Hk,
|
||
Hq,
|
||
D,
|
||
QG,
|
||
num_partitions,
|
||
num_partitions_actual,
|
||
MAX_PARTITIONS,
|
||
tile_groups_per_partition,
|
||
max_bps,
|
||
int(block_size),
|
||
int(max_seq_len),
|
||
use_hw_tr,
|
||
num_partitions * tile_groups_per_partition * kv_compute_block,
|
||
worst_case_max_seq_len,
|
||
)
|
||
launch(
|
||
segm_out,
|
||
segm_sum,
|
||
segm_max,
|
||
q_for_kernel,
|
||
kv_cache,
|
||
centroids_c,
|
||
block_table,
|
||
seq_lens,
|
||
B,
|
||
Hk,
|
||
num_partitions,
|
||
torch.cuda.current_stream(),
|
||
)
|
||
|
||
# ---- Reduce partitions -> [B, Hq, D] --------------------------------
|
||
_reduce_partitions[(B, Hq)](
|
||
output_ptr=output,
|
||
segm_out_ptr=segm_out,
|
||
segm_max_ptr=segm_max,
|
||
segm_sum_ptr=segm_sum,
|
||
out_stride_n=output.stride(0),
|
||
out_stride_h=output.stride(1),
|
||
NUM_KV_HEADS=Hk,
|
||
QG=QG,
|
||
NUM_PARTS=num_partitions,
|
||
HEAD_SIZE=D,
|
||
)
|
||
if sinks is not None and not _LOG_SINKS_WARNED:
|
||
_LOG_SINKS_WARNED = True
|
||
logger.warning(
|
||
"FlyDSL launcher: sinks ignored (NYI). Disable sinks if they "
|
||
"are required for correctness."
|
||
)
|
||
# ── norm_correction is honored IMPLICITLY ───────────────────────────
|
||
# When the model was stored with norm_correction=True (the *_nc presets
|
||
# turboquant_4bit_nc / k3v4_nc / 3bit_nc), the per-token K-norm scalar
|
||
# was pre-folded to ||k_t|| / ||c_t|| at store time by the TurboQuant
|
||
# store path. The decode kernel just multiplies `c_vals * stored_knorm`,
|
||
# which then equals `(c_vals / ||c_t||) * ||k_t||` — exactly the
|
||
# unit-norm-renormalized centroid times the original key norm. The
|
||
# Triton decode does the identical multiply, so both decoders honor
|
||
# norm_correction equivalently and there is nothing to "do" at decode
|
||
# time. The launcher's `norm_correction` arg is only kept for API
|
||
# parity; we emit a one-shot INFO log to make the contract explicit.
|
||
if norm_correction and not _LOG_NORM_WARNED:
|
||
_LOG_NORM_WARNED = True
|
||
logger.info(
|
||
"FlyDSL launcher: norm_correction honored implicitly via "
|
||
"pre-folded stored K-norm (cf. triton_turboquant_store step 3); "
|
||
"no decode-time work required, identical to the SoA decode behavior."
|
||
)
|
||
return output
|