# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """FlyDSL launcher for the optimized UltraQuant 4-bit D=256 decode kernel. The production kernel uses scaled FP4×E4M3 QK MFMA, native V conversion, query hoisting, in-kernel Walsh-Hadamard rotation for GQA-6/8/16, and strided tile-group scheduling. These choices are fixed and are not environment-tunable. """ from __future__ import annotations from typing import Any import torch from vllm.logger import init_logger from vllm.triton_utils import triton logger = init_logger(__name__) _FLYDSL_AVAILABLE: bool | None = None _ULTRAQUANT_MOD: Any = None _FLYC: Any = None _FX: Any = None _TYPING_T: Any = None _CC: Any = None _IR: Any = None def is_ultraquant_flydsl_available() -> bool: """Return whether the gfx950 UltraQuant FlyDSL kernel can be loaded.""" global _FLYDSL_AVAILABLE, _ULTRAQUANT_MOD global _FLYC, _FX, _TYPING_T, _CC, _IR if _FLYDSL_AVAILABLE is not None: return _FLYDSL_AVAILABLE try: from vllm.platforms.rocm import on_gfx950 if not on_gfx950(): _FLYDSL_AVAILABLE = False return False import flydsl.compiler as flyc import flydsl.expr as fx from flydsl._mlir import ir from flydsl.compiler.kernel_function import CompilationContext from flydsl.expr.typing import T from vllm.v1.attention.ops.flydsl_kernels import ( ultraquant_decode_hd256 as ultraquant_mod, ) _FLYC = flyc _FX = fx _TYPING_T = T _CC = CompilationContext _IR = ir _ULTRAQUANT_MOD = ultraquant_mod _FLYDSL_AVAILABLE = True logger.info_once("FlyDSL UltraQuant D=256 decode is available") except Exception as ex: # noqa: BLE001 _FLYDSL_AVAILABLE = False logger.warning_once( "FlyDSL UltraQuant decode is unavailable (%s); using Triton fallback.", ex, ) return _FLYDSL_AVAILABLE _ULTRAQUANT_FLYDSL_GQA = (6, 8, 16) def ultraquant_flydsl_decode_eligible( *, head_size: int, num_kv_groups: int, has_sinks: bool, sliding_window: int | None, flydsl_loaded: bool, ) -> bool: """Return whether decode should use the gfx950 FlyDSL kernel. Production FlyDSL covers D=256 and GQA in {6, 8, 16}. Sinks and sliding window fall back to unified Triton. ``flydsl_loaded`` is the process-level compiler/hardware probe so this policy can be unit-tested without a GPU. """ return ( flydsl_loaded and head_size == 256 and num_kv_groups in _ULTRAQUANT_FLYDSL_GQA and not has_sinks and not (sliding_window and sliding_window > 0) ) # -- Kernel module cache ------------------------------------------------------- # Compiling a FlyDSL kernel is expensive; cache by the constexprs that # parameterize stride math: num_kv_heads, num_partitions, max_blocks_per_seq, # scale, query_group_size, kv_block_size, padded_slot, hw_v_transpose, # tile_groups_per_partition. _KERN_CACHE: dict[tuple, Any] = {} _LOG_INVOKED_ONCE: bool = False # Per-device cached FP4 E2M1 value table (used as the kernel "centroids" LUT). _FP4_LUT_CACHE: dict[tuple[str, torch.dtype], torch.Tensor] = {} def _fp4_value_lut(device: torch.device) -> torch.Tensor: """Return the fixed 16-entry FP4 E2M1 value table as fp32 [16] on device. Indexed by the stored 4-bit E2M1 code, so ``lut[nibble]`` is the dequant value directly (matches ``format.FP4_BITS_TO_VALUE``). """ from vllm.v1.attention.ops.ultraquant.format import FP4_BITS_TO_VALUE key = (str(device), torch.float32) t = _FP4_LUT_CACHE.get(key) if t is None: t = torch.tensor( FP4_BITS_TO_VALUE, dtype=torch.float32, device=device ).contiguous() _FP4_LUT_CACHE[key] = t return t # Per-device cached qperm head-dim permutation for scaled QK MFMA. # qperm[phys] = the natural head-dim placed at physical operand position phys. # It realigns the fixed FP4xFP8 scaled-MFMA contraction so that, with K fed # native-contiguous, Q's columns pair correctly. Folded into PiT on the host # (PiT[:, qperm]) so the Q rotation matmul emits already-permuted q_rot at zero # extra runtime cost. Derived + validated in tmp_qk_scaled_mfma.py (contigK). _QPERM_CACHE: dict[str, torch.Tensor] = {} def _kmap_phys(group: int, s: int) -> int: # Physical operand position of group's element s for the K=128 fp4/fp8 # scaled MFMA (matches tmp_qk_scaled_mfma.py::_kmap). return 32 * (s >> 4) + 64 * (group % 2) + 16 * (group // 2) + (s & 15) def _qperm_index( device: torch.device, D: int = 128, group_size: int = 32 ) -> torch.Tensor: # _kmap_phys describes ONE K=128 contraction (4 groups of 32). A head dim # wider than 128 is covered by ceil(D/128) back-to-back MFMA issues, each # contracting its own independent 128-wide slice, so the same map repeats # per slice: qperm = [qperm128, qperm128 + 128, ...]. Applying _kmap_phys # with group >= 4 instead yields positions outside the slice and is not a # permutation at all. key = f"{device}:{D}:{group_size}" t = _QPERM_CACHE.get(key) if t is None: k_per_mfma = 128 assert D % k_per_mfma == 0, ( f"scaled QK MFMA needs D to be a multiple of {k_per_mfma}, got {D}" ) groups_per_mfma = k_per_mfma // group_size qperm = [0] * D for half in range(D // k_per_mfma): base = half * k_per_mfma for g in range(groups_per_mfma): for s in range(group_size): qperm[base + _kmap_phys(g, s)] = base + g * group_size + s assert sorted(qperm) == list(range(D)), "qperm is not a permutation" t = torch.tensor(qperm, dtype=torch.long, device=device).contiguous() _QPERM_CACHE[key] = t return t def _detect_max_capture_B() -> int: try: from vllm.config import get_current_vllm_config cfg = get_current_vllm_config() candidates: list[int] = [] sizes = cfg.compilation_config.cudagraph_capture_sizes if sizes: candidates.append(int(max(sizes))) sched = getattr(cfg, "scheduler_config", None) if sched is not None and getattr(sched, "max_num_seqs", None): candidates.append(int(sched.max_num_seqs)) if candidates: return max(candidates) except Exception: # noqa: BLE001 pass return 512 class _SegmBufPool: """Single-bucket buffer pool for segm_out/segm_max/segm_sum/output + the pooled Q-rotation intermediates. Identical to the TQ pool except that ultraquant needs an extra ``q_fp8`` (float8_e4m3fn) slot for the Q precision haircut (``q_rot_fp32 -> fp8_e4m3 -> bf16``) so that no fresh allocation lands in the HIP graph memory pool post-capture. """ __slots__ = ("_bufs", "_max_B") def __init__(self) -> None: self._bufs: dict[tuple, dict[str, torch.Tensor]] = {} self._max_B: int | None = None def get( self, B: int, Hk: int, Hq: int, num_partitions: int, QG: int, D: int, device: torch.device, q_dtype: torch.dtype, ) -> dict[str, torch.Tensor]: if self._max_B is None: self._max_B = _detect_max_capture_B() B_bucket = max(self._max_B, int(B)) key = ( int(Hk), int(Hq), int(num_partitions), int(QG), int(D), str(device), q_dtype, ) bufs = self._bufs.get(key) if bufs is None or bufs["segm_out"].shape[0] < B_bucket: if bufs is not None and bufs["segm_out"].shape[0] < B_bucket: # First-time grow: warn so user knows cudagraphs may be # invalidated. This should only happen during warmup or in # always-eager configs (a batch larger than any captured # cudagraph size). logger.warning_once( "FlyDSL UltraQuant buffer pool growing from %d " "to %d (B=%d). If this happens AFTER cudagraph warmup the " "captured graphs hold stale pointers and will GPU-fault.", bufs["segm_out"].shape[0], B_bucket, B, ) bufs = { "segm_out": torch.empty( (B_bucket, Hk, num_partitions, QG, D), dtype=torch.bfloat16, device=device, ), "segm_max": torch.empty( (B_bucket, Hk, num_partitions, QG), dtype=torch.float32, device=device, ), "segm_sum": torch.empty( (B_bucket, Hk, num_partitions, QG), dtype=torch.float32, device=device, ), "output": torch.empty( (B_bucket, Hq, D), dtype=q_dtype, device=device, ), "q_rot": torch.empty( (B_bucket, Hq, D), dtype=q_dtype, device=device, ), "q_float": torch.empty( (B_bucket, Hq, D), dtype=torch.float32, device=device, ), "q_rot_fp32": torch.empty( (B_bucket, Hq, D), dtype=torch.float32, device=device, ), # ultraquant-specific: E4M3 haircut intermediate (pooled so the # cast does not allocate a fresh tensor post-capture). "q_fp8": torch.empty( (B_bucket, Hq, D), dtype=torch.float8_e4m3fn, device=device, ), } self._bufs[key] = bufs self._max_B = B_bucket logger.info_once( "FlyDSL UltraQuant buffer pool allocated " "shape=(Hk=%d, Hq=%d, P=%d, QG=%d, D=%d, dtype=%s) " "B_bucket=%d. VRAM = %.1f MiB / shape.", Hk, Hq, num_partitions, QG, D, q_dtype, B_bucket, sum(t.numel() * t.element_size() for t in bufs.values()) / (1 << 20), ) return { "segm_out": bufs["segm_out"][:B], "segm_max": bufs["segm_max"][:B], "segm_sum": bufs["segm_sum"][:B], "output": bufs["output"][:B], "q_rot": bufs["q_rot"][:B], "q_float": bufs["q_float"][:B], "q_rot_fp32": bufs["q_rot_fp32"][:B], "q_fp8": bufs["q_fp8"][:B], } _SEGM_POOL = _SegmBufPool() _GET_KERNEL_STATS = {"hits": 0, "misses": 0, "build_total_s": 0.0} def _get_kernel( num_kv_heads: int, num_partitions: int, max_blocks_per_seq: int, scale: float, query_group_size: int, kv_block_size: int, padded_slot: int, fuse_qrot: bool, num_seqs_hint: int, tile_groups_per_partition: int, stride_q_seq: int, stride_q_head: int, ): key = ( num_kv_heads, num_partitions, max_blocks_per_seq, round(scale, 8), query_group_size, kv_block_size, padded_slot, fuse_qrot, tile_groups_per_partition, stride_q_seq, stride_q_head, ) cached = _KERN_CACHE.get(key) if cached is not None: _GET_KERNEL_STATS["hits"] += 1 return cached assert is_ultraquant_flydsl_available() import time build_start = time.perf_counter() kmod = _ULTRAQUANT_MOD assert kmod is not None kfn = kmod.build_ultraquant_decode_hd256_module( num_seqs=num_seqs_hint, num_kv_heads=num_kv_heads, num_partitions=num_partitions, padded_slot=padded_slot, max_blocks_per_seq=max_blocks_per_seq, softmax_scale=scale, query_group_size=query_group_size, kv_block_size=kv_block_size, tile_groups_per_partition=tile_groups_per_partition, fuse_qrot=fuse_qrot, stride_q_seq=stride_q_seq, stride_q_head=stride_q_head, ) allocator = kmod.allocator flyc = _FLYC fx = _FX T = _TYPING_T CompilationContext = _CC ir_mod = _IR @flyc.jit def _launch( out, es, ml, q, kvc, cents, bt, sl, gx: fx.Int32, # type: ignore[name-defined] gy: fx.Int32, # type: ignore[name-defined] gz: fx.Int32, # type: ignore[name-defined] stream: fx.Stream, # type: ignore[name-defined] ): allocator.finalized = False ctx = CompilationContext.get_current() with ir_mod.InsertionPoint(ctx.gpu_module_body): allocator.finalize() from flydsl.expr import arith grid_x = arith.index_cast(T.index, gx.ir_value()) grid_y = arith.index_cast(T.index, gy.ir_value()) grid_z = arith.index_cast(T.index, gz.ir_value()) kfn(out, es, ml, q, kvc, cents, bt, sl).launch( grid=(grid_x, grid_y, grid_z), block=(kmod.BLOCK_THREADS, 1, 1), stream=stream, ) _KERN_CACHE[key] = _launch build_time = time.perf_counter() - build_start _GET_KERNEL_STATS["misses"] += 1 _GET_KERNEL_STATS["build_total_s"] += build_time logger.info( "Built FlyDSL UltraQuant kernel #%d in %.2fs (total %.1fs), key=%s", _GET_KERNEL_STATS["misses"], build_time, _GET_KERNEL_STATS["build_total_s"], key, ) return _launch # -- Partition reducer --------------------------------------------------------- from vllm.triton_utils import tl # noqa: E402 @triton.jit def _reduce_partitions( output_ptr, # [N, Hq, D] in OUT_DTYPE segm_out_ptr, # [N, Hk, P, QG, D] bf16 segm_max_ptr, # [N, Hk, P, QG] fp32 segm_sum_ptr, # [N, Hk, P, QG] fp32 out_stride_n: tl.int64, out_stride_h: tl.int64, NUM_KV_HEADS: tl.constexpr, QG: tl.constexpr, NUM_PARTS: tl.constexpr, HEAD_SIZE: tl.constexpr, BLOCK_D: tl.constexpr, ): n = tl.program_id(0) hq = tl.program_id(1) kv_h = hq // QG qg = hq % QG # Splitting the head-dim across programs is what gives this kernel enough # parallelism to use the GPU: the reduction runs over PARTITIONS, so every # head-dim column is independent. A (B, Hq) grid is only 32 workgroups at # B=4 -- ~12% of a 256-CU part -- which is why it moved 8.4 MB at just # ~995 GB/s. The partition reduction itself is untouched (same values, same # order), so the result is bit-identical; only the columns each program # owns change. The [P] max/sum vectors get re-read per D-block, which is # 4 KB of duplicated traffic against a 262 KB tile. msum_base = n * (NUM_KV_HEADS * NUM_PARTS * QG) + kv_h * (NUM_PARTS * QG) + qg so_base = ( n * (NUM_KV_HEADS * NUM_PARTS * QG * HEAD_SIZE) + kv_h * (NUM_PARTS * QG * HEAD_SIZE) + qg * HEAD_SIZE ) p_off = tl.arange(0, NUM_PARTS) d_off = tl.program_id(2) * BLOCK_D + tl.arange(0, BLOCK_D) m_idx = msum_base + p_off * QG seg_max = tl.load(segm_max_ptr + m_idx) seg_sum = tl.load(segm_sum_ptr + m_idx) valid = seg_max > float("-inf") overall_max = tl.max(tl.where(valid, seg_max, float("-inf"))) rescale = tl.where(valid, tl.exp(seg_max - overall_max), 0.0) seg_sum_rescaled = seg_sum * rescale overall_sum = tl.sum(seg_sum_rescaled) so_idx = so_base + p_off[:, None] * (QG * HEAD_SIZE) + d_off[None, :] seg_out = tl.load(segm_out_ptr + so_idx).to(tl.float32) weighted = seg_out * seg_sum_rescaled[:, None] acc_sum = tl.sum(weighted, axis=0) acc = tl.where(overall_sum > 0.0, acc_sum / overall_sum, 0.0) out_off = n * out_stride_n + hq * out_stride_h + d_off tl.store(output_ptr + out_off, acc.to(output_ptr.dtype.element_ty)) # -- Fused Q rotation + E4M3 haircut (Triton) --------------------------------- # Replaces the 4-op torch prologue (copy bf16->fp32, mm by PiT, copy fp32->e4m3, # copy e4m3->bf16) with ONE kernel. The win is kernel COUNT, not bytes: each of # those small ops costs ~2-4us of fixed GPU-side cost here largely independent of # element count (verified: sweeping the pool's B_bucket 512->1, i.e. 512x fewer # elements in the copies, moved the floor 0%). Under CUDA graphs the host launch # cost is already gone, so only per-kernel GPU cost remains and collapsing 4 # kernels into 1 is the lever that is left. @triton.jit def _fused_q_rot_haircut( q_ptr, # [B, Hq, D] query dtype (bf16/fp16), strided pit_ptr, # [D, D] fp32 contiguous (already qperm-permuted) out_ptr, # [B_bucket, Hq, D] out dtype, contiguous M, # B * Hq (runtime) q_stride_n: tl.int64, q_stride_h: tl.int64, HQ: tl.constexpr, D: tl.constexpr, HAIRCUT: tl.constexpr, # round through E4M3 (fp8-consuming QK MFMA paths) BLOCK_M: tl.constexpr, BLOCK_K: tl.constexpr, BLOCK_D: tl.constexpr, ): pid_m = tl.program_id(0) pid_d = tl.program_id(1) m_off = pid_m * BLOCK_M + tl.arange(0, BLOCK_M) m_mask = m_off < M n = m_off // HQ h = m_off % HQ # Splitting the OUTPUT columns across programs (not just the rows) is what # gives this kernel parallelism at small batch: M = B*Hq is only 8 at B=1, # so a row-only grid launches ONE workgroup and serializes the whole # 256x256 rotation on a single CU. The K reduction is untouched by the # split, so the result stays bit-identical. d_off = pid_d * BLOCK_D + tl.arange(0, BLOCK_D) acc = tl.zeros([BLOCK_M, BLOCK_D], dtype=tl.float32) for k0 in range(0, D, BLOCK_K): k_off = k0 + tl.arange(0, BLOCK_K) q_idx = n[:, None] * q_stride_n + h[:, None] * q_stride_h + k_off[None, :] q_blk = tl.load(q_ptr + q_idx, mask=m_mask[:, None], other=0.0) p_blk = tl.load(pit_ptr + k_off[:, None] * D + d_off[None, :]) acc = tl.dot(q_blk.to(tl.float32), p_blk, acc) if HAIRCUT: # fp32 -> E4M3 -> fp32. Mirrors the torch path's # q_rot_fp32 -> float8_e4m3fn -> bf16 round trip (e4m3 is a subset of # bf16, so the second hop is exact and folds away here). acc = acc.to(tl.float8e4nv).to(tl.float32) out_idx = m_off[:, None] * D + d_off[None, :] tl.store(out_ptr + out_idx, acc.to(out_ptr.dtype.element_ty), mask=m_mask[:, None]) # Prologue Q-rotation fusion (fallback for non-hd256 / in-kernel-WHT-off paths). # Cleared the accuracy ladder (offline parity -> autoregressive drift -> paired # GSM8K); pinned ON as part of the validated production config. _FUSE_Q_ROT = True # Stronger form: do the rotation INSIDE the decode kernel as a Walsh-Hadamard # butterfly and drop the prologue dispatch entirely. The prologue is ~7us at # B=4 and that is almost all fixed per-dispatch cost (sweeping its grid 4x moves # it 0%), so deleting the launch -- not optimizing it -- is the win. # Default ON: verified bit-exact (torch.equal) against the prologue path across # B={1,2,3,4,8,16} x seq={4k,8k,16k,32k,50021,75k}, including non-power-of-2 # batch and seq. _FUSE_QROT_INKERNEL = True def _run_fused_q_rot(query, PiT_used, out, B, Hq, D, haircut): """Launch the fused prologue. ``out`` is the pooled [B_bucket, Hq, D] buf.""" M = B * Hq BLOCK_M = 16 BLOCK_K = 32 BLOCK_D = 64 grid = (triton.cdiv(M, BLOCK_M), triton.cdiv(D, BLOCK_D)) _fused_q_rot_haircut[grid]( query, PiT_used, out, M, query.stride(0), query.stride(1), HQ=int(Hq), D=int(D), HAIRCUT=bool(haircut), BLOCK_M=BLOCK_M, BLOCK_K=BLOCK_K, BLOCK_D=BLOCK_D, ) # -- Public launcher ---------------------------------------------------------- def flydsl_ultraquant_decode_attention( query: torch.Tensor, kv_cache: torch.Tensor, block_table: torch.Tensor, seq_lens: torch.Tensor, scale: float, PiT: torch.Tensor | None = None, max_seq_len: int = 0, output_buf: torch.Tensor | None = None, buf_holder: Any = None, max_num_kv_splits: int = 32, sinks: torch.Tensor | None = None, ) -> torch.Tensor: """Launch the optimized FlyDSL UltraQuant D=256 decode.""" if not is_ultraquant_flydsl_available(): raise RuntimeError( "FlyDSL UltraQuant decode requires gfx950 and importable FlyDSL." ) if sinks is not None: raise NotImplementedError("FlyDSL UltraQuant decode does not support sinks") B, Hq, D = query.shape Hk = kv_cache.shape[2] block_size = kv_cache.shape[1] padded_slot = int(kv_cache.shape[3]) QG = Hq // Hk assert D == 256, f"UltraQuant FlyDSL requires head_dim=256, got {D}" assert block_size in (16, 32, 64, 128, 256), ( f"UltraQuant FlyDSL supports kv_block_size 16/32/64/128/256, got {block_size}" ) assert QG in _ULTRAQUANT_FLYDSL_GQA, ( f"UltraQuant FlyDSL supports GQA factor " f"{', '.join(map(str, _ULTRAQUANT_FLYDSL_GQA))}, got {QG}" ) device = query.device # ---- PiT (Hadamard) — cache the contiguous fp32 form on the layer ----- PiT_f32: torch.Tensor if buf_holder is not None: PiT_f32 = getattr(buf_holder, "_ultraquant_PiT_f32", None) if PiT_f32 is None: assert PiT is not None, ( "UltraQuant launcher requires PiT (the Hadamard rotation)" ) PiT_f32 = PiT if PiT.dtype == torch.float32 else PiT.to(torch.float32) if not PiT_f32.is_contiguous(): PiT_f32 = PiT_f32.contiguous() buf_holder._ultraquant_PiT_f32 = PiT_f32 else: assert PiT is not None, ( "UltraQuant launcher requires PiT (the Hadamard rotation)" ) PiT_f32 = PiT if PiT.dtype == torch.float32 else PiT.to(torch.float32) if not PiT_f32.is_contiguous(): PiT_f32 = PiT_f32.contiguous() centroids_c = _fp4_value_lut(device) # ---- Partition count (FA2 split-KV) ----------------------------------- kv_compute_block = _ULTRAQUANT_MOD.KV_COMPUTE_BLOCK worst_case_max_seq_len = int(block_table.shape[1]) * int(block_size) if max_seq_len <= 0: max_seq_len = worst_case_max_seq_len sizing_max_seq_len = worst_case_max_seq_len # Split-KV parallelism cap. The old default (32) silently serialized long # contexts: once required_num_partitions exceeds the cap, TGPP grows and each # workgroup walks MORE tile-groups instead of the grid getting wider. The # decode kernel is latency-bound at ~1-2 waves/CU (grid = B*Hk*P workgroups # of a single wavefront), so that serialization dominated long-context decode. # Raising the cap to 256 keeps TGPP ~1 and measured (B=16, rocprof, per step # decode+reduce+store): 16k 82.7->40.0us, 32k 152.0->55.1us, 64k 339.4->95.7us, # 131k 671.3->179.0us (2.1x - 3.8x, growing with context). 512 regresses # because _reduce_partitions cost grows with partition count. # Cost: the segm pool scales with P (~8x vs P=32). # BATCH-ADAPTIVE split-KV cap (low-concurrency occupancy fix). # rocprof (B=4, 75k): the decode kernel is occupancy/latency-bound, not # compute-bound -- OccupancyPercent 5.8%, MfmaUtil 4.5%, VALUBusy 14.5% # (stalled ~85% on HBM), VALUUtilization 90% (no divergence). The grid is # B*Hk*num_partitions single-wavefront workgroups; at B=4/Hk=1/P=256 that's # only 1024 wg = 4/CU, too few to hide memory latency. Splitting MORE at low # batch adds resident wavefronts and cuts ITL; at high batch the grid is # already full so extra partitions only inflate _reduce_partitions + empty- # partition overhead. Measured end-to-end (graphs on, TP=8, uniform 75k, # median ITL vs kv8): cap512 vs cap256 -> C4 +9.0%->+4.9%, C8 +6.6%->+2.5% # (better), C16 +2.8%->+4.3% (worse). So use 512 at low batch, 256 at high. # This is only SAFE because the build-local-allocator fix in # ultraquant_decode_hd256.py lets the two num_partitions variants (256 & 512) # coexist across CUDA-graph batch buckets without the @flyc.jit # "global 'allocator' changed since first compile" drift crash. MAX_PARTITIONS = 512 if B <= 8 else 256 required_num_partitions = ( sizing_max_seq_len + kv_compute_block - 1 ) // kv_compute_block parallelism_floor = min(MAX_PARTITIONS, max(1, max_num_kv_splits)) num_partitions_actual = max( parallelism_floor, min(MAX_PARTITIONS, required_num_partitions), ) # Cap partitions by how much work there actually IS to split. Nothing above # forces a partition to hold a useful number of tokens -- max_num_kv_splits # sets a parallelism FLOOR -- so a short context gets split into partitions # of a handful of tokens each, where every partition is almost entirely # fixed cost (and each one still writes a full QG x D partial for the # reduce to read back). Measured sweet spot is ~130-147 tokens/partition, # roughly flat in between; next_power_of_2 below rounds this target up, so # the effective range lands at ~96-192 tokens. At B=4 this is worth 27.9 -> # 24.4 us at seq=2048 and 27.0 -> 24.8 us at seq=16384, and is inert once # the context is long enough to want every partition (seq >= ~64k). _tok_per_part = 192 if _tok_per_part > 0: _useful_parts = max( 1, (int(sizing_max_seq_len) + _tok_per_part - 1) // _tok_per_part ) num_partitions_actual = min(num_partitions_actual, _useful_parts) num_partitions = max(2, triton.next_power_of_2(num_partitions_actual)) _tgpp_required = max( 1, (required_num_partitions + num_partitions - 1) // num_partitions, ) tile_groups_per_partition = int(triton.next_power_of_2(_tgpp_required)) # ---- Pooled buffers --------------------------------------------------- pool_bufs = _SEGM_POOL.get( B, Hk, Hq, num_partitions, QG, D, device, query.dtype, ) segm_out = pool_bufs["segm_out"] segm_max = pool_bufs["segm_max"] segm_sum = pool_bufs["segm_sum"] if output_buf is None: output = pool_bufs["output"] else: output = output_buf[:B] if output_buf.shape[0] != B else output_buf # Fold the scaled-MFMA head-dim permutation into PiT's columns so the # WHT / prologue produces native operand order. Always apply the E4M3 # haircut: scaled QK consumes Q as E4M3. PiT_used = PiT_f32 qperm_idx = _qperm_index(device, D=D) if buf_holder is not None: PiT_used = getattr(buf_holder, "_ultraquant_PiT_perm_f32", None) if PiT_used is None: PiT_used = PiT_f32.index_select(1, qperm_idx).contiguous() buf_holder._ultraquant_PiT_perm_f32 = PiT_used else: PiT_used = PiT_f32.index_select(1, qperm_idx).contiguous() _q_float = pool_bufs["q_float"] _q_rot_f32 = pool_bufs["q_rot_fp32"] _q_fp8 = pool_bufs["q_fp8"] _q_rot_out = pool_bufs["q_rot"] _inkernel_qrot = ( _FUSE_QROT_INKERNEL and int(D) == 256 and QG in _ULTRAQUANT_FLYDSL_GQA and query.dim() == 3 and query.stride(2) == 1 and query.stride(1) == int(D) ) if _inkernel_qrot: q_for_kernel = query # The fused kernel indexes the head-dim contiguously; fall back if not. elif _FUSE_Q_ROT and query.stride(-1) == 1: # One kernel for the whole prologue instead of four ops. _run_fused_q_rot(query, PiT_used, _q_rot_out, B, Hq, D, haircut=True) else: _q_float.copy_(query) torch.mm( _q_float.view(B * Hq, D), PiT_used, out=_q_rot_f32.view(B * Hq, D), ) _q_fp8.copy_(_q_rot_f32) # fp32 -> e4m3 _q_rot_out.copy_(_q_fp8) # e4m3 -> bf16 if not _inkernel_qrot: q_for_kernel = _q_rot_out # ---- FlyDSL kernel launch -------------------------------------------- max_bps = int(block_table.shape[1]) launch = _get_kernel( Hk, num_partitions, max_bps, scale, QG, block_size, padded_slot, fuse_qrot=_inkernel_qrot, num_seqs_hint=int(B), tile_groups_per_partition=int(tile_groups_per_partition), stride_q_seq=int(q_for_kernel.stride(0)), stride_q_head=int(q_for_kernel.stride(1)), ) global _LOG_INVOKED_ONCE if not _LOG_INVOKED_ONCE: _LOG_INVOKED_ONCE = True logger.info( "FlyDSL UltraQuant launcher invoked: B=%d Hk=%d Hq=%d D=%d QG=%d " "num_partitions=%d (actual=%d, cap=%d) TGPP=%d max_bps=%d " "block_size=%d padded_slot=%d max_seq_len=%d " "(coverage=%d tokens, worst_case=%d tokens)", B, Hk, Hq, D, QG, num_partitions, num_partitions_actual, MAX_PARTITIONS, tile_groups_per_partition, max_bps, int(block_size), padded_slot, int(max_seq_len), num_partitions * tile_groups_per_partition * kv_compute_block, worst_case_max_seq_len, ) launch( segm_out, segm_sum, segm_max, q_for_kernel, kv_cache, centroids_c, block_table, seq_lens, B, Hk, num_partitions, torch.cuda.current_stream(), ) # ---- Reduce partitions -> [B, Hq, D] --------------------------------- # 64 splits the head-dim 4 ways, taking the grid from 32 to 128 workgroups. # Bit-identical by construction (the partition reduction is per-column). _red_block_d = min(int(D), 64) _reduce_partitions[(B, Hq, triton.cdiv(int(D), _red_block_d))]( output_ptr=output, segm_out_ptr=segm_out, segm_max_ptr=segm_max, segm_sum_ptr=segm_sum, out_stride_n=output.stride(0), out_stride_h=output.stride(1), NUM_KV_HEADS=Hk, QG=QG, NUM_PARTS=num_partitions, HEAD_SIZE=D, BLOCK_D=_red_block_d, ) return output