[Qwen3.8-Flash-Next] Improve QSA sparse GQA for prefill and short-ctx decode (#54873)

Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg>
Co-authored-by: Kimi <noreply@moonshot.cn>
This commit is contained in:
Thien Tran
2026-09-04 06:08:02 -07:00
committed by GitHub
co-authored by Kimi
parent 8340fe1bb9
commit 31a8a26662
6 changed files with 365 additions and 106 deletions
+24 -4
View File
@@ -167,8 +167,19 @@ class QSAIndexer(nn.Module):
@property
def output_width(self) -> int:
"""Selection (index) columns per row."""
return self.token_topk + self.compress_ratio - 1
@property
def packed_output_width(self) -> int:
"""Packed selection-buffer width: output_width + 1.
The trailing column holds each row's valid-entry count (written by
the expand kernel) — never a token index; the sparse attention
kernel reads it as its tile-loop bound.
"""
return self.output_width + 1
def _metadata(
self,
) -> tuple[QSAForwardMetadata, QSAForwardMetadata] | None:
@@ -211,7 +222,14 @@ class QSAIndexer(nn.Module):
positions: torch.Tensor,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Return fixed-width request-relative token indices padded with ``-1``."""
"""Select each query row's token indices.
Returns the packed buffer of shape [num_tokens, output_width + 1]:
the leading ``output_width`` columns are ``-1``-padded
request-relative token indices, and the trailing column is the row's
valid-entry count (the attention kernel's loop bound, never a token
index).
"""
metadata = self._metadata()
if metadata is None:
@@ -219,11 +237,13 @@ class QSAIndexer(nn.Module):
if self.skip_topk and out is not None:
return out
result = torch.full(
(hidden_states.shape[0], self.output_width),
(hidden_states.shape[0], self.packed_output_width),
-1,
dtype=torch.int32,
device=hidden_states.device,
)
# Inert rows carry a zero valid count (empty loop bound), not -1.
result[:, -1] = 0
if out is not None:
out.copy_(result)
return out
@@ -357,11 +377,11 @@ class QSAIndexer(nn.Module):
if out is None:
out = torch.empty(
num_tokens,
self.output_width,
self.packed_output_width,
dtype=torch.int32,
device=q.device,
)
elif out.shape != (num_tokens, self.output_width):
elif out.shape != (num_tokens, self.packed_output_width):
raise ValueError("QSA selection output has an invalid shape")
num_decode_tokens = compressed_metadata.num_decode_tokens
+201 -43
View File
@@ -6,10 +6,14 @@ from __future__ import annotations
import torch
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
TritonWarmupTensor,
triton_scalar_specialization_rep,
)
from vllm.triton_utils import HAS_TRITON, tl, triton
@triton.jit
@triton.jit(do_not_specialize=["num_rows", "num_requests"])
def _qsa_sparse_paged_gqa_splitk_kernel(
q_ptr,
k_cache_ptr,
@@ -52,6 +56,12 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
request = tl.load(token_to_req_ptr + row)
safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
# The packed selection buffer carries one TRAILING COUNT COLUMN per row
# (column TOPK of a TOPK+1-wide buffer): the row's valid-entry count,
# written by the expand kernel. It is never a token index — the tile loop
# and the index load below only ever cover columns [0, TOPK).
valid_count = tl.load(indices_ptr + row * stride_indices_row + TOPK)
head_offsets = tl.arange(0, BLOCK_M)
dim_offsets = tl.arange(0, HEAD_DIM)
column_offsets = tl.arange(0, BLOCK_N)
@@ -70,10 +80,9 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
accumulator = tl.zeros((BLOCK_M, HEAD_DIM), dtype=tl.float32)
softmax_scale_log2: tl.constexpr = (HEAD_DIM**-0.5) * 1.4426950408889634
# Dynamic bounds avoid padded main-loop iterations for uneven splits.
split_tile_start = split_id * NUM_TILES // NUM_SPLITS
split_tile_end = (split_id + 1) * NUM_TILES // NUM_SPLITS
for tile in range(split_tile_start, split_tile_end):
tile_end = tl.minimum(NUM_TILES, tl.cdiv(tl.minimum(valid_count, TOPK), BLOCK_N))
for tile in range(split_id, tile_end, NUM_SPLITS):
columns = tile * BLOCK_N + column_offsets
logical_token = tl.load(
indices_ptr + row * stride_indices_row + columns,
@@ -156,29 +165,23 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
max_value + tl.math.log2(tl.maximum(normalizer, 1.0e-20)),
-float("inf"),
)
partial_row = (split_id * num_rows + row).to(tl.int64)
tl.store(
partial_output_ptr
+ (
(split_id * num_rows + row) * NUM_QUERY_HEADS
+ first_head
+ head_offsets[:, None]
)
+ (partial_row * NUM_QUERY_HEADS + first_head + head_offsets[:, None])
* HEAD_DIM
+ dim_offsets[None, :],
normalized_output,
mask=output_mask,
)
tl.store(
partial_lse_ptr
+ (split_id * num_rows + row) * NUM_QUERY_HEADS
+ first_head
+ head_offsets,
partial_lse_ptr + partial_row * NUM_QUERY_HEADS + first_head + head_offsets,
partial_lse,
mask=head_offsets < GROUP_SIZE,
)
@triton.jit
@triton.jit(do_not_specialize=["num_rows"])
def _qsa_merge_splitk_kernel(
partial_output_ptr,
partial_lse_ptr,
@@ -206,10 +209,10 @@ def _qsa_merge_splitk_kernel(
shifted = tl.where(split_mask & has_values, lse - lse_max, -float("inf"))
weights = tl.math.exp2(shifted)
denominator = tl.sum(weights, axis=0)
split_rows = split_offsets.to(tl.int64) * num_rows + row
partial_output = tl.load(
partial_output_ptr
+ ((split_offsets[:, None] * num_rows + row) * NUM_QUERY_HEADS + head)
* HEAD_DIM
+ (split_rows[:, None] * NUM_QUERY_HEADS + head) * HEAD_DIM
+ dim_offsets[None, :],
mask=split_mask[:, None],
other=0.0,
@@ -408,6 +411,41 @@ def _compress_qsa_groups_kernel(
)
def _select_config(
num_rows: int, num_kv_heads: int, use_prefill_config: bool, num_columns: int
) -> tuple[int, int, int, int]:
"""Select (block_n, num_warps, num_tiles, num_splits) for the kernel.
Tuned on GB300 for the Qwen3.8-Flash-Next TP1/TP2/TP4 shapes, keyed on
base_programs = num_rows * num_kv_heads. The bp > 2048 region splits on
use_prefill_config (capture-stable: at FULL-graph capture max_query_len is the
uniform decode/verify length).
"""
base_programs = num_rows * num_kv_heads
if base_programs > 2048:
BLOCK_N, target_splits, num_warps = (
(32, 1, 1) if use_prefill_config else (64, 1, 2)
)
elif base_programs <= 24:
BLOCK_N, target_splits, num_warps = 32, 64, 4
elif base_programs <= 32:
BLOCK_N, target_splits, num_warps = 32, 16, 1
elif base_programs <= 64:
BLOCK_N, target_splits, num_warps = 32, 8, 1
elif base_programs <= 128:
BLOCK_N, target_splits, num_warps = 32, 4, 1
elif base_programs <= 256:
BLOCK_N, target_splits, num_warps = 32, 8, 1
elif base_programs <= 512:
BLOCK_N, target_splits, num_warps = 64, 4, 2
else:
BLOCK_N, target_splits, num_warps = 64, 1, 2
num_tiles = triton.cdiv(num_columns, BLOCK_N)
# Never more splits than tiles, never empty.
num_splits = min(target_splits, num_tiles)
return BLOCK_N, num_warps, num_tiles, num_splits
def qsa_sparse_paged_attention(
q: torch.Tensor,
k_cache: torch.Tensor,
@@ -415,12 +453,17 @@ def qsa_sparse_paged_attention(
logical_indices: torch.Tensor,
block_table: torch.Tensor,
token_to_req: torch.Tensor,
use_prefill_config: bool,
out: torch.Tensor | None = None,
) -> torch.Tensor:
"""Run sparse GQA directly over paged BF16 K/V caches."""
"""Run sparse GQA directly over paged BF16 K/V caches.
if not q.is_cuda or not HAS_TRITON:
raise RuntimeError("paged QSA sparse attention requires CUDA and Triton")
logical_indices is the PACKED selection buffer: [rows, selection_width + 1]
with the trailing column holding each row's valid-entry count (written by
the expand kernel; never a token index). The kernel reads it as the
tile-loop bound. use_prefill_config only steers the top of the config table; see
_select_config.
"""
if q.ndim != 3 or k_cache.ndim != 4 or v_cache.shape != k_cache.shape:
raise ValueError("QSA sparse attention received invalid Q/K/V shapes")
if logical_indices.ndim != 2 or logical_indices.shape[0] != q.shape[0]:
@@ -429,8 +472,10 @@ def qsa_sparse_paged_attention(
raise ValueError("QSA sparse attention metadata has invalid shapes")
if not all(k_cache.shape[:3]) or not all(block_table.shape):
raise ValueError("QSA sparse attention cache and block table must be nonempty")
if logical_indices.shape[1] <= 0:
raise ValueError("QSA sparse attention requires a positive selection width")
if logical_indices.shape[1] < 2:
raise ValueError(
"QSA packed indices need selection columns plus the count column"
)
if q.shape[2] != k_cache.shape[3] or q.shape[1] % k_cache.shape[2]:
raise ValueError("QSA sparse attention requires valid grouped-query heads")
head_dim = q.shape[2]
@@ -444,6 +489,7 @@ def qsa_sparse_paged_attention(
assert q.stride(2) == k_cache.stride(3) == v_cache.stride(3) == 1
assert logical_indices.stride(1) == block_table.stride(1) == 1
assert token_to_req.stride(0) == 1
if out is None:
out = torch.empty_like(q)
if out.shape != q.shape:
@@ -455,26 +501,10 @@ def qsa_sparse_paged_attention(
group_size = q.shape[1] // k_cache.shape[2]
block_m = triton.next_power_of_2(group_size)
base_programs = q.shape[0] * k_cache.shape[2]
small_profile_limit = 8 if block_m <= 8 else 4
# Tuned on GB300 for the Qwen-Air TP1, TP2, and TP4 attention shapes.
# Narrow tiles favor decode; wide tiles improve throughput for prefill.
if base_programs <= small_profile_limit:
block_n, target_splits, partial_warps = 16, 64, 4
elif base_programs < 32:
block_n, target_splits, partial_warps = 16, 32, 4
elif base_programs <= 256:
block_n, target_splits, partial_warps = 64, 8, 2
elif base_programs <= 512:
block_n, target_splits, partial_warps = 64, 4, 2
else:
block_n, target_splits, partial_warps = 64, 1, 2
num_tiles = triton.cdiv(logical_indices.shape[1], block_n)
# Avoid empty splits when the selection width is smaller than the profile.
max_useful_splits = 1 << (num_tiles.bit_length() - 1)
num_splits = min(max_useful_splits, target_splits)
selection_width = logical_indices.shape[1] - 1 # trailing column is the count
block_n, partial_warps, num_tiles, num_splits = _select_config(
q.shape[0], k_cache.shape[2], use_prefill_config, selection_width
)
# Split=1 writes output directly and compiles out all workspace accesses.
if num_splits == 1:
@@ -518,7 +548,7 @@ def qsa_sparse_paged_attention(
q.shape[0],
k_cache.shape[0],
block_table.shape[0],
TOPK=logical_indices.shape[1],
TOPK=selection_width,
PAGE_SIZE=k_cache.shape[1],
PAGE_TABLE_WIDTH=block_table.shape[1],
GROUP_SIZE=group_size,
@@ -551,6 +581,133 @@ def qsa_sparse_paged_attention(
return out
def warmup_qsa_sparse_paged_attention(
kv_cache: torch.Tensor,
block_table: torch.Tensor,
*,
num_query_heads: int,
selection_width: int,
) -> tuple[tuple[int, int, int], ...]:
"""Compile every production-reachable split-K/merge specialization."""
head_dim = kv_cache.shape[-1] // 2
key_cache, value_cache = kv_cache.transpose(1, 2).split(head_dim, dim=-1)
num_kv_heads = key_cache.shape[2]
group_size = num_query_heads // num_kv_heads
block_m = triton.next_power_of_2(group_size)
# Every config the dispatch can pick for this group size.
profiles = {
_select_config(num_rows, num_kv_heads, use_prefill_config, selection_width)
for num_rows in range(1, 8193)
for use_prefill_config in (False, True)
}
# Scalars constant per deployment get their real values (their divisibility
# specialization is wanted); the batch-varying ones are do_not_specialize'd
# on the kernels, so any value here compiles the only variant.
num_rows = 16
num_requests = 16
q_ptr = TritonWarmupTensor(
torch.bfloat16, shape=(num_rows, num_query_heads, head_dim)
)
k_cache_ptr = TritonWarmupTensor(
key_cache.dtype,
shape=tuple(key_cache.shape),
strides=tuple(key_cache.stride()),
)
v_cache_ptr = TritonWarmupTensor(
value_cache.dtype,
shape=tuple(value_cache.shape),
strides=tuple(value_cache.stride()),
)
# +1: the packed buffer's trailing count column.
indices_ptr = TritonWarmupTensor(torch.int32, shape=(num_rows, selection_width + 1))
block_table_ptr = TritonWarmupTensor(
block_table.dtype,
shape=tuple(block_table.shape),
strides=tuple(block_table.stride()),
)
token_to_req_ptr = TritonWarmupTensor(torch.int32)
output_ptr = TritonWarmupTensor(
torch.bfloat16, shape=(num_rows, num_query_heads, head_dim)
)
head_stride = head_dim
row_stride = num_query_heads * head_dim
num_cache_blocks = triton_scalar_specialization_rep(kv_cache.shape[0])
warmed = []
for block_n, warps, num_tiles, num_splits in sorted(profiles):
if num_splits == 1:
partial_output_ptr = output_ptr
partial_lse_ptr = output_ptr
else:
partial_output_ptr = TritonWarmupTensor(
torch.float32,
shape=(num_splits, num_rows, num_query_heads, head_dim),
)
partial_lse_ptr = TritonWarmupTensor(
torch.float32, shape=(num_splits, num_rows, num_query_heads)
)
_qsa_sparse_paged_gqa_splitk_kernel.warmup(
q_ptr,
k_cache_ptr,
v_cache_ptr,
indices_ptr,
block_table_ptr,
token_to_req_ptr,
partial_output_ptr,
partial_lse_ptr,
output_ptr,
row_stride,
head_stride,
key_cache.stride(0),
key_cache.stride(1),
key_cache.stride(2),
value_cache.stride(0),
value_cache.stride(1),
value_cache.stride(2),
selection_width + 1,
block_table.stride(0),
row_stride,
head_stride,
num_rows,
num_cache_blocks,
num_requests,
TOPK=selection_width,
PAGE_SIZE=key_cache.shape[1],
PAGE_TABLE_WIDTH=block_table.shape[1],
GROUP_SIZE=group_size,
HEAD_DIM=head_dim,
NUM_QUERY_HEADS=num_query_heads,
NUM_SPLITS=num_splits,
NUM_TILES=num_tiles,
BLOCK_M=block_m,
BLOCK_N=block_n,
num_warps=warps,
num_stages=2,
grid=(num_rows, num_kv_heads, num_splits),
)
if num_splits > 1:
_qsa_merge_splitk_kernel.warmup(
partial_output_ptr,
partial_lse_ptr,
output_ptr,
row_stride,
head_stride,
num_rows,
HEAD_DIM=head_dim,
NUM_QUERY_HEADS=num_query_heads,
NUM_SPLITS=num_splits,
BLOCK_SPLITS=triton.next_power_of_2(num_splits),
num_warps=2,
num_stages=1,
grid=(num_rows, num_query_heads),
)
warmed.append((block_n, num_splits, warps))
return tuple(warmed)
def qsa_store_cache_rows(
cache: torch.Tensor,
slot_mapping: torch.Tensor,
@@ -708,4 +865,5 @@ __all__ = [
"qsa_compress_groups_with_ratio",
"qsa_sparse_paged_attention",
"qsa_store_cache_rows",
"warmup_qsa_sparse_paged_attention",
]
@@ -262,6 +262,15 @@ def _expand_qsa_indices_kernel(
tl.where(valid, token, -1),
mask=columns < OUTPUT_WIDTH,
)
# TRAILING COUNT COLUMN: the packed buffer is OUTPUT_WIDTH+1 wide; the
# last column holds this row's valid-entry count (expanded blocks plus
# causal tail). It is NOT a token index — the sparse attention kernel
# reads it as the row's tile-loop bound.
if tl.program_id(1) == 0:
tl.store(
output_ptr + row * stride_output_row + OUTPUT_WIDTH * stride_output_column,
expanded_count + tail_count,
)
def _decode_tiles_per_program(num_requests: int, columns: int) -> int:
@@ -433,7 +442,10 @@ def expand_qsa_block_indices(
output_width = token_topk + compress_ratio - 1
assert block_indices.shape == (query_positions.numel(), block_topk)
assert visible_blocks.shape == query_positions.shape
assert out.shape == (block_indices.shape[0], output_width)
# +1: the packed buffer's trailing column holds each row's valid-entry
# count (never a token index); the index region below only writes
# columns [0, output_width).
assert out.shape == (block_indices.shape[0], output_width + 1)
column_block = 256
grid = (block_indices.shape[0], triton.cdiv(output_width, column_block))
_expand_qsa_indices_kernel[grid](
+15 -8
View File
@@ -30,7 +30,6 @@ from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
_resolve_layer_name,
canonicalize_singleton_dim_strides,
direct_register_custom_op,
kv_cache_dtype_str_to_dtype,
)
@@ -124,6 +123,7 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
attn_metadata: FlashAttentionMetadata,
output: torch.Tensor,
token_to_req: torch.Tensor,
use_prefill_config: bool,
output_scale: torch.Tensor | None = None,
output_block_scale: torch.Tensor | None = None,
) -> torch.Tensor:
@@ -146,8 +146,6 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
logical_indices = topk_buffer[:num_tokens]
token_to_req = token_to_req[:num_tokens]
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
key_cache = canonicalize_singleton_dim_strides(key_cache)
value_cache = canonicalize_singleton_dim_strides(value_cache)
if key_cache.dtype != torch.bfloat16 or query.dtype != torch.bfloat16:
raise NotImplementedError("Qwen4Exp QSA requires BF16 Q/K/V")
@@ -160,6 +158,7 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
logical_indices,
attn_metadata.block_table,
token_to_req,
use_prefill_config,
output[:num_tokens],
)
return output
@@ -209,6 +208,11 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
if self.total_num_heads % tp_size:
raise ValueError("QSA attention heads must be divisible by TP size")
self.num_heads = self.total_num_heads // tp_size
# Decode/verify batches have at most 1 + num_spec query tokens per
# request; use_prefill_config (max_query_len > this) steers the
# config table. Shorter batches take the decode profile — harmless,
# the difference is tile-shape tuning, not correctness.
self._max_decode_query_len = 1 + vllm_config.num_speculative_tokens
self.total_num_kv_heads = int(config.num_key_value_heads)
if self.total_num_kv_heads >= tp_size:
if self.total_num_kv_heads % tp_size:
@@ -310,11 +314,16 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
prefix=f"{prefix}.indexer",
)
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
# PACKED selection buffer: the trailing column holds each row's
# valid-entry count (written by the expand kernel) — never a token
# index; the sparse attention kernel reads it as its loop bound.
# MTP skip_topk steps reuse rows frozen from step 0; the count is
# a row column, so compaction/reuse keep it paired with the content.
self.register_buffer(
"topk_indices_buffer",
torch.empty(
max_tokens,
self.indexer.output_width,
self.indexer.packed_output_width,
dtype=torch.int32,
),
persistent=False,
@@ -369,10 +378,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
positions,
self.topk_indices_buffer[:num_tokens],
)
if selected.shape != (
num_tokens,
self.indexer.output_width,
):
if selected.shape != (num_tokens, self.indexer.packed_output_width):
raise RuntimeError("QSA indexer returned an invalid selection shape")
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
impl.do_kv_cache_update(
@@ -391,6 +397,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
main_metadata,
output,
token_to_req=side_metadata.token_to_req,
use_prefill_config=main_metadata.max_query_len > self._max_decode_query_len,
)
def forward(