[Qwen3.8-Flash-Next] Improve QSA sparse GQA for prefill and short-ctx decode (#54873)
Signed-off-by: Thien Tran <gau.nernst@yahoo.com.sg> Co-authored-by: Kimi <noreply@moonshot.cn>
This commit is contained in:
@@ -167,8 +167,19 @@ class QSAIndexer(nn.Module):
|
||||
|
||||
@property
|
||||
def output_width(self) -> int:
|
||||
"""Selection (index) columns per row."""
|
||||
return self.token_topk + self.compress_ratio - 1
|
||||
|
||||
@property
|
||||
def packed_output_width(self) -> int:
|
||||
"""Packed selection-buffer width: output_width + 1.
|
||||
|
||||
The trailing column holds each row's valid-entry count (written by
|
||||
the expand kernel) — never a token index; the sparse attention
|
||||
kernel reads it as its tile-loop bound.
|
||||
"""
|
||||
return self.output_width + 1
|
||||
|
||||
def _metadata(
|
||||
self,
|
||||
) -> tuple[QSAForwardMetadata, QSAForwardMetadata] | None:
|
||||
@@ -211,7 +222,14 @@ class QSAIndexer(nn.Module):
|
||||
positions: torch.Tensor,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Return fixed-width request-relative token indices padded with ``-1``."""
|
||||
"""Select each query row's token indices.
|
||||
|
||||
Returns the packed buffer of shape [num_tokens, output_width + 1]:
|
||||
the leading ``output_width`` columns are ``-1``-padded
|
||||
request-relative token indices, and the trailing column is the row's
|
||||
valid-entry count (the attention kernel's loop bound, never a token
|
||||
index).
|
||||
"""
|
||||
|
||||
metadata = self._metadata()
|
||||
if metadata is None:
|
||||
@@ -219,11 +237,13 @@ class QSAIndexer(nn.Module):
|
||||
if self.skip_topk and out is not None:
|
||||
return out
|
||||
result = torch.full(
|
||||
(hidden_states.shape[0], self.output_width),
|
||||
(hidden_states.shape[0], self.packed_output_width),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=hidden_states.device,
|
||||
)
|
||||
# Inert rows carry a zero valid count (empty loop bound), not -1.
|
||||
result[:, -1] = 0
|
||||
if out is not None:
|
||||
out.copy_(result)
|
||||
return out
|
||||
@@ -357,11 +377,11 @@ class QSAIndexer(nn.Module):
|
||||
if out is None:
|
||||
out = torch.empty(
|
||||
num_tokens,
|
||||
self.output_width,
|
||||
self.packed_output_width,
|
||||
dtype=torch.int32,
|
||||
device=q.device,
|
||||
)
|
||||
elif out.shape != (num_tokens, self.output_width):
|
||||
elif out.shape != (num_tokens, self.packed_output_width):
|
||||
raise ValueError("QSA selection output has an invalid shape")
|
||||
|
||||
num_decode_tokens = compressed_metadata.num_decode_tokens
|
||||
|
||||
@@ -6,10 +6,14 @@ from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
TritonWarmupTensor,
|
||||
triton_scalar_specialization_rep,
|
||||
)
|
||||
from vllm.triton_utils import HAS_TRITON, tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
@triton.jit(do_not_specialize=["num_rows", "num_requests"])
|
||||
def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
q_ptr,
|
||||
k_cache_ptr,
|
||||
@@ -52,6 +56,12 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
request = tl.load(token_to_req_ptr + row)
|
||||
safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
|
||||
|
||||
# The packed selection buffer carries one TRAILING COUNT COLUMN per row
|
||||
# (column TOPK of a TOPK+1-wide buffer): the row's valid-entry count,
|
||||
# written by the expand kernel. It is never a token index — the tile loop
|
||||
# and the index load below only ever cover columns [0, TOPK).
|
||||
valid_count = tl.load(indices_ptr + row * stride_indices_row + TOPK)
|
||||
|
||||
head_offsets = tl.arange(0, BLOCK_M)
|
||||
dim_offsets = tl.arange(0, HEAD_DIM)
|
||||
column_offsets = tl.arange(0, BLOCK_N)
|
||||
@@ -70,10 +80,9 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
accumulator = tl.zeros((BLOCK_M, HEAD_DIM), dtype=tl.float32)
|
||||
softmax_scale_log2: tl.constexpr = (HEAD_DIM**-0.5) * 1.4426950408889634
|
||||
|
||||
# Dynamic bounds avoid padded main-loop iterations for uneven splits.
|
||||
split_tile_start = split_id * NUM_TILES // NUM_SPLITS
|
||||
split_tile_end = (split_id + 1) * NUM_TILES // NUM_SPLITS
|
||||
for tile in range(split_tile_start, split_tile_end):
|
||||
tile_end = tl.minimum(NUM_TILES, tl.cdiv(tl.minimum(valid_count, TOPK), BLOCK_N))
|
||||
|
||||
for tile in range(split_id, tile_end, NUM_SPLITS):
|
||||
columns = tile * BLOCK_N + column_offsets
|
||||
logical_token = tl.load(
|
||||
indices_ptr + row * stride_indices_row + columns,
|
||||
@@ -156,29 +165,23 @@ def _qsa_sparse_paged_gqa_splitk_kernel(
|
||||
max_value + tl.math.log2(tl.maximum(normalizer, 1.0e-20)),
|
||||
-float("inf"),
|
||||
)
|
||||
partial_row = (split_id * num_rows + row).to(tl.int64)
|
||||
tl.store(
|
||||
partial_output_ptr
|
||||
+ (
|
||||
(split_id * num_rows + row) * NUM_QUERY_HEADS
|
||||
+ first_head
|
||||
+ head_offsets[:, None]
|
||||
)
|
||||
+ (partial_row * NUM_QUERY_HEADS + first_head + head_offsets[:, None])
|
||||
* HEAD_DIM
|
||||
+ dim_offsets[None, :],
|
||||
normalized_output,
|
||||
mask=output_mask,
|
||||
)
|
||||
tl.store(
|
||||
partial_lse_ptr
|
||||
+ (split_id * num_rows + row) * NUM_QUERY_HEADS
|
||||
+ first_head
|
||||
+ head_offsets,
|
||||
partial_lse_ptr + partial_row * NUM_QUERY_HEADS + first_head + head_offsets,
|
||||
partial_lse,
|
||||
mask=head_offsets < GROUP_SIZE,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
@triton.jit(do_not_specialize=["num_rows"])
|
||||
def _qsa_merge_splitk_kernel(
|
||||
partial_output_ptr,
|
||||
partial_lse_ptr,
|
||||
@@ -206,10 +209,10 @@ def _qsa_merge_splitk_kernel(
|
||||
shifted = tl.where(split_mask & has_values, lse - lse_max, -float("inf"))
|
||||
weights = tl.math.exp2(shifted)
|
||||
denominator = tl.sum(weights, axis=0)
|
||||
split_rows = split_offsets.to(tl.int64) * num_rows + row
|
||||
partial_output = tl.load(
|
||||
partial_output_ptr
|
||||
+ ((split_offsets[:, None] * num_rows + row) * NUM_QUERY_HEADS + head)
|
||||
* HEAD_DIM
|
||||
+ (split_rows[:, None] * NUM_QUERY_HEADS + head) * HEAD_DIM
|
||||
+ dim_offsets[None, :],
|
||||
mask=split_mask[:, None],
|
||||
other=0.0,
|
||||
@@ -408,6 +411,41 @@ def _compress_qsa_groups_kernel(
|
||||
)
|
||||
|
||||
|
||||
def _select_config(
|
||||
num_rows: int, num_kv_heads: int, use_prefill_config: bool, num_columns: int
|
||||
) -> tuple[int, int, int, int]:
|
||||
"""Select (block_n, num_warps, num_tiles, num_splits) for the kernel.
|
||||
|
||||
Tuned on GB300 for the Qwen3.8-Flash-Next TP1/TP2/TP4 shapes, keyed on
|
||||
base_programs = num_rows * num_kv_heads. The bp > 2048 region splits on
|
||||
use_prefill_config (capture-stable: at FULL-graph capture max_query_len is the
|
||||
uniform decode/verify length).
|
||||
"""
|
||||
base_programs = num_rows * num_kv_heads
|
||||
if base_programs > 2048:
|
||||
BLOCK_N, target_splits, num_warps = (
|
||||
(32, 1, 1) if use_prefill_config else (64, 1, 2)
|
||||
)
|
||||
elif base_programs <= 24:
|
||||
BLOCK_N, target_splits, num_warps = 32, 64, 4
|
||||
elif base_programs <= 32:
|
||||
BLOCK_N, target_splits, num_warps = 32, 16, 1
|
||||
elif base_programs <= 64:
|
||||
BLOCK_N, target_splits, num_warps = 32, 8, 1
|
||||
elif base_programs <= 128:
|
||||
BLOCK_N, target_splits, num_warps = 32, 4, 1
|
||||
elif base_programs <= 256:
|
||||
BLOCK_N, target_splits, num_warps = 32, 8, 1
|
||||
elif base_programs <= 512:
|
||||
BLOCK_N, target_splits, num_warps = 64, 4, 2
|
||||
else:
|
||||
BLOCK_N, target_splits, num_warps = 64, 1, 2
|
||||
num_tiles = triton.cdiv(num_columns, BLOCK_N)
|
||||
# Never more splits than tiles, never empty.
|
||||
num_splits = min(target_splits, num_tiles)
|
||||
return BLOCK_N, num_warps, num_tiles, num_splits
|
||||
|
||||
|
||||
def qsa_sparse_paged_attention(
|
||||
q: torch.Tensor,
|
||||
k_cache: torch.Tensor,
|
||||
@@ -415,12 +453,17 @@ def qsa_sparse_paged_attention(
|
||||
logical_indices: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
use_prefill_config: bool,
|
||||
out: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Run sparse GQA directly over paged BF16 K/V caches."""
|
||||
"""Run sparse GQA directly over paged BF16 K/V caches.
|
||||
|
||||
if not q.is_cuda or not HAS_TRITON:
|
||||
raise RuntimeError("paged QSA sparse attention requires CUDA and Triton")
|
||||
logical_indices is the PACKED selection buffer: [rows, selection_width + 1]
|
||||
with the trailing column holding each row's valid-entry count (written by
|
||||
the expand kernel; never a token index). The kernel reads it as the
|
||||
tile-loop bound. use_prefill_config only steers the top of the config table; see
|
||||
_select_config.
|
||||
"""
|
||||
if q.ndim != 3 or k_cache.ndim != 4 or v_cache.shape != k_cache.shape:
|
||||
raise ValueError("QSA sparse attention received invalid Q/K/V shapes")
|
||||
if logical_indices.ndim != 2 or logical_indices.shape[0] != q.shape[0]:
|
||||
@@ -429,8 +472,10 @@ def qsa_sparse_paged_attention(
|
||||
raise ValueError("QSA sparse attention metadata has invalid shapes")
|
||||
if not all(k_cache.shape[:3]) or not all(block_table.shape):
|
||||
raise ValueError("QSA sparse attention cache and block table must be nonempty")
|
||||
if logical_indices.shape[1] <= 0:
|
||||
raise ValueError("QSA sparse attention requires a positive selection width")
|
||||
if logical_indices.shape[1] < 2:
|
||||
raise ValueError(
|
||||
"QSA packed indices need selection columns plus the count column"
|
||||
)
|
||||
if q.shape[2] != k_cache.shape[3] or q.shape[1] % k_cache.shape[2]:
|
||||
raise ValueError("QSA sparse attention requires valid grouped-query heads")
|
||||
head_dim = q.shape[2]
|
||||
@@ -444,6 +489,7 @@ def qsa_sparse_paged_attention(
|
||||
assert q.stride(2) == k_cache.stride(3) == v_cache.stride(3) == 1
|
||||
assert logical_indices.stride(1) == block_table.stride(1) == 1
|
||||
assert token_to_req.stride(0) == 1
|
||||
|
||||
if out is None:
|
||||
out = torch.empty_like(q)
|
||||
if out.shape != q.shape:
|
||||
@@ -455,26 +501,10 @@ def qsa_sparse_paged_attention(
|
||||
|
||||
group_size = q.shape[1] // k_cache.shape[2]
|
||||
block_m = triton.next_power_of_2(group_size)
|
||||
base_programs = q.shape[0] * k_cache.shape[2]
|
||||
small_profile_limit = 8 if block_m <= 8 else 4
|
||||
|
||||
# Tuned on GB300 for the Qwen-Air TP1, TP2, and TP4 attention shapes.
|
||||
# Narrow tiles favor decode; wide tiles improve throughput for prefill.
|
||||
if base_programs <= small_profile_limit:
|
||||
block_n, target_splits, partial_warps = 16, 64, 4
|
||||
elif base_programs < 32:
|
||||
block_n, target_splits, partial_warps = 16, 32, 4
|
||||
elif base_programs <= 256:
|
||||
block_n, target_splits, partial_warps = 64, 8, 2
|
||||
elif base_programs <= 512:
|
||||
block_n, target_splits, partial_warps = 64, 4, 2
|
||||
else:
|
||||
block_n, target_splits, partial_warps = 64, 1, 2
|
||||
|
||||
num_tiles = triton.cdiv(logical_indices.shape[1], block_n)
|
||||
# Avoid empty splits when the selection width is smaller than the profile.
|
||||
max_useful_splits = 1 << (num_tiles.bit_length() - 1)
|
||||
num_splits = min(max_useful_splits, target_splits)
|
||||
selection_width = logical_indices.shape[1] - 1 # trailing column is the count
|
||||
block_n, partial_warps, num_tiles, num_splits = _select_config(
|
||||
q.shape[0], k_cache.shape[2], use_prefill_config, selection_width
|
||||
)
|
||||
|
||||
# Split=1 writes output directly and compiles out all workspace accesses.
|
||||
if num_splits == 1:
|
||||
@@ -518,7 +548,7 @@ def qsa_sparse_paged_attention(
|
||||
q.shape[0],
|
||||
k_cache.shape[0],
|
||||
block_table.shape[0],
|
||||
TOPK=logical_indices.shape[1],
|
||||
TOPK=selection_width,
|
||||
PAGE_SIZE=k_cache.shape[1],
|
||||
PAGE_TABLE_WIDTH=block_table.shape[1],
|
||||
GROUP_SIZE=group_size,
|
||||
@@ -551,6 +581,133 @@ def qsa_sparse_paged_attention(
|
||||
return out
|
||||
|
||||
|
||||
def warmup_qsa_sparse_paged_attention(
|
||||
kv_cache: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
*,
|
||||
num_query_heads: int,
|
||||
selection_width: int,
|
||||
) -> tuple[tuple[int, int, int], ...]:
|
||||
"""Compile every production-reachable split-K/merge specialization."""
|
||||
|
||||
head_dim = kv_cache.shape[-1] // 2
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(head_dim, dim=-1)
|
||||
num_kv_heads = key_cache.shape[2]
|
||||
group_size = num_query_heads // num_kv_heads
|
||||
block_m = triton.next_power_of_2(group_size)
|
||||
|
||||
# Every config the dispatch can pick for this group size.
|
||||
profiles = {
|
||||
_select_config(num_rows, num_kv_heads, use_prefill_config, selection_width)
|
||||
for num_rows in range(1, 8193)
|
||||
for use_prefill_config in (False, True)
|
||||
}
|
||||
|
||||
# Scalars constant per deployment get their real values (their divisibility
|
||||
# specialization is wanted); the batch-varying ones are do_not_specialize'd
|
||||
# on the kernels, so any value here compiles the only variant.
|
||||
num_rows = 16
|
||||
num_requests = 16
|
||||
q_ptr = TritonWarmupTensor(
|
||||
torch.bfloat16, shape=(num_rows, num_query_heads, head_dim)
|
||||
)
|
||||
k_cache_ptr = TritonWarmupTensor(
|
||||
key_cache.dtype,
|
||||
shape=tuple(key_cache.shape),
|
||||
strides=tuple(key_cache.stride()),
|
||||
)
|
||||
v_cache_ptr = TritonWarmupTensor(
|
||||
value_cache.dtype,
|
||||
shape=tuple(value_cache.shape),
|
||||
strides=tuple(value_cache.stride()),
|
||||
)
|
||||
# +1: the packed buffer's trailing count column.
|
||||
indices_ptr = TritonWarmupTensor(torch.int32, shape=(num_rows, selection_width + 1))
|
||||
block_table_ptr = TritonWarmupTensor(
|
||||
block_table.dtype,
|
||||
shape=tuple(block_table.shape),
|
||||
strides=tuple(block_table.stride()),
|
||||
)
|
||||
token_to_req_ptr = TritonWarmupTensor(torch.int32)
|
||||
output_ptr = TritonWarmupTensor(
|
||||
torch.bfloat16, shape=(num_rows, num_query_heads, head_dim)
|
||||
)
|
||||
head_stride = head_dim
|
||||
row_stride = num_query_heads * head_dim
|
||||
num_cache_blocks = triton_scalar_specialization_rep(kv_cache.shape[0])
|
||||
|
||||
warmed = []
|
||||
for block_n, warps, num_tiles, num_splits in sorted(profiles):
|
||||
if num_splits == 1:
|
||||
partial_output_ptr = output_ptr
|
||||
partial_lse_ptr = output_ptr
|
||||
else:
|
||||
partial_output_ptr = TritonWarmupTensor(
|
||||
torch.float32,
|
||||
shape=(num_splits, num_rows, num_query_heads, head_dim),
|
||||
)
|
||||
partial_lse_ptr = TritonWarmupTensor(
|
||||
torch.float32, shape=(num_splits, num_rows, num_query_heads)
|
||||
)
|
||||
_qsa_sparse_paged_gqa_splitk_kernel.warmup(
|
||||
q_ptr,
|
||||
k_cache_ptr,
|
||||
v_cache_ptr,
|
||||
indices_ptr,
|
||||
block_table_ptr,
|
||||
token_to_req_ptr,
|
||||
partial_output_ptr,
|
||||
partial_lse_ptr,
|
||||
output_ptr,
|
||||
row_stride,
|
||||
head_stride,
|
||||
key_cache.stride(0),
|
||||
key_cache.stride(1),
|
||||
key_cache.stride(2),
|
||||
value_cache.stride(0),
|
||||
value_cache.stride(1),
|
||||
value_cache.stride(2),
|
||||
selection_width + 1,
|
||||
block_table.stride(0),
|
||||
row_stride,
|
||||
head_stride,
|
||||
num_rows,
|
||||
num_cache_blocks,
|
||||
num_requests,
|
||||
TOPK=selection_width,
|
||||
PAGE_SIZE=key_cache.shape[1],
|
||||
PAGE_TABLE_WIDTH=block_table.shape[1],
|
||||
GROUP_SIZE=group_size,
|
||||
HEAD_DIM=head_dim,
|
||||
NUM_QUERY_HEADS=num_query_heads,
|
||||
NUM_SPLITS=num_splits,
|
||||
NUM_TILES=num_tiles,
|
||||
BLOCK_M=block_m,
|
||||
BLOCK_N=block_n,
|
||||
num_warps=warps,
|
||||
num_stages=2,
|
||||
grid=(num_rows, num_kv_heads, num_splits),
|
||||
)
|
||||
if num_splits > 1:
|
||||
_qsa_merge_splitk_kernel.warmup(
|
||||
partial_output_ptr,
|
||||
partial_lse_ptr,
|
||||
output_ptr,
|
||||
row_stride,
|
||||
head_stride,
|
||||
num_rows,
|
||||
HEAD_DIM=head_dim,
|
||||
NUM_QUERY_HEADS=num_query_heads,
|
||||
NUM_SPLITS=num_splits,
|
||||
BLOCK_SPLITS=triton.next_power_of_2(num_splits),
|
||||
num_warps=2,
|
||||
num_stages=1,
|
||||
grid=(num_rows, num_query_heads),
|
||||
)
|
||||
warmed.append((block_n, num_splits, warps))
|
||||
return tuple(warmed)
|
||||
|
||||
|
||||
def qsa_store_cache_rows(
|
||||
cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
@@ -708,4 +865,5 @@ __all__ = [
|
||||
"qsa_compress_groups_with_ratio",
|
||||
"qsa_sparse_paged_attention",
|
||||
"qsa_store_cache_rows",
|
||||
"warmup_qsa_sparse_paged_attention",
|
||||
]
|
||||
|
||||
@@ -262,6 +262,15 @@ def _expand_qsa_indices_kernel(
|
||||
tl.where(valid, token, -1),
|
||||
mask=columns < OUTPUT_WIDTH,
|
||||
)
|
||||
# TRAILING COUNT COLUMN: the packed buffer is OUTPUT_WIDTH+1 wide; the
|
||||
# last column holds this row's valid-entry count (expanded blocks plus
|
||||
# causal tail). It is NOT a token index — the sparse attention kernel
|
||||
# reads it as the row's tile-loop bound.
|
||||
if tl.program_id(1) == 0:
|
||||
tl.store(
|
||||
output_ptr + row * stride_output_row + OUTPUT_WIDTH * stride_output_column,
|
||||
expanded_count + tail_count,
|
||||
)
|
||||
|
||||
|
||||
def _decode_tiles_per_program(num_requests: int, columns: int) -> int:
|
||||
@@ -433,7 +442,10 @@ def expand_qsa_block_indices(
|
||||
output_width = token_topk + compress_ratio - 1
|
||||
assert block_indices.shape == (query_positions.numel(), block_topk)
|
||||
assert visible_blocks.shape == query_positions.shape
|
||||
assert out.shape == (block_indices.shape[0], output_width)
|
||||
# +1: the packed buffer's trailing column holds each row's valid-entry
|
||||
# count (never a token index); the index region below only writes
|
||||
# columns [0, output_width).
|
||||
assert out.shape == (block_indices.shape[0], output_width + 1)
|
||||
column_block = 256
|
||||
grid = (block_indices.shape[0], triton.cdiv(output_width, column_block))
|
||||
_expand_qsa_indices_kernel[grid](
|
||||
|
||||
@@ -30,7 +30,6 @@ from vllm.utils.torch_utils import (
|
||||
LayerNameType,
|
||||
_encode_layer_name,
|
||||
_resolve_layer_name,
|
||||
canonicalize_singleton_dim_strides,
|
||||
direct_register_custom_op,
|
||||
kv_cache_dtype_str_to_dtype,
|
||||
)
|
||||
@@ -124,6 +123,7 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
attn_metadata: FlashAttentionMetadata,
|
||||
output: torch.Tensor,
|
||||
token_to_req: torch.Tensor,
|
||||
use_prefill_config: bool,
|
||||
output_scale: torch.Tensor | None = None,
|
||||
output_block_scale: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
@@ -146,8 +146,6 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
logical_indices = topk_buffer[:num_tokens]
|
||||
token_to_req = token_to_req[:num_tokens]
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(self.head_size, dim=-1)
|
||||
key_cache = canonicalize_singleton_dim_strides(key_cache)
|
||||
value_cache = canonicalize_singleton_dim_strides(value_cache)
|
||||
if key_cache.dtype != torch.bfloat16 or query.dtype != torch.bfloat16:
|
||||
raise NotImplementedError("Qwen4Exp QSA requires BF16 Q/K/V")
|
||||
|
||||
@@ -160,6 +158,7 @@ class Qwen4ExpQSAFlashAttentionImpl(FlashAttentionImpl):
|
||||
logical_indices,
|
||||
attn_metadata.block_table,
|
||||
token_to_req,
|
||||
use_prefill_config,
|
||||
output[:num_tokens],
|
||||
)
|
||||
return output
|
||||
@@ -209,6 +208,11 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
if self.total_num_heads % tp_size:
|
||||
raise ValueError("QSA attention heads must be divisible by TP size")
|
||||
self.num_heads = self.total_num_heads // tp_size
|
||||
# Decode/verify batches have at most 1 + num_spec query tokens per
|
||||
# request; use_prefill_config (max_query_len > this) steers the
|
||||
# config table. Shorter batches take the decode profile — harmless,
|
||||
# the difference is tile-shape tuning, not correctness.
|
||||
self._max_decode_query_len = 1 + vllm_config.num_speculative_tokens
|
||||
self.total_num_kv_heads = int(config.num_key_value_heads)
|
||||
if self.total_num_kv_heads >= tp_size:
|
||||
if self.total_num_kv_heads % tp_size:
|
||||
@@ -310,11 +314,16 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
prefix=f"{prefix}.indexer",
|
||||
)
|
||||
max_tokens = vllm_config.scheduler_config.max_num_batched_tokens
|
||||
# PACKED selection buffer: the trailing column holds each row's
|
||||
# valid-entry count (written by the expand kernel) — never a token
|
||||
# index; the sparse attention kernel reads it as its loop bound.
|
||||
# MTP skip_topk steps reuse rows frozen from step 0; the count is
|
||||
# a row column, so compaction/reuse keep it paired with the content.
|
||||
self.register_buffer(
|
||||
"topk_indices_buffer",
|
||||
torch.empty(
|
||||
max_tokens,
|
||||
self.indexer.output_width,
|
||||
self.indexer.packed_output_width,
|
||||
dtype=torch.int32,
|
||||
),
|
||||
persistent=False,
|
||||
@@ -369,10 +378,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
positions,
|
||||
self.topk_indices_buffer[:num_tokens],
|
||||
)
|
||||
if selected.shape != (
|
||||
num_tokens,
|
||||
self.indexer.output_width,
|
||||
):
|
||||
if selected.shape != (num_tokens, self.indexer.packed_output_width):
|
||||
raise RuntimeError("QSA indexer returned an invalid selection shape")
|
||||
impl = cast(Qwen4ExpQSAFlashAttentionImpl, self.impl)
|
||||
impl.do_kv_cache_update(
|
||||
@@ -391,6 +397,7 @@ class Qwen4ExpQSAAttention(Qwen3NextAttention, AttentionLayerBase):
|
||||
main_metadata,
|
||||
output,
|
||||
token_to_req=side_metadata.token_to_req,
|
||||
use_prefill_config=main_metadata.max_query_len > self._max_decode_query_len,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
Reference in New Issue
Block a user