Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
1123 lines
38 KiB
Python
1123 lines
38 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Triton kernels for the Qwen4Exp weight-free QSA path."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
|
|
import torch
|
|
|
|
from vllm import _custom_ops as ops
|
|
from vllm.platforms import current_platform
|
|
from vllm.triton_utils import HAS_TRITON, tl, triton
|
|
|
|
_LOGITS_WORKSPACE_BYTES = 128 * 1024 * 1024
|
|
_TOPK_WORKSPACE_BYTES = 1024 * 1024
|
|
|
|
|
|
@triton.jit
|
|
def _qsa_mqa_paged_kernel(
|
|
q_ptr,
|
|
k_cache_ptr,
|
|
page_table_ptr,
|
|
token_to_req_ptr,
|
|
query_positions_ptr,
|
|
sequence_lengths_ptr,
|
|
visible_blocks_ptr,
|
|
logits_ptr,
|
|
stride_q_row,
|
|
stride_q_head,
|
|
stride_q_dim,
|
|
stride_cache_block,
|
|
stride_cache_token,
|
|
stride_cache_dim,
|
|
stride_table_req,
|
|
stride_table_page,
|
|
stride_logits_row,
|
|
num_rows,
|
|
num_columns,
|
|
num_pages,
|
|
num_requests,
|
|
score_divisor,
|
|
PAGE_SIZE: tl.constexpr,
|
|
PAGE_TABLE_WIDTH: tl.constexpr,
|
|
NUM_HEADS: tl.constexpr,
|
|
HEAD_DIM: tl.constexpr,
|
|
BLOCK_N: tl.constexpr,
|
|
BLOCK_D: tl.constexpr,
|
|
COMPRESS_RATIO: tl.constexpr,
|
|
) -> None:
|
|
row = tl.program_id(0)
|
|
columns = tl.program_id(1) * BLOCK_N + tl.arange(0, BLOCK_N)
|
|
dims = tl.arange(0, BLOCK_D)
|
|
request = tl.load(token_to_req_ptr + row)
|
|
safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
|
|
query_position = tl.load(query_positions_ptr + row)
|
|
sequence_length = tl.load(
|
|
sequence_lengths_ptr + safe_request,
|
|
mask=(request >= 0) & (request < num_requests),
|
|
other=0,
|
|
)
|
|
visible = tl.minimum(
|
|
(query_position + 1) // COMPRESS_RATIO,
|
|
sequence_length // COMPRESS_RATIO,
|
|
)
|
|
if tl.program_id(1) == 0:
|
|
tl.store(visible_blocks_ptr + row, visible)
|
|
logical_page = columns // PAGE_SIZE
|
|
page_offset = columns % PAGE_SIZE
|
|
valid = (
|
|
(row < num_rows)
|
|
& (columns < num_columns)
|
|
& (columns < visible)
|
|
& (request >= 0)
|
|
& (request < num_requests)
|
|
& (logical_page < PAGE_TABLE_WIDTH)
|
|
)
|
|
safe_logical_page = tl.minimum(logical_page, PAGE_TABLE_WIDTH - 1)
|
|
physical_page = tl.load(
|
|
page_table_ptr
|
|
+ safe_request * stride_table_req
|
|
+ safe_logical_page * stride_table_page,
|
|
mask=valid,
|
|
other=-1,
|
|
)
|
|
valid &= (physical_page >= 0) & (physical_page < num_pages)
|
|
# physical_page * block stride can overflow int32 for large caches.
|
|
safe_physical_page = tl.maximum(physical_page, 0).to(tl.int64)
|
|
score = tl.zeros((BLOCK_N,), dtype=tl.float32)
|
|
|
|
for head in tl.static_range(0, NUM_HEADS):
|
|
query = tl.load(
|
|
q_ptr + row * stride_q_row + head * stride_q_head + dims * stride_q_dim,
|
|
mask=dims < HEAD_DIM,
|
|
other=0.0,
|
|
).to(tl.float32)
|
|
keys = tl.load(
|
|
k_cache_ptr
|
|
+ safe_physical_page[:, None] * stride_cache_block
|
|
+ page_offset[:, None] * stride_cache_token
|
|
+ dims[None, :] * stride_cache_dim,
|
|
mask=valid[:, None] & (dims[None, :] < HEAD_DIM),
|
|
other=0.0,
|
|
).to(tl.float32)
|
|
dot = tl.sum(keys * query[None, :], axis=1)
|
|
score += tl.maximum(dot, 0.0)
|
|
|
|
score /= score_divisor
|
|
tl.store(
|
|
logits_ptr + row * stride_logits_row + columns,
|
|
tl.where(valid, score, -float("inf")),
|
|
mask=(row < num_rows) & (columns < num_columns),
|
|
)
|
|
|
|
|
|
@triton.jit
|
|
def _expand_qsa_indices_kernel(
|
|
block_indices_ptr,
|
|
query_positions_ptr,
|
|
sequence_lengths_ptr,
|
|
token_to_req_ptr,
|
|
output_ptr,
|
|
stride_blocks_row,
|
|
stride_blocks_column,
|
|
stride_output_row,
|
|
stride_output_column,
|
|
rows,
|
|
num_requests,
|
|
BLOCK_TOPK: tl.constexpr,
|
|
COMPRESS_RATIO: tl.constexpr,
|
|
TOKEN_TOPK: tl.constexpr,
|
|
OUTPUT_WIDTH: tl.constexpr,
|
|
COLUMN_BLOCK: tl.constexpr,
|
|
) -> None:
|
|
row = tl.program_id(0)
|
|
columns = tl.program_id(1) * COLUMN_BLOCK + tl.arange(0, COLUMN_BLOCK)
|
|
query_position = tl.load(query_positions_ptr + row)
|
|
request = tl.load(token_to_req_ptr + row)
|
|
safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
|
|
sequence_length = tl.load(
|
|
sequence_lengths_ptr + safe_request,
|
|
mask=(request >= 0) & (request < num_requests),
|
|
other=0,
|
|
)
|
|
complete_blocks = tl.minimum(
|
|
tl.minimum(
|
|
(query_position + 1) // COMPRESS_RATIO,
|
|
sequence_length // COMPRESS_RATIO,
|
|
),
|
|
BLOCK_TOPK,
|
|
)
|
|
expanded_count = complete_blocks * COMPRESS_RATIO
|
|
tail_start = ((query_position + 1) // COMPRESS_RATIO) * COMPRESS_RATIO
|
|
tail_count = (query_position + 1) - tail_start
|
|
|
|
is_expanded = columns < expanded_count
|
|
block_rank = columns // COMPRESS_RATIO
|
|
offset = columns % COMPRESS_RATIO
|
|
safe_rank = tl.minimum(block_rank, BLOCK_TOPK - 1)
|
|
block = tl.load(
|
|
block_indices_ptr + row * stride_blocks_row + safe_rank * stride_blocks_column,
|
|
mask=(row < rows) & is_expanded,
|
|
other=-1,
|
|
)
|
|
expanded = block * COMPRESS_RATIO + offset
|
|
tail_offset = columns - expanded_count
|
|
is_tail = (
|
|
(columns >= expanded_count)
|
|
& (tail_offset < tail_count)
|
|
& (tail_offset < COMPRESS_RATIO - 1)
|
|
)
|
|
token = tl.where(is_expanded, expanded, tail_start + tail_offset)
|
|
valid = (
|
|
(row < rows)
|
|
& (columns < OUTPUT_WIDTH)
|
|
& (is_expanded | is_tail)
|
|
& (token >= 0)
|
|
& (token < sequence_length)
|
|
)
|
|
tl.store(
|
|
output_ptr + row * stride_output_row + columns * stride_output_column,
|
|
tl.where(valid, token, -1),
|
|
mask=(row < rows) & (columns < OUTPUT_WIDTH),
|
|
)
|
|
|
|
|
|
@triton.jit
|
|
def _qsa_sparse_paged_gqa_splitk_kernel(
|
|
q_ptr,
|
|
k_cache_ptr,
|
|
v_cache_ptr,
|
|
indices_ptr,
|
|
block_table_ptr,
|
|
token_to_req_ptr,
|
|
partial_output_ptr,
|
|
partial_lse_ptr,
|
|
output_ptr,
|
|
stride_q_row,
|
|
stride_q_head,
|
|
stride_k_block,
|
|
stride_k_token,
|
|
stride_k_head,
|
|
stride_v_block,
|
|
stride_v_token,
|
|
stride_v_head,
|
|
stride_indices_row,
|
|
stride_table_req,
|
|
stride_output_row,
|
|
stride_output_head,
|
|
num_rows,
|
|
num_cache_blocks,
|
|
num_requests,
|
|
TOPK: tl.constexpr,
|
|
PAGE_SIZE: tl.constexpr,
|
|
PAGE_TABLE_WIDTH: tl.constexpr,
|
|
GROUP_SIZE: tl.constexpr,
|
|
HEAD_DIM: tl.constexpr,
|
|
NUM_QUERY_HEADS: tl.constexpr,
|
|
NUM_SPLITS: tl.constexpr,
|
|
NUM_TILES: tl.constexpr,
|
|
BLOCK_M: tl.constexpr,
|
|
BLOCK_N: tl.constexpr,
|
|
) -> None:
|
|
row = tl.program_id(0)
|
|
kv_head = tl.program_id(1)
|
|
split_id = tl.program_id(2)
|
|
request = tl.load(token_to_req_ptr + row)
|
|
safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
|
|
|
|
head_offsets = tl.arange(0, BLOCK_M)
|
|
dim_offsets = tl.arange(0, HEAD_DIM)
|
|
column_offsets = tl.arange(0, BLOCK_N)
|
|
first_head = kv_head * GROUP_SIZE
|
|
query = tl.load(
|
|
q_ptr
|
|
+ row * stride_q_row
|
|
+ (first_head + head_offsets[:, None]) * stride_q_head
|
|
+ dim_offsets[None, :],
|
|
mask=head_offsets[:, None] < GROUP_SIZE,
|
|
other=0.0,
|
|
)
|
|
|
|
max_value = tl.full((BLOCK_M,), -1.0e20, dtype=tl.float32)
|
|
normalizer = tl.zeros((BLOCK_M,), dtype=tl.float32)
|
|
accumulator = tl.zeros((BLOCK_M, HEAD_DIM), dtype=tl.float32)
|
|
softmax_scale_log2: tl.constexpr = (HEAD_DIM**-0.5) * 1.4426950408889634
|
|
|
|
# Dynamic bounds avoid padded main-loop iterations for uneven splits.
|
|
split_tile_start = split_id * NUM_TILES // NUM_SPLITS
|
|
split_tile_end = (split_id + 1) * NUM_TILES // NUM_SPLITS
|
|
for tile in range(split_tile_start, split_tile_end):
|
|
columns = tile * BLOCK_N + column_offsets
|
|
logical_token = tl.load(
|
|
indices_ptr + row * stride_indices_row + columns,
|
|
mask=columns < TOPK,
|
|
other=-1,
|
|
)
|
|
safe_token = tl.maximum(logical_token, 0)
|
|
logical_page = safe_token // PAGE_SIZE
|
|
page_offset = safe_token % PAGE_SIZE
|
|
valid = (
|
|
(request >= 0)
|
|
& (request < num_requests)
|
|
& (logical_token >= 0)
|
|
& (logical_page < PAGE_TABLE_WIDTH)
|
|
)
|
|
physical_page = tl.load(
|
|
block_table_ptr
|
|
+ safe_request * stride_table_req
|
|
+ tl.minimum(logical_page, PAGE_TABLE_WIDTH - 1),
|
|
mask=valid,
|
|
other=-1,
|
|
)
|
|
valid &= (physical_page >= 0) & (physical_page < num_cache_blocks)
|
|
# physical_page * block stride can overflow int32 for large caches.
|
|
safe_page = tl.maximum(physical_page, 0).to(tl.int64)
|
|
keys = tl.load(
|
|
k_cache_ptr
|
|
+ safe_page[None, :] * stride_k_block
|
|
+ page_offset[None, :] * stride_k_token
|
|
+ kv_head * stride_k_head
|
|
+ dim_offsets[:, None],
|
|
mask=valid[None, :],
|
|
other=0.0,
|
|
)
|
|
values = tl.load(
|
|
v_cache_ptr
|
|
+ safe_page[:, None] * stride_v_block
|
|
+ page_offset[:, None] * stride_v_token
|
|
+ kv_head * stride_v_head
|
|
+ dim_offsets[None, :],
|
|
mask=valid[:, None],
|
|
other=0.0,
|
|
)
|
|
scores = tl.dot(query, keys)
|
|
# Scaling scores avoids re-quantizing a scaled query to BF16.
|
|
scores *= softmax_scale_log2
|
|
scores = tl.where(valid[None, :], scores, -1.0e20)
|
|
next_max = tl.maximum(max_value, tl.max(scores, axis=1))
|
|
alpha = tl.math.exp2(max_value - next_max)
|
|
probabilities = tl.where(
|
|
valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0
|
|
)
|
|
accumulator = tl.dot(
|
|
probabilities.to(values.dtype),
|
|
values,
|
|
acc=accumulator * alpha[:, None],
|
|
)
|
|
normalizer = normalizer * alpha + tl.sum(probabilities, axis=1)
|
|
max_value = next_max
|
|
|
|
has_values = normalizer > 0
|
|
normalized_output = tl.where(
|
|
has_values[:, None],
|
|
accumulator / tl.maximum(normalizer[:, None], 1.0e-20),
|
|
0.0,
|
|
)
|
|
output_mask = head_offsets[:, None] < GROUP_SIZE
|
|
if NUM_SPLITS == 1:
|
|
tl.store(
|
|
output_ptr
|
|
+ row * stride_output_row
|
|
+ (first_head + head_offsets[:, None]) * stride_output_head
|
|
+ dim_offsets[None, :],
|
|
normalized_output,
|
|
mask=output_mask,
|
|
)
|
|
else:
|
|
partial_lse = tl.where(
|
|
has_values,
|
|
max_value + tl.math.log2(tl.maximum(normalizer, 1.0e-20)),
|
|
-float("inf"),
|
|
)
|
|
tl.store(
|
|
partial_output_ptr
|
|
+ (
|
|
(split_id * num_rows + row) * NUM_QUERY_HEADS
|
|
+ first_head
|
|
+ head_offsets[:, None]
|
|
)
|
|
* HEAD_DIM
|
|
+ dim_offsets[None, :],
|
|
normalized_output,
|
|
mask=output_mask,
|
|
)
|
|
tl.store(
|
|
partial_lse_ptr
|
|
+ (split_id * num_rows + row) * NUM_QUERY_HEADS
|
|
+ first_head
|
|
+ head_offsets,
|
|
partial_lse,
|
|
mask=head_offsets < GROUP_SIZE,
|
|
)
|
|
|
|
|
|
@triton.jit
|
|
def _qsa_merge_splitk_kernel(
|
|
partial_output_ptr,
|
|
partial_lse_ptr,
|
|
output_ptr,
|
|
stride_output_row,
|
|
stride_output_head,
|
|
num_rows,
|
|
HEAD_DIM: tl.constexpr,
|
|
NUM_QUERY_HEADS: tl.constexpr,
|
|
NUM_SPLITS: tl.constexpr,
|
|
BLOCK_SPLITS: tl.constexpr,
|
|
) -> None:
|
|
row = tl.program_id(0)
|
|
head = tl.program_id(1)
|
|
split_offsets = tl.arange(0, BLOCK_SPLITS)
|
|
dim_offsets = tl.arange(0, HEAD_DIM)
|
|
split_mask = split_offsets < NUM_SPLITS
|
|
lse = tl.load(
|
|
partial_lse_ptr + (split_offsets * num_rows + row) * NUM_QUERY_HEADS + head,
|
|
mask=split_mask,
|
|
other=-float("inf"),
|
|
)
|
|
lse_max = tl.max(lse, axis=0)
|
|
has_values = lse_max > -float("inf")
|
|
shifted = tl.where(split_mask & has_values, lse - lse_max, -float("inf"))
|
|
weights = tl.math.exp2(shifted)
|
|
denominator = tl.sum(weights, axis=0)
|
|
partial_output = tl.load(
|
|
partial_output_ptr
|
|
+ ((split_offsets[:, None] * num_rows + row) * NUM_QUERY_HEADS + head)
|
|
* HEAD_DIM
|
|
+ dim_offsets[None, :],
|
|
mask=split_mask[:, None],
|
|
other=0.0,
|
|
)
|
|
merged = tl.sum(partial_output * weights[:, None], axis=0)
|
|
merged = tl.where(denominator > 0, merged / denominator, 0.0)
|
|
tl.store(
|
|
output_ptr + row * stride_output_row + head * stride_output_head + dim_offsets,
|
|
merged,
|
|
)
|
|
|
|
|
|
@triton.jit
|
|
def _store_qsa_rows_kernel(
|
|
cache_ptr,
|
|
slots_ptr,
|
|
rows_ptr,
|
|
stride_cache_block,
|
|
stride_cache_token,
|
|
stride_cache_dim,
|
|
stride_rows_row,
|
|
stride_rows_dim,
|
|
num_rows,
|
|
num_blocks,
|
|
PAGE_SIZE: tl.constexpr,
|
|
WIDTH: tl.constexpr,
|
|
BLOCK_D: tl.constexpr,
|
|
) -> None:
|
|
row = tl.program_id(0)
|
|
dims = tl.arange(0, BLOCK_D)
|
|
slot = tl.load(slots_ptr + row)
|
|
valid = (row < num_rows) & (slot >= 0) & (slot < num_blocks * PAGE_SIZE)
|
|
block = tl.maximum(slot, 0) // PAGE_SIZE
|
|
token = tl.maximum(slot, 0) % PAGE_SIZE
|
|
values = tl.load(
|
|
rows_ptr + row * stride_rows_row + dims * stride_rows_dim,
|
|
mask=valid & (dims < WIDTH),
|
|
other=0,
|
|
)
|
|
tl.store(
|
|
cache_ptr
|
|
+ block * stride_cache_block
|
|
+ token * stride_cache_token
|
|
+ dims * stride_cache_dim,
|
|
values,
|
|
mask=valid & (dims < WIDTH),
|
|
)
|
|
|
|
|
|
@triton.jit
|
|
def _compress_qsa_groups_kernel(
|
|
raw_keys_ptr, # this step's raw key rows, straight from activations
|
|
raw_positions_ptr, # this step's per-token positions
|
|
compressor_state_cache_ptr, # per-request ring of previous raw keys
|
|
rope_cache_ptr, # packed RoPE position tail of the ring
|
|
compressor_state_table_ptr,
|
|
token_to_req_ptr,
|
|
query_start_loc_ptr,
|
|
logical_positions_ptr,
|
|
compressed_slots_ptr,
|
|
pooled_ptr,
|
|
first_positions_ptr,
|
|
stride_raw_row,
|
|
stride_raw_dim,
|
|
stride_raw_positions_row,
|
|
stride_raw_positions_dim,
|
|
stride_compressor_state_block,
|
|
stride_compressor_state_token,
|
|
stride_compressor_state_dim,
|
|
stride_rope_block,
|
|
stride_rope_token,
|
|
stride_rope_dim,
|
|
stride_compressor_state_table_req,
|
|
stride_pooled_row,
|
|
stride_pooled_dim,
|
|
stride_positions_row,
|
|
stride_positions_dim,
|
|
num_rows,
|
|
num_compressor_state_blocks,
|
|
num_requests,
|
|
COMPRESSOR_STATE_SIZE: tl.constexpr,
|
|
COMPRESS_RATIO: tl.constexpr,
|
|
HEAD_DIM: tl.constexpr,
|
|
BLOCK_D: tl.constexpr,
|
|
LOAD_ROPE_POSITIONS: tl.constexpr,
|
|
) -> None:
|
|
row = tl.program_id(0)
|
|
dims = tl.arange(0, BLOCK_D)
|
|
request = tl.load(token_to_req_ptr + row)
|
|
end_position = tl.load(logical_positions_ptr + row)
|
|
compressed_slot = tl.load(compressed_slots_ptr + row)
|
|
valid_request = (request >= 0) & (request < num_requests)
|
|
safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
|
|
query_row_start = tl.load(
|
|
query_start_loc_ptr + safe_request, mask=valid_request, other=0
|
|
)
|
|
query_row_end = tl.load(
|
|
query_start_loc_ptr + safe_request + 1, mask=valid_request, other=0
|
|
)
|
|
chunk_start_position = end_position - (row - query_row_start)
|
|
compressor_state_block = tl.load(
|
|
compressor_state_table_ptr + safe_request * stride_compressor_state_table_req,
|
|
mask=valid_request,
|
|
other=-1,
|
|
)
|
|
valid_compressor_state_block = (compressor_state_block >= 0) & (
|
|
compressor_state_block < num_compressor_state_blocks
|
|
)
|
|
valid_row = (
|
|
(row < num_rows)
|
|
& valid_request
|
|
& (row >= query_row_start)
|
|
& (row < query_row_end)
|
|
& (end_position >= COMPRESS_RATIO - 1)
|
|
& (compressed_slot >= 0)
|
|
)
|
|
accumulator = tl.zeros((BLOCK_D,), dtype=tl.float32)
|
|
|
|
# A group can span the compressor-state ring (older members) and this
|
|
# step's raw rows (members at positions >= chunk_start_position).
|
|
for group_offset in tl.range(0, COMPRESS_RATIO):
|
|
position = end_position - (COMPRESS_RATIO - 1 - group_offset)
|
|
use_raw = position >= chunk_start_position
|
|
raw_row = query_row_start + position - chunk_start_position
|
|
raw_values = tl.load(
|
|
raw_keys_ptr + raw_row * stride_raw_row + dims * stride_raw_dim,
|
|
mask=valid_row
|
|
& use_raw
|
|
& (raw_row >= query_row_start)
|
|
& (raw_row < query_row_end)
|
|
& (raw_row < num_rows)
|
|
& (dims < HEAD_DIM),
|
|
other=0.0,
|
|
).to(tl.float32)
|
|
compressor_state_values = tl.load(
|
|
compressor_state_cache_ptr
|
|
+ tl.maximum(compressor_state_block, 0).to(tl.int64)
|
|
* stride_compressor_state_block
|
|
+ (position % COMPRESSOR_STATE_SIZE) * stride_compressor_state_token
|
|
+ dims * stride_compressor_state_dim,
|
|
mask=valid_row
|
|
& ~use_raw
|
|
& valid_compressor_state_block
|
|
& (dims < HEAD_DIM),
|
|
other=0.0,
|
|
).to(tl.float32)
|
|
accumulator += tl.where(use_raw, raw_values, compressor_state_values)
|
|
|
|
tl.store(
|
|
pooled_ptr + row * stride_pooled_row + dims * stride_pooled_dim,
|
|
accumulator / COMPRESS_RATIO,
|
|
mask=(row < num_rows) & (dims < HEAD_DIM),
|
|
)
|
|
|
|
position_dims = tl.arange(0, 4)
|
|
first_position = end_position - COMPRESS_RATIO + 1
|
|
if LOAD_ROPE_POSITIONS:
|
|
first_from_raw = first_position >= chunk_start_position
|
|
raw_first_row = query_row_start + first_position - chunk_start_position
|
|
raw_position_values = tl.load(
|
|
raw_positions_ptr
|
|
+ raw_first_row * stride_raw_positions_row
|
|
+ position_dims * stride_raw_positions_dim,
|
|
mask=valid_row
|
|
& first_from_raw
|
|
& (raw_first_row >= query_row_start)
|
|
& (raw_first_row < query_row_end)
|
|
& (raw_first_row < num_rows)
|
|
& (position_dims < 3),
|
|
other=0,
|
|
)
|
|
compressor_state_position_values = tl.load(
|
|
rope_cache_ptr
|
|
+ tl.maximum(compressor_state_block, 0).to(tl.int64) * stride_rope_block
|
|
+ (first_position % COMPRESSOR_STATE_SIZE) * stride_rope_token
|
|
+ position_dims * stride_rope_dim,
|
|
mask=valid_row
|
|
& ~first_from_raw
|
|
& valid_compressor_state_block
|
|
& (position_dims < 3),
|
|
other=0,
|
|
)
|
|
position_values = tl.where(
|
|
first_from_raw,
|
|
raw_position_values,
|
|
compressor_state_position_values,
|
|
)
|
|
else:
|
|
position_values = tl.where(valid_row, first_position, 0)
|
|
tl.store(
|
|
first_positions_ptr
|
|
+ row * stride_positions_row
|
|
+ position_dims * stride_positions_dim,
|
|
position_values,
|
|
mask=(row < num_rows) & (position_dims < 3),
|
|
)
|
|
|
|
|
|
def _validate_mqa(q: torch.Tensor) -> None:
|
|
if q.ndim != 3 or q.shape[1] <= 0 or q.shape[2] <= 0:
|
|
raise ValueError("QSA query must be [rows, heads, head_dim]")
|
|
|
|
|
|
def qsa_mqa_paged(
|
|
q: torch.Tensor,
|
|
k_cache: torch.Tensor,
|
|
page_table: torch.Tensor,
|
|
token_to_req: torch.Tensor,
|
|
query_positions: torch.Tensor,
|
|
sequence_lengths: torch.Tensor,
|
|
compress_ratio: int,
|
|
num_columns: int | None = None,
|
|
score_scale: float | None = None,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Compute QSA scores directly from a paged compressed-key cache."""
|
|
_validate_mqa(q)
|
|
if not q.is_cuda or not HAS_TRITON:
|
|
raise RuntimeError("paged QSA scoring requires a GPU and Triton")
|
|
if k_cache.ndim != 4 or k_cache.shape[2] != 1:
|
|
raise ValueError("QSA cache must be [pages, page_size, 1, head_dim]")
|
|
if k_cache.shape[3] != q.shape[2]:
|
|
raise ValueError("QSA query and cache dimensions must match")
|
|
if page_table.ndim != 2:
|
|
raise ValueError("QSA page table must be two-dimensional")
|
|
if q.shape[0] and (not all(k_cache.shape[:2]) or not all(page_table.shape)):
|
|
raise ValueError("QSA paged scoring cache and page table must be nonempty")
|
|
if token_to_req.shape != (q.shape[0],):
|
|
raise ValueError("QSA request mapping must match query rows")
|
|
if query_positions.shape != (q.shape[0],):
|
|
raise ValueError("QSA query positions must match query rows")
|
|
if sequence_lengths.shape != (page_table.shape[0],):
|
|
raise ValueError("QSA sequence lengths must match page-table requests")
|
|
if compress_ratio <= 0:
|
|
raise ValueError("QSA compression ratio must be positive")
|
|
score_divisor = math.sqrt(q.shape[2]) if score_scale is None else score_scale
|
|
if score_divisor <= 0:
|
|
raise ValueError("QSA score scale must be positive")
|
|
|
|
capacity = page_table.shape[1] * k_cache.shape[1]
|
|
columns = capacity if num_columns is None else num_columns
|
|
if columns < 0:
|
|
raise ValueError("QSA score width must be non-negative")
|
|
logits = torch.empty((q.shape[0], columns), dtype=torch.float32, device=q.device)
|
|
visible_blocks = torch.empty(q.shape[0], dtype=torch.int32, device=q.device)
|
|
if not q.shape[0] or not columns:
|
|
return logits, visible_blocks
|
|
block_n = 32
|
|
_qsa_mqa_paged_kernel[(q.shape[0], triton.cdiv(columns, block_n))](
|
|
q,
|
|
k_cache,
|
|
page_table,
|
|
token_to_req,
|
|
query_positions,
|
|
sequence_lengths,
|
|
visible_blocks,
|
|
logits,
|
|
q.stride(0),
|
|
q.stride(1),
|
|
q.stride(2),
|
|
k_cache.stride(0),
|
|
k_cache.stride(1),
|
|
k_cache.stride(3),
|
|
page_table.stride(0),
|
|
page_table.stride(1),
|
|
logits.stride(0),
|
|
q.shape[0],
|
|
columns,
|
|
k_cache.shape[0],
|
|
page_table.shape[0],
|
|
float(score_divisor),
|
|
PAGE_SIZE=k_cache.shape[1],
|
|
PAGE_TABLE_WIDTH=page_table.shape[1],
|
|
NUM_HEADS=q.shape[1],
|
|
HEAD_DIM=q.shape[2],
|
|
BLOCK_N=block_n,
|
|
BLOCK_D=triton.next_power_of_2(q.shape[2]),
|
|
COMPRESS_RATIO=compress_ratio,
|
|
num_warps=4,
|
|
)
|
|
return logits, visible_blocks
|
|
|
|
|
|
def expand_qsa_block_indices_cuda(
|
|
block_indices: torch.Tensor,
|
|
query_positions: torch.Tensor,
|
|
sequence_lengths: torch.Tensor,
|
|
token_to_req: torch.Tensor,
|
|
compress_ratio: int,
|
|
token_topk: int,
|
|
out: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
"""Expand compressed blocks and compact the causal tail of the open group."""
|
|
if not block_indices.is_cuda or not HAS_TRITON:
|
|
raise RuntimeError("QSA index expansion requires a GPU and Triton")
|
|
if token_topk % compress_ratio:
|
|
raise ValueError("QSA token top-k must be divisible by compression ratio")
|
|
block_topk = token_topk // compress_ratio
|
|
output_width = token_topk + compress_ratio - 1
|
|
if block_indices.shape != (query_positions.numel(), block_topk):
|
|
raise ValueError("QSA compressed top-k has an invalid shape")
|
|
if token_to_req.shape != query_positions.shape:
|
|
raise ValueError("QSA request mapping must match query positions")
|
|
if sequence_lengths.ndim != 1 or not sequence_lengths.shape[0]:
|
|
raise ValueError("QSA request sequence lengths must be nonempty")
|
|
if out is None:
|
|
out = torch.empty(
|
|
(block_indices.shape[0], output_width),
|
|
dtype=torch.int32,
|
|
device=block_indices.device,
|
|
)
|
|
elif out.shape != (block_indices.shape[0], output_width):
|
|
raise ValueError("QSA expansion output has an invalid shape")
|
|
if not block_indices.shape[0]:
|
|
return out
|
|
column_block = 256
|
|
_expand_qsa_indices_kernel[
|
|
(block_indices.shape[0], triton.cdiv(output_width, column_block))
|
|
](
|
|
block_indices,
|
|
query_positions,
|
|
sequence_lengths,
|
|
token_to_req,
|
|
out,
|
|
block_indices.stride(0),
|
|
block_indices.stride(1),
|
|
out.stride(0),
|
|
out.stride(1),
|
|
block_indices.shape[0],
|
|
sequence_lengths.shape[0],
|
|
BLOCK_TOPK=block_topk,
|
|
COMPRESS_RATIO=compress_ratio,
|
|
TOKEN_TOPK=token_topk,
|
|
OUTPUT_WIDTH=output_width,
|
|
COLUMN_BLOCK=column_block,
|
|
num_warps=4,
|
|
)
|
|
return out
|
|
|
|
|
|
def qsa_select_paged_tokens(
|
|
q: torch.Tensor,
|
|
k_cache: torch.Tensor,
|
|
page_table: torch.Tensor,
|
|
token_to_req: torch.Tensor,
|
|
query_positions: torch.Tensor,
|
|
sequence_lengths: torch.Tensor,
|
|
token_topk: int,
|
|
compress_ratio: int,
|
|
out: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
"""Score, select, and expand QSA indices without host synchronization."""
|
|
rows = q.shape[0]
|
|
output_width = token_topk + compress_ratio - 1
|
|
if out is None:
|
|
out = torch.empty((rows, output_width), dtype=torch.int32, device=q.device)
|
|
if out.shape != (rows, output_width):
|
|
raise ValueError("QSA selection output has an invalid shape")
|
|
if not rows:
|
|
return out
|
|
|
|
columns = page_table.shape[1] * k_cache.shape[1]
|
|
block_topk = token_topk // compress_ratio
|
|
rows_per_chunk = max(1, _LOGITS_WORKSPACE_BYTES // max(columns * 4, 1))
|
|
chunk_rows = min(rows, rows_per_chunk)
|
|
blocks_buffer = torch.empty(
|
|
(chunk_rows, block_topk), dtype=torch.int32, device=q.device
|
|
)
|
|
topk_workspace = torch.empty(
|
|
(_TOPK_WORKSPACE_BYTES,), dtype=torch.uint8, device=q.device
|
|
)
|
|
for row_start in range(0, rows, rows_per_chunk):
|
|
row_end = min(row_start + rows_per_chunk, rows)
|
|
row_slice = slice(row_start, row_end)
|
|
logits, visible_blocks = qsa_mqa_paged(
|
|
q[row_slice],
|
|
k_cache,
|
|
page_table,
|
|
token_to_req[row_slice],
|
|
query_positions[row_slice],
|
|
sequence_lengths,
|
|
compress_ratio,
|
|
)
|
|
blocks = blocks_buffer[: row_end - row_start]
|
|
use_cooperative_topk = (
|
|
current_platform.is_cuda()
|
|
and blocks.shape[0] <= 32
|
|
and logits.stride(0) % 4 == 0
|
|
and current_platform.has_device_capability(90)
|
|
and not current_platform.is_device_capability_family(120)
|
|
)
|
|
if use_cooperative_topk:
|
|
torch.ops._C.cooperative_topk(
|
|
logits,
|
|
visible_blocks,
|
|
blocks,
|
|
topk_workspace,
|
|
block_topk,
|
|
columns,
|
|
)
|
|
elif current_platform.is_cuda():
|
|
torch.ops._C.persistent_topk(
|
|
logits,
|
|
visible_blocks,
|
|
blocks,
|
|
topk_workspace,
|
|
block_topk,
|
|
columns,
|
|
)
|
|
else:
|
|
ops.top_k_per_row_decode(
|
|
logits,
|
|
1,
|
|
visible_blocks,
|
|
blocks,
|
|
blocks.shape[0],
|
|
logits.stride(0),
|
|
logits.stride(1),
|
|
block_topk,
|
|
)
|
|
expand_qsa_block_indices_cuda(
|
|
blocks,
|
|
query_positions[row_slice],
|
|
sequence_lengths,
|
|
token_to_req[row_slice],
|
|
compress_ratio,
|
|
token_topk,
|
|
out[row_slice],
|
|
)
|
|
return out
|
|
|
|
|
|
def qsa_sparse_paged_attention(
|
|
q: torch.Tensor,
|
|
k_cache: torch.Tensor,
|
|
v_cache: torch.Tensor,
|
|
logical_indices: torch.Tensor,
|
|
block_table: torch.Tensor,
|
|
token_to_req: torch.Tensor,
|
|
out: torch.Tensor | None = None,
|
|
) -> torch.Tensor:
|
|
"""Run sparse GQA directly over paged BF16 K/V caches."""
|
|
if not q.is_cuda or not HAS_TRITON:
|
|
raise RuntimeError("paged QSA sparse attention requires a GPU and Triton")
|
|
if q.ndim != 3 or k_cache.ndim != 4 or v_cache.shape != k_cache.shape:
|
|
raise ValueError("QSA sparse attention received invalid Q/K/V shapes")
|
|
if logical_indices.ndim != 2 or logical_indices.shape[0] != q.shape[0]:
|
|
raise ValueError("QSA indices must have one row per query")
|
|
if token_to_req.shape != (q.shape[0],) or block_table.ndim != 2:
|
|
raise ValueError("QSA sparse attention metadata has invalid shapes")
|
|
if not all(k_cache.shape[:3]) or not all(block_table.shape):
|
|
raise ValueError("QSA sparse attention cache and block table must be nonempty")
|
|
if logical_indices.shape[1] <= 0:
|
|
raise ValueError("QSA sparse attention requires a positive selection width")
|
|
if q.shape[2] != k_cache.shape[3] or q.shape[1] % k_cache.shape[2]:
|
|
raise ValueError("QSA sparse attention requires valid grouped-query heads")
|
|
head_dim = q.shape[2]
|
|
assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0
|
|
assert q.dtype == k_cache.dtype == v_cache.dtype == torch.bfloat16
|
|
assert logical_indices.dtype == block_table.dtype == torch.int32
|
|
assert token_to_req.dtype == torch.int32
|
|
assert q.device == k_cache.device == v_cache.device
|
|
assert q.device == logical_indices.device == block_table.device
|
|
assert q.device == token_to_req.device
|
|
assert q.stride(2) == k_cache.stride(3) == v_cache.stride(3) == 1
|
|
assert logical_indices.stride(1) == block_table.stride(1) == 1
|
|
assert token_to_req.stride(0) == 1
|
|
if out is None:
|
|
out = torch.empty_like(q)
|
|
if out.shape != q.shape:
|
|
raise ValueError("QSA sparse output must match its query")
|
|
assert out.dtype == q.dtype and out.device == q.device
|
|
assert out.stride(2) == 1
|
|
if not q.shape[0]:
|
|
return out
|
|
|
|
group_size = q.shape[1] // k_cache.shape[2]
|
|
block_m = triton.next_power_of_2(group_size)
|
|
base_programs = q.shape[0] * k_cache.shape[2]
|
|
small_profile_limit = 8 if block_m <= 8 else 4
|
|
|
|
# Tuned on GB300 for the Qwen-Air TP1, TP2, and TP4 attention shapes.
|
|
# Narrow tiles favor decode; wide tiles improve throughput for prefill.
|
|
if base_programs <= small_profile_limit:
|
|
block_n, target_splits, partial_warps = 16, 64, 4
|
|
elif base_programs < 32:
|
|
block_n, target_splits, partial_warps = 16, 32, 4
|
|
elif base_programs <= 256:
|
|
block_n, target_splits, partial_warps = 64, 8, 2
|
|
elif base_programs <= 512:
|
|
block_n, target_splits, partial_warps = 64, 4, 2
|
|
else:
|
|
block_n, target_splits, partial_warps = 64, 1, 2
|
|
# gfx942 and gfx950 have a 64 KiB LDS limit. One software-pipelining
|
|
# stage keeps the wide TP4 tile within that shared-memory budget.
|
|
partial_stages = 1 if current_platform.is_rocm() else 2
|
|
|
|
num_tiles = triton.cdiv(logical_indices.shape[1], block_n)
|
|
# Avoid empty splits when the selection width is smaller than the profile.
|
|
max_useful_splits = 1 << (num_tiles.bit_length() - 1)
|
|
num_splits = min(max_useful_splits, target_splits)
|
|
|
|
# Split=1 writes output directly and compiles out all workspace accesses.
|
|
if num_splits == 1:
|
|
partial_output = out
|
|
partial_lse = out
|
|
else:
|
|
# FP32 partials preserve accuracy when merging independently normalized
|
|
# splits.
|
|
partial_output = torch.empty(
|
|
(num_splits, *q.shape), dtype=torch.float32, device=q.device
|
|
)
|
|
partial_lse = torch.empty(
|
|
(num_splits, q.shape[0], q.shape[1]),
|
|
dtype=torch.float32,
|
|
device=q.device,
|
|
)
|
|
|
|
partial_grid = (q.shape[0], k_cache.shape[2], num_splits)
|
|
_qsa_sparse_paged_gqa_splitk_kernel[partial_grid](
|
|
q,
|
|
k_cache,
|
|
v_cache,
|
|
logical_indices,
|
|
block_table,
|
|
token_to_req,
|
|
partial_output,
|
|
partial_lse,
|
|
out,
|
|
q.stride(0),
|
|
q.stride(1),
|
|
k_cache.stride(0),
|
|
k_cache.stride(1),
|
|
k_cache.stride(2),
|
|
v_cache.stride(0),
|
|
v_cache.stride(1),
|
|
v_cache.stride(2),
|
|
logical_indices.stride(0),
|
|
block_table.stride(0),
|
|
out.stride(0),
|
|
out.stride(1),
|
|
q.shape[0],
|
|
k_cache.shape[0],
|
|
block_table.shape[0],
|
|
TOPK=logical_indices.shape[1],
|
|
PAGE_SIZE=k_cache.shape[1],
|
|
PAGE_TABLE_WIDTH=block_table.shape[1],
|
|
GROUP_SIZE=group_size,
|
|
HEAD_DIM=q.shape[2],
|
|
NUM_QUERY_HEADS=q.shape[1],
|
|
NUM_SPLITS=num_splits,
|
|
NUM_TILES=num_tiles,
|
|
BLOCK_M=block_m,
|
|
BLOCK_N=block_n,
|
|
num_warps=partial_warps,
|
|
num_stages=partial_stages,
|
|
)
|
|
if num_splits == 1:
|
|
return out
|
|
|
|
_qsa_merge_splitk_kernel[(q.shape[0], q.shape[1])](
|
|
partial_output,
|
|
partial_lse,
|
|
out,
|
|
out.stride(0),
|
|
out.stride(1),
|
|
q.shape[0],
|
|
HEAD_DIM=q.shape[2],
|
|
NUM_QUERY_HEADS=q.shape[1],
|
|
NUM_SPLITS=num_splits,
|
|
BLOCK_SPLITS=triton.next_power_of_2(num_splits),
|
|
num_warps=2,
|
|
num_stages=1,
|
|
)
|
|
return out
|
|
|
|
|
|
def qsa_store_cache_rows(
|
|
cache: torch.Tensor,
|
|
slot_mapping: torch.Tensor,
|
|
rows: torch.Tensor,
|
|
) -> None:
|
|
"""Store fixed-width rows in a QSA cache without boolean indexing."""
|
|
if not cache.is_cuda or not HAS_TRITON:
|
|
raise RuntimeError("QSA cache stores require a GPU and Triton")
|
|
if cache.ndim != 4 or cache.shape[2] != 1:
|
|
raise ValueError("QSA cache must be [pages, page_size, 1, width]")
|
|
if not all(cache.shape):
|
|
raise ValueError("QSA cache dimensions must be nonzero")
|
|
if rows.ndim == 3:
|
|
if rows.shape[1] != 1:
|
|
raise ValueError("QSA cache rows must have one head")
|
|
rows = rows[:, 0]
|
|
if rows.shape != (slot_mapping.numel(), cache.shape[3]):
|
|
raise ValueError("QSA cache rows and slots have incompatible shapes")
|
|
if not rows.shape[0]:
|
|
return
|
|
_store_qsa_rows_kernel[(rows.shape[0],)](
|
|
cache,
|
|
slot_mapping,
|
|
rows,
|
|
cache.stride(0),
|
|
cache.stride(1),
|
|
cache.stride(3),
|
|
rows.stride(0),
|
|
rows.stride(1),
|
|
rows.shape[0],
|
|
cache.shape[0],
|
|
PAGE_SIZE=cache.shape[1],
|
|
WIDTH=cache.shape[3],
|
|
BLOCK_D=triton.next_power_of_2(cache.shape[3]),
|
|
num_warps=4,
|
|
)
|
|
|
|
|
|
def qsa_compress_groups_with_ratio(
|
|
raw_keys: torch.Tensor, # this step's raw key rows [rows, 1, head_size]
|
|
raw_positions: torch.Tensor, # this step's positions [rows, 1, 3] int64
|
|
compressor_state_cache: torch.Tensor,
|
|
compressor_state_block_table: torch.Tensor,
|
|
token_to_req: torch.Tensor,
|
|
query_start_loc: torch.Tensor,
|
|
logical_positions: torch.Tensor,
|
|
compressed_slots: torch.Tensor,
|
|
compress_ratio: int,
|
|
rope_cache: torch.Tensor | None = None,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Pool completed groups from the compressor-state ring and raw token rows."""
|
|
if not raw_keys.is_cuda or not HAS_TRITON:
|
|
raise RuntimeError("QSA compression requires a GPU and Triton")
|
|
rows = token_to_req.numel()
|
|
if compress_ratio <= 0:
|
|
raise ValueError("QSA compression ratio must be positive")
|
|
if raw_keys.ndim != 3 or raw_keys.shape[:2] != (rows, 1):
|
|
raise ValueError("QSA raw keys must be [rows, 1, head_size]")
|
|
if raw_positions.shape != (rows, 1, 3) or raw_positions.dtype != torch.int64:
|
|
raise ValueError("QSA raw positions must be [rows, 1, 3] int64")
|
|
if logical_positions.shape != (rows,) or compressed_slots.shape != (rows,):
|
|
raise ValueError("QSA compression metadata must match token rows")
|
|
if compressor_state_cache.ndim != 4 or compressor_state_cache.shape[2] != 1:
|
|
raise ValueError("QSA compressor-state cache has an invalid shape")
|
|
if (
|
|
# The ring is wider than one group so speculative rows cannot alias
|
|
# onto the committed keys of the group still being collected.
|
|
compressor_state_cache.shape[1] < compress_ratio
|
|
or compressor_state_cache.shape[3] != raw_keys.shape[2]
|
|
or compressor_state_cache.dtype != raw_keys.dtype
|
|
):
|
|
raise ValueError(
|
|
"QSA compressor-state cache does not match the compression layout"
|
|
)
|
|
if (
|
|
compressor_state_block_table.ndim != 2
|
|
or compressor_state_block_table.shape[1] < 1
|
|
):
|
|
raise ValueError(
|
|
"QSA compressor-state block table must contain one block per request"
|
|
)
|
|
if query_start_loc.ndim != 1 or query_start_loc.shape[0] < 2:
|
|
raise ValueError("QSA query starts must contain a terminal offset")
|
|
num_requests = query_start_loc.shape[0] - 1
|
|
if compressor_state_block_table.shape[0] < num_requests:
|
|
raise ValueError("QSA compressor-state block table has too few request rows")
|
|
if rope_cache is not None and (
|
|
rope_cache.ndim != 4
|
|
or rope_cache.shape[:3] != compressor_state_cache.shape[:3]
|
|
or rope_cache.shape[3] != 3
|
|
or rope_cache.dtype != torch.int64
|
|
):
|
|
raise ValueError("QSA packed position view has an invalid shape or dtype")
|
|
if rows and (
|
|
not all(compressor_state_cache.shape)
|
|
or not all(compressor_state_block_table.shape)
|
|
):
|
|
raise ValueError("QSA compressor-state cache and block table must be nonempty")
|
|
pooled = torch.empty(
|
|
(rows, 1, raw_keys.shape[2]),
|
|
dtype=raw_keys.dtype,
|
|
device=raw_keys.device,
|
|
)
|
|
first_positions = torch.empty((rows, 3), dtype=torch.int64, device=raw_keys.device)
|
|
if not rows:
|
|
return pooled, first_positions
|
|
if rope_cache is None:
|
|
rope_cache = compressor_state_cache
|
|
load_rope_positions = False
|
|
else:
|
|
load_rope_positions = True
|
|
_compress_qsa_groups_kernel[(rows,)](
|
|
raw_keys,
|
|
raw_positions,
|
|
compressor_state_cache,
|
|
rope_cache,
|
|
compressor_state_block_table,
|
|
token_to_req,
|
|
query_start_loc,
|
|
logical_positions,
|
|
compressed_slots,
|
|
pooled,
|
|
first_positions,
|
|
raw_keys.stride(0),
|
|
raw_keys.stride(2),
|
|
raw_positions.stride(0),
|
|
raw_positions.stride(2),
|
|
compressor_state_cache.stride(0),
|
|
compressor_state_cache.stride(1),
|
|
compressor_state_cache.stride(3),
|
|
rope_cache.stride(0),
|
|
rope_cache.stride(1),
|
|
rope_cache.stride(3),
|
|
compressor_state_block_table.stride(0),
|
|
pooled.stride(0),
|
|
pooled.stride(2),
|
|
first_positions.stride(0),
|
|
first_positions.stride(1),
|
|
rows,
|
|
compressor_state_cache.shape[0],
|
|
num_requests,
|
|
COMPRESSOR_STATE_SIZE=compressor_state_cache.shape[1],
|
|
COMPRESS_RATIO=compress_ratio,
|
|
HEAD_DIM=raw_keys.shape[2],
|
|
LOAD_ROPE_POSITIONS=load_rope_positions,
|
|
BLOCK_D=triton.next_power_of_2(raw_keys.shape[2]),
|
|
num_warps=4,
|
|
)
|
|
return pooled, first_positions
|
|
|
|
|
|
__all__ = [
|
|
"expand_qsa_block_indices_cuda",
|
|
"qsa_compress_groups_with_ratio",
|
|
"qsa_mqa_paged",
|
|
"qsa_select_paged_tokens",
|
|
"qsa_sparse_paged_attention",
|
|
"qsa_store_cache_rows",
|
|
]
|