Signed-off-by: kimseunghyun <hellonypcteam@gmail.com> Co-authored-by: kimseunghyun <hellonypcteam@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Thien Tran <gau.nernst@yahoo.com.sg>
1117 lines
39 KiB
Python
1117 lines
39 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Triton kernels for Qwen4Exp QSA sparse attention and cache updates."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from functools import lru_cache
|
|
|
|
import torch
|
|
|
|
from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
|
TritonWarmupTensor,
|
|
triton_scalar_specialization_rep,
|
|
)
|
|
from vllm.platforms import current_platform
|
|
from vllm.triton_utils import HAS_TRITON, tl, triton
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def _is_sm120() -> bool:
|
|
"""True on sm_120 (RTX PRO 6000 Blackwell): selects the sm_120 tuning table."""
|
|
return current_platform.get_device_capability() == (12, 0)
|
|
|
|
|
|
@triton.jit
|
|
def _maybe_dequant(tile):
|
|
if tile.dtype == tl.uint8:
|
|
# e4m3 bytes below sm_89 (no Triton fp8e4nv pointer type): sign to bit
|
|
# 15, exp+mantissa to bits 13..7 gives the fp16 of x * 2^-8. Exact for
|
|
# every finite code; the two NaN codes decode to +-480 after the host
|
|
# scale instead of NaN. Decodes four bytes per 32-bit register: the
|
|
# rotate moves each byte's bits 6..1 to 5..0 and its bit 0 to bit 7 of
|
|
# the byte below, lop3 restores the sign bits, prmt pairs high and low
|
|
# bytes, and the mask clears the sign copy at bit 14 and bits 6..0.
|
|
# pack=4 puts element k in byte k of $2 and half k of ($0, $1), the
|
|
# same lane mapping as Triton's own sm<89 fp8 -> fp16 asm.
|
|
return tl.inline_asm_elementwise(
|
|
asm="""
|
|
{
|
|
.reg .b32 rot, high;
|
|
shf.r.wrap.b32 rot, $2, $2, 1;
|
|
lop3.b32 high, $2, rot, 0x80808080, 0xE4;
|
|
prmt.b32 $0, rot, high, 0x5043;
|
|
prmt.b32 $1, rot, high, 0x7261;
|
|
and.b32 $0, $0, 0xBF80BF80;
|
|
and.b32 $1, $1, 0xBF80BF80;
|
|
}
|
|
""",
|
|
constraints="=r,=r,r",
|
|
args=[tile],
|
|
dtype=tl.float16,
|
|
is_pure=True,
|
|
pack=4,
|
|
)
|
|
elif tile.dtype == tl.float8e4nv:
|
|
return tile.to(tl.float16)
|
|
else:
|
|
return tile
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def _is_sm80() -> bool:
|
|
"""True on sm_80 (A100/A30): selects the wide FP8 byte-decode tiles."""
|
|
return current_platform.get_device_capability() == (8, 0)
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def _is_sm90() -> bool:
|
|
"""True on sm_90 (H100/H200/H20): selects the sm_90 tuning table."""
|
|
return current_platform.get_device_capability() == (9, 0)
|
|
|
|
|
|
@triton.jit(do_not_specialize=["num_rows", "num_requests"])
|
|
def _qsa_sparse_paged_gqa_splitk_kernel(
|
|
q_ptr,
|
|
k_cache_ptr,
|
|
v_cache_ptr,
|
|
indices_ptr,
|
|
block_table_ptr,
|
|
token_to_req_ptr,
|
|
partial_output_ptr,
|
|
partial_lse_ptr,
|
|
output_ptr,
|
|
softmax_scale,
|
|
output_scale,
|
|
output_gate_ptr,
|
|
stride_q_row,
|
|
stride_q_head,
|
|
stride_k_block,
|
|
stride_k_token,
|
|
stride_k_head,
|
|
stride_v_block,
|
|
stride_v_token,
|
|
stride_v_head,
|
|
stride_indices_row,
|
|
stride_table_req,
|
|
stride_output_row,
|
|
stride_output_head,
|
|
stride_output_gate_row,
|
|
stride_output_gate_head,
|
|
num_rows,
|
|
num_cache_blocks,
|
|
num_requests,
|
|
TOPK: tl.constexpr,
|
|
PAGE_SIZE: tl.constexpr,
|
|
PAGE_TABLE_WIDTH: tl.constexpr,
|
|
GROUP_SIZE: tl.constexpr,
|
|
HEAD_DIM: tl.constexpr,
|
|
NUM_QUERY_HEADS: tl.constexpr,
|
|
NUM_SPLITS: tl.constexpr,
|
|
NUM_TILES: tl.constexpr,
|
|
BLOCK_M: tl.constexpr,
|
|
BLOCK_N: tl.constexpr,
|
|
) -> None:
|
|
row = tl.program_id(0)
|
|
kv_head = tl.program_id(1)
|
|
split_id = tl.program_id(2)
|
|
request = tl.load(token_to_req_ptr + row)
|
|
safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
|
|
|
|
# The packed selection buffer carries one TRAILING COUNT COLUMN per row
|
|
# (column TOPK of a TOPK+1-wide buffer): the row's valid-entry count,
|
|
# written by the expand kernel. It is never a token index — the tile loop
|
|
# and the index load below only ever cover columns [0, TOPK).
|
|
valid_count = tl.load(indices_ptr + row * stride_indices_row + TOPK)
|
|
|
|
head_offsets = tl.arange(0, BLOCK_M)
|
|
dim_offsets = tl.arange(0, HEAD_DIM)
|
|
column_offsets = tl.arange(0, BLOCK_N)
|
|
first_head = kv_head * GROUP_SIZE
|
|
query = tl.load(
|
|
q_ptr
|
|
+ row * stride_q_row
|
|
+ (first_head + head_offsets[:, None]) * stride_q_head
|
|
+ dim_offsets[None, :],
|
|
mask=head_offsets[:, None] < GROUP_SIZE,
|
|
other=0.0,
|
|
)
|
|
|
|
max_value = tl.full((BLOCK_M,), -1.0e20, dtype=tl.float32)
|
|
normalizer = tl.zeros((BLOCK_M,), dtype=tl.float32)
|
|
accumulator = tl.zeros((BLOCK_M, HEAD_DIM), dtype=tl.float32)
|
|
# softmax_scale is the host-side attention scale (1/sqrt(head_dim), with the
|
|
# fp8 K dequant scale already pre-multiplied in); convert to log2 units once
|
|
# here for the exp2-based online softmax.
|
|
score_scale = softmax_scale * 1.4426950408889634
|
|
|
|
tile_end = tl.minimum(NUM_TILES, tl.cdiv(tl.minimum(valid_count, TOPK), BLOCK_N))
|
|
|
|
for tile in range(split_id, tile_end, NUM_SPLITS):
|
|
columns = tile * BLOCK_N + column_offsets
|
|
logical_token = tl.load(
|
|
indices_ptr + row * stride_indices_row + columns,
|
|
mask=columns < TOPK,
|
|
other=-1,
|
|
)
|
|
safe_token = tl.maximum(logical_token, 0)
|
|
logical_page = safe_token // PAGE_SIZE
|
|
page_offset = safe_token % PAGE_SIZE
|
|
valid = (
|
|
(request >= 0)
|
|
& (request < num_requests)
|
|
& (logical_token >= 0)
|
|
& (logical_page < PAGE_TABLE_WIDTH)
|
|
)
|
|
physical_page = tl.load(
|
|
block_table_ptr
|
|
+ safe_request * stride_table_req
|
|
+ tl.minimum(logical_page, PAGE_TABLE_WIDTH - 1),
|
|
mask=valid,
|
|
other=-1,
|
|
)
|
|
valid &= (physical_page >= 0) & (physical_page < num_cache_blocks)
|
|
# physical_page * block stride can overflow int32 for large caches.
|
|
safe_page = tl.maximum(physical_page, 0).to(tl.int64)
|
|
keys = tl.load(
|
|
k_cache_ptr
|
|
+ safe_page[None, :] * stride_k_block
|
|
+ page_offset[None, :] * stride_k_token
|
|
+ kv_head * stride_k_head
|
|
+ dim_offsets[:, None],
|
|
mask=valid[None, :],
|
|
other=0.0,
|
|
)
|
|
# e4m3 -> Q dtype is exact; keep the QK dot in Q's dtype (fp8 QK
|
|
# measured slower here and less accurate).
|
|
keys = _maybe_dequant(keys).to(query.dtype)
|
|
values = tl.load(
|
|
v_cache_ptr
|
|
+ safe_page[:, None] * stride_v_block
|
|
+ page_offset[:, None] * stride_v_token
|
|
+ kv_head * stride_v_head
|
|
+ dim_offsets[None, :],
|
|
mask=valid[:, None],
|
|
other=0.0,
|
|
)
|
|
# Dequant V to fp16 (not bf16) for the PV dot: P <= 1 (online
|
|
# softmax) so fp16 has the range, its wider mantissa is more
|
|
# accurate, and the fp8->fp16 upcast with an fp16 PV dot is faster.
|
|
values = _maybe_dequant(values)
|
|
scores = tl.dot(query, keys)
|
|
# Scaling scores avoids re-quantizing a scaled query to BF16; for fp8
|
|
# caches the K dequant scale is already folded into softmax_scale on the
|
|
# host.
|
|
scores *= score_scale
|
|
scores = tl.where(valid[None, :], scores, -1.0e20)
|
|
next_max = tl.maximum(max_value, tl.max(scores, axis=1))
|
|
alpha = tl.math.exp2(max_value - next_max)
|
|
probabilities = tl.where(
|
|
valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0
|
|
)
|
|
accumulator = tl.dot(
|
|
probabilities.to(values.dtype),
|
|
values,
|
|
acc=accumulator * alpha[:, None],
|
|
)
|
|
normalizer = normalizer * alpha + tl.sum(probabilities, axis=1)
|
|
max_value = next_max
|
|
|
|
has_values = normalizer > 0
|
|
# Fold the fp8 V dequant scale (output_scale, 1.0 for bf16) into a per-row
|
|
# reciprocal normalizer, so the output is a per-row multiply rather than a
|
|
# HEAD_DIM-wide scale. The split-K LSE below keeps the unscaled normalizer,
|
|
# so the merge stays correct.
|
|
inv_normalizer = output_scale / tl.maximum(normalizer, 1.0e-20)
|
|
normalized_output = tl.where(
|
|
has_values[:, None],
|
|
accumulator * inv_normalizer[:, None],
|
|
0.0,
|
|
)
|
|
output_mask = head_offsets[:, None] < GROUP_SIZE
|
|
if NUM_SPLITS == 1:
|
|
# Preserve the unfused path's BF16 attention-output rounding before
|
|
# applying the gate in FP32.
|
|
normalized_output = normalized_output.to(output_ptr.dtype.element_ty)
|
|
output_gate = tl.load(
|
|
output_gate_ptr
|
|
+ row * stride_output_gate_row
|
|
+ (first_head + head_offsets[:, None]) * stride_output_gate_head
|
|
+ dim_offsets[None, :],
|
|
mask=output_mask,
|
|
other=0.0,
|
|
).to(tl.float32)
|
|
normalized_output = normalized_output.to(tl.float32) * tl.sigmoid(output_gate)
|
|
tl.store(
|
|
output_ptr
|
|
+ row * stride_output_row
|
|
+ (first_head + head_offsets[:, None]) * stride_output_head
|
|
+ dim_offsets[None, :],
|
|
normalized_output,
|
|
mask=output_mask,
|
|
)
|
|
else:
|
|
partial_lse = tl.where(
|
|
has_values,
|
|
max_value + tl.math.log2(tl.maximum(normalizer, 1.0e-20)),
|
|
-float("inf"),
|
|
)
|
|
partial_row = (split_id * num_rows + row).to(tl.int64)
|
|
tl.store(
|
|
partial_output_ptr
|
|
+ (partial_row * NUM_QUERY_HEADS + first_head + head_offsets[:, None])
|
|
* HEAD_DIM
|
|
+ dim_offsets[None, :],
|
|
normalized_output,
|
|
mask=output_mask,
|
|
)
|
|
tl.store(
|
|
partial_lse_ptr + partial_row * NUM_QUERY_HEADS + first_head + head_offsets,
|
|
partial_lse,
|
|
mask=head_offsets < GROUP_SIZE,
|
|
)
|
|
|
|
|
|
@triton.jit(do_not_specialize=["num_rows"])
|
|
def _qsa_merge_splitk_kernel(
|
|
partial_output_ptr,
|
|
partial_lse_ptr,
|
|
output_ptr,
|
|
output_gate_ptr,
|
|
stride_output_row,
|
|
stride_output_head,
|
|
stride_output_gate_row,
|
|
stride_output_gate_head,
|
|
num_rows,
|
|
HEAD_DIM: tl.constexpr,
|
|
NUM_QUERY_HEADS: tl.constexpr,
|
|
NUM_SPLITS: tl.constexpr,
|
|
BLOCK_SPLITS: tl.constexpr,
|
|
) -> None:
|
|
row = tl.program_id(0)
|
|
head = tl.program_id(1)
|
|
split_offsets = tl.arange(0, BLOCK_SPLITS)
|
|
dim_offsets = tl.arange(0, HEAD_DIM)
|
|
split_mask = split_offsets < NUM_SPLITS
|
|
lse = tl.load(
|
|
partial_lse_ptr + (split_offsets * num_rows + row) * NUM_QUERY_HEADS + head,
|
|
mask=split_mask,
|
|
other=-float("inf"),
|
|
)
|
|
lse_max = tl.max(lse, axis=0)
|
|
has_values = lse_max > -float("inf")
|
|
shifted = tl.where(split_mask & has_values, lse - lse_max, -float("inf"))
|
|
weights = tl.math.exp2(shifted)
|
|
denominator = tl.sum(weights, axis=0)
|
|
split_rows = split_offsets.to(tl.int64) * num_rows + row
|
|
partial_output = tl.load(
|
|
partial_output_ptr
|
|
+ (split_rows[:, None] * NUM_QUERY_HEADS + head) * HEAD_DIM
|
|
+ dim_offsets[None, :],
|
|
mask=split_mask[:, None],
|
|
other=0.0,
|
|
)
|
|
merged = tl.sum(partial_output * weights[:, None], axis=0)
|
|
merged = tl.where(denominator > 0, merged / denominator, 0.0)
|
|
# Preserve the unfused path's BF16 attention-output rounding before
|
|
# applying the gate in FP32.
|
|
merged = merged.to(output_ptr.dtype.element_ty)
|
|
output_gate = tl.load(
|
|
output_gate_ptr
|
|
+ row * stride_output_gate_row
|
|
+ head * stride_output_gate_head
|
|
+ dim_offsets
|
|
).to(tl.float32)
|
|
merged = merged.to(tl.float32) * tl.sigmoid(output_gate)
|
|
tl.store(
|
|
output_ptr + row * stride_output_row + head * stride_output_head + dim_offsets,
|
|
merged,
|
|
)
|
|
|
|
|
|
@triton.jit
|
|
def _store_qsa_rows_kernel(
|
|
cache_ptr,
|
|
slots_ptr,
|
|
rows_ptr,
|
|
stride_cache_block,
|
|
stride_cache_token,
|
|
stride_cache_dim,
|
|
stride_rows_row,
|
|
stride_rows_dim,
|
|
num_rows,
|
|
num_blocks,
|
|
PAGE_SIZE: tl.constexpr,
|
|
WIDTH: tl.constexpr,
|
|
BLOCK_D: tl.constexpr,
|
|
) -> None:
|
|
row = tl.program_id(0)
|
|
dims = tl.arange(0, BLOCK_D)
|
|
slot = tl.load(slots_ptr + row)
|
|
valid = (row < num_rows) & (slot >= 0) & (slot < num_blocks * PAGE_SIZE)
|
|
block = tl.maximum(slot, 0) // PAGE_SIZE
|
|
token = tl.maximum(slot, 0) % PAGE_SIZE
|
|
values = tl.load(
|
|
rows_ptr + row * stride_rows_row + dims * stride_rows_dim,
|
|
mask=valid & (dims < WIDTH),
|
|
other=0,
|
|
)
|
|
tl.store(
|
|
cache_ptr
|
|
+ block * stride_cache_block
|
|
+ token * stride_cache_token
|
|
+ dims * stride_cache_dim,
|
|
values,
|
|
mask=valid & (dims < WIDTH),
|
|
)
|
|
|
|
|
|
@triton.jit
|
|
def _compress_qsa_groups_kernel(
|
|
raw_keys_ptr, # this step's raw key rows, straight from activations
|
|
raw_positions_ptr, # this step's per-token positions
|
|
compressor_state_cache_ptr, # per-request ring of previous raw keys
|
|
rope_cache_ptr, # packed RoPE position tail of the ring
|
|
compressor_state_table_ptr,
|
|
token_to_req_ptr,
|
|
query_start_loc_ptr,
|
|
logical_positions_ptr,
|
|
compressed_slots_ptr,
|
|
pooled_ptr,
|
|
first_positions_ptr,
|
|
stride_raw_row,
|
|
stride_raw_dim,
|
|
stride_raw_positions_row,
|
|
stride_raw_positions_dim,
|
|
stride_compressor_state_block,
|
|
stride_compressor_state_token,
|
|
stride_compressor_state_dim,
|
|
stride_rope_block,
|
|
stride_rope_token,
|
|
stride_rope_dim,
|
|
stride_compressor_state_table_req,
|
|
stride_pooled_row,
|
|
stride_pooled_dim,
|
|
stride_positions_row,
|
|
stride_positions_dim,
|
|
num_rows,
|
|
num_compressor_state_blocks,
|
|
num_requests,
|
|
COMPRESSOR_STATE_SIZE: tl.constexpr,
|
|
COMPRESS_RATIO: tl.constexpr,
|
|
HEAD_DIM: tl.constexpr,
|
|
BLOCK_D: tl.constexpr,
|
|
LOAD_ROPE_POSITIONS: tl.constexpr,
|
|
) -> None:
|
|
row = tl.program_id(0)
|
|
dims = tl.arange(0, BLOCK_D)
|
|
request = tl.load(token_to_req_ptr + row)
|
|
end_position = tl.load(logical_positions_ptr + row)
|
|
compressed_slot = tl.load(compressed_slots_ptr + row)
|
|
valid_request = (request >= 0) & (request < num_requests)
|
|
safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
|
|
query_row_start = tl.load(
|
|
query_start_loc_ptr + safe_request, mask=valid_request, other=0
|
|
)
|
|
query_row_end = tl.load(
|
|
query_start_loc_ptr + safe_request + 1, mask=valid_request, other=0
|
|
)
|
|
chunk_start_position = end_position - (row - query_row_start)
|
|
compressor_state_block = tl.load(
|
|
compressor_state_table_ptr + safe_request * stride_compressor_state_table_req,
|
|
mask=valid_request,
|
|
other=-1,
|
|
)
|
|
valid_compressor_state_block = (compressor_state_block >= 0) & (
|
|
compressor_state_block < num_compressor_state_blocks
|
|
)
|
|
valid_row = (
|
|
(row < num_rows)
|
|
& valid_request
|
|
& (row >= query_row_start)
|
|
& (row < query_row_end)
|
|
& (end_position >= COMPRESS_RATIO - 1)
|
|
& (compressed_slot >= 0)
|
|
)
|
|
accumulator = tl.zeros((BLOCK_D,), dtype=tl.float32)
|
|
|
|
# A group can span the compressor-state ring (older members) and this
|
|
# step's raw rows (members at positions >= chunk_start_position).
|
|
for group_offset in tl.range(0, COMPRESS_RATIO):
|
|
position = end_position - (COMPRESS_RATIO - 1 - group_offset)
|
|
use_raw = position >= chunk_start_position
|
|
raw_row = query_row_start + position - chunk_start_position
|
|
raw_values = tl.load(
|
|
raw_keys_ptr + raw_row * stride_raw_row + dims * stride_raw_dim,
|
|
mask=valid_row
|
|
& use_raw
|
|
& (raw_row >= query_row_start)
|
|
& (raw_row < query_row_end)
|
|
& (raw_row < num_rows)
|
|
& (dims < HEAD_DIM),
|
|
other=0.0,
|
|
).to(tl.float32)
|
|
compressor_state_values = tl.load(
|
|
compressor_state_cache_ptr
|
|
+ tl.maximum(compressor_state_block, 0).to(tl.int64)
|
|
* stride_compressor_state_block
|
|
+ (position % COMPRESSOR_STATE_SIZE) * stride_compressor_state_token
|
|
+ dims * stride_compressor_state_dim,
|
|
mask=valid_row
|
|
& ~use_raw
|
|
& valid_compressor_state_block
|
|
& (dims < HEAD_DIM),
|
|
other=0.0,
|
|
).to(tl.float32)
|
|
accumulator += tl.where(use_raw, raw_values, compressor_state_values)
|
|
|
|
tl.store(
|
|
pooled_ptr + row * stride_pooled_row + dims * stride_pooled_dim,
|
|
accumulator / COMPRESS_RATIO,
|
|
mask=(row < num_rows) & (dims < HEAD_DIM),
|
|
)
|
|
|
|
position_dims = tl.arange(0, 4)
|
|
first_position = end_position - COMPRESS_RATIO + 1
|
|
if LOAD_ROPE_POSITIONS:
|
|
first_from_raw = first_position >= chunk_start_position
|
|
raw_first_row = query_row_start + first_position - chunk_start_position
|
|
raw_position_values = tl.load(
|
|
raw_positions_ptr
|
|
+ raw_first_row * stride_raw_positions_row
|
|
+ position_dims * stride_raw_positions_dim,
|
|
mask=valid_row
|
|
& first_from_raw
|
|
& (raw_first_row >= query_row_start)
|
|
& (raw_first_row < query_row_end)
|
|
& (raw_first_row < num_rows)
|
|
& (position_dims < 3),
|
|
other=0,
|
|
)
|
|
compressor_state_position_values = tl.load(
|
|
rope_cache_ptr
|
|
+ tl.maximum(compressor_state_block, 0).to(tl.int64) * stride_rope_block
|
|
+ (first_position % COMPRESSOR_STATE_SIZE) * stride_rope_token
|
|
+ position_dims * stride_rope_dim,
|
|
mask=valid_row
|
|
& ~first_from_raw
|
|
& valid_compressor_state_block
|
|
& (position_dims < 3),
|
|
other=0,
|
|
)
|
|
position_values = tl.where(
|
|
first_from_raw,
|
|
raw_position_values,
|
|
compressor_state_position_values,
|
|
)
|
|
else:
|
|
position_values = tl.where(valid_row, first_position, 0)
|
|
tl.store(
|
|
first_positions_ptr
|
|
+ row * stride_positions_row
|
|
+ position_dims * stride_positions_dim,
|
|
position_values,
|
|
mask=(row < num_rows) & (position_dims < 3),
|
|
)
|
|
|
|
|
|
def _select_sm120_config(
|
|
base_programs: int, use_prefill_config: bool, is_fp8: bool
|
|
) -> tuple[int, int, int]:
|
|
"""(block_n, target_splits, num_warps) retuned on sm_120 (RTX PRO 6000
|
|
Blackwell), split by cache dtype: bf16 and fp8 favour different configs on
|
|
sm_120, most visibly on the large-prefill region. Each entry is the fastest
|
|
config that stays correct on its own path; the main win is more warps.
|
|
"""
|
|
if is_fp8:
|
|
if base_programs > 2048:
|
|
return (32, 2, 1) if use_prefill_config else (64, 1, 2)
|
|
if base_programs <= 24:
|
|
return 64, 64, 8
|
|
if base_programs <= 32:
|
|
return 128, 8, 4
|
|
if base_programs <= 64:
|
|
return 64, 8, 4
|
|
if base_programs <= 128:
|
|
return 32, 4, 4
|
|
if base_programs <= 256:
|
|
return 128, 4, 4
|
|
if base_programs <= 512:
|
|
return 64, 4, 4
|
|
return 32, 2, 1
|
|
# bf16 K/V cache.
|
|
if base_programs > 2048:
|
|
return 64, 2, 2
|
|
if base_programs <= 24:
|
|
return 64, 64, 8
|
|
if base_programs <= 64:
|
|
return 64, 8, 8
|
|
if base_programs <= 128:
|
|
return 32, 8, 8
|
|
if base_programs <= 256:
|
|
return 32, 8, 1
|
|
if base_programs <= 512:
|
|
return 64, 4, 2
|
|
return 32, 4, 1
|
|
|
|
|
|
def _select_sm90_config(
|
|
base_programs: int, use_prefill_config: bool, is_fp8: bool
|
|
) -> tuple[int, int, int]:
|
|
"""(block_n, target_splits, num_warps) tuned on sm_90 (H20)."""
|
|
if base_programs > 2048:
|
|
return (32, 1, 1) if use_prefill_config else (64, 1, 2)
|
|
if is_fp8:
|
|
if base_programs <= 24:
|
|
return 64, 64, 2
|
|
if base_programs <= 32:
|
|
return 32, 16, 1
|
|
if base_programs <= 64:
|
|
return 32, 8, 1
|
|
if base_programs <= 128:
|
|
return 32, 4, 1
|
|
if base_programs <= 256:
|
|
return 32, 8, 1
|
|
return 32, 4, 1
|
|
if base_programs <= 32:
|
|
return 64, 64, 2
|
|
if base_programs <= 64:
|
|
return 32, 16, 1
|
|
if base_programs <= 256:
|
|
return 32, 8, 1
|
|
return 32, 4, 1
|
|
|
|
|
|
def _select_config(
|
|
num_rows: int,
|
|
num_kv_heads: int,
|
|
use_prefill_config: bool,
|
|
num_columns: int,
|
|
is_fp8: bool = False,
|
|
) -> tuple[int, int, int, int]:
|
|
"""Select (block_n, num_warps, num_tiles, num_splits) for the kernel.
|
|
|
|
Keyed on base_programs = num_rows * num_kv_heads. The bp > 2048 region splits
|
|
on use_prefill_config (capture-stable: at FULL-graph capture max_query_len is
|
|
the uniform decode/verify length). This default table was tuned on GB300;
|
|
sm_120 (RTX PRO 6000 Blackwell) dispatches to _select_sm120_config instead,
|
|
and sm_90 (Hopper) to _select_sm90_config.
|
|
"""
|
|
base_programs = num_rows * num_kv_heads
|
|
if _is_sm120():
|
|
BLOCK_N, target_splits, num_warps = _select_sm120_config(
|
|
base_programs, use_prefill_config, is_fp8
|
|
)
|
|
elif _is_sm90():
|
|
BLOCK_N, target_splits, num_warps = _select_sm90_config(
|
|
base_programs, use_prefill_config, is_fp8
|
|
)
|
|
elif base_programs > 2048:
|
|
BLOCK_N, target_splits, num_warps = (
|
|
(32, 1, 1) if use_prefill_config else (64, 1, 2)
|
|
)
|
|
elif base_programs <= 24:
|
|
BLOCK_N, target_splits, num_warps = 32, 64, 4
|
|
elif base_programs <= 32:
|
|
BLOCK_N, target_splits, num_warps = 32, 16, 1
|
|
elif base_programs <= 64:
|
|
BLOCK_N, target_splits, num_warps = 32, 8, 1
|
|
elif base_programs <= 128:
|
|
BLOCK_N, target_splits, num_warps = 32, 4, 1
|
|
elif base_programs <= 256:
|
|
BLOCK_N, target_splits, num_warps = 32, 8, 1
|
|
elif base_programs <= 512:
|
|
BLOCK_N, target_splits, num_warps = 64, 4, 2
|
|
else:
|
|
BLOCK_N, target_splits, num_warps = 64, 1, 2
|
|
if is_fp8 and _is_sm80():
|
|
# Wider tiles measured faster for the byte-decode path on A100;
|
|
# unmeasured on sm_86, which keeps the table's tiles.
|
|
BLOCK_N, num_warps = 128, 4
|
|
num_tiles = triton.cdiv(num_columns, BLOCK_N)
|
|
# Never more splits than tiles, never empty.
|
|
num_splits = min(target_splits, num_tiles)
|
|
return BLOCK_N, num_warps, num_tiles, num_splits
|
|
|
|
|
|
def qsa_sparse_paged_attention(
|
|
q: torch.Tensor,
|
|
k_cache: torch.Tensor,
|
|
v_cache: torch.Tensor,
|
|
logical_indices: torch.Tensor,
|
|
block_table: torch.Tensor,
|
|
token_to_req: torch.Tensor,
|
|
use_prefill_config: bool,
|
|
out: torch.Tensor | None = None,
|
|
k_scale: float | None = None,
|
|
v_scale: float | None = None,
|
|
*,
|
|
output_gate: torch.Tensor,
|
|
) -> torch.Tensor:
|
|
"""Run sparse GQA directly over paged BF16 or FP8-e4m3 K/V caches.
|
|
|
|
An fp8 cache is a native e4m3 view on sm_89+ and the raw uint8 storage
|
|
below sm_89 (no Triton fp8e4nv pointer type there); the kernel dequantizes
|
|
either on load. k_scale/v_scale are the layer's per-tensor dequant scales
|
|
as host floats (e.g. layer._k_scale_float/_v_scale_float): k_scale is
|
|
pre-multiplied into the softmax scale and v_scale becomes the kernel's
|
|
output scale, so the kernel needs no device scale buffers.
|
|
|
|
logical_indices is the PACKED selection buffer: [rows, selection_width + 1]
|
|
with the trailing column holding each row's valid-entry count (written by
|
|
the expand kernel; never a token index). The kernel reads it as the
|
|
tile-loop bound. use_prefill_config only steers the top of the config table; see
|
|
_select_config.
|
|
"""
|
|
if q.ndim != 3 or k_cache.ndim != 4 or v_cache.shape != k_cache.shape:
|
|
raise ValueError("QSA sparse attention received invalid Q/K/V shapes")
|
|
if logical_indices.ndim != 2 or logical_indices.shape[0] != q.shape[0]:
|
|
raise ValueError("QSA indices must have one row per query")
|
|
if token_to_req.shape != (q.shape[0],) or block_table.ndim != 2:
|
|
raise ValueError("QSA sparse attention metadata has invalid shapes")
|
|
if not all(k_cache.shape[:3]) or not all(block_table.shape):
|
|
raise ValueError("QSA sparse attention cache and block table must be nonempty")
|
|
if logical_indices.shape[1] < 2:
|
|
raise ValueError(
|
|
"QSA packed indices need selection columns plus the count column"
|
|
)
|
|
if q.shape[2] != k_cache.shape[3] or q.shape[1] % k_cache.shape[2]:
|
|
raise ValueError("QSA sparse attention requires valid grouped-query heads")
|
|
head_dim = q.shape[2]
|
|
assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0
|
|
assert q.dtype == torch.bfloat16
|
|
assert k_cache.dtype == v_cache.dtype
|
|
is_fp8 = k_cache.dtype != torch.bfloat16
|
|
if is_fp8:
|
|
assert k_scale is not None and v_scale is not None
|
|
# Host pre-multiply: fold the K dequant scale into the attention scale
|
|
# and pass V's dequant scale as the kernel's output scale.
|
|
softmax_scale = (head_dim**-0.5) * float(k_scale)
|
|
output_scale = float(v_scale)
|
|
if k_cache.dtype == torch.uint8:
|
|
# _maybe_dequant's byte decode yields x * 2^-8; fold 2^8 into the
|
|
# scales so the decode needs no multiply.
|
|
softmax_scale *= 256.0
|
|
output_scale *= 256.0
|
|
else:
|
|
assert k_cache.dtype == torch.float8_e4m3fn
|
|
else:
|
|
softmax_scale = head_dim**-0.5
|
|
output_scale = 1.0
|
|
assert logical_indices.dtype == block_table.dtype == torch.int32
|
|
assert token_to_req.dtype == torch.int32
|
|
assert q.device == k_cache.device == v_cache.device
|
|
assert q.device == logical_indices.device == block_table.device
|
|
assert q.device == token_to_req.device
|
|
assert q.stride(2) == k_cache.stride(3) == v_cache.stride(3) == 1
|
|
assert logical_indices.stride(1) == block_table.stride(1) == 1
|
|
assert token_to_req.stride(0) == 1
|
|
|
|
if out is None:
|
|
out = torch.empty_like(q)
|
|
if out.shape != q.shape:
|
|
raise ValueError("QSA sparse output must match its query")
|
|
assert out.dtype == q.dtype and out.device == q.device
|
|
assert out.stride(2) == 1
|
|
assert output_gate.is_contiguous()
|
|
output_gate_view = output_gate.view_as(q)
|
|
if not q.shape[0]:
|
|
return out
|
|
|
|
group_size = q.shape[1] // k_cache.shape[2]
|
|
block_m = triton.next_power_of_2(group_size)
|
|
selection_width = logical_indices.shape[1] - 1 # trailing column is the count
|
|
block_n, partial_warps, num_tiles, num_splits = _select_config(
|
|
q.shape[0], k_cache.shape[2], use_prefill_config, selection_width, is_fp8
|
|
)
|
|
|
|
# Split=1 writes output directly and compiles out all workspace accesses.
|
|
if num_splits == 1:
|
|
partial_output = out
|
|
partial_lse = out
|
|
else:
|
|
# FP32 partials preserve accuracy when merging independently normalized
|
|
# splits.
|
|
partial_output = torch.empty(
|
|
(num_splits, *q.shape), dtype=torch.float32, device=q.device
|
|
)
|
|
partial_lse = torch.empty(
|
|
(num_splits, q.shape[0], q.shape[1]),
|
|
dtype=torch.float32,
|
|
device=q.device,
|
|
)
|
|
|
|
partial_grid = (q.shape[0], k_cache.shape[2], num_splits)
|
|
_qsa_sparse_paged_gqa_splitk_kernel[partial_grid](
|
|
q,
|
|
k_cache,
|
|
v_cache,
|
|
logical_indices,
|
|
block_table,
|
|
token_to_req,
|
|
partial_output,
|
|
partial_lse,
|
|
out,
|
|
softmax_scale,
|
|
output_scale,
|
|
output_gate_view,
|
|
q.stride(0),
|
|
q.stride(1),
|
|
k_cache.stride(0),
|
|
k_cache.stride(1),
|
|
k_cache.stride(2),
|
|
v_cache.stride(0),
|
|
v_cache.stride(1),
|
|
v_cache.stride(2),
|
|
logical_indices.stride(0),
|
|
block_table.stride(0),
|
|
out.stride(0),
|
|
out.stride(1),
|
|
output_gate_view.stride(0),
|
|
output_gate_view.stride(1),
|
|
q.shape[0],
|
|
k_cache.shape[0],
|
|
block_table.shape[0],
|
|
TOPK=selection_width,
|
|
PAGE_SIZE=k_cache.shape[1],
|
|
PAGE_TABLE_WIDTH=block_table.shape[1],
|
|
GROUP_SIZE=group_size,
|
|
HEAD_DIM=q.shape[2],
|
|
NUM_QUERY_HEADS=q.shape[1],
|
|
NUM_SPLITS=num_splits,
|
|
NUM_TILES=num_tiles,
|
|
BLOCK_M=block_m,
|
|
BLOCK_N=block_n,
|
|
num_warps=partial_warps,
|
|
num_stages=2,
|
|
)
|
|
if num_splits == 1:
|
|
return out
|
|
|
|
_qsa_merge_splitk_kernel[(q.shape[0], q.shape[1])](
|
|
partial_output,
|
|
partial_lse,
|
|
out,
|
|
output_gate_view,
|
|
out.stride(0),
|
|
out.stride(1),
|
|
output_gate_view.stride(0),
|
|
output_gate_view.stride(1),
|
|
q.shape[0],
|
|
HEAD_DIM=q.shape[2],
|
|
NUM_QUERY_HEADS=q.shape[1],
|
|
NUM_SPLITS=num_splits,
|
|
BLOCK_SPLITS=triton.next_power_of_2(num_splits),
|
|
num_warps=2,
|
|
num_stages=1,
|
|
)
|
|
return out
|
|
|
|
|
|
def warmup_qsa_sparse_paged_attention(
|
|
kv_cache: torch.Tensor,
|
|
block_table: torch.Tensor,
|
|
*,
|
|
num_query_heads: int,
|
|
selection_width: int,
|
|
) -> tuple[tuple[int, int, int], ...]:
|
|
"""Compile every production-reachable split-K/merge specialization."""
|
|
head_dim = kv_cache.shape[-1] // 2
|
|
key_cache, value_cache = kv_cache.transpose(1, 2).split(head_dim, dim=-1)
|
|
# An fp8 cache is allocated as uint8 storage; warm the dtype the impl hands
|
|
# over: an e4m3 view on sm_89+, the raw bytes below.
|
|
is_fp8 = kv_cache.dtype == torch.uint8
|
|
fp8_dtype = (
|
|
torch.float8_e4m3fn
|
|
if current_platform.has_device_capability(89)
|
|
else torch.uint8
|
|
)
|
|
cache_dtype = fp8_dtype if is_fp8 else key_cache.dtype
|
|
num_kv_heads = key_cache.shape[2]
|
|
group_size = num_query_heads // num_kv_heads
|
|
block_m = triton.next_power_of_2(group_size)
|
|
|
|
# Every config the dispatch can pick for this group size.
|
|
profiles = {
|
|
_select_config(
|
|
num_rows, num_kv_heads, use_prefill_config, selection_width, is_fp8
|
|
)
|
|
for num_rows in range(1, 8193)
|
|
for use_prefill_config in (False, True)
|
|
}
|
|
|
|
# Scalars constant per deployment get their real values (their divisibility
|
|
# specialization is wanted); the batch-varying ones are do_not_specialize'd
|
|
# on the kernels, so any value here compiles the only variant.
|
|
num_rows = 16
|
|
num_requests = 16
|
|
q_ptr = TritonWarmupTensor(
|
|
torch.bfloat16, shape=(num_rows, num_query_heads, head_dim)
|
|
)
|
|
k_cache_ptr = TritonWarmupTensor(
|
|
cache_dtype,
|
|
shape=tuple(key_cache.shape),
|
|
strides=tuple(key_cache.stride()),
|
|
)
|
|
v_cache_ptr = TritonWarmupTensor(
|
|
cache_dtype,
|
|
shape=tuple(value_cache.shape),
|
|
strides=tuple(value_cache.stride()),
|
|
)
|
|
# +1: the packed buffer's trailing count column.
|
|
indices_ptr = TritonWarmupTensor(torch.int32, shape=(num_rows, selection_width + 1))
|
|
block_table_ptr = TritonWarmupTensor(
|
|
block_table.dtype,
|
|
shape=tuple(block_table.shape),
|
|
strides=tuple(block_table.stride()),
|
|
)
|
|
token_to_req_ptr = TritonWarmupTensor(torch.int32)
|
|
output_ptr = TritonWarmupTensor(
|
|
torch.bfloat16, shape=(num_rows, num_query_heads, head_dim)
|
|
)
|
|
# The output gate is mandatory at runtime; warm the gated specialization.
|
|
output_gate_ptr = TritonWarmupTensor(
|
|
torch.bfloat16, shape=(num_rows, num_query_heads, head_dim)
|
|
)
|
|
head_stride = head_dim
|
|
row_stride = num_query_heads * head_dim
|
|
num_cache_blocks = triton_scalar_specialization_rep(kv_cache.shape[0])
|
|
|
|
warmed = []
|
|
for block_n, warps, num_tiles, num_splits in sorted(profiles):
|
|
if num_splits == 1:
|
|
partial_output_ptr = output_ptr
|
|
partial_lse_ptr = output_ptr
|
|
else:
|
|
partial_output_ptr = TritonWarmupTensor(
|
|
torch.float32,
|
|
shape=(num_splits, num_rows, num_query_heads, head_dim),
|
|
)
|
|
partial_lse_ptr = TritonWarmupTensor(
|
|
torch.float32, shape=(num_splits, num_rows, num_query_heads)
|
|
)
|
|
_qsa_sparse_paged_gqa_splitk_kernel.warmup(
|
|
q_ptr,
|
|
k_cache_ptr,
|
|
v_cache_ptr,
|
|
indices_ptr,
|
|
block_table_ptr,
|
|
token_to_req_ptr,
|
|
partial_output_ptr,
|
|
partial_lse_ptr,
|
|
output_ptr,
|
|
1.0,
|
|
1.0,
|
|
output_gate_ptr,
|
|
row_stride,
|
|
head_stride,
|
|
key_cache.stride(0),
|
|
key_cache.stride(1),
|
|
key_cache.stride(2),
|
|
value_cache.stride(0),
|
|
value_cache.stride(1),
|
|
value_cache.stride(2),
|
|
selection_width + 1,
|
|
block_table.stride(0),
|
|
row_stride,
|
|
head_stride,
|
|
row_stride,
|
|
head_stride,
|
|
num_rows,
|
|
num_cache_blocks,
|
|
num_requests,
|
|
TOPK=selection_width,
|
|
PAGE_SIZE=key_cache.shape[1],
|
|
PAGE_TABLE_WIDTH=block_table.shape[1],
|
|
GROUP_SIZE=group_size,
|
|
HEAD_DIM=head_dim,
|
|
NUM_QUERY_HEADS=num_query_heads,
|
|
NUM_SPLITS=num_splits,
|
|
NUM_TILES=num_tiles,
|
|
BLOCK_M=block_m,
|
|
BLOCK_N=block_n,
|
|
num_warps=warps,
|
|
num_stages=2,
|
|
grid=(num_rows, num_kv_heads, num_splits),
|
|
)
|
|
if num_splits > 1:
|
|
_qsa_merge_splitk_kernel.warmup(
|
|
partial_output_ptr,
|
|
partial_lse_ptr,
|
|
output_ptr,
|
|
output_gate_ptr,
|
|
row_stride,
|
|
head_stride,
|
|
row_stride,
|
|
head_stride,
|
|
num_rows,
|
|
HEAD_DIM=head_dim,
|
|
NUM_QUERY_HEADS=num_query_heads,
|
|
NUM_SPLITS=num_splits,
|
|
BLOCK_SPLITS=triton.next_power_of_2(num_splits),
|
|
num_warps=2,
|
|
num_stages=1,
|
|
grid=(num_rows, num_query_heads),
|
|
)
|
|
warmed.append((block_n, num_splits, warps))
|
|
return tuple(warmed)
|
|
|
|
|
|
def qsa_store_cache_rows(
|
|
cache: torch.Tensor,
|
|
slot_mapping: torch.Tensor,
|
|
rows: torch.Tensor,
|
|
) -> None:
|
|
"""Store fixed-width rows in a QSA cache without boolean indexing."""
|
|
if not cache.is_cuda or not HAS_TRITON:
|
|
raise RuntimeError("QSA CUDA cache stores require Triton")
|
|
if cache.ndim != 4 or cache.shape[2] != 1:
|
|
raise ValueError("QSA cache must be [pages, page_size, 1, width]")
|
|
if not all(cache.shape):
|
|
raise ValueError("QSA cache dimensions must be nonzero")
|
|
if rows.ndim == 3:
|
|
if rows.shape[1] != 1:
|
|
raise ValueError("QSA cache rows must have one head")
|
|
rows = rows[:, 0]
|
|
if rows.shape != (slot_mapping.numel(), cache.shape[3]):
|
|
raise ValueError("QSA cache rows and slots have incompatible shapes")
|
|
if not rows.shape[0]:
|
|
return
|
|
_store_qsa_rows_kernel[(rows.shape[0],)](
|
|
cache,
|
|
slot_mapping,
|
|
rows,
|
|
cache.stride(0),
|
|
cache.stride(1),
|
|
cache.stride(3),
|
|
rows.stride(0),
|
|
rows.stride(1),
|
|
rows.shape[0],
|
|
cache.shape[0],
|
|
PAGE_SIZE=cache.shape[1],
|
|
WIDTH=cache.shape[3],
|
|
BLOCK_D=triton.next_power_of_2(cache.shape[3]),
|
|
num_warps=4,
|
|
)
|
|
|
|
|
|
def qsa_compress_groups_with_ratio(
|
|
raw_keys: torch.Tensor, # this step's raw key rows [rows, 1, head_size]
|
|
raw_positions: torch.Tensor, # this step's positions [rows, 1, 3] int64
|
|
compressor_state_cache: torch.Tensor,
|
|
compressor_state_block_table: torch.Tensor,
|
|
token_to_req: torch.Tensor,
|
|
query_start_loc: torch.Tensor,
|
|
logical_positions: torch.Tensor,
|
|
compressed_slots: torch.Tensor,
|
|
compress_ratio: int,
|
|
rope_cache: torch.Tensor | None = None,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
"""Pool completed groups from the compressor-state ring and raw token rows."""
|
|
if not raw_keys.is_cuda or not HAS_TRITON:
|
|
raise RuntimeError("QSA CUDA compression requires Triton")
|
|
rows = token_to_req.numel()
|
|
if compress_ratio <= 0:
|
|
raise ValueError("QSA compression ratio must be positive")
|
|
if raw_keys.ndim != 3 or raw_keys.shape[:2] != (rows, 1):
|
|
raise ValueError("QSA raw keys must be [rows, 1, head_size]")
|
|
if raw_positions.shape != (rows, 1, 3) or raw_positions.dtype != torch.int64:
|
|
raise ValueError("QSA raw positions must be [rows, 1, 3] int64")
|
|
if logical_positions.shape != (rows,) or compressed_slots.shape != (rows,):
|
|
raise ValueError("QSA compression metadata must match token rows")
|
|
if compressor_state_cache.ndim != 4 or compressor_state_cache.shape[2] != 1:
|
|
raise ValueError("QSA compressor-state cache has an invalid shape")
|
|
if (
|
|
# The ring is wider than one group so speculative rows cannot alias
|
|
# onto the committed keys of the group still being collected.
|
|
compressor_state_cache.shape[1] < compress_ratio
|
|
or compressor_state_cache.shape[3] != raw_keys.shape[2]
|
|
or compressor_state_cache.dtype != raw_keys.dtype
|
|
):
|
|
raise ValueError(
|
|
"QSA compressor-state cache does not match the compression layout"
|
|
)
|
|
if (
|
|
compressor_state_block_table.ndim != 2
|
|
or compressor_state_block_table.shape[1] < 1
|
|
):
|
|
raise ValueError(
|
|
"QSA compressor-state block table must contain one block per request"
|
|
)
|
|
if query_start_loc.ndim != 1 or query_start_loc.shape[0] < 2:
|
|
raise ValueError("QSA query starts must contain a terminal offset")
|
|
num_requests = query_start_loc.shape[0] - 1
|
|
if compressor_state_block_table.shape[0] < num_requests:
|
|
raise ValueError("QSA compressor-state block table has too few request rows")
|
|
if rope_cache is not None and (
|
|
rope_cache.ndim != 4
|
|
or rope_cache.shape[:3] != compressor_state_cache.shape[:3]
|
|
or rope_cache.shape[3] != 3
|
|
or rope_cache.dtype != torch.int64
|
|
):
|
|
raise ValueError("QSA packed position view has an invalid shape or dtype")
|
|
if rows and (
|
|
not all(compressor_state_cache.shape)
|
|
or not all(compressor_state_block_table.shape)
|
|
):
|
|
raise ValueError("QSA compressor-state cache and block table must be nonempty")
|
|
pooled = torch.empty(
|
|
(rows, 1, raw_keys.shape[2]),
|
|
dtype=raw_keys.dtype,
|
|
device=raw_keys.device,
|
|
)
|
|
first_positions = torch.empty((rows, 3), dtype=torch.int64, device=raw_keys.device)
|
|
if not rows:
|
|
return pooled, first_positions
|
|
if rope_cache is None:
|
|
rope_cache = compressor_state_cache
|
|
load_rope_positions = False
|
|
else:
|
|
load_rope_positions = True
|
|
_compress_qsa_groups_kernel[(rows,)](
|
|
raw_keys,
|
|
raw_positions,
|
|
compressor_state_cache,
|
|
rope_cache,
|
|
compressor_state_block_table,
|
|
token_to_req,
|
|
query_start_loc,
|
|
logical_positions,
|
|
compressed_slots,
|
|
pooled,
|
|
first_positions,
|
|
raw_keys.stride(0),
|
|
raw_keys.stride(2),
|
|
raw_positions.stride(0),
|
|
raw_positions.stride(2),
|
|
compressor_state_cache.stride(0),
|
|
compressor_state_cache.stride(1),
|
|
compressor_state_cache.stride(3),
|
|
rope_cache.stride(0),
|
|
rope_cache.stride(1),
|
|
rope_cache.stride(3),
|
|
compressor_state_block_table.stride(0),
|
|
pooled.stride(0),
|
|
pooled.stride(2),
|
|
first_positions.stride(0),
|
|
first_positions.stride(1),
|
|
rows,
|
|
compressor_state_cache.shape[0],
|
|
num_requests,
|
|
COMPRESSOR_STATE_SIZE=compressor_state_cache.shape[1],
|
|
COMPRESS_RATIO=compress_ratio,
|
|
HEAD_DIM=raw_keys.shape[2],
|
|
LOAD_ROPE_POSITIONS=load_rope_positions,
|
|
BLOCK_D=triton.next_power_of_2(raw_keys.shape[2]),
|
|
num_warps=4,
|
|
)
|
|
return pooled, first_positions
|
|
|
|
|
|
__all__ = [
|
|
"qsa_compress_groups_with_ratio",
|
|
"qsa_sparse_paged_attention",
|
|
"qsa_store_cache_rows",
|
|
"warmup_qsa_sparse_paged_attention",
|
|
]
|