[ROCm][Perf] Reach the fused QSA pre-indexer from the AMD path (#57947)

Signed-off-by: mjkvaak-amd <mjkvaak-amd@users.noreply.github.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Mikko Tukiainen
2026-10-05 16:22:29 +00:00
committed by GitHub
co-authored by Cursor
parent 528772a4bf
commit d23af12cb1
5 changed files with 1136 additions and 14 deletions
+83 -13
View File
@@ -23,6 +23,7 @@ from ..common.qsa_cache import (
QSAKeyStateCache,
canonical_qsa_rope_positions,
)
from .ops.qsa_pre_indexer import qsa_pre_indexer, supports_fused_pre_indexer
def apply_qsa_rope(
@@ -98,6 +99,12 @@ class QSAIndexer(nn.Module):
self.token_topk = int(config.indexer_budget)
self.compress_ratio = int(config.indexer_compress_ratio)
self.rotary_emb = rotary_emb
self.use_fused_pre_indexer = supports_fused_pre_indexer(
rotary_emb,
self.index_head_dim,
self.index_kv_heads,
self.compress_ratio,
)
self.prefix = prefix
# MTP step 0 selects the target-aligned rows; later steps reuse them
# while continuing to update the QSA side cache.
@@ -143,20 +150,27 @@ class QSAIndexer(nn.Module):
def output_width(self) -> int:
return self.token_topk + self.compress_ratio - 1
def project_qk(
def project(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Project replicated Q/K, normalize+rotate Q, and preserve raw K."""
"""Project replicated Q/K and split, leaving both operands unrotated."""
qk, _ = self.index_qk_proj(hidden_states)
q_raw, token_k = qk.split(
return qk.split(
(
self.index_n_heads * self.index_head_dim,
self.index_kv_heads * self.index_head_dim,
),
dim=-1,
)
def project_qk(
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Project replicated Q/K, normalize+rotate Q, and preserve raw K."""
q_raw, token_k = self.project(hidden_states)
q = q_raw.reshape(-1, self.index_n_heads, self.index_head_dim)
q = apply_qsa_rmsnorm(
self.q_layernorm,
@@ -251,6 +265,52 @@ class QSAIndexer(nn.Module):
position_rows,
)
def _fused_pre_indexer(
self,
projected_q: torch.Tensor,
raw_keys: torch.Tensor,
positions: torch.Tensor,
raw_metadata: QSAForwardMetadata,
compressed_metadata: QSAForwardMetadata,
) -> torch.Tensor:
"""Normalize and rotate Q, compress K, and write both caches in one launch.
Replaces ``project_qk``'s norm/RoPE half and the whole of
``_update_and_compress``; the two must stay numerically interchangeable.
"""
raw_key_cache = self.raw_key_cache
q = projected_q.new_empty(
raw_metadata.num_actual_tokens,
self.index_n_heads,
self.index_head_dim,
)
qsa_pre_indexer(
projected_q,
raw_keys,
positions,
self.rotary_emb.cos_sin_cache,
self.q_layernorm.weight,
self.k_layernorm.weight,
self.q_layernorm.variance_epsilon,
q,
raw_key_cache.kv_cache,
raw_metadata.slot_mapping,
raw_metadata.block_table,
raw_metadata.query_start_loc,
raw_metadata.logical_positions,
self.compressed_key_cache.kv_cache,
compressed_metadata.slot_mapping,
compressed_metadata.k_work_metadata,
compress_ratio=self.compress_ratio,
mrope_section=getattr(self.rotary_emb, "mrope_section", None),
rope_pos_offset=(
raw_key_cache.rope_position_offset
if raw_key_cache.cache_rope_positions
else None
),
)
return q
def _select(
self,
q: torch.Tensor,
@@ -295,15 +355,25 @@ class QSAIndexer(nn.Module):
return result
raw_metadata, compressed_metadata = metadata
num_tokens = raw_metadata.num_actual_tokens
q, token_k = self.project_qk(
hidden_states[:num_tokens], positions[..., :num_tokens]
)
self._update_and_compress(
token_k,
positions[..., :num_tokens],
raw_metadata,
compressed_metadata,
)
hidden_states = hidden_states[:num_tokens]
positions = positions[..., :num_tokens]
if self.use_fused_pre_indexer:
projected_q, raw_keys = self.project(hidden_states)
q = self._fused_pre_indexer(
projected_q,
raw_keys,
positions,
raw_metadata,
compressed_metadata,
)
else:
q, token_k = self.project_qk(hidden_states, positions)
self._update_and_compress(
token_k,
positions,
raw_metadata,
compressed_metadata,
)
if self.skip_topk:
if out is None:
raise RuntimeError("QSA top-k reuse requires an output buffer")
@@ -0,0 +1,558 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fused QSA pre-indexer kernel for the AMD Qwen4Exp path.
Started as a copy of the NVIDIA kernel (``nvidia/ops/qsa_prepare.py``) and is
kept separate so either vendor can change its own copy without re-validating
the other. The state source select is two masked loads rather than a pointer
select, which ROCm Triton rejects.
"""
import torch
from torch import nn
from vllm.triton_utils import tl, triton
def supports_fused_pre_indexer(
rotary_emb: nn.Module,
head_dim: int,
num_kv_heads: int,
compress_ratio: int,
) -> bool:
"""Report whether this indexer's shapes match the fused kernel's assumptions.
The kernel hard-codes the rotary layout and the single-KV-head group
compression it was written for; everything it rejects has a working unfused
path.
"""
rotary_dim = int(rotary_emb.rotary_dim)
mrope_section = getattr(rotary_emb, "mrope_section", None)
return (
bool(getattr(rotary_emb, "is_neox_style", False))
and (
not mrope_section
or (
len(mrope_section) == 3
and sum(mrope_section) == rotary_dim // 2
and bool(getattr(rotary_emb, "mrope_interleaved", False))
)
)
and head_dim == 128
and rotary_dim == 64
and num_kv_heads == 1
and compress_ratio > 1
and compress_ratio & (compress_ratio - 1) == 0
)
@triton.jit
def _norm_rope(
x,
pos_t,
pos_h,
pos_w,
cos_sin_ptr,
cos_sin_stride,
norm_weight_ptr,
eps,
IS_MROPE: tl.constexpr,
MROPE_H: tl.constexpr,
MROPE_W: tl.constexpr,
):
"""Apply Gemma RMSNorm and selected-axis NeoX RoPE to register rows."""
TILE_T: tl.constexpr = x.shape[0]
TILE_H: tl.constexpr = x.shape[1]
D: tl.constexpr = x.shape[2]
ROWS: tl.constexpr = TILE_T * TILE_H
HALF: tl.constexpr = D // 2
QUARTER: tl.constexpr = D // 4
pairs = tl.arange(0, QUARTER)
if IS_MROPE:
# Qwen interleaves temporal, height, and width rotary pairs. Each axis
# still indexes the same position-major cos/sin table.
h_mask = ((pairs % 3) == 1) & (pairs <= 3 * MROPE_H)
w_mask = ((pairs % 3) == 2) & (pairs <= 3 * MROPE_W)
t_mask = ~(h_mask | w_mask)
base = cos_sin_ptr + pairs[None, :]
pos_rows = (pos_t, pos_h, pos_w)
axis_masks = (t_mask, h_mask, w_mask)
cos = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty)
sin = tl.zeros((TILE_T, QUARTER), dtype=cos_sin_ptr.dtype.element_ty)
for axis in tl.static_range(3):
cos += tl.load(
base + pos_rows[axis][:, None] * cos_sin_stride,
mask=axis_masks[axis][None, :],
other=0,
)
sin += tl.load(
base + pos_rows[axis][:, None] * cos_sin_stride + QUARTER,
mask=axis_masks[axis][None, :],
other=0,
)
else:
cos = tl.load(cos_sin_ptr + pos_t[:, None] * cos_sin_stride + pairs[None, :])
sin = tl.load(
cos_sin_ptr + pos_t[:, None] * cos_sin_stride + QUARTER + pairs[None, :]
)
cos = tl.reshape(
tl.broadcast_to(cos[:, None, :], (TILE_T, TILE_H, QUARTER)),
(ROWS, QUARTER),
)
sin = tl.reshape(
tl.broadcast_to(sin[:, None, :], (TILE_T, TILE_H, QUARTER)),
(ROWS, QUARTER),
)
x = tl.reshape(x, (ROWS, D)).to(tl.float32)
weight = tl.load(norm_weight_ptr + tl.arange(0, D)).to(tl.float32) + 1.0
rrms = tl.rsqrt(tl.sum(x * x, axis=1) / D + eps)
y = (x * rrms[:, None] * weight[None, :]).to(cos.dtype)
rotated, passthrough = tl.split(
tl.permute(tl.reshape(y, (ROWS, 2, HALF)), (0, 2, 1))
)
r0, r1 = tl.split(tl.permute(tl.reshape(rotated, (ROWS, 2, QUARTER)), (0, 2, 1)))
out0 = r0 * cos - r1 * sin
out1 = r1 * cos + r0 * sin
rotated = tl.reshape(tl.permute(tl.join(out0, out1), (0, 2, 1)), (ROWS, HALF))
result = tl.reshape(tl.permute(tl.join(rotated, passthrough), (0, 2, 1)), (ROWS, D))
return tl.reshape(result, (TILE_T, TILE_H, D))
@triton.jit(
do_not_specialize=[
"num_tokens",
"num_state_blocks",
"num_compressed_blocks",
"num_k_work",
]
)
def _qsa_pre_indexer_kernel(
q_ptr,
q_stride_token,
k_ptr,
k_stride_token,
pos_ptr,
pos_stride_axis,
pos_stride_token,
cos_sin_ptr,
q_norm_weight_ptr,
k_norm_weight_ptr,
eps,
q_out_ptr,
q_out_stride_token,
q_out_stride_head,
state_cache_ptr,
state_cache_stride_block,
state_cache_stride_token,
state_slots_ptr,
state_table_ptr,
state_table_stride_req,
query_start_loc_ptr,
logical_positions_ptr,
compressed_slots_ptr,
k_work_metadata_ptr,
compressed_cache_ptr,
compressed_cache_stride_block,
compressed_cache_stride_token,
num_tokens,
num_state_blocks,
num_compressed_blocks,
num_k_work,
HQ: tl.constexpr,
D: tl.constexpr,
TILE_T_Q: tl.constexpr,
TILE_H_Q: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
STATE_SIZE: tl.constexpr,
COMP_PAGE_SIZE: tl.constexpr,
IS_2D_POSITIONS: tl.constexpr,
IS_K_MROPE: tl.constexpr,
CACHE_HAS_ROPE_POS: tl.constexpr,
MROPE_H: tl.constexpr,
MROPE_W: tl.constexpr,
):
pid = tl.program_id(0)
# K work occupies the first programs; the remaining programs tile Q. This
# keeps both paths in one launch while leaving their register shapes
# independent.
if pid >= num_k_work:
q_pid = pid - num_k_work
num_head_tiles: tl.constexpr = tl.cdiv(HQ, TILE_H_Q)
token_tile = q_pid // num_head_tiles
head_tile = q_pid % num_head_tiles
tokens = token_tile * TILE_T_Q + tl.arange(0, TILE_T_Q)
heads = head_tile * TILE_H_Q + tl.arange(0, TILE_H_Q)
valid_tokens = tokens < num_tokens
valid_heads = heads < HQ
dims = tl.arange(0, D)
mask = valid_tokens[:, None, None] & valid_heads[None, :, None]
x = tl.load(
q_ptr
+ tokens[:, None, None] * q_stride_token
+ heads[None, :, None] * D
+ dims[None, None, :],
mask=mask,
other=0.0,
)
pos_t = tl.load(pos_ptr + tokens * pos_stride_token, mask=valid_tokens, other=0)
if IS_2D_POSITIONS:
pos_h = tl.load(
pos_ptr + pos_stride_axis + tokens * pos_stride_token,
mask=valid_tokens,
other=0,
)
pos_w = tl.load(
pos_ptr + 2 * pos_stride_axis + tokens * pos_stride_token,
mask=valid_tokens,
other=0,
)
else:
pos_h = pos_t
pos_w = pos_t
y = _norm_rope(
x,
pos_t,
pos_h,
pos_w,
cos_sin_ptr,
D // 2,
q_norm_weight_ptr,
eps,
IS_2D_POSITIONS,
MROPE_H,
MROPE_W,
)
tl.store(
q_out_ptr
+ tokens[:, None, None] * q_out_stride_token
+ heads[None, :, None] * q_out_stride_head
+ dims[None, None, :],
y,
mask=mask,
)
if pid < num_k_work:
# One work item owns one completed compression group. Work item zero
# additionally commits the request's current raw-K suffix below.
work_metadata = tl.load(k_work_metadata_ptr + pid * 2 + tl.arange(0, 2))
request, work_in_request = tl.split(work_metadata)
if request < 0:
return
query_start = tl.load(query_start_loc_ptr + request)
query_end = tl.load(query_start_loc_ptr + request + 1)
query_len = query_end - query_start
chunk_end = tl.load(logical_positions_ptr + query_end - 1)
chunk_start = chunk_end - query_len + 1
num_groups = (chunk_end + 1) // COMPRESS_RATIO - chunk_start // COMPRESS_RATIO
dims = tl.arange(0, D)
if work_in_request < num_groups:
first_boundary = (
(chunk_start + COMPRESS_RATIO) // COMPRESS_RATIO
) * COMPRESS_RATIO - 1
end_position = first_boundary + work_in_request * COMPRESS_RATIO
boundary_token = query_start + end_position - chunk_start
valid_token = (
(boundary_token >= query_start)
& (boundary_token < query_end)
& (boundary_token < num_tokens)
)
compressed_slot = tl.load(
compressed_slots_ptr + boundary_token,
mask=valid_token,
other=-1,
)
valid = (
valid_token
& (compressed_slot >= 0)
& (compressed_slot < num_compressed_blocks * COMP_PAGE_SIZE)
)
state_block = tl.load(state_table_ptr + request * state_table_stride_req)
state_block_valid = (state_block >= 0) & (state_block < num_state_blocks)
safe_state_block = tl.maximum(state_block, 0).to(tl.int64)
group_offsets = tl.arange(0, COMPRESS_RATIO)
source_positions = end_position - (COMPRESS_RATIO - 1) + group_offsets
source_in_chunk = source_positions >= chunk_start
source_in_cache = source_positions < chunk_start
source_tokens = query_start + source_positions - chunk_start
source_tokens_valid = (
(source_tokens >= query_start)
& (source_tokens < query_end)
& (source_tokens < num_tokens)
)
current_base = (
k_ptr + tl.maximum(source_tokens, 0)[:, None] * k_stride_token
)
cached_base = (
state_cache_ptr
+ safe_state_block * state_cache_stride_block
+ (source_positions % STATE_SIZE)[:, None] * state_cache_stride_token
)
# Only the first completed group can cross the chunk boundary, so
# every row comes from exactly one of the two sources. Selecting
# between the base pointers would need a pointer-typed tl.where,
# which ROCm's Triton rejects ("'tt.addptr' op failed to verify
# that result type matches ptr type"), so select the loaded values
# instead. The two masks are complementary, so no lane fetches from
# both and memory traffic is unchanged.
#
# Both bases still need the alignment stated explicitly. Without it
# Triton vectorizes these loads differently, which reorders the
# pooling reduction below; float addition is not associative, so the
# compressed rows stop matching the unfused path bit for bit and an
# FP8 cache rounds a near-zero pooled value to the wrong code.
current_base = tl.multiple_of(current_base, (8, 8))
cached_base = tl.multiple_of(cached_base, (8, 8))
from_chunk = valid & source_in_chunk[:, None] & source_tokens_valid[:, None]
from_cache = valid & source_in_cache[:, None] & state_block_valid
current = tl.load(current_base + dims[None, :], mask=from_chunk, other=0.0)
cached = tl.load(cached_base + dims[None, :], mask=from_cache, other=0.0)
source = tl.where(source_in_chunk[:, None], current, cached).to(tl.float32)
# Match the unfused path's BF16 pooled tensor before RMSNorm.
pooled = (
(tl.sum(source, axis=0) / COMPRESS_RATIO).to(tl.bfloat16).to(tl.float32)
)
first_position = end_position - (COMPRESS_RATIO - 1)
if CACHE_HAS_ROPE_POS:
# RoPE uses the first token in the pooled group. Its exact MRoPE
# coordinates may live in this chunk or the raw-state ring.
first_in_chunk = first_position >= chunk_start
first_token = query_start + first_position - chunk_start
first_token_valid = (
(first_token >= query_start)
& (first_token < query_end)
& (first_token < num_tokens)
)
safe_first_token = tl.maximum(first_token, 0)
load_current_position = first_in_chunk & first_token_valid
first_pos_t = tl.load(
pos_ptr + safe_first_token * pos_stride_token,
mask=load_current_position,
other=0,
)
if IS_2D_POSITIONS:
first_pos_h = tl.load(
pos_ptr + pos_stride_axis + safe_first_token * pos_stride_token,
mask=load_current_position,
other=0,
)
first_pos_w = tl.load(
pos_ptr
+ 2 * pos_stride_axis
+ safe_first_token * pos_stride_token,
mask=load_current_position,
other=0,
)
else:
first_pos_h = first_pos_t
first_pos_w = first_pos_t
tail = (
state_cache_ptr
+ safe_state_block * state_cache_stride_block
+ (first_position % STATE_SIZE) * state_cache_stride_token
+ D
).to(tl.pointer_type(tl.int64))
load_cached_position = ~first_in_chunk & state_block_valid
cached_pos_t = tl.load(tail, mask=load_cached_position, other=0)
cached_pos_h = tl.load(tail + 1, mask=load_cached_position, other=0)
cached_pos_w = tl.load(tail + 2, mask=load_cached_position, other=0)
pos_t = tl.where(first_in_chunk, first_pos_t.to(tl.int64), cached_pos_t)
pos_h = tl.where(first_in_chunk, first_pos_h.to(tl.int64), cached_pos_h)
pos_w = tl.where(first_in_chunk, first_pos_w.to(tl.int64), cached_pos_w)
else:
pos_t = first_position
pos_h = first_position
pos_w = first_position
y = _norm_rope(
tl.reshape(pooled, (1, 1, D)),
pos_t + tl.arange(0, 1),
pos_h + tl.arange(0, 1),
pos_w + tl.arange(0, 1),
cos_sin_ptr,
D // 2,
k_norm_weight_ptr,
eps,
IS_K_MROPE,
MROPE_H,
MROPE_W,
)
compressed_block = (compressed_slot // COMP_PAGE_SIZE).to(tl.int64)
compressed_row = compressed_slot % COMP_PAGE_SIZE
tl.store(
compressed_cache_ptr
+ compressed_block * compressed_cache_stride_block
+ compressed_row * compressed_cache_stride_token
+ dims,
tl.reshape(y, (D,)),
mask=valid,
)
if work_in_request == 0:
# This CTA may have just read historical rows from the circular buffer.
# Keep every lane past those loads before overwriting the same ring.
tl.debug_barrier()
# One CTA per request commits only the suffix retained by the ring.
num_state_rows = tl.minimum(query_len, STATE_SIZE)
for state_offset in tl.range(0, num_state_rows):
token = query_end - num_state_rows + state_offset
valid_token = (
(token >= query_start) & (token < query_end) & (token < num_tokens)
)
slot = tl.load(state_slots_ptr + token, mask=valid_token, other=-1)
valid_slot = (
valid_token & (slot >= 0) & (slot < num_state_blocks * STATE_SIZE)
)
safe_slot = tl.maximum(slot, 0)
state_row = (
state_cache_ptr
+ (safe_slot // STATE_SIZE).to(tl.int64) * state_cache_stride_block
+ (safe_slot % STATE_SIZE) * state_cache_stride_token
)
k = tl.load(
k_ptr + tl.maximum(token, 0) * k_stride_token + dims,
mask=valid_slot,
other=0.0,
)
tl.store(state_row + dims, k, mask=valid_slot)
if CACHE_HAS_ROPE_POS:
pos_t = tl.load(
pos_ptr + tl.maximum(token, 0) * pos_stride_token,
mask=valid_slot,
other=0,
)
if IS_2D_POSITIONS:
pos_h = tl.load(
pos_ptr
+ pos_stride_axis
+ tl.maximum(token, 0) * pos_stride_token,
mask=valid_slot,
other=0,
)
pos_w = tl.load(
pos_ptr
+ 2 * pos_stride_axis
+ tl.maximum(token, 0) * pos_stride_token,
mask=valid_slot,
other=0,
)
else:
pos_h = pos_t
pos_w = pos_t
tail = (state_row + D).to(tl.pointer_type(tl.int64))
tl.store(tail, pos_t.to(tl.int64), mask=valid_slot)
tl.store(tail + 1, pos_h.to(tl.int64), mask=valid_slot)
tl.store(tail + 2, pos_w.to(tl.int64), mask=valid_slot)
def qsa_pre_indexer(
q: torch.Tensor,
k: torch.Tensor,
positions: torch.Tensor,
cos_sin_cache: torch.Tensor,
q_norm_weight: torch.Tensor,
k_norm_weight: torch.Tensor,
eps: float,
q_out: torch.Tensor,
state_cache: torch.Tensor,
state_slots: torch.Tensor,
state_block_table: torch.Tensor,
query_start_loc: torch.Tensor,
logical_positions: torch.Tensor,
compressed_cache: torch.Tensor,
compressed_slots: torch.Tensor,
k_work_metadata: torch.Tensor,
*,
compress_ratio: int,
mrope_section: tuple[int, int, int] | None,
rope_pos_offset: int | None,
) -> None:
"""Normalize Q, compress K, then update the circular raw state."""
num_tokens = q.shape[0]
if num_tokens == 0:
return
num_q_heads, head_dim = q_out.shape[1:]
assert cos_sin_cache.shape[-1] * 2 == head_dim
assert q.shape == (num_tokens, num_q_heads * head_dim)
assert k.shape == (num_tokens, head_dim)
assert q.stride(-1) == 1
assert k.stride(-1) == 1
assert q_out.stride(-1) == 1
assert cos_sin_cache.is_contiguous()
assert state_cache.stride(-1) == 1
assert compressed_cache.stride(-1) == 1
assert k_work_metadata.ndim == 2 and k_work_metadata.shape[1] == 2
is_2d_positions = positions.ndim == 2
is_k_mrope = bool(mrope_section)
cache_has_rope_pos = rope_pos_offset is not None
assert rope_pos_offset is None or rope_pos_offset == head_dim
if is_2d_positions:
assert positions.shape == (3, num_tokens)
assert is_k_mrope
pos_stride_axis, pos_stride_token = positions.stride()
else:
assert positions.shape == (num_tokens,)
pos_stride_axis, pos_stride_token = 0, positions.stride(0)
section = mrope_section if mrope_section is not None else (0, 0, 0)
assert len(section) == 3
# Swept on MI355X (64-lane wavefronts) over TILE_T_Q 1-16, TILE_H_Q 1-4 and
# num_warps 1/2/4, from 4 to 16384 tokens: no config beat these by more
# than noise, and 2 or 4 warps were slower everywhere.
if num_tokens <= 4096:
TILE_T_Q, TILE_H_Q = 2, 2
else:
TILE_T_Q, TILE_H_Q = 2, 4
num_k_work = k_work_metadata.shape[0]
num_q_work = triton.cdiv(num_tokens, TILE_T_Q) * triton.cdiv(num_q_heads, TILE_H_Q)
_qsa_pre_indexer_kernel[(num_k_work + num_q_work,)](
q,
q.stride(0),
k,
k.stride(0),
positions,
pos_stride_axis,
pos_stride_token,
cos_sin_cache,
q_norm_weight,
k_norm_weight,
eps,
q_out,
q_out.stride(0),
q_out.stride(1),
state_cache,
state_cache.stride(0),
state_cache.stride(1),
state_slots,
state_block_table,
state_block_table.stride(0),
query_start_loc,
logical_positions,
compressed_slots,
k_work_metadata,
compressed_cache,
compressed_cache.stride(0),
compressed_cache.stride(1),
num_tokens,
state_cache.shape[0],
compressed_cache.shape[0],
num_k_work,
HQ=num_q_heads,
D=head_dim,
TILE_T_Q=TILE_T_Q,
TILE_H_Q=TILE_H_Q,
COMPRESS_RATIO=compress_ratio,
STATE_SIZE=state_cache.shape[1],
COMP_PAGE_SIZE=compressed_cache.shape[1],
IS_2D_POSITIONS=is_2d_positions,
IS_K_MROPE=is_k_mrope,
CACHE_HAS_ROPE_POS=cache_has_rope_pos,
MROPE_H=section[1],
MROPE_W=section[2],
num_warps=1,
)
__all__ = ["qsa_pre_indexer", "supports_fused_pre_indexer"]