2ce8012fa0
Signed-off-by: Sriram Kumar <sriramkumar.kishorekumar@amd.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
4707 lines
161 KiB
Python
4707 lines
161 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||
import functools
|
||
import importlib
|
||
import math
|
||
from collections.abc import Callable
|
||
from importlib.util import find_spec
|
||
|
||
import torch
|
||
import torch.nn.functional as F
|
||
|
||
import vllm.envs as envs
|
||
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
|
||
from vllm.config import CUDAGraphMode, get_current_vllm_config
|
||
from vllm.forward_context import get_forward_context
|
||
from vllm.logger import init_logger
|
||
from vllm.platforms import current_platform
|
||
from vllm.triton_utils import tl, triton
|
||
from vllm.utils.torch_utils import LayerNameType, direct_register_custom_op
|
||
from vllm.v1.attention.backends.mla.indexer import DeepseekV32IndexerMetadata
|
||
from vllm.v1.attention.ops.common import pack_seq_triton, unpack_seq_triton
|
||
from vllm.v1.worker.workspace import current_workspace_manager
|
||
|
||
if current_platform.is_rocm():
|
||
from vllm.platforms.rocm import _ON_GFX942, _ON_GFX950
|
||
else:
|
||
_ON_GFX942 = False
|
||
_ON_GFX950 = False
|
||
|
||
logger = init_logger(__name__)
|
||
|
||
FP8_DTYPE = current_platform.fp8_dtype()
|
||
|
||
|
||
@functools.cache
|
||
def _get_aiter_sparse_prefill_opus() -> Callable[..., torch.Tensor] | None:
|
||
from vllm._aiter_ops import rocm_aiter_ops
|
||
|
||
if not rocm_aiter_ops.is_mla_enabled():
|
||
return None
|
||
try:
|
||
from aiter.ops.pa_sparse_prefill_opus import pa_sparse_prefill_opus
|
||
except ImportError:
|
||
return None
|
||
logger.info_once("Using AITER OPUS for large sparse MLA prefill on gfx950")
|
||
return pa_sparse_prefill_opus
|
||
|
||
|
||
# Conservative perf gate, not a correctness bound: OPUS is correct for any query
|
||
# count, but Triton stays faster below this measured crossover.
|
||
_GFX950_AITER_SPARSE_PREFILL_OPUS_MIN_QUERIES = 1024
|
||
|
||
|
||
def _indexer_k_is_c4a_block_flat(compress_ratio: int) -> bool:
|
||
"""V4.0 C4A is block-flat (NORMAL). Ratio 1 and 2 are 16×16 SHUFFLE."""
|
||
return compress_ratio == 4
|
||
|
||
|
||
@triton.jit
|
||
def _localize_aiter_prefill_topk_kernel(
|
||
indices_ptr,
|
||
row_starts_ptr,
|
||
indices_stride,
|
||
num_topk,
|
||
BLOCK_SIZE: tl.constexpr,
|
||
):
|
||
row_idx = tl.program_id(0)
|
||
offsets = tl.program_id(1) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||
mask = offsets < num_topk
|
||
indices = tl.load(indices_ptr + row_idx * indices_stride + offsets, mask=mask)
|
||
row_start = tl.load(row_starts_ptr + row_idx)
|
||
localized = tl.where(indices < 0, indices, indices - row_start)
|
||
tl.store(
|
||
indices_ptr + row_idx * indices_stride + offsets,
|
||
localized,
|
||
mask=mask,
|
||
)
|
||
|
||
|
||
def _localize_aiter_prefill_topk(
|
||
indices: torch.Tensor,
|
||
row_starts: torch.Tensor,
|
||
) -> None:
|
||
num_rows, num_topk = indices.shape
|
||
block_size = 256
|
||
_localize_aiter_prefill_topk_kernel[(num_rows, triton.cdiv(num_topk, block_size))](
|
||
indices,
|
||
row_starts,
|
||
indices.stride(0),
|
||
num_topk,
|
||
BLOCK_SIZE=block_size,
|
||
)
|
||
|
||
|
||
def _launch_aiter_top_k_per_row_prefill(
|
||
logits: torch.Tensor,
|
||
row_starts: torch.Tensor,
|
||
row_ends: torch.Tensor,
|
||
indices: torch.Tensor,
|
||
topk_tokens: int,
|
||
) -> None:
|
||
from vllm._aiter_ops import rocm_aiter_ops
|
||
|
||
rocm_aiter_ops.indexer_top_k_prefill(
|
||
logits, row_starts, row_ends, indices, topk_tokens
|
||
)
|
||
_localize_aiter_prefill_topk(indices, row_starts)
|
||
|
||
|
||
@triton.jit
|
||
def _indexer_k_quant_and_cache_kernel(
|
||
k_ptr, # [num_tokens, head_dim]
|
||
kv_cache_ptr, # [n_blks, blk_size//tile_block, head_dim // 16B, tile_block, 16B]
|
||
# [n_blocks, blk_size, head_dim]
|
||
kv_cache_scale_ptr, # [n_blks, blk_size]
|
||
slot_mapping_ptr, # [num_tokens]
|
||
kv_cache_scale_stride,
|
||
kv_cache_value_stride,
|
||
block_size,
|
||
num_tokens,
|
||
head_dim: tl.constexpr,
|
||
LAYOUT: tl.constexpr,
|
||
BLOCK_TILE_SIZE: tl.constexpr,
|
||
HEAD_TILE_SIZE: tl.constexpr,
|
||
IS_FNUZ: tl.constexpr,
|
||
USE_UE8M0: tl.constexpr,
|
||
):
|
||
tid = tl.program_id(0)
|
||
offset = tl.arange(0, head_dim)
|
||
if LAYOUT == "SHUFFLE":
|
||
tile_offset = (
|
||
offset // HEAD_TILE_SIZE * BLOCK_TILE_SIZE * HEAD_TILE_SIZE
|
||
+ offset % HEAD_TILE_SIZE
|
||
)
|
||
else:
|
||
tile_offset = offset
|
||
tile_store_offset = tile_offset
|
||
# for idx in tl.range(tid, num_tokens, n_program):
|
||
src_ptr = k_ptr + tid * head_dim
|
||
slot_id = tl.load(slot_mapping_ptr + tid)
|
||
if slot_id < 0:
|
||
return
|
||
# The packed KV layout makes per-block strides large
|
||
# enough that block_id * stride can exceed 32-bit range.
|
||
block_id = (slot_id // block_size).to(tl.int64)
|
||
block_offset = slot_id % block_size
|
||
tile_block_id = block_offset // BLOCK_TILE_SIZE
|
||
tile_block_offset = block_offset % BLOCK_TILE_SIZE
|
||
val = tl.load(src_ptr + offset)
|
||
amax = tl.max(val.abs(), axis=-1).to(tl.float32)
|
||
if IS_FNUZ:
|
||
scale = tl.maximum(1e-4, amax) / 224.0
|
||
else:
|
||
scale = tl.maximum(1e-4, amax) / 448.0
|
||
|
||
if USE_UE8M0:
|
||
scale = tl.exp2(tl.ceil(tl.log2(scale)))
|
||
|
||
fp8_val = (val.to(tl.float32) / scale).to(kv_cache_ptr.type.element_ty)
|
||
if LAYOUT == "SHUFFLE":
|
||
dst_ptr = (
|
||
kv_cache_ptr
|
||
+ block_id * kv_cache_value_stride
|
||
+ tile_block_id * BLOCK_TILE_SIZE * head_dim
|
||
+ tile_block_offset * HEAD_TILE_SIZE
|
||
)
|
||
else:
|
||
dst_ptr = (
|
||
kv_cache_ptr + block_id * kv_cache_value_stride + block_offset * head_dim
|
||
)
|
||
tl.store(dst_ptr + tile_store_offset, fp8_val)
|
||
dst_scale_ptr = kv_cache_scale_ptr + block_id * kv_cache_scale_stride + block_offset
|
||
tl.store(dst_scale_ptr, scale)
|
||
|
||
|
||
def indexer_k_quant_and_cache_triton(
|
||
k: torch.Tensor,
|
||
kv_cache: torch.Tensor, # [num_blocks, block_size, head_dim + 4]
|
||
slot_mapping: torch.Tensor,
|
||
quant_block_size,
|
||
scale_fmt,
|
||
block_tile_size=16,
|
||
head_tile_size=16,
|
||
):
|
||
num_blocks = kv_cache.shape[0]
|
||
head_dim = k.shape[-1]
|
||
num_tokens = slot_mapping.shape[0]
|
||
block_size = kv_cache.shape[1]
|
||
# In real layout, we store the first portion as kv cache value
|
||
# and second portion as kv cache scale
|
||
kv_cache = kv_cache.view(num_blocks, -1)
|
||
kv_cache_value = kv_cache[:, : block_size * head_dim].view(FP8_DTYPE)
|
||
kv_cache_scale = kv_cache[:, block_size * head_dim :].view(torch.float32)
|
||
head_tile_size = head_tile_size // kv_cache.element_size()
|
||
layout = "NORMAL" if block_size == 1 else "SHUFFLE"
|
||
grid = (num_tokens,)
|
||
_indexer_k_quant_and_cache_kernel[grid](
|
||
k,
|
||
kv_cache_value,
|
||
kv_cache_scale,
|
||
slot_mapping,
|
||
kv_cache_scale.stride(0),
|
||
kv_cache_value.stride(0),
|
||
block_size,
|
||
num_tokens,
|
||
head_dim,
|
||
layout,
|
||
block_tile_size,
|
||
head_tile_size,
|
||
IS_FNUZ=torch.float8_e4m3fnuz == FP8_DTYPE,
|
||
USE_UE8M0=scale_fmt == "ue8m0",
|
||
)
|
||
|
||
|
||
@triton.jit
|
||
def _cp_gather_indexer_quant_cache_kernel(
|
||
kv_cache_ptr, # [n_blks,blk_size//tile_blk,head_dim//16B,tile_blk,16B]
|
||
# [n_blks, blk_size, head_dim]
|
||
kv_cache_scale_ptr, # [n_blks, blk_size]
|
||
k_fp8_ptr, # [num_tokens, head_dim]
|
||
k_scale_ptr, # [num_tokens]
|
||
block_table_ptr, # [batch_size, block_table_stride]
|
||
cu_seqlen_ptr, # [batch_size + 1]
|
||
token_to_seq_ptr, # [num_tokens]
|
||
block_size,
|
||
block_table_stride,
|
||
kv_cache_stride,
|
||
kv_cache_scale_stride,
|
||
LAYOUT: tl.constexpr,
|
||
HEAD_DIM: tl.constexpr,
|
||
BLOCK_TILE_SIZE: tl.constexpr,
|
||
HEAD_TILE_SIZE: tl.constexpr,
|
||
num_tokens,
|
||
num_batches,
|
||
block_table_width,
|
||
num_blocks,
|
||
):
|
||
tid = tl.program_id(0)
|
||
offset = tl.arange(0, HEAD_DIM)
|
||
valid_tid = tid < num_tokens
|
||
batch_id = tl.load(token_to_seq_ptr + tid, mask=valid_tid, other=-1)
|
||
valid_batch = (batch_id >= 0) & (batch_id < num_batches)
|
||
safe_batch_id = tl.where(valid_batch, batch_id, 0)
|
||
batch_start = tl.load(cu_seqlen_ptr + safe_batch_id, mask=valid_batch, other=0)
|
||
batch_end = tl.load(cu_seqlen_ptr + safe_batch_id + 1, mask=valid_batch, other=0)
|
||
batch_offset = tid - batch_start
|
||
valid_token = valid_tid & valid_batch & (tid >= batch_start) & (tid < batch_end)
|
||
if not valid_token:
|
||
return
|
||
block_table_id = batch_offset // block_size
|
||
block_offset = batch_offset % block_size
|
||
valid_block_table = (
|
||
valid_token
|
||
& (block_table_id >= 0)
|
||
& (block_table_id < block_table_width)
|
||
& (block_offset >= 0)
|
||
& (block_offset < block_size)
|
||
)
|
||
safe_block_table_id = tl.where(valid_block_table, block_table_id, 0)
|
||
block_table_offset = safe_batch_id * block_table_stride + safe_block_table_id
|
||
block_id = tl.load(
|
||
block_table_ptr + block_table_offset, mask=valid_block_table, other=-1
|
||
)
|
||
valid_block = valid_block_table & (block_id >= 0) & (block_id < num_blocks)
|
||
# The packed KV layout makes per-block strides large
|
||
# enough that block_id * stride can exceed 32-bit range.
|
||
safe_block_id = tl.where(valid_block, block_id, 0).to(tl.int64)
|
||
safe_block_offset = tl.where(valid_block, block_offset, 0)
|
||
tiled_block_offset = safe_block_offset % BLOCK_TILE_SIZE
|
||
if LAYOUT == "SHUFFLE":
|
||
src_cache_offset = (
|
||
safe_block_id * kv_cache_stride
|
||
+ (safe_block_offset // BLOCK_TILE_SIZE) * HEAD_DIM * BLOCK_TILE_SIZE
|
||
+ tiled_block_offset * HEAD_TILE_SIZE
|
||
)
|
||
else:
|
||
src_cache_offset = (
|
||
safe_block_id * kv_cache_stride + safe_block_offset * HEAD_DIM
|
||
)
|
||
src_scale_offset = safe_block_id * kv_cache_scale_stride + safe_block_offset
|
||
dst_offset = tid * HEAD_DIM
|
||
src_scale_ptr = kv_cache_scale_ptr + src_scale_offset
|
||
src_cache_ptr = kv_cache_ptr + src_cache_offset
|
||
dst_k_ptr = k_fp8_ptr + dst_offset
|
||
scale_val = tl.load(src_scale_ptr, mask=valid_block, other=0.0)
|
||
tl.store(k_scale_ptr + tid, scale_val)
|
||
if LAYOUT == "SHUFFLE":
|
||
tiled_src_offset = (
|
||
offset // HEAD_TILE_SIZE * HEAD_TILE_SIZE * BLOCK_TILE_SIZE
|
||
+ offset % HEAD_TILE_SIZE
|
||
)
|
||
else:
|
||
tiled_src_offset = offset
|
||
val = tl.load(src_cache_ptr + tiled_src_offset)
|
||
tl.store(dst_k_ptr + offset, val, mask=valid_block)
|
||
|
||
|
||
@triton.jit(do_not_specialize=["num_batches"])
|
||
def _cp_gather_indexer_quant_cache_gfx950_kernel(
|
||
kv_cache_ptr, # [n_blks,blk_size//tile_blk,head_dim//16B,tile_blk,16B]
|
||
# [n_blks, blk_size, head_dim]
|
||
kv_cache_scale_ptr, # [n_blks, blk_size]
|
||
k_fp8_ptr, # [num_tokens, head_dim]
|
||
k_scale_ptr, # [num_tokens]
|
||
block_table_ptr, # [batch_size, block_table_stride]
|
||
cu_seqlen_ptr, # [batch_size + 1]
|
||
token_to_seq_ptr, # [num_tokens]
|
||
block_size,
|
||
block_table_stride,
|
||
kv_cache_stride,
|
||
kv_cache_scale_stride,
|
||
LAYOUT: tl.constexpr,
|
||
HEAD_DIM: tl.constexpr,
|
||
BLOCK_TILE_SIZE: tl.constexpr,
|
||
HEAD_TILE_SIZE: tl.constexpr,
|
||
num_batches,
|
||
BLOCK_TABLE_WIDTH: tl.constexpr,
|
||
NUM_BLOCKS: tl.constexpr,
|
||
):
|
||
tid = tl.program_id(0)
|
||
offset = tl.arange(0, HEAD_DIM)
|
||
batch_id = tl.load(token_to_seq_ptr + tid)
|
||
valid_batch = (batch_id >= 0) & (batch_id < num_batches)
|
||
safe_batch_id = tl.where(valid_batch, batch_id, 0)
|
||
batch_start = tl.load(cu_seqlen_ptr + safe_batch_id, mask=valid_batch, other=0)
|
||
batch_end = tl.load(cu_seqlen_ptr + safe_batch_id + 1, mask=valid_batch, other=0)
|
||
batch_offset = tid - batch_start
|
||
valid_token = valid_batch & (tid >= batch_start) & (tid < batch_end)
|
||
if not valid_token:
|
||
return
|
||
block_table_id = batch_offset // block_size
|
||
block_offset = batch_offset % block_size
|
||
valid_block_table = (
|
||
valid_token
|
||
& (block_table_id >= 0)
|
||
& (block_table_id < BLOCK_TABLE_WIDTH)
|
||
& (block_offset >= 0)
|
||
& (block_offset < block_size)
|
||
)
|
||
safe_block_table_id = tl.where(valid_block_table, block_table_id, 0)
|
||
block_table_offset = safe_batch_id * block_table_stride + safe_block_table_id
|
||
block_id = tl.load(
|
||
block_table_ptr + block_table_offset, mask=valid_block_table, other=-1
|
||
)
|
||
valid_block = valid_block_table & (block_id >= 0) & (block_id < NUM_BLOCKS)
|
||
# The packed KV layout makes per-block strides large
|
||
# enough that block_id * stride can exceed 32-bit range.
|
||
safe_block_id = tl.where(valid_block, block_id, 0).to(tl.int64)
|
||
safe_block_offset = tl.where(valid_block, block_offset, 0)
|
||
tiled_block_offset = safe_block_offset % BLOCK_TILE_SIZE
|
||
if LAYOUT == "SHUFFLE":
|
||
src_cache_offset = (
|
||
safe_block_id * kv_cache_stride
|
||
+ (safe_block_offset // BLOCK_TILE_SIZE) * HEAD_DIM * BLOCK_TILE_SIZE
|
||
+ tiled_block_offset * HEAD_TILE_SIZE
|
||
)
|
||
else:
|
||
src_cache_offset = (
|
||
safe_block_id * kv_cache_stride + safe_block_offset * HEAD_DIM
|
||
)
|
||
src_scale_offset = safe_block_id * kv_cache_scale_stride + safe_block_offset
|
||
dst_offset = tid * HEAD_DIM
|
||
src_scale_ptr = kv_cache_scale_ptr + src_scale_offset
|
||
src_cache_ptr = kv_cache_ptr + src_cache_offset
|
||
dst_k_ptr = k_fp8_ptr + dst_offset
|
||
scale_val = tl.load(src_scale_ptr, mask=valid_block, other=0.0)
|
||
tl.store(k_scale_ptr + tid, scale_val)
|
||
if LAYOUT == "SHUFFLE":
|
||
tiled_src_offset = (
|
||
offset // HEAD_TILE_SIZE * HEAD_TILE_SIZE * BLOCK_TILE_SIZE
|
||
+ offset % HEAD_TILE_SIZE
|
||
)
|
||
else:
|
||
tiled_src_offset = offset
|
||
val = tl.load(src_cache_ptr + tiled_src_offset)
|
||
tl.store(dst_k_ptr + offset, val, mask=valid_block)
|
||
|
||
|
||
def cp_gather_indexer_k_quant_cache_triton(
|
||
k_cache: torch.Tensor, # [num_blocks, block_size, head_dim + 4]
|
||
k_fp8: torch.Tensor,
|
||
k_fp8_scale: torch.Tensor,
|
||
block_table: torch.Tensor,
|
||
cu_seqlen: torch.Tensor,
|
||
token_to_seq: torch.Tensor,
|
||
block_tile_size: int = 16,
|
||
head_tile_size: int = 16,
|
||
cache_layout: str | None = None,
|
||
):
|
||
num_tokens = k_fp8.size(0)
|
||
block_size = k_cache.size(1)
|
||
block_table_stride = block_table.stride(0)
|
||
head_dim = k_fp8.shape[-1]
|
||
num_blocks = k_cache.shape[0]
|
||
# we assume the kv cache already been split to 2 portion
|
||
k_cache = k_cache.view(num_blocks, -1)
|
||
k_cache_value = k_cache[:, : block_size * head_dim].view(FP8_DTYPE)
|
||
k_cache_scale = k_cache[:, block_size * head_dim :].view(torch.float32)
|
||
grid = (num_tokens,)
|
||
k_fp8_scale = k_fp8_scale.view(torch.float32)
|
||
if cache_layout is None:
|
||
layout = "NORMAL" if block_size == 1 else "SHUFFLE"
|
||
else:
|
||
layout = cache_layout
|
||
kernel_args = (
|
||
k_cache_value,
|
||
k_cache_scale,
|
||
k_fp8,
|
||
k_fp8_scale,
|
||
block_table,
|
||
cu_seqlen,
|
||
token_to_seq,
|
||
block_size,
|
||
block_table_stride,
|
||
k_cache_value.stride(0),
|
||
k_cache_scale.stride(0),
|
||
layout,
|
||
head_dim,
|
||
block_tile_size,
|
||
head_tile_size,
|
||
)
|
||
if _ON_GFX950:
|
||
_cp_gather_indexer_quant_cache_gfx950_kernel[grid](
|
||
*kernel_args,
|
||
cu_seqlen.shape[0] - 1,
|
||
block_table.shape[1],
|
||
num_blocks,
|
||
)
|
||
else:
|
||
_cp_gather_indexer_quant_cache_kernel[grid](
|
||
*kernel_args,
|
||
num_tokens,
|
||
cu_seqlen.shape[0] - 1,
|
||
block_table.shape[1],
|
||
num_blocks,
|
||
)
|
||
|
||
|
||
# Taken from https://github.com/deepseek-ai/DeepGEMM/blob/main/tests/test_attention.py#L156
|
||
def fp8_paged_mqa_logits_torch(
|
||
q: torch.Tensor,
|
||
kv_cache: torch.Tensor,
|
||
weights: torch.Tensor,
|
||
context_lens: torch.Tensor,
|
||
block_tables: torch.Tensor,
|
||
max_model_len: int,
|
||
):
|
||
from vllm.utils.math_utils import cdiv
|
||
|
||
batch_size, next_n, _, dim = q.size()
|
||
if next_n == 1:
|
||
block_size = kv_cache.shape[1]
|
||
logits = torch.full(
|
||
[batch_size, max_model_len],
|
||
float("-inf"),
|
||
device=q.device,
|
||
dtype=torch.float32,
|
||
)
|
||
if context_lens.dim() > 1:
|
||
context_lens = context_lens.squeeze(-1)
|
||
kv_cache_flat = kv_cache.view(-1, block_size * (dim + 4))
|
||
for i in range(batch_size):
|
||
q_i = q[i, 0].to(torch.float32)
|
||
q_scale = weights[i]
|
||
seq_len = int(context_lens[i].item())
|
||
assert seq_len <= max_model_len
|
||
num_pages = cdiv(seq_len, block_size)
|
||
padded_seq_len = num_pages * block_size
|
||
pages = block_tables[i, :num_pages]
|
||
cache = kv_cache_flat[pages]
|
||
scale_offset = block_size * dim
|
||
cache_value = (
|
||
cache[..., :scale_offset].view(dtype=FP8_DTYPE).to(torch.float32)
|
||
)
|
||
cache_scale = (
|
||
cache[..., scale_offset:].view(dtype=torch.float32).contiguous()
|
||
)
|
||
cache_value = cache_value.view(padded_seq_len, dim)
|
||
cache_scale = cache_scale.view(padded_seq_len)
|
||
score = F.linear(cache_value, q_i)
|
||
score = F.relu(score)
|
||
score *= q_scale[None, :]
|
||
score = score.sum(dim=1)
|
||
score *= cache_scale
|
||
logits[i, :seq_len] = score[:seq_len]
|
||
return logits
|
||
|
||
kv_cache, scale = kv_cache[..., :dim], kv_cache[..., dim:]
|
||
scale = scale.contiguous().view(torch.float)
|
||
q = q.float()
|
||
kv_cache = kv_cache.view(FP8_DTYPE).float() * scale
|
||
num_block, block_size, _, dim = kv_cache.size()
|
||
logits = torch.full(
|
||
[batch_size * next_n, max_model_len],
|
||
float("-inf"),
|
||
device=q.device,
|
||
dtype=torch.float32,
|
||
)
|
||
for i in range(batch_size):
|
||
context_len = context_lens[i]
|
||
if context_len.ndim == 0:
|
||
context_len_i = int(context_len.item())
|
||
q_offsets = torch.arange(
|
||
context_len_i - next_n, context_len_i, device=q.device
|
||
)
|
||
context_limit = torch.full(
|
||
(next_n,), context_len_i, dtype=torch.int32, device=q.device
|
||
)
|
||
else:
|
||
context_limit = context_len.to(device=q.device, dtype=torch.int32)
|
||
q_offsets = context_limit - 1
|
||
weight_slice = (
|
||
weights[i * next_n : (i + 1) * next_n, :].transpose(0, 1).contiguous()
|
||
)
|
||
max_context_len = int(context_limit.max().item())
|
||
for block_rk in range(cdiv(max_context_len, block_size)):
|
||
block_idx = block_tables[i][block_rk]
|
||
qx, kx = q[i], kv_cache[block_idx]
|
||
k_offsets = torch.arange(
|
||
block_rk * block_size, (block_rk + 1) * block_size, device=q.device
|
||
)
|
||
mask = (k_offsets[None, :] < context_limit[:, None]) & (
|
||
k_offsets[None, :] <= q_offsets[:, None]
|
||
)
|
||
s = torch.where(
|
||
mask[None, :, :],
|
||
(qx.transpose(0, 1) @ kx.transpose(0, 1).transpose(1, 2)).to(
|
||
logits.dtype
|
||
),
|
||
float("-inf"),
|
||
)
|
||
s = torch.relu(s) * weight_slice[..., None]
|
||
s = s.sum(dim=0)
|
||
logits[
|
||
i * next_n : (i + 1) * next_n,
|
||
block_rk * block_size : (block_rk + 1) * block_size,
|
||
] = torch.where(k_offsets[None, :] <= q_offsets[:, None], s, float("-inf"))
|
||
return logits
|
||
|
||
|
||
@triton.jit
|
||
def _fp8_paged_mqa_logits_decode_kernel(
|
||
q_ptr, # fp8 [B, NEXT_N, H, D]
|
||
kv_val_ptr, # fp8, block-flat: [num_blocks, block_size*(D+4)]
|
||
kv_scale_ptr, # fp32, block-flat: [num_blocks, block_size*(D+4)//4]
|
||
weights_ptr, # fp32 [B*NEXT_N, H]
|
||
ctx_lens_ptr, # int32 [B*NEXT_N] if CTX_PER_ROW else [B]
|
||
block_tables_ptr, # int32 [B, max_blocks]
|
||
logits_ptr, # fp32 [B*NEXT_N, max_model_len]
|
||
stride_q_b,
|
||
stride_q_n,
|
||
stride_q_h,
|
||
stride_w_row,
|
||
stride_kvblk_fp8,
|
||
stride_kvblk_f32,
|
||
scale_region_off, # block_size*D // 4 (fp32 offset of scale region within a block)
|
||
stride_bt_b,
|
||
stride_logits_row,
|
||
max_blocks, # block_tables width; guards the block-table gather
|
||
max_model_len,
|
||
NUM_HEADS: tl.constexpr,
|
||
HEAD_SIZE: tl.constexpr,
|
||
BLOCK_SIZE: tl.constexpr, # paged-cache page size
|
||
BLOCK_KV: tl.constexpr, # positions per tile; must divide BLOCK_SIZE
|
||
N_SPLITS: tl.constexpr, # KV-tile parallelism factor (grid dim 1)
|
||
NEXT_N: tl.constexpr, # query positions per batch (1 = decode; >1 = MTP verify)
|
||
CTX_PER_ROW: tl.constexpr, # ctx_lens is already per-(b, n)
|
||
):
|
||
# scale_region_off = block_size*D//4 needs D % 4 == 0 (D=128).
|
||
tl.static_assert(HEAD_SIZE % 4 == 0)
|
||
# Grid (B*NEXT_N, N_SPLITS): disjoint KV tiles per program, no host sync.
|
||
row = tl.program_id(0)
|
||
split = tl.program_id(1)
|
||
b = row // NEXT_N
|
||
n = row % NEXT_N
|
||
if CTX_PER_ROW:
|
||
seq_len = tl.load(ctx_lens_ptr + row)
|
||
else:
|
||
seq_len = tl.load(ctx_lens_ptr + b) - NEXT_N + n + 1
|
||
seq_len = tl.minimum(tl.maximum(seq_len, 0), max_model_len)
|
||
|
||
h = tl.arange(0, NUM_HEADS)
|
||
d = tl.arange(0, HEAD_SIZE)
|
||
# keep q/kv fp8 -> fp8 MFMA (f32 accum), not the slow f32 path
|
||
q = tl.load(
|
||
q_ptr + b * stride_q_b + n * stride_q_n + h[:, None] * stride_q_h + d[None, :]
|
||
)
|
||
w = tl.load(weights_ptr + row * stride_w_row + h).to(tl.float32) # [H]
|
||
|
||
kv_col = tl.arange(0, BLOCK_KV)
|
||
for kv_start in tl.range(split * BLOCK_KV, seq_len, N_SPLITS * BLOCK_KV):
|
||
pos = kv_start + kv_col
|
||
# Tiles are page-aligned (BLOCK_KV | BLOCK_SIZE), so a tile is one page.
|
||
logical_blk = kv_start // BLOCK_SIZE
|
||
blk_ok = logical_blk < max_blocks # guard against a wild page -> fault
|
||
mask_pos = (pos < seq_len) & blk_ok
|
||
page = tl.load(
|
||
block_tables_ptr + b * stride_bt_b + logical_blk, mask=blk_ok, other=0
|
||
).to(tl.int64)
|
||
pos_in_blk = pos - logical_blk * BLOCK_SIZE # [BLOCK_KV] == pos % BLOCK_SIZE
|
||
|
||
# block-flat layout: values region, then scales region.
|
||
val_off = page * stride_kvblk_fp8 + pos_in_blk[:, None] * HEAD_SIZE + d[None, :]
|
||
kv = tl.load(
|
||
kv_val_ptr + val_off, mask=mask_pos[:, None], other=0.0
|
||
) # [BLOCK_KV, D] fp8
|
||
sc = tl.load(
|
||
kv_scale_ptr + page * stride_kvblk_f32 + scale_region_off + pos_in_blk,
|
||
mask=mask_pos,
|
||
other=0.0,
|
||
) # [BLOCK_KV]
|
||
|
||
# fp8 upcasts exactly; bit-identical on gfx942, ~1e-5 rel on gfx950
|
||
# (accum order), top-k unchanged.
|
||
s = tl.dot(kv, tl.trans(q)) # [BLOCK_KV, H]
|
||
s = tl.maximum(s, 0.0)
|
||
s = s * w[None, :]
|
||
s = tl.sum(s, axis=1) # [BLOCK_KV]
|
||
s = s * sc
|
||
tl.store(logits_ptr + row * stride_logits_row + pos, s, mask=mask_pos)
|
||
|
||
|
||
def rocm_fp8_paged_mqa_logits_triton(
|
||
q_fp8: torch.Tensor,
|
||
kv_cache_fp8: torch.Tensor,
|
||
weights: torch.Tensor,
|
||
context_lens: torch.Tensor,
|
||
block_tables: torch.Tensor,
|
||
max_model_len: int,
|
||
) -> torch.Tensor:
|
||
"""Triton paged MQA-logits for decode and MTP; matches the torch ref but
|
||
has no host sync, so it is safe to capture under a full CUDA graph."""
|
||
batch_size, next_n, num_heads, head_size = q_fp8.shape
|
||
block_size = kv_cache_fp8.shape[1]
|
||
BLOCK_KV = 64
|
||
assert block_size % BLOCK_KV == 0
|
||
|
||
fp8_dtype = current_platform.fp8_dtype()
|
||
num_blocks = kv_cache_fp8.shape[0]
|
||
kv_flat = kv_cache_fp8.reshape(
|
||
num_blocks, -1
|
||
) # uint8 [num_blocks, block_size*(D+4)]
|
||
kv_val = kv_flat.view(fp8_dtype) # [num_blocks, block_size*(D+4)] fp8
|
||
kv_scale = kv_flat.view(torch.float32) # [num_blocks, block_size*(D+4)//4] fp32
|
||
|
||
cl = context_lens.reshape(-1)
|
||
ctx_per_row = not (next_n > 1 and cl.numel() == batch_size)
|
||
|
||
max_blocks = block_tables.shape[1]
|
||
(out_logits,) = current_workspace_manager().get_simultaneous(
|
||
((batch_size * next_n, max_model_len), torch.float32),
|
||
)
|
||
|
||
# Memory-bound over the KV range: split each row's keys across programs so
|
||
# few-row / long-context launches still fill the GPU. Cap splits at the
|
||
# device CU count (304 on gfx942, 256 on gfx950) rather than a gfx950-sized
|
||
# constant. All terms are static at launch, so the grid stays CUDA-graph-safe.
|
||
rows = batch_size * next_n
|
||
tiles_cap = (max_model_len + BLOCK_KV - 1) // BLOCK_KV
|
||
N_SPLITS = max(1, min(max(1, _decode_cu_count()), tiles_cap, 1024 // rows))
|
||
_fp8_paged_mqa_logits_decode_kernel[(rows, N_SPLITS)](
|
||
q_fp8,
|
||
kv_val,
|
||
kv_scale,
|
||
weights,
|
||
cl,
|
||
block_tables,
|
||
out_logits,
|
||
q_fp8.stride(0),
|
||
q_fp8.stride(1),
|
||
q_fp8.stride(2),
|
||
weights.stride(0),
|
||
kv_val.stride(0),
|
||
kv_scale.stride(0),
|
||
(block_size * head_size) // 4,
|
||
block_tables.stride(0),
|
||
out_logits.stride(0),
|
||
max_blocks,
|
||
max_model_len,
|
||
NUM_HEADS=num_heads,
|
||
HEAD_SIZE=head_size,
|
||
BLOCK_SIZE=block_size,
|
||
BLOCK_KV=BLOCK_KV,
|
||
N_SPLITS=N_SPLITS,
|
||
NEXT_N=next_n,
|
||
CTX_PER_ROW=ctx_per_row,
|
||
num_warps=4,
|
||
num_stages=2,
|
||
)
|
||
return out_logits
|
||
|
||
|
||
@functools.lru_cache
|
||
def paged_mqa_logits_module():
|
||
paged_mqa_logits_module_path = None
|
||
if find_spec("aiter.ops.triton.pa_mqa_logits") is not None:
|
||
paged_mqa_logits_module_path = "aiter.ops.triton.pa_mqa_logits"
|
||
elif find_spec("aiter.ops.triton.attention.pa_mqa_logits") is not None:
|
||
paged_mqa_logits_module_path = "aiter.ops.triton.attention.pa_mqa_logits"
|
||
|
||
if paged_mqa_logits_module_path is not None:
|
||
try:
|
||
module = importlib.import_module(paged_mqa_logits_module_path)
|
||
return module
|
||
except ImportError:
|
||
return None
|
||
return None
|
||
|
||
|
||
def _per_sequence_context_lens(context_lens: torch.Tensor) -> torch.Tensor:
|
||
# AITER takes one length per sequence: the last column of a (B, next_n) table.
|
||
# TODO: remove once AITER accepts (B, next_n) context_lens (ROCm/aiter#6153).
|
||
if context_lens.dim() == 2 and context_lens.shape[1] > 1:
|
||
return context_lens[:, -1].contiguous()
|
||
return context_lens
|
||
|
||
|
||
def rocm_fp8_paged_mqa_logits(
|
||
q_fp8: torch.Tensor,
|
||
kv_cache_fp8: torch.Tensor,
|
||
weights: torch.Tensor,
|
||
context_lens: torch.Tensor,
|
||
block_tables: torch.Tensor,
|
||
schedule_metadata: torch.Tensor,
|
||
max_model_len: int,
|
||
*,
|
||
compress_ratio: int = 1,
|
||
) -> torch.Tensor:
|
||
"""Compute FP8 MQA logits using paged KV-cache.
|
||
|
||
Args:
|
||
q_fp8: Query tensor of shape [B, next_n, H, D]. Casted to
|
||
`torch.float8_e4m3fn` by caller.
|
||
kv_cache_fp8: Paged KV-cache in packed FP8+scale layout with shape
|
||
[num_blocks, block_size, 1, D+4], dtype `torch.uint8`.
|
||
weights: Tensor of shape [B * next_n, H], dtype `torch.float32`.
|
||
context_lens: Tensor of shape [B] or [B, next_n], dtype int32; effective
|
||
context length for each batch element, or for each of its Q rows.
|
||
block_tables: Tensor of shape [B, max_blocks], dtype int32; maps logical
|
||
block indices to physical blocks in the paged cache.
|
||
schedule_metadata: Returned by `get_paged_mqa_logits_metadata`;
|
||
used to distribute work across SMs.
|
||
max_model_len: Maximum sequence length used to size the logits output.
|
||
compress_ratio: C4A (4) takes block-flat Triton; 1 and 2 stay on AITER.
|
||
|
||
Returns:
|
||
Logits tensor of shape [B * next_n, max_model_len], dtype
|
||
`torch.float32`.
|
||
|
||
"""
|
||
from vllm._aiter_ops import rocm_aiter_ops
|
||
|
||
batch_size, next_n = q_fp8.shape[:2]
|
||
block_size = kv_cache_fp8.shape[1]
|
||
|
||
# C4A only: Flash/DSv3.2 also skip insert but still write SHUFFLE.
|
||
if (
|
||
(_ON_GFX950 or _ON_GFX942)
|
||
and _indexer_k_is_c4a_block_flat(compress_ratio)
|
||
and block_size > 1
|
||
):
|
||
if block_size % 64 == 0:
|
||
return rocm_fp8_paged_mqa_logits_triton(
|
||
q_fp8, kv_cache_fp8, weights, context_lens, block_tables, max_model_len
|
||
)
|
||
# Non 64-aligned page size (not used in prod): eager torch ref.
|
||
return fp8_paged_mqa_logits_torch(
|
||
q_fp8, kv_cache_fp8, weights, context_lens, block_tables, max_model_len
|
||
)
|
||
|
||
aiter_paged_mqa_logits_module = None
|
||
|
||
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
|
||
aiter_paged_mqa_logits_module = paged_mqa_logits_module()
|
||
|
||
if aiter_paged_mqa_logits_module is not None:
|
||
if _ON_GFX942 or _ON_GFX950:
|
||
deepgemm_fp8_paged_mqa_logits = (
|
||
aiter_paged_mqa_logits_module.deepgemm_fp8_paged_mqa_logits
|
||
)
|
||
batch_size, next_n, heads, _ = q_fp8.shape
|
||
(out_logits,) = current_workspace_manager().get_simultaneous(
|
||
((batch_size * next_n, max_model_len), torch.float32),
|
||
)
|
||
deepgemm_fp8_paged_mqa_logits(
|
||
q_fp8,
|
||
kv_cache_fp8,
|
||
weights,
|
||
out_logits,
|
||
_per_sequence_context_lens(context_lens),
|
||
block_tables,
|
||
max_model_len,
|
||
ChunkK=256,
|
||
Preshuffle=block_size > 1,
|
||
KVBlockSize=block_size,
|
||
WavePerEU=2,
|
||
)
|
||
return out_logits
|
||
deepgemm_fp8_paged_mqa_logits_stage1 = (
|
||
aiter_paged_mqa_logits_module.deepgemm_fp8_paged_mqa_logits_stage1
|
||
)
|
||
batch_size, next_n, heads, _ = q_fp8.shape
|
||
(out_qk,) = current_workspace_manager().get_simultaneous(
|
||
((heads, batch_size * next_n, max_model_len), torch.float32),
|
||
)
|
||
out_qk.fill_(float("-inf"))
|
||
deepgemm_fp8_paged_mqa_logits_stage1(
|
||
q_fp8,
|
||
kv_cache_fp8,
|
||
weights,
|
||
out_qk,
|
||
_per_sequence_context_lens(context_lens),
|
||
block_tables,
|
||
max_model_len,
|
||
ChunkQ=heads,
|
||
)
|
||
return out_qk.sum(dim=0)
|
||
else:
|
||
return fp8_paged_mqa_logits_torch(
|
||
q_fp8, kv_cache_fp8, weights, context_lens, block_tables, max_model_len
|
||
)
|
||
|
||
|
||
# Take from https://github.com/deepseek-ai/DeepGEMM/blob/main/tests/test_attention.py#L84
|
||
def fp8_mqa_logits_torch(
|
||
q: torch.Tensor,
|
||
kv: tuple[torch.Tensor, torch.Tensor],
|
||
weights: torch.Tensor,
|
||
cu_seqlen_ks: torch.Tensor,
|
||
cu_seqlen_ke: torch.Tensor,
|
||
) -> torch.Tensor:
|
||
"""Compute FP8 MQA logits for a single sequence without KV paging.
|
||
|
||
Args:
|
||
q: Query tensor of shape [M, H, D]. Casted to
|
||
`torch.float8_e4m3fn` by caller.
|
||
kv: Tuple `(k_fp8, k_scales)` where `k_fp8` has shape [N, D] with
|
||
dtype `torch.float8_e4m3fn` and `k_scales` has shape [N] (or
|
||
[N, 1]) with dtype `torch.float32`.
|
||
weights: weights of shape [M, H], dtype `torch.float32`.
|
||
cu_seqlen_ks: Start indices (inclusive) for valid K per query position,
|
||
shape [M], dtype int32.
|
||
cu_seqlen_ke: End indices (exclusive) for valid K per query position,
|
||
shape [M], dtype int32.
|
||
|
||
Returns:
|
||
Logits tensor of shape [M, N], dtype `torch.float32`.
|
||
|
||
"""
|
||
k_fp8, scale = kv
|
||
seq_len_kv = k_fp8.shape[0]
|
||
k = k_fp8.to(torch.bfloat16)
|
||
q = q.to(torch.bfloat16)
|
||
device = q.device
|
||
|
||
mask_lo = (
|
||
torch.arange(0, seq_len_kv, device=device)[None, :] >= cu_seqlen_ks[:, None]
|
||
)
|
||
mask_hi = (
|
||
torch.arange(0, seq_len_kv, device=device)[None, :] < cu_seqlen_ke[:, None]
|
||
)
|
||
mask = mask_lo & mask_hi
|
||
|
||
# ``score`` is [H, M, N]; ``scale`` is the per-KV-token scale, which
|
||
# vLLM callers hand us as ``[N, 1]`` (a ``[N, 4]`` uint8 buffer cast
|
||
# to fp32). PyTorch right-aligns dimensions for broadcasting, so a
|
||
# naked ``score * scale`` would align ``scale``'s leading dim with
|
||
# ``score``'s M dim and raise a shape mismatch. Flatten to ``[N]`` so
|
||
# broadcasting lines up with the last dim of ``score``.
|
||
score = torch.einsum("mhd,nd->hmn", q, k).float() * scale.reshape(-1)
|
||
logits = (score.relu() * weights.unsqueeze(-1).transpose(0, 1)).sum(dim=0)
|
||
logits = logits.masked_fill(~mask, float("-inf"))
|
||
|
||
return logits
|
||
|
||
|
||
@functools.lru_cache
|
||
def mqa_logits_module():
|
||
mqa_logits_module_path = None
|
||
if find_spec("aiter.ops.triton.fp8_mqa_logits") is not None:
|
||
mqa_logits_module_path = "aiter.ops.triton.fp8_mqa_logits"
|
||
elif find_spec("aiter.ops.triton.attention.fp8_mqa_logits") is not None:
|
||
mqa_logits_module_path = "aiter.ops.triton.attention.fp8_mqa_logits"
|
||
|
||
if mqa_logits_module_path is not None:
|
||
try:
|
||
module = importlib.import_module(mqa_logits_module_path)
|
||
return module
|
||
except ImportError:
|
||
return None
|
||
return None
|
||
|
||
|
||
def rocm_fp8_mqa_logits(
|
||
q: torch.Tensor,
|
||
kv: tuple[torch.Tensor, torch.Tensor],
|
||
weights: torch.Tensor,
|
||
cu_seqlen_ks: torch.Tensor,
|
||
cu_seqlen_ke: torch.Tensor,
|
||
) -> torch.Tensor:
|
||
"""Compute FP8 MQA logits for a single sequence without KV paging.
|
||
|
||
Args:
|
||
q: Query tensor of shape [M, H, D]. Casted to
|
||
`torch.float8_e4m3fn` by caller.
|
||
kv: Tuple `(k_fp8, k_scales)` where `k_fp8` has shape [N, D] with
|
||
dtype `torch.float8_e4m3fn` and `k_scales` has shape [N] (or
|
||
[N, 1]) with dtype `torch.float32`.
|
||
weights: weights of shape [M, H], dtype `torch.float32`.
|
||
cu_seqlen_ks: Start indices (inclusive) for valid K per query position,
|
||
shape [M], dtype int32.
|
||
cu_seqlen_ke: End indices (exclusive) for valid K per query position,
|
||
shape [M], dtype int32.
|
||
|
||
Returns:
|
||
Logits tensor of shape [M, N], dtype `torch.float32`.
|
||
|
||
"""
|
||
from vllm._aiter_ops import rocm_aiter_ops
|
||
|
||
k_fp8, scale = kv
|
||
|
||
if _ON_GFX942 and rocm_aiter_ops.is_enabled():
|
||
from aiter.ops.flydsl import flydsl_fp8_mqa_logits
|
||
|
||
return flydsl_fp8_mqa_logits(
|
||
q, k_fp8, scale, weights, cu_seqlen_ks, cu_seqlen_ke
|
||
)
|
||
|
||
aiter_mqa_logits_module = None
|
||
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
|
||
aiter_mqa_logits_module = mqa_logits_module()
|
||
|
||
if aiter_mqa_logits_module is not None:
|
||
fp8_mqa_logits = aiter_mqa_logits_module.fp8_mqa_logits
|
||
return fp8_mqa_logits(
|
||
q, k_fp8, scale, weights, cu_seqlen_ks, cu_seqlen_ke, clean_logits=False
|
||
)
|
||
else:
|
||
return fp8_mqa_logits_torch(q, kv, weights, cu_seqlen_ks, cu_seqlen_ke)
|
||
|
||
|
||
# Programs along the column axis of the decode mask grid. The shared kernel in
|
||
# model_executor/kernels/attention/dsa/candidate_blocks.py launches one program
|
||
# per 1024-column tile, which is 1024 of them per row at max_model_len 1048576
|
||
# with only the first few doing work. This is a fixed count that strides
|
||
# instead, measured on gfx950; it is not a portable choice, which is why this
|
||
# variant lives here rather than replacing the shared one.
|
||
_MASK_GRID_COLS = 128
|
||
# Columns each program handles per iteration. Work stops at the tile boundary
|
||
# containing a row's end rather than at the end itself; the overshoot is
|
||
# masked the same way the shared kernel masks it, so it costs a tile and
|
||
# changes nothing.
|
||
_MASK_TILE = 1024
|
||
|
||
|
||
@triton.jit(do_not_specialize=["width", "nblocks"])
|
||
def _mask_candidates_strided_kernel(
|
||
logits,
|
||
starts,
|
||
ends,
|
||
flags,
|
||
stride_row,
|
||
stride_col,
|
||
stride_start,
|
||
stride_end,
|
||
width,
|
||
nblocks,
|
||
BLOCK_SIZE: tl.constexpr,
|
||
HAS_STARTS: tl.constexpr,
|
||
ROW_REPEAT: tl.constexpr,
|
||
TILE: tl.constexpr,
|
||
):
|
||
row = tl.program_id(0).to(tl.int64)
|
||
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
|
||
end = tl.load(ends + row // ROW_REPEAT * stride_end)
|
||
edge = tl.load(flags + row * (nblocks + 1) + nblocks)
|
||
step = tl.num_programs(1) * TILE
|
||
tile_start = tl.program_id(1) * TILE
|
||
# The shared kernel also sanitizes columns at or past `end`. Nothing reads
|
||
# them: top_k_per_row_decode bounds its scan by the same row ends passed
|
||
# here, and persistent_topk/cooperative_topk clamp by the same lengths.
|
||
# Stopping at `end` is what makes the cost track the live context rather
|
||
# than max_model_len.
|
||
while tile_start < end:
|
||
cols = tile_start + tl.arange(0, TILE)
|
||
valid = (cols >= start) & (cols < end) & (cols < width)
|
||
block = (cols - start) // BLOCK_SIZE
|
||
keep = tl.load(flags + row * (nblocks + 1) + block, valid, other=0)
|
||
keep = (keep != 0) | ((cols == width - 1) & (edge != 0))
|
||
tl.store(
|
||
logits + row * stride_row + cols * stride_col,
|
||
-float("inf"),
|
||
(cols < width) & ~(valid & keep),
|
||
)
|
||
tile_start += step
|
||
|
||
|
||
def _apply_candidate_mask_strided(
|
||
logits: torch.Tensor,
|
||
row_ks: torch.Tensor | None,
|
||
row_ke: torch.Tensor,
|
||
candidate_blocks: torch.Tensor,
|
||
block_size: int,
|
||
row_repeat: int = 1,
|
||
) -> None:
|
||
"""ROCm decode variant of ``apply_candidate_mask``.
|
||
|
||
Same masking semantics over ``[0, end)``, but the grid is sized by a fixed
|
||
program count rather than by the logits width. Only worth using where the
|
||
width is the ``max_model_len`` workspace and the live context is far
|
||
shorter, i.e. the paged decode path below; the prefill chunks pass
|
||
chunk-sized logits and stay on the shared kernel.
|
||
"""
|
||
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
|
||
_candidate_flags_kernel,
|
||
)
|
||
|
||
rows, width = logits.shape
|
||
if not rows or not width:
|
||
return
|
||
nblocks = triton.cdiv(width, block_size)
|
||
flags = torch.empty((rows, nblocks + 1), device=logits.device, dtype=torch.uint8)
|
||
start_stride = row_ks.stride(0) if row_ks is not None else 0
|
||
_candidate_flags_kernel[(rows,)](
|
||
candidate_blocks,
|
||
row_ks,
|
||
flags,
|
||
*candidate_blocks.stride(),
|
||
start_stride,
|
||
width,
|
||
nblocks,
|
||
block_size,
|
||
candidate_blocks.shape[1],
|
||
row_ks is not None,
|
||
row_repeat,
|
||
)
|
||
# Derived from width, which is a tensor shape, so the grid stays static and
|
||
# a FULL cudagraph capture remains valid across replays; only the loop trip
|
||
# count inside the kernel is data-dependent. The min keeps narrow widths
|
||
# from launching programs that would only fall through.
|
||
grid_cols = min(_MASK_GRID_COLS, triton.cdiv(width, _MASK_TILE))
|
||
_mask_candidates_strided_kernel[(rows, grid_cols)](
|
||
logits,
|
||
row_ks,
|
||
row_ke,
|
||
flags,
|
||
*logits.stride(),
|
||
start_stride,
|
||
row_ke.stride(0),
|
||
width,
|
||
nblocks,
|
||
block_size,
|
||
row_ks is not None,
|
||
row_repeat,
|
||
_MASK_TILE,
|
||
)
|
||
|
||
|
||
def _max_decode_logits_rows(num_batched_tokens: int) -> int:
|
||
"""Upper bound on decode rows the paged-MQA logits buffer can ever hold.
|
||
|
||
``rocm_fp8_paged_mqa_logits`` sizes its workspace as
|
||
``(batch_size * next_n, max_model_len)``. ``batch_size`` is bounded by
|
||
``max_num_seqs`` and ``next_n`` by ``1 + num_speculative_tokens``, which is
|
||
far tighter than ``max_num_batched_tokens`` -- 192 vs 16384 for a typical
|
||
32-seq DSpark-5 deployment. The loose bound is harmless at short contexts
|
||
but scales with ``max_model_len``, so at the model's full context it asks
|
||
for tens of TiB and the engine cannot start. Take whichever valid bound is
|
||
smaller; the workspace is locked after profiling, so it must not be under-
|
||
estimated.
|
||
"""
|
||
try:
|
||
vllm_config = get_current_vllm_config()
|
||
except Exception:
|
||
return num_batched_tokens
|
||
scheduler_config = getattr(vllm_config, "scheduler_config", None)
|
||
max_num_seqs = getattr(scheduler_config, "max_num_seqs", None)
|
||
if not max_num_seqs:
|
||
return num_batched_tokens
|
||
speculative_config = getattr(vllm_config, "speculative_config", None)
|
||
num_spec = getattr(speculative_config, "num_speculative_tokens", 0) or 0
|
||
return min(num_batched_tokens, max_num_seqs * (1 + num_spec))
|
||
|
||
|
||
def rocm_aiter_sparse_attn_indexer_fake(
|
||
hidden_states: torch.Tensor,
|
||
k_cache_prefix: LayerNameType,
|
||
kv_cache: torch.Tensor,
|
||
q_fp8: torch.Tensor,
|
||
k: torch.Tensor | None,
|
||
weights: torch.Tensor,
|
||
quant_block_size: int,
|
||
scale_fmt: str | None,
|
||
topk_tokens: int,
|
||
head_dim: int,
|
||
max_model_len: int,
|
||
total_seq_lens: int,
|
||
topk_indices_buffer: torch.Tensor | None,
|
||
skip_k_cache_insert: bool = False,
|
||
compress_ratio: int = 1,
|
||
candidate_blocks: torch.Tensor | None = None,
|
||
candidate_block_size: int = 0,
|
||
candidate_write: bool = False,
|
||
) -> torch.Tensor:
|
||
return topk_indices_buffer
|
||
|
||
|
||
@eager_break_during_capture
|
||
def rocm_aiter_sparse_attn_indexer(
|
||
hidden_states: torch.Tensor,
|
||
k_cache_prefix: LayerNameType,
|
||
kv_cache: torch.Tensor,
|
||
q_fp8: torch.Tensor,
|
||
k: torch.Tensor | None,
|
||
weights: torch.Tensor,
|
||
quant_block_size: int,
|
||
scale_fmt: str | None,
|
||
topk_tokens: int,
|
||
head_dim: int,
|
||
max_model_len: int,
|
||
total_seq_lens: int,
|
||
topk_indices_buffer: torch.Tensor | None,
|
||
skip_k_cache_insert: bool = False,
|
||
compress_ratio: int = 1,
|
||
candidate_blocks: torch.Tensor | None = None,
|
||
candidate_block_size: int = 0,
|
||
candidate_write: bool = False,
|
||
) -> torch.Tensor:
|
||
from vllm._aiter_ops import rocm_aiter_ops
|
||
|
||
# careful! this will be None in dummy run
|
||
forward_context = get_forward_context()
|
||
attn_metadata = forward_context.attn_metadata
|
||
from vllm.utils.torch_utils import _resolve_layer_name
|
||
|
||
k_cache_prefix = _resolve_layer_name(k_cache_prefix)
|
||
# assert isinstance(attn_metadata, dict)
|
||
if not isinstance(attn_metadata, dict):
|
||
# Profiling early-exit: reserve memory to account for runtime
|
||
# allocations. Must be in the real impl, not the fake impl —
|
||
# torch.compile calls the fake impl under FakeTensor mode where
|
||
# workspace manager operations on the locked real workspace
|
||
# would corrupt PyTorch's dispatch state.
|
||
workspace_manager = current_workspace_manager()
|
||
|
||
# Prefill k_fp8 and k_scale buffers, used by
|
||
# rocm_aiter_sparse_attn_indexer's prefill path
|
||
workspace_manager.get_simultaneous(
|
||
((total_seq_lens, head_dim), FP8_DTYPE),
|
||
((total_seq_lens, 4), torch.uint8),
|
||
)
|
||
|
||
# Decode logits buffer, used by rocm_fp8_paged_mqa_logits.
|
||
decode_rows = _max_decode_logits_rows(hidden_states.shape[0])
|
||
if _ON_GFX942 or _ON_GFX950:
|
||
workspace_manager.get_simultaneous(
|
||
((decode_rows, max_model_len), torch.float32),
|
||
)
|
||
else:
|
||
workspace_manager.get_simultaneous(
|
||
(
|
||
(q_fp8.shape[1], decode_rows, max_model_len),
|
||
torch.float32,
|
||
),
|
||
)
|
||
# Transient logits tensor peak memory, produced by
|
||
# rocm_fp8_mqa_logits (prefill) and rocm_fp8_paged_mqa_logits
|
||
# (decode). Prefill logits are bounded by
|
||
# VLLM_SPARSE_INDEXER_MAX_LOGITS_MB via chunking in
|
||
# split_indexer_prefill_chunks; decode logits are smaller.
|
||
max_logits_elems = envs.VLLM_SPARSE_INDEXER_MAX_LOGITS_MB * 1024 * 1024
|
||
_ = torch.empty(
|
||
max_logits_elems, dtype=torch.uint8, device=hidden_states.device
|
||
)
|
||
|
||
return rocm_aiter_sparse_attn_indexer_fake(
|
||
hidden_states,
|
||
k_cache_prefix,
|
||
kv_cache,
|
||
q_fp8,
|
||
k,
|
||
weights,
|
||
quant_block_size,
|
||
scale_fmt,
|
||
topk_tokens,
|
||
head_dim,
|
||
max_model_len,
|
||
total_seq_lens,
|
||
topk_indices_buffer,
|
||
skip_k_cache_insert,
|
||
compress_ratio,
|
||
candidate_blocks,
|
||
candidate_block_size,
|
||
candidate_write,
|
||
)
|
||
layer_attn_metadata = attn_metadata[k_cache_prefix]
|
||
assert isinstance(layer_attn_metadata, DeepseekV32IndexerMetadata)
|
||
assert topk_indices_buffer is not None
|
||
assert scale_fmt is not None
|
||
slot_mapping = layer_attn_metadata.slot_mapping
|
||
has_decode = layer_attn_metadata.num_decodes > 0
|
||
has_prefill = layer_attn_metadata.num_prefills > 0
|
||
num_decode_tokens = layer_attn_metadata.num_decode_tokens
|
||
topk_indices_buffer[: hidden_states.shape[0]] = -1
|
||
|
||
# during speculative decoding, k may be padded to the CUDA graph batch
|
||
# size while slot_mapping only covers actual tokens.
|
||
num_tokens = slot_mapping.shape[0]
|
||
if k is not None:
|
||
k = k[:num_tokens]
|
||
elif not skip_k_cache_insert:
|
||
raise ValueError("k must be provided when skip_k_cache_insert is False")
|
||
|
||
if not skip_k_cache_insert:
|
||
indexer_k_quant_and_cache_triton(
|
||
k,
|
||
kv_cache,
|
||
slot_mapping,
|
||
quant_block_size,
|
||
scale_fmt,
|
||
)
|
||
|
||
if has_prefill:
|
||
prefill_metadata = layer_attn_metadata.prefill
|
||
assert prefill_metadata is not None
|
||
|
||
workspace_manager = current_workspace_manager()
|
||
k_fp8_full, k_scale_full = workspace_manager.get_simultaneous(
|
||
((total_seq_lens, head_dim), FP8_DTYPE),
|
||
((total_seq_lens, 4), torch.uint8),
|
||
)
|
||
for chunk in prefill_metadata.chunks:
|
||
k_fp8 = k_fp8_full[: chunk.total_seq_lens]
|
||
k_scale = k_scale_full[: chunk.total_seq_lens]
|
||
cp_gather_indexer_k_quant_cache_triton(
|
||
kv_cache,
|
||
k_fp8,
|
||
k_scale,
|
||
chunk.block_table,
|
||
chunk.cu_seq_lens,
|
||
token_to_seq=chunk.token_to_seq,
|
||
cache_layout=(
|
||
"NORMAL" if _indexer_k_is_c4a_block_flat(compress_ratio) else None
|
||
),
|
||
)
|
||
logits = rocm_fp8_mqa_logits(
|
||
q_fp8[chunk.token_start : chunk.token_end],
|
||
(k_fp8, k_scale.view(torch.float32)),
|
||
weights[chunk.token_start : chunk.token_end],
|
||
chunk.cu_seqlen_ks,
|
||
chunk.cu_seqlen_ke,
|
||
)
|
||
if candidate_blocks is not None:
|
||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||
_apply_candidate_mask,
|
||
_select_candidate_blocks,
|
||
)
|
||
|
||
chunk_candidates = candidate_blocks[chunk.token_start : chunk.token_end]
|
||
if candidate_write:
|
||
_select_candidate_blocks(
|
||
logits,
|
||
chunk.cu_seqlen_ks,
|
||
chunk.cu_seqlen_ke,
|
||
chunk_candidates.shape[1],
|
||
candidate_block_size,
|
||
chunk_candidates,
|
||
)
|
||
else:
|
||
_apply_candidate_mask(
|
||
logits,
|
||
chunk.cu_seqlen_ks,
|
||
chunk.cu_seqlen_ke,
|
||
chunk_candidates,
|
||
candidate_block_size,
|
||
)
|
||
topk_indices = topk_indices_buffer[
|
||
chunk.token_start : chunk.token_end, :topk_tokens
|
||
]
|
||
|
||
num_rows = logits.shape[0]
|
||
|
||
if rocm_aiter_ops.is_indexer_top_k_supported(
|
||
is_prefill=True,
|
||
compress_ratio=compress_ratio,
|
||
num_rows=num_rows,
|
||
):
|
||
_launch_aiter_top_k_per_row_prefill(
|
||
logits,
|
||
chunk.cu_seqlen_ks,
|
||
chunk.cu_seqlen_ke,
|
||
topk_indices,
|
||
topk_tokens,
|
||
)
|
||
else:
|
||
torch.ops._C.top_k_per_row_prefill(
|
||
logits,
|
||
chunk.cu_seqlen_ks,
|
||
chunk.cu_seqlen_ke,
|
||
topk_indices,
|
||
num_rows,
|
||
logits.stride(0),
|
||
logits.stride(1),
|
||
topk_tokens,
|
||
)
|
||
|
||
if has_decode:
|
||
decode_metadata = layer_attn_metadata.decode
|
||
assert decode_metadata is not None
|
||
# kv_cache size requirement [num_block, block_size, n_head, head_dim],
|
||
# we only have [num_block, block_size, head_dim],
|
||
kv_cache = kv_cache.unsqueeze(-2)
|
||
decode_lens = decode_metadata.decode_lens
|
||
if decode_metadata.requires_padding:
|
||
# pad in edge case where we have short chunked prefill length <
|
||
# decode_threshold since we unstrictly split
|
||
# prefill and decode by decode_threshold
|
||
# (currently set to 1 + speculative tokens)
|
||
padded_q_fp8_decode_tokens = pack_seq_triton(
|
||
q_fp8[:num_decode_tokens], decode_lens
|
||
)
|
||
else:
|
||
padded_q_fp8_decode_tokens = q_fp8[:num_decode_tokens].reshape(
|
||
decode_lens.shape[0], -1, *q_fp8.shape[1:]
|
||
)
|
||
# TODO: move and optimize below logic with triton kernels
|
||
batch_size = padded_q_fp8_decode_tokens.shape[0]
|
||
next_n = padded_q_fp8_decode_tokens.shape[1]
|
||
assert batch_size == decode_metadata.seq_lens.shape[0]
|
||
num_padded_tokens = batch_size * next_n
|
||
|
||
logits = rocm_fp8_paged_mqa_logits(
|
||
padded_q_fp8_decode_tokens,
|
||
kv_cache,
|
||
weights[:num_padded_tokens],
|
||
decode_metadata.seq_lens,
|
||
decode_metadata.block_table,
|
||
decode_metadata.schedule_metadata,
|
||
max_model_len=max_model_len,
|
||
compress_ratio=compress_ratio,
|
||
)
|
||
|
||
if candidate_blocks is not None:
|
||
from vllm.model_executor.layers.sparse_attn_indexer import (
|
||
_select_candidate_blocks,
|
||
)
|
||
|
||
num_rows = logits.shape[0]
|
||
visible = decode_metadata.seq_lens.reshape(-1)
|
||
if visible.numel() != num_rows:
|
||
visible = visible.repeat_interleave(next_n)
|
||
visible = visible[:num_rows].to(torch.int64)
|
||
row_starts = torch.zeros_like(visible)
|
||
decode_candidates = candidate_blocks[:num_rows]
|
||
if candidate_write:
|
||
_select_candidate_blocks(
|
||
logits,
|
||
row_starts,
|
||
visible,
|
||
decode_candidates.shape[1],
|
||
candidate_block_size,
|
||
decode_candidates,
|
||
)
|
||
else:
|
||
_apply_candidate_mask_strided(
|
||
logits,
|
||
row_starts,
|
||
visible,
|
||
decode_candidates,
|
||
candidate_block_size,
|
||
)
|
||
|
||
topk_indices = topk_indices_buffer[:num_padded_tokens, :topk_tokens]
|
||
num_rows = logits.shape[0]
|
||
|
||
# FULL graphs are not keyed by context length, so use a replay-safe
|
||
# upper bound when choosing the captured kernel.
|
||
if forward_context.cudagraph_runtime_mode == CUDAGraphMode.FULL:
|
||
max_compressed_seq_len = max_model_len
|
||
else:
|
||
max_compressed_seq_len = layer_attn_metadata.max_seq_len // compress_ratio
|
||
if rocm_aiter_ops.is_indexer_top_k_supported(
|
||
is_prefill=False,
|
||
compress_ratio=compress_ratio,
|
||
num_rows=num_rows,
|
||
max_valid_seq_len=max_compressed_seq_len,
|
||
) and not rocm_aiter_ops.dsv4_indexer_prefers_native_top_k(
|
||
num_rows=num_rows,
|
||
num_columns=logits.shape[1],
|
||
topk_tokens=topk_tokens,
|
||
):
|
||
rocm_aiter_ops.indexer_top_k_decode(
|
||
logits,
|
||
1,
|
||
decode_metadata.seq_lens.reshape(-1),
|
||
topk_indices,
|
||
topk_tokens,
|
||
)
|
||
else:
|
||
torch.ops._C.top_k_per_row_decode(
|
||
logits,
|
||
next_n,
|
||
decode_metadata.seq_lens,
|
||
topk_indices,
|
||
num_rows,
|
||
logits.stride(0),
|
||
logits.stride(1),
|
||
topk_tokens,
|
||
)
|
||
|
||
if decode_metadata.requires_padding:
|
||
# if padded, we need to unpack
|
||
# the topk indices removing padded tokens
|
||
topk_indices = unpack_seq_triton(
|
||
topk_indices.reshape(batch_size, next_n, topk_indices.shape[-1]),
|
||
decode_lens,
|
||
)
|
||
topk_indices_buffer[:num_decode_tokens, : topk_indices.shape[-1]] = (
|
||
topk_indices
|
||
)
|
||
|
||
return topk_indices_buffer
|
||
|
||
|
||
def _decode_e8m0_scales(scale: torch.Tensor) -> torch.Tensor:
|
||
if scale.dtype == torch.float8_e8m0fnu:
|
||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||
_upcast_e8m0_to_fp32,
|
||
)
|
||
|
||
return _upcast_e8m0_to_fp32(scale).contiguous()
|
||
if scale.dtype == torch.uint8:
|
||
# MXFP8 parameters preserve E8M0 scales as their raw exponent bytes.
|
||
# They are biased exponents, not numeric uint8 scale values.
|
||
return torch.exp2(scale.to(torch.int16).to(torch.float32) - 127.0)
|
||
return scale.to(torch.float32)
|
||
|
||
|
||
def _expand_2d_block_scales(
|
||
scale: torch.Tensor,
|
||
rows: int,
|
||
cols: int,
|
||
) -> torch.Tensor:
|
||
scale = _decode_e8m0_scales(scale)
|
||
row_blocks, col_blocks = scale.shape[-2:]
|
||
row_block = math.ceil(rows / row_blocks)
|
||
col_block = math.ceil(cols / col_blocks)
|
||
scale = torch.repeat_interleave(scale, row_block, dim=-2)[..., :rows, :]
|
||
scale = torch.repeat_interleave(scale, col_block, dim=-1)[..., :, :cols]
|
||
return scale
|
||
|
||
|
||
@triton.jit
|
||
def _inverse_rope_gptj_kernel(
|
||
o_ptr, # [T, H, D] input
|
||
out_ptr, # [T, H, D] bf16 output
|
||
pos_ptr, # [T] positions
|
||
cos_sin_ptr, # [P, rope_dim] fp32 (cos[:half] | sin[half:])
|
||
s_t,
|
||
s_h, # input row strides (last dim contiguous)
|
||
os_t,
|
||
os_h, # output row strides
|
||
cs_stride, # cos_sin_cache row stride
|
||
NOPE: tl.constexpr, # non-rope head dims (passed through)
|
||
HALF: tl.constexpr, # rope_dim // 2
|
||
BLOCK_NOPE: tl.constexpr,
|
||
BLOCK_HALF: tl.constexpr,
|
||
):
|
||
"""Fused inverse GPT-J RoPE on the trailing rope_dim of each (token, head).
|
||
|
||
Mirrors ``DeepseekV4ScalingRotaryEmbedding.forward_native(inverse=True)``
|
||
for the GPT-J (non-neox) layout, writing bf16 directly. Replaces the
|
||
clone + index_select + repeat_interleave + neg + stack + cat + cast chain
|
||
(~10 small kernels) with a single launch.
|
||
"""
|
||
t = tl.program_id(0)
|
||
h = tl.program_id(1)
|
||
in_base = t * s_t + h * s_h
|
||
out_base = t * os_t + h * os_h
|
||
|
||
# NoPE lanes pass through unchanged (only cast to bf16).
|
||
n = tl.arange(0, BLOCK_NOPE)
|
||
nmask = n < NOPE
|
||
vals = tl.load(o_ptr + in_base + n, mask=nmask)
|
||
tl.store(out_ptr + out_base + n, vals.to(tl.bfloat16), mask=nmask)
|
||
|
||
# RoPE lanes: out_even = a*cos + b*sin, out_odd = b*cos - a*sin
|
||
# (a = even lane, b = odd lane; sin negated for the inverse rotation).
|
||
pos = tl.load(pos_ptr + t).to(tl.int64)
|
||
k = tl.arange(0, BLOCK_HALF)
|
||
kmask = k < HALF
|
||
a = tl.load(o_ptr + in_base + NOPE + 2 * k, mask=kmask).to(tl.float32)
|
||
b = tl.load(o_ptr + in_base + NOPE + 2 * k + 1, mask=kmask).to(tl.float32)
|
||
cos = tl.load(cos_sin_ptr + pos * cs_stride + k, mask=kmask)
|
||
sin = tl.load(cos_sin_ptr + pos * cs_stride + HALF + k, mask=kmask)
|
||
out_even = a * cos + b * sin
|
||
out_odd = b * cos - a * sin
|
||
tl.store(out_ptr + out_base + NOPE + 2 * k, out_even.to(tl.bfloat16), mask=kmask)
|
||
tl.store(out_ptr + out_base + NOPE + 2 * k + 1, out_odd.to(tl.bfloat16), mask=kmask)
|
||
|
||
|
||
def _fused_inverse_rope_gptj(
|
||
o: torch.Tensor,
|
||
positions: torch.Tensor,
|
||
cos_sin_cache: torch.Tensor,
|
||
rope_head_dim: int,
|
||
out: torch.Tensor | None = None,
|
||
) -> torch.Tensor:
|
||
"""bf16 inverse GPT-J RoPE via a single fused Triton kernel.
|
||
|
||
``out`` may alias ``o``: the rotation is a per-row bijection whose kernel
|
||
reads both lanes of a pair before storing either.
|
||
"""
|
||
assert o.dim() == 3 and o.stride(-1) == 1, (
|
||
"_fused_inverse_rope_gptj expects a [T, H, D] input with a contiguous last dim"
|
||
)
|
||
assert rope_head_dim > 0 and rope_head_dim % 2 == 0, (
|
||
f"_fused_inverse_rope_gptj expects an even rope_head_dim, got {rope_head_dim}"
|
||
)
|
||
assert cos_sin_cache.shape[-1] == rope_head_dim, (
|
||
"_fused_inverse_rope_gptj expects cos_sin_cache laid out as "
|
||
f"[P, {rope_head_dim}] = cos | sin, got {tuple(cos_sin_cache.shape)}"
|
||
)
|
||
num_tokens, num_heads, head_dim = o.shape
|
||
if out is None:
|
||
out = torch.empty(
|
||
(num_tokens, num_heads, head_dim), dtype=torch.bfloat16, device=o.device
|
||
)
|
||
else:
|
||
assert out.dtype == torch.bfloat16, (
|
||
f"inverse RoPE writes bf16, got an output buffer of {out.dtype}"
|
||
)
|
||
if num_tokens == 0:
|
||
return out
|
||
_inverse_rope_gptj_kernel[(num_tokens, num_heads)](
|
||
o,
|
||
out,
|
||
positions,
|
||
cos_sin_cache,
|
||
o.stride(0),
|
||
o.stride(1),
|
||
out.stride(0),
|
||
out.stride(1),
|
||
cos_sin_cache.stride(0),
|
||
NOPE=head_dim - rope_head_dim,
|
||
HALF=rope_head_dim // 2,
|
||
BLOCK_NOPE=triton.next_power_of_2(head_dim - rope_head_dim),
|
||
BLOCK_HALF=triton.next_power_of_2(rope_head_dim // 2),
|
||
)
|
||
return out
|
||
|
||
|
||
def rocm_inverse_rope_rows_(
|
||
o: torch.Tensor,
|
||
positions: torch.Tensor,
|
||
cos_sin_cache: torch.Tensor,
|
||
rope_head_dim: int,
|
||
) -> None:
|
||
"""Inverse-RoPE attention output rows in place.
|
||
|
||
For rows no attention kernel rotated in its epilogue. Call it from the
|
||
eager attention segment: which rows still owe a rotation depends on the
|
||
prefill/decode split, and the o_proj that used to do this runs inside the
|
||
compiled region, where a batch-dependent Python value would be frozen at
|
||
trace time.
|
||
"""
|
||
if o.shape[0] == 0:
|
||
return
|
||
_fused_inverse_rope_gptj(o, positions, cos_sin_cache, rope_head_dim, out=o)
|
||
|
||
|
||
@triton.jit
|
||
def _inverse_rope_mxfp8_quant_kernel(
|
||
o_ptr, # [T, H, D] bf16
|
||
q_ptr, # [T, H * D] e4m3
|
||
s_ptr, # [T, H * D // 32] uint8 (E8M0)
|
||
pos_ptr,
|
||
cos_sin_ptr,
|
||
s_t,
|
||
s_h,
|
||
qs_t,
|
||
ss_t,
|
||
cs_stride,
|
||
HEAD_DIM: tl.constexpr,
|
||
NOPE: tl.constexpr,
|
||
HALF: tl.constexpr,
|
||
):
|
||
t = tl.program_id(0)
|
||
h = tl.program_id(1)
|
||
lanes = tl.arange(0, HEAD_DIM)
|
||
x = tl.load(o_ptr + t * s_t + h * s_h + lanes).to(tl.float32)
|
||
# Same lane-pair formulation as the reduce epilogue: NoPE pairs take
|
||
# cos=1 / sin=0, so one expression rotates the whole row.
|
||
pos = tl.load(pos_ptr + t).to(tl.int64)
|
||
pair_idx = tl.arange(0, HEAD_DIM // 2) - (NOPE // 2)
|
||
is_rope = pair_idx >= 0
|
||
k = tl.where(is_rope, pair_idx, 0)
|
||
cos = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + k), 1.0)
|
||
sin = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + HALF + k), 0.0)
|
||
even, odd = tl.split(tl.reshape(x, (HEAD_DIM // 2, 2)))
|
||
x = tl.reshape(
|
||
tl.join(even * cos + odd * sin, odd * cos - even * sin), (1, HEAD_DIM)
|
||
)
|
||
xq, bits = _mxfp8_quantize_rows(x, 1, HEAD_DIM)
|
||
tl.store(
|
||
q_ptr + t * qs_t + h * HEAD_DIM + lanes,
|
||
tl.reshape(xq, (HEAD_DIM,)).to(q_ptr.dtype.element_ty),
|
||
)
|
||
scale_offsets = tl.arange(0, HEAD_DIM // 32)
|
||
tl.store(
|
||
s_ptr + t * ss_t + h * (HEAD_DIM // 32) + scale_offsets,
|
||
tl.reshape(bits, (HEAD_DIM // 32,)).to(tl.uint8),
|
||
)
|
||
|
||
|
||
def rocm_inverse_rope_mxfp8_rows(
|
||
o: torch.Tensor,
|
||
positions: torch.Tensor,
|
||
cos_sin_cache: torch.Tensor,
|
||
rope_head_dim: int,
|
||
out_data: torch.Tensor,
|
||
out_scale: torch.Tensor,
|
||
) -> None:
|
||
"""Inverse-RoPE bf16 attention rows and MXFP8-quantize them for wo_a.
|
||
|
||
The counterpart of ``rocm_inverse_rope_rows_`` for layers whose attention
|
||
output is MXFP8: rows the decode reduce did not emit (prefill) go through
|
||
here. ``o`` is [T, H, D]; ``out_data`` [T, H * D] e4m3 and ``out_scale``
|
||
[T, H * D // 32] E8M0, the layout the reduce epilogue writes.
|
||
"""
|
||
num_tokens, num_heads, head_dim = o.shape
|
||
if num_tokens == 0:
|
||
return
|
||
assert o.stride(-1) == 1 and out_data.stride(-1) == 1 and out_scale.stride(-1) == 1
|
||
assert out_data.shape == (num_tokens, num_heads * head_dim)
|
||
assert out_scale.shape == (num_tokens, num_heads * head_dim // 32)
|
||
assert cos_sin_cache.shape[-1] == rope_head_dim
|
||
_inverse_rope_mxfp8_quant_kernel[(num_tokens, num_heads)](
|
||
o,
|
||
out_data,
|
||
out_scale,
|
||
positions,
|
||
cos_sin_cache,
|
||
o.stride(0),
|
||
o.stride(1),
|
||
out_data.stride(0),
|
||
out_scale.stride(0),
|
||
cos_sin_cache.stride(0),
|
||
HEAD_DIM=head_dim,
|
||
NOPE=head_dim - rope_head_dim,
|
||
HALF=rope_head_dim // 2,
|
||
num_warps=4,
|
||
)
|
||
|
||
|
||
def _get_cached_wo_a_bf16(
|
||
wo_a: torch.nn.Module,
|
||
n_local_groups: int,
|
||
o_lora_rank: int,
|
||
hidden_dim: int,
|
||
) -> torch.Tensor:
|
||
"""Dequantize wo_a to bf16 once and cache it on the module.
|
||
|
||
wo_a weights are static, so the fp8 -> fp32 -> (* block scale) -> bf16
|
||
dequant only needs to run once. Recomputing it every decode step shows up
|
||
in the profile as the largest copy/mul kernels (``direct_copy float`` ~55us
|
||
and ``MulFunctor float`` ~31us per two layers). SGLang / ATOM keep wo_a in
|
||
bf16 and feed a plain bf16 GEMM; this mirrors that.
|
||
"""
|
||
cached = getattr(wo_a, "_dsv4_wo_a_bf16", None)
|
||
if cached is not None:
|
||
return cached
|
||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||
get_fp8_block_weight_scale,
|
||
)
|
||
|
||
wo_a_scale_param = get_fp8_block_weight_scale(wo_a)
|
||
if wo_a_scale_param is None:
|
||
# ModelOpt MXFP8 stores the multiplicative E8M0 scale without the
|
||
# historical ``_inv`` suffix.
|
||
wo_a_scale_param = getattr(wo_a, "weight_scale", None)
|
||
# Emulated MXFP8 kernels can replace the original one-byte weight with an
|
||
# already-dequantized BF16 tensor while retaining the scale attribute for
|
||
# metadata. Applying that retained scale again would double-dequantize the
|
||
# weight. Block scaling is only valid while the one-byte FP8 storage remains.
|
||
if wo_a_scale_param is not None and wo_a.weight.element_size() == 1:
|
||
wo_a_weight = wo_a.weight.view(n_local_groups, o_lora_rank, hidden_dim).to(
|
||
torch.float32
|
||
)
|
||
wo_a_scale = _expand_2d_block_scales(
|
||
wo_a_scale_param.view(n_local_groups, -1, wo_a_scale_param.shape[-1]),
|
||
o_lora_rank,
|
||
hidden_dim,
|
||
)
|
||
cached = (wo_a_weight * wo_a_scale).to(torch.bfloat16)
|
||
else:
|
||
cached = wo_a.weight.view(n_local_groups, o_lora_rank, hidden_dim).to(
|
||
torch.bfloat16
|
||
)
|
||
wo_a._dsv4_wo_a_bf16 = cached
|
||
return cached
|
||
|
||
|
||
def rocm_inv_rope_einsum(
|
||
rotary_emb: torch.nn.Module,
|
||
o: torch.Tensor,
|
||
positions: torch.Tensor,
|
||
rope_head_dim: int,
|
||
n_local_groups: int,
|
||
o_lora_rank: int,
|
||
wo_a: torch.nn.Module,
|
||
inverse_rope: bool = True,
|
||
) -> torch.Tensor:
|
||
"""Inverse-RoPE + WO_A bmm path used on ROCm.
|
||
|
||
Fuses the inverse GPT-J RoPE into one Triton kernel and caches the bf16
|
||
wo_a weight so the per-step dequant disappears. Callers whose attention
|
||
already rotated every row pass ``inverse_rope=False``; that is a property
|
||
of the attention backend, not of the batch, so it stays constant across
|
||
steps and is safe to read from compiled code.
|
||
"""
|
||
if inverse_rope:
|
||
o_ref = _fused_inverse_rope_gptj(
|
||
o, positions, rotary_emb.cos_sin_cache, rope_head_dim
|
||
)
|
||
else:
|
||
assert o.dtype == torch.bfloat16, (
|
||
"a pre-rotated attention output feeds the wo_a bmm directly, so it "
|
||
f"must already be bf16, got {o.dtype}"
|
||
)
|
||
o_ref = o
|
||
o_ref = o_ref.reshape(o.shape[0], n_local_groups, -1)
|
||
|
||
wo_a_weight = _get_cached_wo_a_bf16(
|
||
wo_a, n_local_groups, o_lora_rank, o_ref.shape[-1]
|
||
)
|
||
|
||
return torch.einsum("tgd,grd->tgr", o_ref, wo_a_weight)
|
||
|
||
|
||
@triton.jit
|
||
def _mxfp8_wo_a_bmm_kernel(
|
||
a_ptr, # [T, G * K] e4m3
|
||
as_ptr, # [T, G * K // 32] E8M0
|
||
w_ptr, # [G * R, K] e4m3
|
||
ws_ptr, # [G * R // SCALE_ROWS, K // 32] E8M0
|
||
out_ptr, # [T, G * R]
|
||
num_tokens,
|
||
stride_at,
|
||
stride_ast,
|
||
stride_wn,
|
||
stride_wsn,
|
||
stride_out,
|
||
R: tl.constexpr,
|
||
K: tl.constexpr,
|
||
SCALE_ROWS: tl.constexpr,
|
||
BLOCK_M: tl.constexpr,
|
||
BLOCK_N: tl.constexpr,
|
||
BLOCK_K: tl.constexpr,
|
||
):
|
||
# Output columns are group-major, so a BLOCK_N tile never straddles two
|
||
# groups (R % BLOCK_N == 0) and picks its group's K slice of A.
|
||
pid_n = tl.program_id(0)
|
||
pid_m = tl.program_id(1)
|
||
group = (pid_n * BLOCK_N) // R
|
||
offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
|
||
offs_m = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
|
||
m_mask = offs_m < num_tokens
|
||
offs_k = tl.arange(0, BLOCK_K)
|
||
offs_sk = tl.arange(0, BLOCK_K // 32)
|
||
|
||
a_ptrs = a_ptr + offs_m[:, None] * stride_at + (group * K + offs_k)[None, :]
|
||
as_ptrs = (
|
||
as_ptr + offs_m[:, None] * stride_ast + (group * (K // 32) + offs_sk)[None, :]
|
||
)
|
||
w_ptrs = w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :]
|
||
ws_ptrs = ws_ptr + (offs_n // SCALE_ROWS)[:, None] * stride_wsn + offs_sk[None, :]
|
||
|
||
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||
for _ in range(K // BLOCK_K):
|
||
a = tl.load(a_ptrs, mask=m_mask[:, None], other=0.0)
|
||
a_s = tl.load(as_ptrs, mask=m_mask[:, None], other=127)
|
||
w = tl.load(w_ptrs)
|
||
w_s = tl.load(ws_ptrs)
|
||
acc += tl.dot_scaled(a, a_s, "e4m3", w.T, w_s, "e4m3")
|
||
a_ptrs += BLOCK_K
|
||
as_ptrs += BLOCK_K // 32
|
||
w_ptrs += BLOCK_K
|
||
ws_ptrs += BLOCK_K // 32
|
||
|
||
tl.store(
|
||
out_ptr + offs_m[:, None] * stride_out + offs_n[None, :],
|
||
acc.to(out_ptr.dtype.element_ty),
|
||
mask=m_mask[:, None],
|
||
)
|
||
|
||
|
||
def _mxfp8_wo_a_bmm_config(num_tokens: int, n_groups: int) -> tuple[int, ...]:
|
||
"""(BLOCK_M, BLOCK_N, BLOCK_K, num_warps, num_stages) for gfx950.
|
||
|
||
Tuned under HIP graphs with a cold weight at G = 4 and 2, over every
|
||
decode shape of conc 1-128 x 0-5 spec tokens plus prefill chunks up to
|
||
8K tokens. The best tile tracks the total work T * G, so the tiers are
|
||
keyed on it.
|
||
|
||
This will be replaced after new GEMM kernel from AITER with proper 32x32 scale
|
||
shape GEMM fp8 enabled.
|
||
"""
|
||
work = num_tokens * n_groups
|
||
if work <= 64:
|
||
return 16, 16, 1024, 2, 3
|
||
if work <= 128:
|
||
return 32, 16, 1024, 2, 3
|
||
if work <= 256:
|
||
return 32, 32, 512, 2, 3
|
||
if work <= 512:
|
||
return 64, 32, 512, 2, 3
|
||
if work <= 1024:
|
||
return 64, 64, 512, 4, 2
|
||
if work <= 2048:
|
||
return 64, 64, 256, 4, 2
|
||
if work <= 3072:
|
||
return 64, 64, 256, 2, 1
|
||
if work <= 4096:
|
||
return 128, 128, 256, 8, 2
|
||
return 128, 128, 128, 4, 2
|
||
|
||
|
||
def _rocm_mxfp8_wo_a_bmm_impl(
|
||
a: torch.Tensor,
|
||
a_scale: torch.Tensor,
|
||
weight: torch.Tensor,
|
||
weight_scale: torch.Tensor,
|
||
n_groups: int,
|
||
o_lora_rank: int,
|
||
) -> torch.Tensor:
|
||
num_tokens = a.shape[0]
|
||
group_dim = a.shape[1] // n_groups
|
||
out = torch.empty(
|
||
(num_tokens, n_groups * o_lora_rank), dtype=torch.bfloat16, device=a.device
|
||
)
|
||
if num_tokens == 0:
|
||
return out
|
||
block_m, block_n, block_k, num_warps, num_stages = _mxfp8_wo_a_bmm_config(
|
||
num_tokens, n_groups
|
||
)
|
||
grid = (n_groups * o_lora_rank // block_n, triton.cdiv(num_tokens, block_m))
|
||
_mxfp8_wo_a_bmm_kernel[grid](
|
||
a,
|
||
a_scale,
|
||
weight,
|
||
weight_scale,
|
||
out,
|
||
num_tokens,
|
||
a.stride(0),
|
||
a_scale.stride(0),
|
||
weight.stride(0),
|
||
weight_scale.stride(0),
|
||
out.stride(0),
|
||
R=o_lora_rank,
|
||
K=group_dim,
|
||
SCALE_ROWS=weight.shape[0] // weight_scale.shape[0],
|
||
BLOCK_M=block_m,
|
||
BLOCK_N=block_n,
|
||
BLOCK_K=block_k,
|
||
num_warps=num_warps,
|
||
num_stages=num_stages,
|
||
)
|
||
return out
|
||
|
||
|
||
def _rocm_mxfp8_wo_a_bmm_fake(
|
||
a: torch.Tensor,
|
||
a_scale: torch.Tensor,
|
||
weight: torch.Tensor,
|
||
weight_scale: torch.Tensor,
|
||
n_groups: int,
|
||
o_lora_rank: int,
|
||
) -> torch.Tensor:
|
||
return a.new_empty((a.shape[0], n_groups * o_lora_rank), dtype=torch.bfloat16)
|
||
|
||
|
||
# An opaque op: the tile choice branches on the token count, which the
|
||
# compiled o_proj must not freeze at its trace-time value.
|
||
direct_register_custom_op(
|
||
op_name="rocm_dsv41_mxfp8_wo_a_bmm",
|
||
op_func=_rocm_mxfp8_wo_a_bmm_impl,
|
||
fake_impl=_rocm_mxfp8_wo_a_bmm_fake,
|
||
)
|
||
|
||
|
||
def rocm_mxfp8_wo_a_bmm(
|
||
a: torch.Tensor,
|
||
a_scale: torch.Tensor,
|
||
wo_a: torch.nn.Module,
|
||
n_groups: int,
|
||
o_lora_rank: int,
|
||
) -> torch.Tensor:
|
||
"""Grouped MXFP8 wo_a: ``out[t, g, :] = a[t, g, :] @ W[g].T``, bf16 out.
|
||
|
||
``a`` is the [T, G * K] e4m3 attention output and ``a_scale`` its
|
||
[T, G * K // 32] E8M0 scales, as the sparse decode reduce writes them.
|
||
The weight is the checkpoint's MXFP8 ``wo_a`` as loaded, [G * R, K] with
|
||
either [G * R // 32, K // 32] block scales or [G * R, K // 32] per-row
|
||
scales, so there is no dequantized copy to keep.
|
||
Returns [T, G * R].
|
||
"""
|
||
return torch.ops.vllm.rocm_dsv41_mxfp8_wo_a_bmm(
|
||
a, a_scale, wo_a.weight, wo_a.weight_scale, n_groups, o_lora_rank
|
||
)
|
||
|
||
|
||
_DSV4_SPARSE_NOPE_DIM = 448
|
||
_DSV4_SPARSE_ROPE_DIM = 64
|
||
|
||
|
||
def _validate_sparse_dims(
|
||
head_dim: int,
|
||
nope_head_dim: int,
|
||
rope_head_dim: int,
|
||
op_name: str,
|
||
) -> None:
|
||
assert head_dim > 0, f"{op_name} expected a positive head_dim, got {head_dim}"
|
||
assert nope_head_dim > 0, (
|
||
f"{op_name} expected a positive NoPE dimension, got {nope_head_dim}"
|
||
)
|
||
assert rope_head_dim >= 0, (
|
||
f"{op_name} expected a non-negative RoPE dimension, got {rope_head_dim}"
|
||
)
|
||
assert head_dim == nope_head_dim + rope_head_dim, (
|
||
f"{op_name} expected head_dim={nope_head_dim + rope_head_dim}, got {head_dim}"
|
||
)
|
||
|
||
|
||
def _validate_dsv4_sparse_dims(
|
||
head_dim: int,
|
||
nope_head_dim: int,
|
||
rope_head_dim: int,
|
||
op_name: str,
|
||
) -> None:
|
||
_validate_sparse_dims(head_dim, nope_head_dim, rope_head_dim, op_name)
|
||
assert (
|
||
nope_head_dim == _DSV4_SPARSE_NOPE_DIM
|
||
and rope_head_dim == _DSV4_SPARSE_ROPE_DIM
|
||
), (
|
||
f"{op_name} expects {_DSV4_SPARSE_NOPE_DIM} NoPE dims and "
|
||
f"{_DSV4_SPARSE_ROPE_DIM} RoPE dims"
|
||
)
|
||
|
||
|
||
@triton.jit
|
||
def _count_valid_row_entries_kernel(
|
||
indices_ptr,
|
||
lengths_ptr,
|
||
counts_ptr,
|
||
indptr_ptr,
|
||
indices_stride0,
|
||
num_rows_limit,
|
||
num_queries,
|
||
row_width,
|
||
BLOCK_R: tl.constexpr,
|
||
BLOCK_W: tl.constexpr,
|
||
):
|
||
pid = tl.program_id(0)
|
||
# The scan fills indptr[1:], so write indptr[0] here and skip a zero fill.
|
||
if pid == 0:
|
||
tl.store(indptr_ptr, 0)
|
||
|
||
rows = pid * BLOCK_R + tl.arange(0, BLOCK_R)
|
||
row_ok = rows < num_queries
|
||
row_len = tl.load(lengths_ptr + rows, mask=row_ok, other=0)
|
||
|
||
acc = tl.zeros((BLOCK_R,), dtype=tl.int32)
|
||
for start in range(0, row_width, BLOCK_W):
|
||
cols = start + tl.arange(0, BLOCK_W)
|
||
in_row = cols[None, :] < row_width
|
||
vals = tl.load(
|
||
indices_ptr + rows[:, None] * indices_stride0 + cols[None, :],
|
||
mask=row_ok[:, None] & in_row,
|
||
other=-1,
|
||
).to(tl.int32)
|
||
keep = (
|
||
row_ok[:, None] & in_row & (cols[None, :] < row_len[:, None]) & (vals >= 0)
|
||
)
|
||
if num_rows_limit >= 0:
|
||
keep = keep & (vals < num_rows_limit)
|
||
acc += tl.sum(keep.to(tl.int32), axis=1)
|
||
|
||
tl.store(counts_ptr + rows, acc, mask=row_ok)
|
||
|
||
|
||
@triton.jit
|
||
def _compact_dense_row_to_ragged_kernel(
|
||
indices_ptr,
|
||
lengths_ptr,
|
||
indptr_ptr,
|
||
out_ptr,
|
||
indices_stride0,
|
||
num_rows_limit,
|
||
row_width,
|
||
BLOCK_W: tl.constexpr,
|
||
):
|
||
row_idx = tl.program_id(0)
|
||
row_len = tl.load(lengths_ptr + row_idx)
|
||
out_start = tl.load(indptr_ptr + row_idx)
|
||
|
||
written = 0
|
||
for start in range(0, row_width, BLOCK_W):
|
||
cols = start + tl.arange(0, BLOCK_W)
|
||
in_row = cols < row_width
|
||
vals = tl.load(
|
||
indices_ptr + row_idx * indices_stride0 + cols, mask=in_row, other=-1
|
||
).to(tl.int32)
|
||
keep = in_row & (cols < row_len) & (vals >= 0)
|
||
if num_rows_limit >= 0:
|
||
keep = keep & (vals < num_rows_limit)
|
||
rank = tl.cumsum(keep.to(tl.int32), axis=0) - 1
|
||
tl.store(out_ptr + out_start + written + rank, vals, mask=keep)
|
||
written += tl.sum(keep.to(tl.int32), axis=0)
|
||
|
||
|
||
def build_ragged_indices_from_dense(
|
||
indices: torch.Tensor,
|
||
lengths: torch.Tensor,
|
||
num_rows: int = -1,
|
||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||
indices = indices.reshape(indices.shape[0], -1)
|
||
lengths = lengths.to(device=indices.device, dtype=torch.int32).reshape(-1)
|
||
assert lengths.numel() == indices.shape[0], (
|
||
f"Expected one length per row, got {lengths.shape} for indices {indices.shape}"
|
||
)
|
||
|
||
max_width = indices.shape[1] if indices.ndim == 2 else 0
|
||
# Both kernels bound the row by row_width and row_len, so no clamp needed.
|
||
lengths = lengths.contiguous()
|
||
|
||
# Dense rows reserve slots they may not fill, leaving -1 anywhere in the
|
||
# first `lengths` entries. Keep only the valid ones, since the consumers
|
||
# index the KV pool without checking the sign.
|
||
num_queries = indices.shape[0]
|
||
# max_width is whatever the caller packed (align(topk + window, 128) for the
|
||
# V4.1 indexer) and nothing caps it, so loop the row rather than size the
|
||
# block by it.
|
||
block = min(triton.next_power_of_2(max_width), 1024) if max_width > 0 else 1
|
||
counts = torch.empty(num_queries, dtype=torch.int32, device=indices.device)
|
||
indptr = torch.empty(num_queries + 1, dtype=torch.int32, device=indices.device)
|
||
if num_queries > 0 and max_width > 0:
|
||
# A program per row only gets row_width elements, which is launch
|
||
# bound for narrow rows. Tile rows to fill the block.
|
||
block_r = max(1, 1024 // block)
|
||
_count_valid_row_entries_kernel[(triton.cdiv(num_queries, block_r),)](
|
||
indices,
|
||
lengths,
|
||
counts,
|
||
indptr,
|
||
indices.stride(0),
|
||
int(num_rows),
|
||
num_queries,
|
||
max_width,
|
||
BLOCK_R=block_r,
|
||
BLOCK_W=block,
|
||
)
|
||
else:
|
||
counts.zero_()
|
||
indptr.zero_()
|
||
|
||
torch.cumsum(counts, dim=0, out=indptr[1:])
|
||
|
||
if indices.numel() == 0:
|
||
flat = torch.empty(0, dtype=torch.int32, device=indices.device)
|
||
else:
|
||
flat = torch.empty(
|
||
indices.shape[0] * max_width,
|
||
dtype=torch.int32,
|
||
device=indices.device,
|
||
)
|
||
if flat.numel() > 0:
|
||
_compact_dense_row_to_ragged_kernel[(indices.shape[0],)](
|
||
indices,
|
||
lengths,
|
||
indptr,
|
||
flat,
|
||
indices.stride(0),
|
||
int(num_rows),
|
||
max_width,
|
||
BLOCK_W=block,
|
||
)
|
||
|
||
return flat, indptr
|
||
|
||
|
||
def _as_int32_contiguous_1d(x: torch.Tensor) -> torch.Tensor:
|
||
if x.dtype == torch.int32 and x.ndim == 1 and x.is_contiguous():
|
||
return x
|
||
return x.to(torch.int32).contiguous()
|
||
|
||
|
||
@triton.jit
|
||
def _sparse_kv_row_offset(slot, stride):
|
||
# A global token slot fits in int32, but its byte/element offset may not.
|
||
return slot.to(tl.int64) * stride
|
||
|
||
|
||
@triton.jit
|
||
def _sparse_attn_prefill_ragged_kernel(
|
||
q_ptr,
|
||
kv_ptr,
|
||
kv_indices_ptr,
|
||
kv_indptr_ptr,
|
||
attn_sink_ptr,
|
||
out_ptr,
|
||
q_stride_t,
|
||
q_stride_h,
|
||
q_stride_d,
|
||
kv_stride_n,
|
||
kv_stride_d,
|
||
out_stride_t,
|
||
out_stride_h,
|
||
out_stride_d,
|
||
num_heads,
|
||
head_dim,
|
||
num_kv,
|
||
scale,
|
||
HAS_ATTN_SINK: tl.constexpr,
|
||
OUT_DV: tl.constexpr,
|
||
BLOCK_H: tl.constexpr,
|
||
BLOCK_D: tl.constexpr,
|
||
BLOCK_K: tl.constexpr,
|
||
):
|
||
query_idx = tl.program_id(0)
|
||
pid_h = tl.program_id(1)
|
||
|
||
head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
|
||
dim_offsets = tl.arange(0, BLOCK_D)
|
||
head_mask = head_offsets < num_heads
|
||
dim_mask = dim_offsets < head_dim
|
||
|
||
q = tl.load(
|
||
q_ptr
|
||
+ query_idx * q_stride_t
|
||
+ head_offsets[:, None] * q_stride_h
|
||
+ dim_offsets[None, :] * q_stride_d,
|
||
mask=head_mask[:, None] & dim_mask[None, :],
|
||
other=0.0,
|
||
)
|
||
|
||
neg_large = -3.4028234663852886e38
|
||
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
|
||
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||
acc = tl.zeros((BLOCK_H, BLOCK_D), dtype=tl.float32)
|
||
|
||
kv_start = tl.load(kv_indptr_ptr + query_idx)
|
||
kv_end = tl.load(kv_indptr_ptr + query_idx + 1)
|
||
kv_len = kv_end - kv_start
|
||
|
||
k_offsets = tl.arange(0, BLOCK_K)
|
||
slot = tl.load(
|
||
kv_indices_ptr + kv_start + k_offsets, mask=k_offsets < kv_len, other=-1
|
||
)
|
||
for k_start in tl.range(0, kv_len, BLOCK_K):
|
||
k_pos = k_start + k_offsets
|
||
in_range = k_pos < kv_len
|
||
valid = in_range & (slot >= 0) & (slot < num_kv)
|
||
safe_slot = tl.where(valid, slot, 0)
|
||
|
||
kv = tl.load(
|
||
kv_ptr
|
||
+ _sparse_kv_row_offset(safe_slot[:, None], kv_stride_n)
|
||
+ dim_offsets[None, :] * kv_stride_d,
|
||
mask=valid[:, None] & dim_mask[None, :],
|
||
other=0.0,
|
||
)
|
||
|
||
next_k_pos = k_start + BLOCK_K + k_offsets
|
||
slot = tl.load(
|
||
kv_indices_ptr + kv_start + next_k_pos, mask=next_k_pos < kv_len, other=-1
|
||
)
|
||
|
||
scores = tl.dot(q, tl.trans(kv)) * scale
|
||
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
|
||
|
||
m_block = tl.max(scores, axis=1)
|
||
m_new = tl.maximum(m_i, m_block)
|
||
alpha = tl.exp(m_i - m_new)
|
||
p = tl.exp(scores - m_new[:, None])
|
||
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
|
||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||
|
||
acc = acc * alpha[:, None] + tl.dot(p.to(kv.dtype), kv)
|
||
m_i = m_new
|
||
l_i = l_new
|
||
|
||
if HAS_ATTN_SINK:
|
||
sink = tl.load(
|
||
attn_sink_ptr + head_offsets, mask=head_mask, other=neg_large
|
||
).to(tl.float32)
|
||
m_final = tl.maximum(m_i, sink)
|
||
alpha = tl.exp(m_i - m_final)
|
||
l_final = l_i * alpha + tl.exp(sink - m_final)
|
||
denom = tl.maximum(l_final, 1.0e-30)
|
||
out = tl.where(
|
||
l_final[:, None] > 0.0,
|
||
(acc * alpha[:, None]) / denom[:, None],
|
||
0.0,
|
||
)
|
||
else:
|
||
denom = tl.maximum(l_i, 1.0e-30)
|
||
out = tl.where(l_i[:, None] > 0.0, acc / denom[:, None], 0.0)
|
||
|
||
tl.store(
|
||
out_ptr
|
||
+ query_idx * out_stride_t
|
||
+ head_offsets[:, None] * out_stride_h
|
||
+ dim_offsets[None, :] * out_stride_d,
|
||
out,
|
||
mask=head_mask[:, None] & (dim_offsets[None, :] < OUT_DV),
|
||
)
|
||
|
||
|
||
@triton.jit
|
||
def _sparse_attn_decode_ragged_bf16_partial_kernel(
|
||
q_ptr,
|
||
kv_ptr,
|
||
kv_indices_ptr,
|
||
kv_indptr_ptr,
|
||
part_m_ptr,
|
||
part_l_ptr,
|
||
part_acc_ptr,
|
||
q_stride_t,
|
||
q_stride_h,
|
||
q_stride_d,
|
||
kv_stride_n,
|
||
kv_stride_d,
|
||
pm_stride0,
|
||
pm_stride_s,
|
||
pa_stride0,
|
||
pa_stride_s,
|
||
pa_stride_h,
|
||
num_heads,
|
||
head_dim,
|
||
num_kv,
|
||
scale,
|
||
BLOCK_H: tl.constexpr,
|
||
BLOCK_D: tl.constexpr,
|
||
BLOCK_K: tl.constexpr,
|
||
NUM_SPLITS: tl.constexpr,
|
||
):
|
||
query_idx = tl.program_id(0)
|
||
split_id = tl.program_id(1)
|
||
pid_h = tl.program_id(2)
|
||
|
||
head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
|
||
dim_offsets = tl.arange(0, BLOCK_D)
|
||
head_mask = head_offsets < num_heads
|
||
dim_mask = dim_offsets < head_dim
|
||
|
||
q = tl.load(
|
||
q_ptr
|
||
+ query_idx * q_stride_t
|
||
+ head_offsets[:, None] * q_stride_h
|
||
+ dim_offsets[None, :] * q_stride_d,
|
||
mask=head_mask[:, None] & dim_mask[None, :],
|
||
other=0.0,
|
||
)
|
||
|
||
neg_large = -3.4028234663852886e38
|
||
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
|
||
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||
acc = tl.zeros((BLOCK_H, BLOCK_D), dtype=tl.float32)
|
||
|
||
kv_start = tl.load(kv_indptr_ptr + query_idx)
|
||
kv_end = tl.load(kv_indptr_ptr + query_idx + 1)
|
||
kv_len = kv_end - kv_start
|
||
chunk = (kv_len + NUM_SPLITS - 1) // NUM_SPLITS
|
||
kv_lo = split_id * chunk
|
||
kv_hi = tl.minimum(kv_lo + chunk, kv_len)
|
||
|
||
k_offsets = tl.arange(0, BLOCK_K)
|
||
slot = tl.load(
|
||
kv_indices_ptr + kv_start + kv_lo + k_offsets,
|
||
mask=kv_lo + k_offsets < kv_hi,
|
||
other=-1,
|
||
)
|
||
for k_start in tl.range(kv_lo, kv_hi, BLOCK_K):
|
||
k_pos = k_start + k_offsets
|
||
in_range = k_pos < kv_hi
|
||
valid = in_range & (slot >= 0) & (slot < num_kv)
|
||
safe_slot = tl.where(valid, slot, 0)
|
||
|
||
kv = tl.load(
|
||
kv_ptr
|
||
+ _sparse_kv_row_offset(safe_slot[:, None], kv_stride_n)
|
||
+ dim_offsets[None, :] * kv_stride_d,
|
||
mask=valid[:, None] & dim_mask[None, :],
|
||
other=0.0,
|
||
)
|
||
|
||
next_k_pos = k_start + BLOCK_K + k_offsets
|
||
slot = tl.load(
|
||
kv_indices_ptr + kv_start + next_k_pos, mask=next_k_pos < kv_hi, other=-1
|
||
)
|
||
|
||
scores = tl.dot(q, tl.trans(kv)) * scale
|
||
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
|
||
|
||
m_block = tl.max(scores, axis=1)
|
||
m_new = tl.maximum(m_i, m_block)
|
||
alpha = tl.exp(m_i - m_new)
|
||
p = tl.exp(scores - m_new[:, None])
|
||
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
|
||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||
|
||
acc = acc * alpha[:, None] + tl.dot(p.to(kv.dtype), kv)
|
||
m_i = m_new
|
||
l_i = l_new
|
||
|
||
# Every launched slot is written; empty splits store the (neg_large, 0.0)
|
||
# sentinel the reduce combines to nothing.
|
||
pm_base = query_idx * pm_stride0 + split_id * pm_stride_s + head_offsets
|
||
tl.store(part_m_ptr + pm_base, m_i, mask=head_mask)
|
||
tl.store(part_l_ptr + pm_base, l_i, mask=head_mask)
|
||
tl.store(
|
||
part_acc_ptr
|
||
+ query_idx * pa_stride0
|
||
+ split_id * pa_stride_s
|
||
+ head_offsets[:, None] * pa_stride_h
|
||
+ dim_offsets[None, :],
|
||
acc,
|
||
mask=head_mask[:, None] & dim_mask[None, :],
|
||
)
|
||
|
||
|
||
@triton.jit
|
||
def _decode_e8m0_scales_triton(encoded_scales):
|
||
scale_bits = encoded_scales.to(tl.int32) << 23
|
||
scale_bits = tl.where(encoded_scales == 0, 1 << 22, scale_bits)
|
||
return scale_bits.to(tl.float32, bitcast=True)
|
||
|
||
|
||
@triton.jit
|
||
def _load_fp8_ds_mla_gfx950_nope_exact_chunk(
|
||
token_data_ptr,
|
||
token_scale_ptr,
|
||
valid,
|
||
CHUNK_START: tl.constexpr,
|
||
CHUNK_SIZE: tl.constexpr,
|
||
BLOCK_K: tl.constexpr,
|
||
IS_FNUZ: tl.constexpr,
|
||
):
|
||
offsets = CHUNK_START + tl.arange(0, CHUNK_SIZE)
|
||
x_uint8 = tl.load(
|
||
token_data_ptr[:, None] + offsets[None, :],
|
||
mask=valid[:, None],
|
||
other=0,
|
||
)
|
||
scale_offsets = CHUNK_START // 64 + tl.arange(0, CHUNK_SIZE // 64)
|
||
encoded_scales = tl.load(
|
||
token_scale_ptr[:, None] + scale_offsets[None, :],
|
||
mask=valid[:, None],
|
||
other=127,
|
||
)
|
||
scales = _decode_e8m0_scales_triton(encoded_scales)
|
||
scales = tl.broadcast_to(scales[:, :, None], (BLOCK_K, CHUNK_SIZE // 64, 64))
|
||
scales = tl.reshape(scales, (BLOCK_K, CHUNK_SIZE))
|
||
if IS_FNUZ:
|
||
x_f32 = x_uint8.to(tl.float8e4b8, bitcast=True).to(tl.bfloat16).to(tl.float32)
|
||
else:
|
||
x_f32 = x_uint8.to(tl.float8e4nv, bitcast=True).to(tl.float32)
|
||
value = (x_f32 * scales).to(tl.bfloat16)
|
||
zero = tl.zeros((BLOCK_K, CHUNK_SIZE), dtype=tl.bfloat16)
|
||
return tl.where(valid[:, None], value, zero)
|
||
|
||
|
||
@triton.jit
|
||
def _load_fp8_ds_mla_gfx950_tail128(
|
||
token_data_ptr,
|
||
token_scale_ptr,
|
||
valid,
|
||
NOPE_DIM: tl.constexpr,
|
||
BLOCK_K: tl.constexpr,
|
||
IS_FNUZ: tl.constexpr,
|
||
):
|
||
tail_offsets = 384 + tl.arange(0, 128)
|
||
nope_mask = tail_offsets < NOPE_DIM
|
||
x_uint8 = tl.load(
|
||
token_data_ptr[:, None] + tail_offsets[None, :],
|
||
mask=valid[:, None] & nope_mask[None, :],
|
||
other=0,
|
||
)
|
||
scale_offsets = 6 + tl.arange(0, 2)
|
||
scale_mask = scale_offsets < NOPE_DIM // 64
|
||
encoded_scales = tl.load(
|
||
token_scale_ptr[:, None] + scale_offsets[None, :],
|
||
mask=valid[:, None] & scale_mask[None, :],
|
||
other=127,
|
||
)
|
||
scales = _decode_e8m0_scales_triton(encoded_scales)
|
||
scales = tl.broadcast_to(scales[:, :, None], (BLOCK_K, 2, 64))
|
||
scales = tl.reshape(scales, (BLOCK_K, 128))
|
||
if IS_FNUZ:
|
||
x_f32 = x_uint8.to(tl.float8e4b8, bitcast=True).to(tl.bfloat16).to(tl.float32)
|
||
else:
|
||
x_f32 = x_uint8.to(tl.float8e4nv, bitcast=True).to(tl.float32)
|
||
nope = (x_f32 * scales).to(tl.bfloat16)
|
||
|
||
rope_ptr = (token_data_ptr + NOPE_DIM).to(tl.pointer_type(tl.bfloat16))
|
||
rope = tl.load(
|
||
rope_ptr[:, None] + (tail_offsets[None, :] - NOPE_DIM),
|
||
mask=valid[:, None] & ~nope_mask[None, :],
|
||
other=0.0,
|
||
)
|
||
value = tl.where(nope_mask[None, :], nope, rope)
|
||
zero = tl.zeros((BLOCK_K, 128), dtype=tl.bfloat16)
|
||
return tl.where(valid[:, None], value, zero)
|
||
|
||
|
||
@triton.jit
|
||
def _sparse_attn_decode_ragged_kernel(
|
||
q_ptr,
|
||
main_cache_ptr,
|
||
main_indices_ptr,
|
||
main_indptr_ptr,
|
||
extra_cache_ptr,
|
||
extra_indices_ptr,
|
||
extra_indptr_ptr,
|
||
attn_sink_ptr,
|
||
out_ptr,
|
||
q_stride0,
|
||
q_stride1,
|
||
out_stride0,
|
||
out_stride1,
|
||
main_cache_stride0,
|
||
extra_cache_stride0,
|
||
main_num_rows,
|
||
extra_num_rows,
|
||
main_block_size,
|
||
extra_block_size,
|
||
scale,
|
||
num_heads,
|
||
HAS_ATTN_SINK: tl.constexpr,
|
||
HAS_EXTRA: tl.constexpr,
|
||
NOPE_DIM: tl.constexpr,
|
||
NOPE_BLOCK: tl.constexpr,
|
||
ROPE_DIM: tl.constexpr,
|
||
# SWA K-cache (main): C++ encoder writes FNUZ on gfx942, OCP on gfx950.
|
||
# Compressed K-cache (extra): Triton encoder writes OCP everywhere.
|
||
IS_FNUZ_MAIN: tl.constexpr,
|
||
IS_FNUZ_EXTRA: tl.constexpr,
|
||
BLOCK_H: tl.constexpr,
|
||
BLOCK_K: tl.constexpr,
|
||
):
|
||
query_idx = tl.program_id(0)
|
||
pid_h = tl.program_id(1)
|
||
|
||
head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
|
||
head_mask = head_offsets < num_heads
|
||
nope_offsets = tl.arange(0, NOPE_BLOCK)
|
||
nope_mask = nope_offsets < NOPE_DIM
|
||
rope_offsets = tl.arange(0, ROPE_DIM)
|
||
|
||
q_row_ptr = q_ptr + query_idx * q_stride0 + head_offsets[:, None] * q_stride1
|
||
q_nope = tl.load(
|
||
q_row_ptr + nope_offsets[None, :],
|
||
mask=head_mask[:, None] & nope_mask[None, :],
|
||
other=0.0,
|
||
)
|
||
q_rope = tl.load(
|
||
q_row_ptr + NOPE_DIM + rope_offsets[None, :],
|
||
mask=head_mask[:, None],
|
||
other=0.0,
|
||
)
|
||
|
||
neg_large = -3.4028234663852886e38
|
||
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
|
||
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||
acc_nope = tl.zeros((BLOCK_H, NOPE_BLOCK), dtype=tl.float32)
|
||
acc_rope = tl.zeros((BLOCK_H, ROPE_DIM), dtype=tl.float32)
|
||
k_offsets = tl.arange(0, BLOCK_K)
|
||
|
||
main_start = tl.load(main_indptr_ptr + query_idx)
|
||
main_end = tl.load(main_indptr_ptr + query_idx + 1)
|
||
main_len = main_end - main_start
|
||
|
||
zero_nope = tl.zeros((BLOCK_K, NOPE_BLOCK), dtype=tl.bfloat16)
|
||
zero_rope = tl.zeros((BLOCK_K, ROPE_DIM), dtype=tl.bfloat16)
|
||
|
||
for k_start in tl.range(0, main_len, BLOCK_K):
|
||
k_pos = k_start + k_offsets
|
||
in_range = k_pos < main_len
|
||
slot = tl.load(main_indices_ptr + main_start + k_pos, mask=in_range, other=-1)
|
||
valid = in_range & (slot >= 0) & (slot < main_num_rows)
|
||
safe_slot = tl.where(valid, slot, 0)
|
||
|
||
block_idx = safe_slot // main_block_size
|
||
pos_in_block = safe_slot % main_block_size
|
||
cache_block_ptr = main_cache_ptr + block_idx.to(tl.int64) * main_cache_stride0
|
||
token_data_ptr = cache_block_ptr + pos_in_block * 576
|
||
token_scale_ptr = cache_block_ptr + main_block_size * 576 + pos_in_block * 8
|
||
|
||
x_uint8 = tl.load(
|
||
token_data_ptr[:, None] + nope_offsets[None, :],
|
||
mask=valid[:, None] & nope_mask[None, :],
|
||
other=0,
|
||
)
|
||
if IS_FNUZ_MAIN:
|
||
x_fp8 = x_uint8.to(tl.float8e4b8, bitcast=True)
|
||
else:
|
||
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
|
||
encoded_scales = tl.load(
|
||
token_scale_ptr[:, None] + nope_offsets[None, :] // 64,
|
||
mask=valid[:, None] & nope_mask[None, :],
|
||
other=127,
|
||
)
|
||
scales = tl.exp2(encoded_scales.to(tl.float32) - 127.0)
|
||
k_nope = x_fp8.to(tl.bfloat16) * scales.to(tl.bfloat16)
|
||
k_nope = tl.where(valid[:, None] & nope_mask[None, :], k_nope, zero_nope)
|
||
k_nope = tl.where(k_nope == k_nope, k_nope, zero_nope)
|
||
|
||
rope_ptr = (token_data_ptr + NOPE_DIM).to(tl.pointer_type(tl.bfloat16))
|
||
k_rope = tl.load(
|
||
rope_ptr[:, None] + rope_offsets[None, :],
|
||
mask=valid[:, None],
|
||
other=0.0,
|
||
)
|
||
k_rope = tl.where(valid[:, None], k_rope, zero_rope)
|
||
k_rope = tl.where(k_rope == k_rope, k_rope, zero_rope)
|
||
|
||
scores = tl.dot(q_nope, tl.trans(k_nope)) + tl.dot(q_rope, tl.trans(k_rope))
|
||
scores *= scale
|
||
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
|
||
|
||
m_block = tl.max(scores, axis=1)
|
||
m_new = tl.maximum(m_i, m_block)
|
||
alpha = tl.exp(m_i - m_new)
|
||
p = tl.exp(scores - m_new[:, None])
|
||
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
|
||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||
|
||
acc_nope = acc_nope * alpha[:, None] + tl.dot(p.to(k_nope.dtype), k_nope)
|
||
acc_rope = acc_rope * alpha[:, None] + tl.dot(p.to(k_rope.dtype), k_rope)
|
||
m_i = m_new
|
||
l_i = l_new
|
||
|
||
if HAS_EXTRA:
|
||
extra_start = tl.load(extra_indptr_ptr + query_idx)
|
||
extra_end = tl.load(extra_indptr_ptr + query_idx + 1)
|
||
extra_len = extra_end - extra_start
|
||
|
||
for k_start in tl.range(0, extra_len, BLOCK_K):
|
||
k_pos = k_start + k_offsets
|
||
in_range = k_pos < extra_len
|
||
slot = tl.load(
|
||
extra_indices_ptr + extra_start + k_pos, mask=in_range, other=-1
|
||
)
|
||
valid = in_range & (slot >= 0) & (slot < extra_num_rows)
|
||
safe_slot = tl.where(valid, slot, 0)
|
||
|
||
block_idx = safe_slot // extra_block_size
|
||
pos_in_block = safe_slot % extra_block_size
|
||
cache_block_ptr = (
|
||
extra_cache_ptr + block_idx.to(tl.int64) * extra_cache_stride0
|
||
)
|
||
token_data_ptr = cache_block_ptr + pos_in_block * 576
|
||
token_scale_ptr = (
|
||
cache_block_ptr + extra_block_size * 576 + pos_in_block * 8
|
||
)
|
||
|
||
x_uint8 = tl.load(
|
||
token_data_ptr[:, None] + nope_offsets[None, :],
|
||
mask=valid[:, None] & nope_mask[None, :],
|
||
other=0,
|
||
)
|
||
if IS_FNUZ_EXTRA:
|
||
x_fp8 = x_uint8.to(tl.float8e4b8, bitcast=True)
|
||
else:
|
||
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
|
||
encoded_scales = tl.load(
|
||
token_scale_ptr[:, None] + nope_offsets[None, :] // 64,
|
||
mask=valid[:, None] & nope_mask[None, :],
|
||
other=127,
|
||
)
|
||
scales = tl.exp2(encoded_scales.to(tl.float32) - 127.0)
|
||
k_nope = x_fp8.to(tl.bfloat16) * scales.to(tl.bfloat16)
|
||
k_nope = tl.where(valid[:, None] & nope_mask[None, :], k_nope, zero_nope)
|
||
k_nope = tl.where(k_nope == k_nope, k_nope, zero_nope)
|
||
|
||
rope_ptr = (token_data_ptr + NOPE_DIM).to(tl.pointer_type(tl.bfloat16))
|
||
k_rope = tl.load(
|
||
rope_ptr[:, None] + rope_offsets[None, :],
|
||
mask=valid[:, None],
|
||
other=0.0,
|
||
)
|
||
k_rope = tl.where(valid[:, None], k_rope, zero_rope)
|
||
k_rope = tl.where(k_rope == k_rope, k_rope, zero_rope)
|
||
|
||
scores = tl.dot(q_nope, tl.trans(k_nope)) + tl.dot(
|
||
q_rope,
|
||
tl.trans(k_rope),
|
||
)
|
||
scores *= scale
|
||
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
|
||
|
||
m_block = tl.max(scores, axis=1)
|
||
m_new = tl.maximum(m_i, m_block)
|
||
alpha = tl.exp(m_i - m_new)
|
||
p = tl.exp(scores - m_new[:, None])
|
||
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
|
||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||
|
||
acc_nope = acc_nope * alpha[:, None] + tl.dot(p.to(k_nope.dtype), k_nope)
|
||
acc_rope = acc_rope * alpha[:, None] + tl.dot(p.to(k_rope.dtype), k_rope)
|
||
m_i = m_new
|
||
l_i = l_new
|
||
|
||
if HAS_ATTN_SINK:
|
||
sink = tl.load(
|
||
attn_sink_ptr + head_offsets, mask=head_mask, other=neg_large
|
||
).to(tl.float32)
|
||
m_final = tl.maximum(m_i, sink)
|
||
alpha = tl.exp(m_i - m_final)
|
||
l_final = l_i * alpha + tl.exp(sink - m_final)
|
||
denom = tl.maximum(l_final, 1.0e-30)
|
||
out_nope = tl.where(
|
||
l_final[:, None] > 0.0,
|
||
(acc_nope * alpha[:, None]) / denom[:, None],
|
||
0.0,
|
||
)
|
||
out_rope = tl.where(
|
||
l_final[:, None] > 0.0,
|
||
(acc_rope * alpha[:, None]) / denom[:, None],
|
||
0.0,
|
||
)
|
||
else:
|
||
denom = tl.maximum(l_i, 1.0e-30)
|
||
out_nope = tl.where(l_i[:, None] > 0.0, acc_nope / denom[:, None], 0.0)
|
||
out_rope = tl.where(l_i[:, None] > 0.0, acc_rope / denom[:, None], 0.0)
|
||
|
||
out_row_ptr = (
|
||
out_ptr + query_idx * out_stride0 + head_offsets[:, None] * out_stride1
|
||
)
|
||
tl.store(
|
||
out_row_ptr + nope_offsets[None, :],
|
||
out_nope,
|
||
mask=head_mask[:, None] & nope_mask[None, :],
|
||
)
|
||
tl.store(
|
||
out_row_ptr + NOPE_DIM + rope_offsets[None, :],
|
||
out_rope,
|
||
mask=head_mask[:, None],
|
||
)
|
||
|
||
|
||
@triton.jit
|
||
def _sparse_attn_decode_partial_kernel(
|
||
q_ptr,
|
||
main_cache_ptr,
|
||
main_indices_ptr,
|
||
main_indptr_ptr,
|
||
extra_cache_ptr,
|
||
extra_indices_ptr,
|
||
extra_indptr_ptr,
|
||
part_m_ptr,
|
||
part_l_ptr,
|
||
part_acc_ptr,
|
||
q_stride0,
|
||
q_stride1,
|
||
main_cache_stride0,
|
||
extra_cache_stride0,
|
||
pm_stride0,
|
||
pm_stride_s,
|
||
pa_stride0,
|
||
pa_stride_s,
|
||
pa_stride_h,
|
||
main_num_rows,
|
||
extra_num_rows,
|
||
main_block_size,
|
||
extra_block_size,
|
||
scale,
|
||
num_heads,
|
||
HAS_EXTRA: tl.constexpr,
|
||
NOPE_DIM: tl.constexpr,
|
||
NOPE_BLOCK: tl.constexpr,
|
||
ROPE_DIM: tl.constexpr,
|
||
# `main_cache` is the SWA K-cache (written by the C++ encoder, FNUZ on
|
||
# gfx942 / OCP on gfx950). `extra_cache` is the compressed K-cache
|
||
# (Triton encoder, OCP on every platform). Reading both with the same
|
||
# `IS_FNUZ` would decode one of them with the wrong FNUZ/OCP scale ratio.
|
||
IS_FNUZ_MAIN: tl.constexpr,
|
||
IS_FNUZ_EXTRA: tl.constexpr,
|
||
BLOCK_H: tl.constexpr,
|
||
BLOCK_K: tl.constexpr,
|
||
NUM_SPLITS: tl.constexpr,
|
||
NUM_STAGES: tl.constexpr,
|
||
):
|
||
query_idx = tl.program_id(0)
|
||
split_id = tl.program_id(1)
|
||
pid_h = tl.program_id(2)
|
||
|
||
head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
|
||
head_mask = head_offsets < num_heads
|
||
nope_offsets = tl.arange(0, NOPE_BLOCK)
|
||
nope_mask = nope_offsets < NOPE_DIM
|
||
rope_offsets = tl.arange(0, ROPE_DIM)
|
||
|
||
q_row_ptr = q_ptr + query_idx * q_stride0 + head_offsets[:, None] * q_stride1
|
||
q_nope = tl.load(
|
||
q_row_ptr + nope_offsets[None, :],
|
||
mask=head_mask[:, None] & nope_mask[None, :],
|
||
other=0.0,
|
||
)
|
||
q_rope = tl.load(
|
||
q_row_ptr + NOPE_DIM + rope_offsets[None, :],
|
||
mask=head_mask[:, None],
|
||
other=0.0,
|
||
)
|
||
|
||
neg_large = -3.4028234663852886e38
|
||
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
|
||
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||
acc_nope = tl.zeros((BLOCK_H, NOPE_BLOCK), dtype=tl.float32)
|
||
acc_rope = tl.zeros((BLOCK_H, ROPE_DIM), dtype=tl.float32)
|
||
k_offsets = tl.arange(0, BLOCK_K)
|
||
|
||
zero_nope = tl.zeros((BLOCK_K, NOPE_BLOCK), dtype=tl.bfloat16)
|
||
zero_rope = tl.zeros((BLOCK_K, ROPE_DIM), dtype=tl.bfloat16)
|
||
|
||
# Each split processes a contiguous slice of this query's main (SWA) and
|
||
# extra (topk) segments. Slices are handled independently so a block never
|
||
# straddles the main/extra boundary.
|
||
main_start = tl.load(main_indptr_ptr + query_idx)
|
||
main_end = tl.load(main_indptr_ptr + query_idx + 1)
|
||
main_len = main_end - main_start
|
||
main_chunk = (main_len + NUM_SPLITS - 1) // NUM_SPLITS
|
||
main_lo = split_id * main_chunk
|
||
main_hi = tl.minimum(main_lo + main_chunk, main_len)
|
||
|
||
for k_start in tl.range(main_lo, main_hi, BLOCK_K, num_stages=NUM_STAGES):
|
||
k_pos = k_start + k_offsets
|
||
in_range = k_pos < main_hi
|
||
slot = tl.load(main_indices_ptr + main_start + k_pos, mask=in_range, other=-1)
|
||
valid = in_range & (slot >= 0) & (slot < main_num_rows)
|
||
safe_slot = tl.where(valid, slot, 0)
|
||
|
||
block_idx = safe_slot // main_block_size
|
||
pos_in_block = safe_slot % main_block_size
|
||
cache_block_ptr = main_cache_ptr + block_idx.to(tl.int64) * main_cache_stride0
|
||
token_data_ptr = cache_block_ptr + pos_in_block * 576
|
||
token_scale_ptr = cache_block_ptr + main_block_size * 576 + pos_in_block * 8
|
||
|
||
x_uint8 = tl.load(
|
||
token_data_ptr[:, None] + nope_offsets[None, :],
|
||
mask=valid[:, None] & nope_mask[None, :],
|
||
other=0,
|
||
)
|
||
if IS_FNUZ_MAIN:
|
||
x_fp8 = x_uint8.to(tl.float8e4b8, bitcast=True)
|
||
else:
|
||
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
|
||
encoded_scales = tl.load(
|
||
token_scale_ptr[:, None] + nope_offsets[None, :] // 64,
|
||
mask=valid[:, None] & nope_mask[None, :],
|
||
other=127,
|
||
)
|
||
scales = tl.exp2(encoded_scales.to(tl.float32) - 127.0)
|
||
k_nope = x_fp8.to(tl.bfloat16) * scales.to(tl.bfloat16)
|
||
k_nope = tl.where(valid[:, None] & nope_mask[None, :], k_nope, zero_nope)
|
||
k_nope = tl.where(k_nope == k_nope, k_nope, zero_nope)
|
||
|
||
rope_ptr = (token_data_ptr + NOPE_DIM).to(tl.pointer_type(tl.bfloat16))
|
||
k_rope = tl.load(
|
||
rope_ptr[:, None] + rope_offsets[None, :],
|
||
mask=valid[:, None],
|
||
other=0.0,
|
||
)
|
||
k_rope = tl.where(valid[:, None], k_rope, zero_rope)
|
||
k_rope = tl.where(k_rope == k_rope, k_rope, zero_rope)
|
||
|
||
scores = tl.dot(q_nope, tl.trans(k_nope)) + tl.dot(q_rope, tl.trans(k_rope))
|
||
scores *= scale
|
||
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
|
||
|
||
m_block = tl.max(scores, axis=1)
|
||
m_new = tl.maximum(m_i, m_block)
|
||
alpha = tl.exp(m_i - m_new)
|
||
p = tl.exp(scores - m_new[:, None])
|
||
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
|
||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||
|
||
acc_nope = acc_nope * alpha[:, None] + tl.dot(p.to(k_nope.dtype), k_nope)
|
||
acc_rope = acc_rope * alpha[:, None] + tl.dot(p.to(k_rope.dtype), k_rope)
|
||
m_i = m_new
|
||
l_i = l_new
|
||
|
||
if HAS_EXTRA:
|
||
extra_start = tl.load(extra_indptr_ptr + query_idx)
|
||
extra_end = tl.load(extra_indptr_ptr + query_idx + 1)
|
||
extra_len = extra_end - extra_start
|
||
extra_chunk = (extra_len + NUM_SPLITS - 1) // NUM_SPLITS
|
||
extra_lo = split_id * extra_chunk
|
||
extra_hi = tl.minimum(extra_lo + extra_chunk, extra_len)
|
||
|
||
for k_start in tl.range(extra_lo, extra_hi, BLOCK_K, num_stages=NUM_STAGES):
|
||
k_pos = k_start + k_offsets
|
||
in_range = k_pos < extra_hi
|
||
slot = tl.load(
|
||
extra_indices_ptr + extra_start + k_pos, mask=in_range, other=-1
|
||
)
|
||
valid = in_range & (slot >= 0) & (slot < extra_num_rows)
|
||
safe_slot = tl.where(valid, slot, 0)
|
||
|
||
block_idx = safe_slot // extra_block_size
|
||
pos_in_block = safe_slot % extra_block_size
|
||
cache_block_ptr = (
|
||
extra_cache_ptr + block_idx.to(tl.int64) * extra_cache_stride0
|
||
)
|
||
token_data_ptr = cache_block_ptr + pos_in_block * 576
|
||
token_scale_ptr = (
|
||
cache_block_ptr + extra_block_size * 576 + pos_in_block * 8
|
||
)
|
||
|
||
x_uint8 = tl.load(
|
||
token_data_ptr[:, None] + nope_offsets[None, :],
|
||
mask=valid[:, None] & nope_mask[None, :],
|
||
other=0,
|
||
)
|
||
if IS_FNUZ_EXTRA:
|
||
x_fp8 = x_uint8.to(tl.float8e4b8, bitcast=True)
|
||
else:
|
||
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
|
||
encoded_scales = tl.load(
|
||
token_scale_ptr[:, None] + nope_offsets[None, :] // 64,
|
||
mask=valid[:, None] & nope_mask[None, :],
|
||
other=127,
|
||
)
|
||
scales = tl.exp2(encoded_scales.to(tl.float32) - 127.0)
|
||
k_nope = x_fp8.to(tl.bfloat16) * scales.to(tl.bfloat16)
|
||
k_nope = tl.where(valid[:, None] & nope_mask[None, :], k_nope, zero_nope)
|
||
k_nope = tl.where(k_nope == k_nope, k_nope, zero_nope)
|
||
|
||
rope_ptr = (token_data_ptr + NOPE_DIM).to(tl.pointer_type(tl.bfloat16))
|
||
k_rope = tl.load(
|
||
rope_ptr[:, None] + rope_offsets[None, :],
|
||
mask=valid[:, None],
|
||
other=0.0,
|
||
)
|
||
k_rope = tl.where(valid[:, None], k_rope, zero_rope)
|
||
k_rope = tl.where(k_rope == k_rope, k_rope, zero_rope)
|
||
|
||
scores = tl.dot(q_nope, tl.trans(k_nope)) + tl.dot(
|
||
q_rope,
|
||
tl.trans(k_rope),
|
||
)
|
||
scores *= scale
|
||
scores = tl.where(head_mask[:, None] & valid[None, :], scores, neg_large)
|
||
|
||
m_block = tl.max(scores, axis=1)
|
||
m_new = tl.maximum(m_i, m_block)
|
||
alpha = tl.exp(m_i - m_new)
|
||
p = tl.exp(scores - m_new[:, None])
|
||
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
|
||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||
|
||
acc_nope = acc_nope * alpha[:, None] + tl.dot(p.to(k_nope.dtype), k_nope)
|
||
acc_rope = acc_rope * alpha[:, None] + tl.dot(p.to(k_rope.dtype), k_rope)
|
||
m_i = m_new
|
||
l_i = l_new
|
||
|
||
# Store raw (un-normalized) partial state for this split. Softmax sink and
|
||
# final normalization happen in the reduce kernel.
|
||
pm_base = query_idx * pm_stride0 + split_id * pm_stride_s + head_offsets
|
||
tl.store(part_m_ptr + pm_base, m_i, mask=head_mask)
|
||
tl.store(part_l_ptr + pm_base, l_i, mask=head_mask)
|
||
acc_base = (
|
||
part_acc_ptr
|
||
+ query_idx * pa_stride0
|
||
+ split_id * pa_stride_s
|
||
+ head_offsets[:, None] * pa_stride_h
|
||
)
|
||
tl.store(
|
||
acc_base + nope_offsets[None, :],
|
||
acc_nope,
|
||
mask=head_mask[:, None] & nope_mask[None, :],
|
||
)
|
||
tl.store(
|
||
acc_base + NOPE_DIM + rope_offsets[None, :],
|
||
acc_rope,
|
||
mask=head_mask[:, None],
|
||
)
|
||
|
||
|
||
@triton.jit
|
||
def _sparse_attn_decode_gfx950_partial_loaded_tile(
|
||
q_combined,
|
||
cache_ptr,
|
||
slot,
|
||
valid,
|
||
cache_stride0,
|
||
scale: tl.constexpr,
|
||
head_mask,
|
||
m_i,
|
||
l_i,
|
||
acc_nope_0a,
|
||
acc_nope_0b,
|
||
acc_nope_1,
|
||
acc_tail,
|
||
BLOCK_SIZE: tl.constexpr,
|
||
NOPE_DIM: tl.constexpr,
|
||
BLOCK_K: tl.constexpr,
|
||
IS_FNUZ: tl.constexpr,
|
||
TRUST_EXTRA_CACHE_NAN_FREE: tl.constexpr,
|
||
):
|
||
safe_slot = tl.where(valid, slot, 0)
|
||
block_idx = safe_slot // BLOCK_SIZE
|
||
pos_in_block = safe_slot % BLOCK_SIZE
|
||
cache_block_ptr = cache_ptr + block_idx.to(tl.int64) * cache_stride0
|
||
token_data_ptr = cache_block_ptr + pos_in_block * 576
|
||
token_scale_ptr = cache_block_ptr + BLOCK_SIZE * 576 + pos_in_block * 8
|
||
k_nope_0a = _load_fp8_ds_mla_gfx950_nope_exact_chunk(
|
||
token_data_ptr,
|
||
token_scale_ptr,
|
||
valid,
|
||
0,
|
||
128,
|
||
BLOCK_K,
|
||
IS_FNUZ,
|
||
)
|
||
k_nope_0b = _load_fp8_ds_mla_gfx950_nope_exact_chunk(
|
||
token_data_ptr,
|
||
token_scale_ptr,
|
||
valid,
|
||
128,
|
||
128,
|
||
BLOCK_K,
|
||
IS_FNUZ,
|
||
)
|
||
k_nope_1 = _load_fp8_ds_mla_gfx950_nope_exact_chunk(
|
||
token_data_ptr,
|
||
token_scale_ptr,
|
||
valid,
|
||
256,
|
||
128,
|
||
BLOCK_K,
|
||
IS_FNUZ,
|
||
)
|
||
k_tail = _load_fp8_ds_mla_gfx950_tail128(
|
||
token_data_ptr,
|
||
token_scale_ptr,
|
||
valid,
|
||
NOPE_DIM,
|
||
BLOCK_K,
|
||
IS_FNUZ,
|
||
)
|
||
if not TRUST_EXTRA_CACHE_NAN_FREE:
|
||
zero = tl.zeros((BLOCK_K, 128), dtype=tl.bfloat16)
|
||
k_nope_0a = tl.where(k_nope_0a == k_nope_0a, k_nope_0a, zero)
|
||
k_nope_0b = tl.where(k_nope_0b == k_nope_0b, k_nope_0b, zero)
|
||
k_nope_1 = tl.where(k_nope_1 == k_nope_1, k_nope_1, zero)
|
||
k_tail = tl.where(k_tail == k_tail, k_tail, zero)
|
||
k_nope_0 = tl.cat(k_nope_0a, k_nope_0b, dim=1)
|
||
k_tail_256 = tl.cat(k_nope_1, k_tail, dim=1)
|
||
k_combined = tl.cat(k_nope_0, k_tail_256, dim=1)
|
||
|
||
scores = tl.dot(q_combined, tl.trans(k_combined))
|
||
scores *= scale * 1.4426950408889634
|
||
scores = tl.where(
|
||
head_mask[:, None] & valid[None, :],
|
||
scores,
|
||
-3.4028234663852886e38,
|
||
)
|
||
m_block = tl.max(scores, axis=1)
|
||
m_new = tl.maximum(m_i, m_block)
|
||
alpha = tl.exp2(m_i - m_new)
|
||
p = tl.exp2(scores - m_new[:, None])
|
||
p = tl.where(head_mask[:, None] & valid[None, :], p, 0.0)
|
||
l_new = l_i * alpha + tl.sum(p, axis=1)
|
||
p_bf16 = p.to(k_nope_0a.dtype)
|
||
acc_nope_0a = acc_nope_0a * alpha[:, None] + tl.dot(p_bf16, k_nope_0a)
|
||
acc_nope_0b = acc_nope_0b * alpha[:, None] + tl.dot(p_bf16, k_nope_0b)
|
||
acc_nope_1 = acc_nope_1 * alpha[:, None] + tl.dot(p_bf16, k_nope_1)
|
||
acc_tail = acc_tail * alpha[:, None] + tl.dot(p_bf16, k_tail)
|
||
return (
|
||
m_new,
|
||
l_new,
|
||
acc_nope_0a,
|
||
acc_nope_0b,
|
||
acc_nope_1,
|
||
acc_tail,
|
||
)
|
||
|
||
|
||
@triton.jit
|
||
def _sparse_attn_decode_gfx950_partial_kernel(
|
||
q_ptr,
|
||
main_cache_ptr,
|
||
main_indices_ptr,
|
||
main_indptr_ptr,
|
||
extra_cache_ptr,
|
||
extra_indices_ptr,
|
||
extra_indptr_ptr,
|
||
part_m_ptr,
|
||
part_l_ptr,
|
||
part_acc_ptr,
|
||
q_stride0: tl.constexpr,
|
||
q_stride1: tl.constexpr,
|
||
main_cache_stride0: tl.constexpr,
|
||
extra_cache_stride0: tl.constexpr,
|
||
main_num_rows,
|
||
extra_num_rows,
|
||
MAIN_BLOCK_SIZE: tl.constexpr,
|
||
EXTRA_BLOCK_SIZE: tl.constexpr,
|
||
scale: tl.constexpr,
|
||
num_heads: tl.constexpr,
|
||
HAS_EXTRA: tl.constexpr,
|
||
NOPE_DIM: tl.constexpr,
|
||
ROPE_DIM: tl.constexpr,
|
||
IS_FNUZ_MAIN: tl.constexpr,
|
||
IS_FNUZ_EXTRA: tl.constexpr,
|
||
TRUST_EXTRA_CACHE_NAN_FREE: tl.constexpr,
|
||
ADAPTIVE_SPLITS: tl.constexpr,
|
||
ONE_WAVE_SPLITS: tl.constexpr,
|
||
BLOCK_H: tl.constexpr,
|
||
BLOCK_K: tl.constexpr,
|
||
NUM_SPLITS: tl.constexpr,
|
||
NUM_STAGES: tl.constexpr,
|
||
):
|
||
query_idx = tl.program_id(0)
|
||
split_id = tl.program_id(1)
|
||
pid_h = tl.program_id(2)
|
||
|
||
tl.static_assert(NOPE_DIM == 448)
|
||
tl.static_assert(ROPE_DIM == 64)
|
||
|
||
head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
|
||
if num_heads % BLOCK_H == 0:
|
||
head_mask = tl.full((BLOCK_H,), True, tl.int1)
|
||
else:
|
||
head_mask = head_offsets < num_heads
|
||
neg_large = -3.4028234663852886e38
|
||
|
||
if ADAPTIVE_SPLITS:
|
||
main_start = tl.load(main_indptr_ptr + query_idx)
|
||
main_end = tl.load(main_indptr_ptr + query_idx + 1)
|
||
main_len = main_end - main_start
|
||
if HAS_EXTRA:
|
||
extra_start = tl.load(extra_indptr_ptr + query_idx)
|
||
extra_end = tl.load(extra_indptr_ptr + query_idx + 1)
|
||
extra_len = extra_end - extra_start
|
||
else:
|
||
extra_start = 0
|
||
extra_len = 0
|
||
split4_span: tl.constexpr = 4 * BLOCK_K
|
||
split4_iters = (main_len + split4_span - 1) // split4_span
|
||
split4_iters += (extra_len + split4_span - 1) // split4_span
|
||
use_four_splits = split4_iters <= 3
|
||
work_splits = NUM_SPLITS
|
||
if ONE_WAVE_SPLITS > 4 and ONE_WAVE_SPLITS < NUM_SPLITS:
|
||
one_wave_span: tl.constexpr = ONE_WAVE_SPLITS * BLOCK_K
|
||
one_wave_iters = (main_len + one_wave_span - 1) // one_wave_span
|
||
one_wave_iters += (extra_len + one_wave_span - 1) // one_wave_span
|
||
work_splits = tl.where(one_wave_iters <= 3, ONE_WAVE_SPLITS, work_splits)
|
||
work_splits = tl.where(use_four_splits, 4, work_splits)
|
||
if split_id >= work_splits:
|
||
pm_base = (query_idx * NUM_SPLITS + split_id) * num_heads + head_offsets
|
||
tl.store(part_m_ptr + pm_base, neg_large, mask=head_mask)
|
||
tl.store(part_l_ptr + pm_base, 0.0, mask=head_mask)
|
||
return
|
||
else:
|
||
work_splits = NUM_SPLITS
|
||
|
||
nope_offsets_0a = tl.arange(0, 128)
|
||
nope_offsets_0b = 128 + tl.arange(0, 128)
|
||
nope_offsets_0 = tl.arange(0, 256)
|
||
tail_offsets = 256 + tl.arange(0, 256)
|
||
nope_offsets_1 = 256 + tl.arange(0, 128)
|
||
tail_offsets_128 = 384 + tl.arange(0, 128)
|
||
|
||
q_row_ptr = q_ptr + query_idx * q_stride0 + head_offsets[:, None] * q_stride1
|
||
q_nope_0 = tl.load(
|
||
q_row_ptr + nope_offsets_0[None, :],
|
||
mask=head_mask[:, None],
|
||
other=0.0,
|
||
)
|
||
q_tail = tl.load(
|
||
q_row_ptr + tail_offsets[None, :],
|
||
mask=head_mask[:, None],
|
||
other=0.0,
|
||
)
|
||
q_combined = tl.cat(q_nope_0, q_tail, dim=1)
|
||
|
||
m_i = tl.full((BLOCK_H,), neg_large, dtype=tl.float32)
|
||
l_i = tl.zeros((BLOCK_H,), dtype=tl.float32)
|
||
acc_nope_0a = tl.zeros((BLOCK_H, 128), dtype=tl.float32)
|
||
acc_nope_0b = tl.zeros((BLOCK_H, 128), dtype=tl.float32)
|
||
acc_nope_1 = tl.zeros((BLOCK_H, 128), dtype=tl.float32)
|
||
acc_tail = tl.zeros((BLOCK_H, 128), dtype=tl.float32)
|
||
k_offsets = tl.arange(0, BLOCK_K)
|
||
|
||
if not ADAPTIVE_SPLITS:
|
||
main_start = tl.load(main_indptr_ptr + query_idx)
|
||
main_end = tl.load(main_indptr_ptr + query_idx + 1)
|
||
main_len = main_end - main_start
|
||
main_chunk = (main_len + work_splits - 1) // work_splits
|
||
main_lo = split_id * main_chunk
|
||
main_hi = tl.minimum(main_lo + main_chunk, main_len)
|
||
|
||
for k_start in tl.range(
|
||
main_lo,
|
||
main_hi,
|
||
BLOCK_K,
|
||
num_stages=NUM_STAGES,
|
||
):
|
||
k_pos = k_start + k_offsets
|
||
in_range = k_pos < main_hi
|
||
slot = tl.load(main_indices_ptr + main_start + k_pos, mask=in_range, other=-1)
|
||
valid = in_range & (slot >= 0) & (slot < main_num_rows)
|
||
(
|
||
m_i,
|
||
l_i,
|
||
acc_nope_0a,
|
||
acc_nope_0b,
|
||
acc_nope_1,
|
||
acc_tail,
|
||
) = _sparse_attn_decode_gfx950_partial_loaded_tile(
|
||
q_combined,
|
||
main_cache_ptr,
|
||
slot,
|
||
valid,
|
||
main_cache_stride0,
|
||
scale,
|
||
head_mask,
|
||
m_i,
|
||
l_i,
|
||
acc_nope_0a,
|
||
acc_nope_0b,
|
||
acc_nope_1,
|
||
acc_tail,
|
||
MAIN_BLOCK_SIZE,
|
||
NOPE_DIM,
|
||
BLOCK_K,
|
||
IS_FNUZ_MAIN,
|
||
False,
|
||
)
|
||
|
||
if HAS_EXTRA:
|
||
if not ADAPTIVE_SPLITS:
|
||
extra_start = tl.load(extra_indptr_ptr + query_idx)
|
||
extra_end = tl.load(extra_indptr_ptr + query_idx + 1)
|
||
extra_len = extra_end - extra_start
|
||
extra_chunk = (extra_len + work_splits - 1) // work_splits
|
||
extra_lo = split_id * extra_chunk
|
||
extra_hi = tl.minimum(extra_lo + extra_chunk, extra_len)
|
||
|
||
outer_block_k: tl.constexpr = 2 * BLOCK_K
|
||
outer_k_offsets = tl.arange(0, outer_block_k)
|
||
extra_hi_full = (
|
||
extra_lo + ((extra_hi - extra_lo) // outer_block_k) * outer_block_k
|
||
)
|
||
for k_start in tl.range(
|
||
extra_lo,
|
||
extra_hi_full,
|
||
outer_block_k,
|
||
num_stages=NUM_STAGES,
|
||
):
|
||
slot = tl.load(extra_indices_ptr + extra_start + k_start + outer_k_offsets)
|
||
valid = (slot >= 0) & (slot < extra_num_rows)
|
||
slot_pairs = tl.trans(tl.reshape(slot, (2, BLOCK_K)))
|
||
valid_pairs = tl.trans(tl.reshape(valid, (2, BLOCK_K)))
|
||
slot_lo, slot_hi = tl.split(slot_pairs)
|
||
valid_lo, valid_hi = tl.split(valid_pairs)
|
||
(
|
||
m_i,
|
||
l_i,
|
||
acc_nope_0a,
|
||
acc_nope_0b,
|
||
acc_nope_1,
|
||
acc_tail,
|
||
) = _sparse_attn_decode_gfx950_partial_loaded_tile(
|
||
q_combined,
|
||
extra_cache_ptr,
|
||
slot_lo,
|
||
valid_lo,
|
||
extra_cache_stride0,
|
||
scale,
|
||
head_mask,
|
||
m_i,
|
||
l_i,
|
||
acc_nope_0a,
|
||
acc_nope_0b,
|
||
acc_nope_1,
|
||
acc_tail,
|
||
EXTRA_BLOCK_SIZE,
|
||
NOPE_DIM,
|
||
BLOCK_K,
|
||
IS_FNUZ_EXTRA,
|
||
TRUST_EXTRA_CACHE_NAN_FREE,
|
||
)
|
||
(
|
||
m_i,
|
||
l_i,
|
||
acc_nope_0a,
|
||
acc_nope_0b,
|
||
acc_nope_1,
|
||
acc_tail,
|
||
) = _sparse_attn_decode_gfx950_partial_loaded_tile(
|
||
q_combined,
|
||
extra_cache_ptr,
|
||
slot_hi,
|
||
valid_hi,
|
||
extra_cache_stride0,
|
||
scale,
|
||
head_mask,
|
||
m_i,
|
||
l_i,
|
||
acc_nope_0a,
|
||
acc_nope_0b,
|
||
acc_nope_1,
|
||
acc_tail,
|
||
EXTRA_BLOCK_SIZE,
|
||
NOPE_DIM,
|
||
BLOCK_K,
|
||
IS_FNUZ_EXTRA,
|
||
TRUST_EXTRA_CACHE_NAN_FREE,
|
||
)
|
||
for tail_idx in tl.static_range(2):
|
||
tail_start = extra_hi_full + tail_idx * BLOCK_K
|
||
if tail_start < extra_hi:
|
||
k_pos = tail_start + k_offsets
|
||
in_range = k_pos < extra_hi
|
||
slot = tl.load(
|
||
extra_indices_ptr + extra_start + k_pos,
|
||
mask=in_range,
|
||
other=-1,
|
||
)
|
||
valid = in_range & (slot >= 0) & (slot < extra_num_rows)
|
||
(
|
||
m_i,
|
||
l_i,
|
||
acc_nope_0a,
|
||
acc_nope_0b,
|
||
acc_nope_1,
|
||
acc_tail,
|
||
) = _sparse_attn_decode_gfx950_partial_loaded_tile(
|
||
q_combined,
|
||
extra_cache_ptr,
|
||
slot,
|
||
valid,
|
||
extra_cache_stride0,
|
||
scale,
|
||
head_mask,
|
||
m_i,
|
||
l_i,
|
||
acc_nope_0a,
|
||
acc_nope_0b,
|
||
acc_nope_1,
|
||
acc_tail,
|
||
EXTRA_BLOCK_SIZE,
|
||
NOPE_DIM,
|
||
BLOCK_K,
|
||
IS_FNUZ_EXTRA,
|
||
TRUST_EXTRA_CACHE_NAN_FREE,
|
||
)
|
||
|
||
pm_base = (query_idx * NUM_SPLITS + split_id) * num_heads + head_offsets
|
||
m_store = tl.where(l_i > 0.0, m_i * 0.6931471805599453, neg_large)
|
||
tl.store(part_m_ptr + pm_base, m_store, mask=head_mask)
|
||
tl.store(part_l_ptr + pm_base, l_i, mask=head_mask)
|
||
acc_base = part_acc_ptr + (
|
||
(query_idx * NUM_SPLITS + split_id) * num_heads + head_offsets[:, None]
|
||
) * (NOPE_DIM + ROPE_DIM)
|
||
tl.store(
|
||
acc_base + nope_offsets_0a[None, :],
|
||
acc_nope_0a,
|
||
mask=head_mask[:, None],
|
||
)
|
||
tl.store(
|
||
acc_base + nope_offsets_0b[None, :],
|
||
acc_nope_0b,
|
||
mask=head_mask[:, None],
|
||
)
|
||
tl.store(
|
||
acc_base + nope_offsets_1[None, :],
|
||
acc_nope_1,
|
||
mask=head_mask[:, None],
|
||
)
|
||
tl.store(
|
||
acc_base + tail_offsets_128[None, :],
|
||
acc_tail,
|
||
mask=head_mask[:, None],
|
||
)
|
||
|
||
|
||
@triton.jit
|
||
def _mxfp8_scale_bits(amax):
|
||
"""Biased E8M0 exponent that puts ``amax`` at the top of the e4m3 range.
|
||
|
||
Same rounding as ``mxfp8_e4m3_quantize``, so the output is bit-identical to
|
||
quantizing the tensor there.
|
||
"""
|
||
amax = tl.maximum(amax, 1.1754943508222875e-38)
|
||
bits = tl.ceil(tl.log2(amax / 448.0)) + 127.0
|
||
return tl.minimum(tl.maximum(bits, 0.0), 254.0)
|
||
|
||
|
||
@triton.jit
|
||
def _mxfp8_quantize_rows(x, ROWS: tl.constexpr, COLS: tl.constexpr):
|
||
"""MXFP8-quantize ``x`` [ROWS, COLS] in registers, one scale per 32 lanes.
|
||
|
||
Returns the rescaled fp32 values (to be cast to e4m3 on store) and the
|
||
[ROWS, COLS // 32] biased E8M0 exponents.
|
||
"""
|
||
blocks = tl.reshape(x, (ROWS, COLS // 32, 32))
|
||
bits = _mxfp8_scale_bits(tl.max(tl.abs(blocks), axis=2))
|
||
# Multiply by the reciprocal: a divisor of 2**-127 would be subnormal and
|
||
# flush to zero, turning an all-zero block into NaN.
|
||
q = blocks * tl.exp2(127.0 - bits)[:, :, None]
|
||
return tl.reshape(q, (ROWS, COLS)), bits
|
||
|
||
|
||
@triton.jit
|
||
def _sparse_attn_decode_reduce_kernel(
|
||
part_m_ptr,
|
||
part_l_ptr,
|
||
part_acc_ptr,
|
||
attn_sink_ptr,
|
||
out_ptr,
|
||
pos_ptr,
|
||
cos_sin_ptr,
|
||
out_scale_ptr,
|
||
out_stride0,
|
||
out_stride1,
|
||
os_stride0,
|
||
os_stride1,
|
||
pm_stride0,
|
||
pm_stride_s,
|
||
pa_stride0,
|
||
pa_stride_s,
|
||
pa_stride_h,
|
||
cs_stride,
|
||
num_heads,
|
||
HAS_ATTN_SINK: tl.constexpr,
|
||
ADAPTIVE_SPLITS: tl.constexpr,
|
||
COMB_DIM: tl.constexpr,
|
||
BLOCK_H: tl.constexpr,
|
||
NUM_SPLITS: tl.constexpr,
|
||
SPLITS_PAD: tl.constexpr,
|
||
FUSE_INV_ROPE: tl.constexpr,
|
||
NOPE: tl.constexpr,
|
||
HALF: tl.constexpr,
|
||
QUANT_OUT: tl.constexpr,
|
||
):
|
||
query_idx = tl.program_id(0)
|
||
pid_h = tl.program_id(1)
|
||
|
||
head_offsets = pid_h * BLOCK_H + tl.arange(0, BLOCK_H)
|
||
head_mask = head_offsets < num_heads
|
||
comb_offsets = tl.arange(0, COMB_DIM)
|
||
# SPLITS_PAD is NUM_SPLITS rounded up to a power of two so the parallel
|
||
# split-axis load is a legal arange for any split count; padding lanes are
|
||
# masked off.
|
||
split_offsets = tl.arange(0, SPLITS_PAD)
|
||
split_mask = split_offsets < NUM_SPLITS
|
||
|
||
neg_large = -3.4028234663852886e38
|
||
|
||
# Phase 1: load every split's running max/sum at once and reduce the max
|
||
# in parallel (tl.max over the split axis) instead of walking the splits
|
||
# serially. This breaks the long online-softmax dependency chain that made
|
||
# the reduce latency-bound.
|
||
load_mask = split_mask[:, None] & head_mask[None, :]
|
||
pm_split = (
|
||
part_m_ptr
|
||
+ query_idx * pm_stride0
|
||
+ split_offsets[:, None] * pm_stride_s
|
||
+ head_offsets[None, :]
|
||
)
|
||
m_all = tl.load(pm_split, mask=load_mask, other=neg_large) # [S, H]
|
||
l_all = tl.load(
|
||
part_l_ptr
|
||
+ query_idx * pm_stride0
|
||
+ split_offsets[:, None] * pm_stride_s
|
||
+ head_offsets[None, :],
|
||
mask=load_mask,
|
||
other=0.0,
|
||
)
|
||
|
||
m_comb = tl.max(m_all, axis=0) # [H]
|
||
if HAS_ATTN_SINK:
|
||
sink = tl.load(
|
||
attn_sink_ptr + head_offsets, mask=head_mask, other=neg_large
|
||
).to(tl.float32)
|
||
m_final = tl.maximum(m_comb, sink)
|
||
else:
|
||
m_final = m_comb
|
||
|
||
w_all = tl.exp(m_all - m_final[None, :]) # [S, H]
|
||
w_all = tl.where(load_mask, w_all, 0.0)
|
||
l_final = tl.sum(w_all * l_all, axis=0) # [H]
|
||
if HAS_ATTN_SINK:
|
||
l_final = l_final + tl.exp(sink - m_final)
|
||
denom = tl.maximum(l_final, 1.0e-30)
|
||
|
||
# Phase 2: weighted sum of the per-split accumulators. The combine weight
|
||
# for each split only depends on the (already known) global max, so the
|
||
# acc loads carry no cross-split dependency and the compiler can pipeline
|
||
# them; only the cheap FMA into `acc` is loop-carried.
|
||
acc = tl.zeros((BLOCK_H, COMB_DIM), dtype=tl.float32)
|
||
for s in tl.static_range(NUM_SPLITS):
|
||
m_s = tl.load(
|
||
part_m_ptr + query_idx * pm_stride0 + s * pm_stride_s + head_offsets,
|
||
mask=head_mask,
|
||
other=neg_large,
|
||
)
|
||
w_s = tl.exp(m_s - m_final)
|
||
if ADAPTIVE_SPLITS:
|
||
active_split = m_s > neg_large
|
||
w_s = tl.where(head_mask & active_split, w_s, 0.0)
|
||
acc_base = (
|
||
part_acc_ptr
|
||
+ query_idx * pa_stride0
|
||
+ s * pa_stride_s
|
||
+ head_offsets[:, None] * pa_stride_h
|
||
)
|
||
if ADAPTIVE_SPLITS:
|
||
acc_s = tl.load(
|
||
acc_base + comb_offsets[None, :],
|
||
mask=head_mask[:, None] & active_split[:, None],
|
||
other=0.0,
|
||
)
|
||
else:
|
||
acc_s = tl.load(
|
||
acc_base + comb_offsets[None, :],
|
||
mask=head_mask[:, None],
|
||
other=0.0,
|
||
)
|
||
acc += w_s[:, None] * acc_s
|
||
|
||
out = tl.where(l_final[:, None] > 0.0, acc / denom[:, None], 0.0)
|
||
|
||
if FUSE_INV_ROPE:
|
||
# Inverse GPT-J RoPE on the trailing rope lanes, straight out of the
|
||
# combine registers: out_even = a*cos + b*sin, out_odd = b*cos - a*sin.
|
||
# NoPE lanes take cos=1/sin=0 so one expression covers the whole row
|
||
# and the o_proj rotation pass disappears for these tokens.
|
||
pos = tl.load(pos_ptr + query_idx).to(tl.int64)
|
||
pair_idx = tl.arange(0, COMB_DIM // 2) - (NOPE // 2)
|
||
is_rope = pair_idx >= 0
|
||
k = tl.where(is_rope, pair_idx, 0)
|
||
cos = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + k), 1.0)
|
||
sin = tl.where(is_rope, tl.load(cos_sin_ptr + pos * cs_stride + HALF + k), 0.0)
|
||
even, odd = tl.split(tl.reshape(out, (BLOCK_H, COMB_DIM // 2, 2)))
|
||
out = tl.reshape(
|
||
tl.join(
|
||
even * cos[None, :] + odd * sin[None, :],
|
||
odd * cos[None, :] - even * sin[None, :],
|
||
),
|
||
(BLOCK_H, COMB_DIM),
|
||
)
|
||
|
||
out_row_ptr = (
|
||
out_ptr + query_idx * out_stride0 + head_offsets[:, None] * out_stride1
|
||
)
|
||
if QUANT_OUT:
|
||
# Emit wo_a's MXFP8 input directly instead of a bf16 row that a
|
||
# separate pass would read back to quantize.
|
||
out, scale_bits = _mxfp8_quantize_rows(out, BLOCK_H, COMB_DIM)
|
||
scale_offsets = tl.arange(0, COMB_DIM // 32)
|
||
tl.store(
|
||
out_scale_ptr
|
||
+ query_idx * os_stride0
|
||
+ head_offsets[:, None] * os_stride1
|
||
+ scale_offsets[None, :],
|
||
scale_bits.to(tl.uint8),
|
||
mask=head_mask[:, None],
|
||
)
|
||
tl.store(
|
||
out_row_ptr + comb_offsets[None, :],
|
||
out.to(out_ptr.dtype.element_ty),
|
||
mask=head_mask[:, None],
|
||
)
|
||
|
||
|
||
def _rocm_sparse_attn_prefill_ragged_triton(
|
||
q: torch.Tensor,
|
||
kv: torch.Tensor,
|
||
indices: torch.Tensor,
|
||
indptr: torch.Tensor,
|
||
scale: float,
|
||
attn_sink: torch.Tensor | None,
|
||
nope_head_dim: int,
|
||
rope_head_dim: int,
|
||
out: torch.Tensor | None = None,
|
||
) -> torch.Tensor:
|
||
assert q.ndim == 3, f"expected q=[sq,h,d], got {q.shape}"
|
||
assert kv.ndim == 2, f"expected kv=[skv,d], got {kv.shape}"
|
||
assert indices.ndim == 1, f"expected indices=[nnz], got {indices.shape}"
|
||
assert indptr.ndim == 1, f"expected indptr=[sq+1], got {indptr.shape}"
|
||
assert not q.is_cpu and not kv.is_cpu and not indices.is_cpu and not indptr.is_cpu
|
||
|
||
indices = _as_int32_contiguous_1d(indices)
|
||
indptr = _as_int32_contiguous_1d(indptr)
|
||
has_attn_sink = attn_sink is not None
|
||
if attn_sink is None:
|
||
attn_sink = torch.empty(1, device=q.device, dtype=torch.float32)
|
||
else:
|
||
attn_sink = attn_sink.contiguous()
|
||
|
||
num_queries, num_heads, head_dim = q.shape
|
||
assert indptr.numel() == num_queries + 1, (
|
||
f"expected indptr shape [{num_queries + 1}], got {indptr.shape}"
|
||
)
|
||
_validate_sparse_dims(
|
||
head_dim,
|
||
nope_head_dim,
|
||
rope_head_dim,
|
||
"_rocm_sparse_attn_prefill_ragged_triton",
|
||
)
|
||
|
||
block_h = 16
|
||
block_d = triton.next_power_of_2(head_dim)
|
||
block_k = 16 if head_dim >= 256 else 32
|
||
num_warps = 4
|
||
if out is None:
|
||
out = torch.empty_like(q)
|
||
else:
|
||
assert out.shape[:2] == q.shape[:2], (
|
||
f"expected out=[{num_queries},{num_heads},*], got {out.shape}"
|
||
)
|
||
assert out.shape[-1] in (nope_head_dim, head_dim), (
|
||
f"expected out width {nope_head_dim} or {head_dim}, got {out.shape[-1]}"
|
||
)
|
||
_sparse_attn_prefill_ragged_kernel[(num_queries, triton.cdiv(num_heads, block_h))](
|
||
q,
|
||
kv,
|
||
indices,
|
||
indptr,
|
||
attn_sink,
|
||
out,
|
||
q.stride(0),
|
||
q.stride(1),
|
||
q.stride(2),
|
||
kv.stride(0),
|
||
kv.stride(1),
|
||
out.stride(0),
|
||
out.stride(1),
|
||
out.stride(2),
|
||
num_heads,
|
||
head_dim,
|
||
kv.shape[0],
|
||
float(scale),
|
||
HAS_ATTN_SINK=has_attn_sink,
|
||
OUT_DV=out.shape[-1],
|
||
BLOCK_H=block_h,
|
||
BLOCK_D=block_d,
|
||
BLOCK_K=block_k,
|
||
num_warps=num_warps,
|
||
)
|
||
return out
|
||
|
||
|
||
def _rocm_sparse_attn_prefill_triton(
|
||
q: torch.Tensor,
|
||
kv: torch.Tensor,
|
||
indices: torch.Tensor,
|
||
scale: float,
|
||
attn_sink: torch.Tensor | None,
|
||
nope_head_dim: int,
|
||
rope_head_dim: int,
|
||
topk_length: torch.Tensor | None = None,
|
||
out: torch.Tensor | None = None,
|
||
) -> torch.Tensor:
|
||
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
|
||
indices,
|
||
topk_length
|
||
if topk_length is not None
|
||
else (indices >= 0).sum(dim=-1, dtype=torch.int32),
|
||
num_rows=kv.shape[0],
|
||
)
|
||
return _rocm_sparse_attn_prefill_ragged_triton(
|
||
q=q,
|
||
kv=kv,
|
||
indices=ragged_indices,
|
||
indptr=ragged_indptr,
|
||
scale=scale,
|
||
attn_sink=attn_sink,
|
||
nope_head_dim=nope_head_dim,
|
||
rope_head_dim=rope_head_dim,
|
||
out=out,
|
||
)
|
||
|
||
|
||
def _can_use_aiter_sparse_prefill_opus(
|
||
q: torch.Tensor,
|
||
kv: torch.Tensor,
|
||
attn_sink: torch.Tensor | None,
|
||
output: torch.Tensor,
|
||
on_gfx950: bool = _ON_GFX950,
|
||
) -> bool:
|
||
return (
|
||
on_gfx950
|
||
and q.shape[0] >= _GFX950_AITER_SPARSE_PREFILL_OPUS_MIN_QUERIES
|
||
and q.is_cuda
|
||
and output.shape == q.shape
|
||
and kv.shape[-1] == q.shape[-1]
|
||
and q.dtype in (torch.bfloat16, torch.float16)
|
||
and kv.dtype == q.dtype
|
||
and output.dtype == q.dtype
|
||
and kv.device == q.device
|
||
and output.device == q.device
|
||
and q.stride(-1) == 1
|
||
and kv.stride(-1) == 1
|
||
and output.stride() == q.stride()
|
||
and attn_sink is not None
|
||
and attn_sink.shape == (q.shape[1],)
|
||
and attn_sink.dtype == torch.float32
|
||
and attn_sink.device == q.device
|
||
)
|
||
|
||
|
||
def _rocm_sparse_attn_prefill_ragged_aiter_opus(
|
||
q: torch.Tensor,
|
||
kv: torch.Tensor,
|
||
indices: torch.Tensor,
|
||
indptr: torch.Tensor,
|
||
scale: float,
|
||
attn_sink: torch.Tensor,
|
||
output: torch.Tensor,
|
||
) -> bool:
|
||
pa_sparse_prefill_opus = _get_aiter_sparse_prefill_opus()
|
||
if pa_sparse_prefill_opus is None:
|
||
return False
|
||
|
||
indices = _as_int32_contiguous_1d(indices)
|
||
indptr = _as_int32_contiguous_1d(indptr)
|
||
empty_indices = indices[:0]
|
||
empty_indptr = torch.zeros_like(indptr)
|
||
pa_sparse_prefill_opus(
|
||
q,
|
||
kv,
|
||
indices,
|
||
indptr,
|
||
kv[:1],
|
||
empty_indices,
|
||
empty_indptr,
|
||
attn_sink.contiguous(),
|
||
float(scale),
|
||
out=output,
|
||
)
|
||
return True
|
||
|
||
|
||
@functools.lru_cache
|
||
def _decode_cu_count() -> int:
|
||
try:
|
||
return torch.cuda.get_device_properties(0).multi_processor_count
|
||
except Exception:
|
||
return 256 # For gfx950 arch, gated behind a fallback path for other archs.
|
||
|
||
|
||
def _decode_partial_iters(
|
||
avg_main_len: float, avg_extra_len: float, splits: int, block_k: int
|
||
) -> int:
|
||
"""BLOCK_K iterations one partial workgroup walks for ``splits`` splits.
|
||
|
||
Each split processes ``ceil(seg_len / splits)`` tokens of a segment, walked
|
||
``BLOCK_K`` at a time, and the main/extra segments are handled separately.
|
||
"""
|
||
main_iters = (
|
||
math.ceil(math.ceil(avg_main_len / splits) / block_k) if avg_main_len > 0 else 0
|
||
)
|
||
extra_iters = (
|
||
math.ceil(math.ceil(avg_extra_len / splits) / block_k)
|
||
if avg_extra_len > 0
|
||
else 0
|
||
)
|
||
return main_iters + extra_iters
|
||
|
||
|
||
def _decode_num_splits(
|
||
num_queries: int,
|
||
heads_blocks: int,
|
||
avg_main_len: float = 0.0,
|
||
avg_extra_len: float = 0.0,
|
||
block_k: int = 32,
|
||
) -> int:
|
||
"""Pick a flash-decode split count to keep the GPU busy across batch sizes.
|
||
|
||
Decode launches only ``num_queries * heads_blocks`` workgroups otherwise,
|
||
which severely under-fills the device for the low-concurrency regime that
|
||
dominates latency. Splitting the KV sequence adds parallelism.
|
||
|
||
We model the relative partial-kernel latency for a given split count ``s``
|
||
as ``waves * (1/s + mu)`` where ``waves = ceil(base * s / CU)`` and ``mu``
|
||
is a small per-wave overhead penalty:
|
||
|
||
- ``waves / s`` captures the partial compute: each wave walks roughly
|
||
``total_tokens / s`` tokens and there are ``waves`` of them, so dividing
|
||
by ``s`` makes more splits cheaper *until* they spill into extra waves.
|
||
- ``mu * waves`` charges per-wave launch/tail overhead so we do not
|
||
over-split into many mostly-idle waves (e.g. batch 224 on 256 CUs is
|
||
best left at 1 split rather than 8 splits across 7 waves).
|
||
|
||
The minimiser naturally prefers split counts that pack the device into full
|
||
waves (``base * s`` near a multiple of ``CU``) and falls back to 1 split
|
||
once the batch already fills the device. Ties favour the smaller split
|
||
count (less reduce work).
|
||
|
||
Finally we "snap down" the chosen split count to the smallest value that
|
||
yields the same wave count *and* the same per-workgroup BLOCK_K iteration
|
||
count. Because latency tracks iteration count (not raw token count), extra
|
||
splits that do not lower the iteration count add only reduce/HBM overhead
|
||
for no parallelism gain (e.g. batch 24: s8 and s10 both walk 4 extra iters
|
||
in one wave, so s8 is strictly better). Snapping needs the average segment
|
||
lengths, which the caller derives sync-free from the ragged index sizes.
|
||
"""
|
||
base = max(1, num_queries * heads_blocks)
|
||
# Target ~1 workgroup per CU: enough to fill the device while keeping the
|
||
# reduce cost (which grows with split count) small. Tuned on gfx950.
|
||
cu = max(1, _decode_cu_count())
|
||
# Per-wave overhead penalty: higher values discourage split counts that
|
||
# spill into extra GPU waves. Tuned on gfx950.
|
||
mu = 0.04
|
||
best_splits = 1
|
||
best_cost = None
|
||
# Search up to 16 splits; beyond that the reduce/HBM overhead dominates.
|
||
for splits in range(1, 17):
|
||
waves = (base * splits + cu - 1) // cu
|
||
cost = waves * (1.0 / splits + mu)
|
||
if best_cost is None or cost < best_cost - 1e-9:
|
||
best_splits = splits
|
||
best_cost = cost
|
||
|
||
if best_splits > 1 and (avg_main_len > 0 or avg_extra_len > 0):
|
||
target_waves = (base * best_splits + cu - 1) // cu
|
||
target_iters = _decode_partial_iters(
|
||
avg_main_len, avg_extra_len, best_splits, block_k
|
||
)
|
||
for splits in range(1, best_splits):
|
||
waves = (base * splits + cu - 1) // cu
|
||
iters = _decode_partial_iters(avg_main_len, avg_extra_len, splits, block_k)
|
||
if waves == target_waves and iters == target_iters:
|
||
best_splits = splits
|
||
break
|
||
return best_splits
|
||
|
||
|
||
def _decode_gfx950_num_splits(
|
||
num_queries: int,
|
||
heads_blocks: int,
|
||
avg_main_len: float = 0.0,
|
||
avg_extra_len: float = 0.0,
|
||
block_k: int = 32,
|
||
) -> int:
|
||
base = max(1, num_queries * heads_blocks)
|
||
cu = max(1, _decode_cu_count())
|
||
target_workgroups = 2 * cu
|
||
num_splits = min(
|
||
32,
|
||
max(
|
||
1,
|
||
math.ceil(target_workgroups / base),
|
||
),
|
||
)
|
||
if (
|
||
base >= 16
|
||
and num_splits > 4
|
||
and _decode_partial_iters(avg_main_len, avg_extra_len, 4, block_k) <= 3
|
||
):
|
||
return 4
|
||
if 16 <= base < 64:
|
||
one_wave_splits = max(1, cu // base)
|
||
one_wave_iters = _decode_partial_iters(
|
||
avg_main_len, avg_extra_len, one_wave_splits, block_k
|
||
)
|
||
target_waves = 1 if one_wave_iters <= 9 else 2
|
||
num_splits = min(num_splits, max(1, target_waves * cu // base))
|
||
if base >= 16 and num_splits > 1:
|
||
target_waves = (base * num_splits + cu - 1) // cu
|
||
target_iters = _decode_partial_iters(
|
||
avg_main_len, avg_extra_len, num_splits, block_k
|
||
)
|
||
for splits in range(1, num_splits):
|
||
waves = (base * splits + cu - 1) // cu
|
||
iters = _decode_partial_iters(avg_main_len, avg_extra_len, splits, block_k)
|
||
if waves == target_waves and iters == target_iters:
|
||
return splits
|
||
return num_splits
|
||
|
||
|
||
# Empirically single-pass bf16 decode is faster than split-K below this number of
|
||
# selected slots per query, so we do not split.
|
||
_SPARSE_DECODE_BF16_MIN_SPLIT_LEN = 512
|
||
_SPARSE_DECODE_BF16_BLOCK_H = 16
|
||
_SPARSE_DECODE_BF16_BLOCK_K = 32
|
||
|
||
|
||
def rocm_sparse_decode_bf16_num_splits(
|
||
num_queries: int, num_heads: int, sparse_len: int
|
||
) -> int:
|
||
"""Number or kv splits in splitK for the sparse bf16 decode, or 1 for single-pass.
|
||
|
||
Args:
|
||
num_queries: Decode rows in the batch.
|
||
num_heads: Query heads per row.
|
||
sparse_len: Longest selected KV run any decode row can walk.
|
||
|
||
Returns:
|
||
The split count, or 1 when the caller should use the single-pass kernel.
|
||
|
||
"""
|
||
if sparse_len < _SPARSE_DECODE_BF16_MIN_SPLIT_LEN:
|
||
return 1
|
||
block_k = _SPARSE_DECODE_BF16_BLOCK_K
|
||
heads_blocks = triton.cdiv(num_heads, _SPARSE_DECODE_BF16_BLOCK_H)
|
||
select = _decode_gfx950_num_splits if _ON_GFX950 else _decode_num_splits
|
||
num_splits = select(num_queries, heads_blocks, sparse_len, 0.0, block_k)
|
||
# Number of splits cannot exceed the available k tiles
|
||
return max(1, min(num_splits, math.ceil(sparse_len / block_k)))
|
||
|
||
|
||
def _rocm_sparse_attn_decode_ragged_bf16_triton(
|
||
q: torch.Tensor,
|
||
kv: torch.Tensor,
|
||
indices: torch.Tensor,
|
||
indptr: torch.Tensor,
|
||
scale: float,
|
||
attn_sink: torch.Tensor | None,
|
||
nope_head_dim: int,
|
||
rope_head_dim: int,
|
||
num_splits: int,
|
||
out: torch.Tensor | None = None,
|
||
) -> torch.Tensor:
|
||
"""Split-K decode over an bf16 ragged KV cache.
|
||
|
||
Partitions each query's selected tokens across different workgroups and combines
|
||
the partials through reduction.
|
||
|
||
Args:
|
||
q: Queries laid out as ``[sq, h, d]``.
|
||
kv: Unquantized KV rows laid out as ``[skv, d]``.
|
||
indices: Flattened per-query KV slots.
|
||
indptr: Segment offsets into ``indices``, ``[sq + 1]``.
|
||
scale: Softmax scale.
|
||
attn_sink: Optional per-head sink logits.
|
||
nope_head_dim: NoPE width of ``d``.
|
||
rope_head_dim: RoPE width of ``d``.
|
||
num_splits: Number of KV splits per query.
|
||
out: Optional destination with ``d`` trailing elements.
|
||
|
||
Returns:
|
||
The attention output, ``out`` when provided.
|
||
|
||
"""
|
||
assert q.ndim == 3, f"expected q=[sq,h,d], got {q.shape}"
|
||
assert kv.ndim == 2, f"expected kv=[skv,d], got {kv.shape}"
|
||
assert indices.ndim == 1, f"expected indices=[nnz], got {indices.shape}"
|
||
assert indptr.ndim == 1, f"expected indptr=[sq+1], got {indptr.shape}"
|
||
assert not q.is_cpu and not kv.is_cpu and not indices.is_cpu and not indptr.is_cpu
|
||
|
||
indices = _as_int32_contiguous_1d(indices)
|
||
indptr = _as_int32_contiguous_1d(indptr)
|
||
has_attn_sink = attn_sink is not None
|
||
if attn_sink is None:
|
||
attn_sink = torch.empty(1, device=q.device, dtype=torch.float32)
|
||
else:
|
||
attn_sink = attn_sink.contiguous()
|
||
|
||
num_queries, num_heads, head_dim = q.shape
|
||
assert indptr.numel() == num_queries + 1, (
|
||
f"expected indptr shape [{num_queries + 1}], got {indptr.shape}"
|
||
)
|
||
_validate_sparse_dims(
|
||
head_dim,
|
||
nope_head_dim,
|
||
rope_head_dim,
|
||
"_rocm_sparse_attn_decode_ragged_bf16_triton",
|
||
)
|
||
if out is None:
|
||
out = torch.empty_like(q)
|
||
assert out.shape[-1] == head_dim, (
|
||
f"expected out trailing dim {head_dim}, got {out.shape[-1]}"
|
||
)
|
||
|
||
block_h = _SPARSE_DECODE_BF16_BLOCK_H
|
||
block_d = triton.next_power_of_2(head_dim)
|
||
block_k = _SPARSE_DECODE_BF16_BLOCK_K
|
||
heads_blocks = triton.cdiv(num_heads, block_h)
|
||
comb_dim = nope_head_dim + rope_head_dim
|
||
|
||
part_m = torch.empty(
|
||
(num_queries, num_splits, num_heads), dtype=torch.float32, device=q.device
|
||
)
|
||
part_l = torch.empty_like(part_m)
|
||
part_acc = torch.empty(
|
||
(num_queries, num_splits, num_heads, comb_dim),
|
||
dtype=torch.float32,
|
||
device=q.device,
|
||
)
|
||
|
||
_sparse_attn_decode_ragged_bf16_partial_kernel[
|
||
(num_queries, num_splits, heads_blocks)
|
||
](
|
||
q,
|
||
kv,
|
||
indices,
|
||
indptr,
|
||
part_m,
|
||
part_l,
|
||
part_acc,
|
||
q.stride(0),
|
||
q.stride(1),
|
||
q.stride(2),
|
||
kv.stride(0),
|
||
kv.stride(1),
|
||
part_m.stride(0),
|
||
part_m.stride(1),
|
||
part_acc.stride(0),
|
||
part_acc.stride(1),
|
||
part_acc.stride(2),
|
||
num_heads,
|
||
head_dim,
|
||
kv.shape[0],
|
||
float(scale),
|
||
BLOCK_H=block_h,
|
||
BLOCK_D=block_d,
|
||
BLOCK_K=block_k,
|
||
NUM_SPLITS=num_splits,
|
||
num_warps=4,
|
||
)
|
||
|
||
_sparse_attn_decode_reduce_kernel[(num_queries, num_heads)](
|
||
part_m,
|
||
part_l,
|
||
part_acc,
|
||
attn_sink,
|
||
out,
|
||
None,
|
||
None,
|
||
None,
|
||
out.stride(0),
|
||
out.stride(1),
|
||
0,
|
||
head_dim // 32,
|
||
part_m.stride(0),
|
||
part_m.stride(1),
|
||
part_acc.stride(0),
|
||
part_acc.stride(1),
|
||
part_acc.stride(2),
|
||
0,
|
||
num_heads,
|
||
HAS_ATTN_SINK=has_attn_sink,
|
||
ADAPTIVE_SPLITS=False,
|
||
COMB_DIM=comb_dim,
|
||
BLOCK_H=1,
|
||
NUM_SPLITS=num_splits,
|
||
SPLITS_PAD=triton.next_power_of_2(num_splits),
|
||
FUSE_INV_ROPE=False,
|
||
NOPE=nope_head_dim,
|
||
HALF=rope_head_dim // 2,
|
||
QUANT_OUT=False,
|
||
num_warps=4,
|
||
)
|
||
return out
|
||
|
||
|
||
def _rocm_sparse_attn_decode_ragged_triton(
|
||
q: torch.Tensor,
|
||
main_cache: torch.Tensor,
|
||
main_indices: torch.Tensor,
|
||
main_indptr: torch.Tensor,
|
||
scale: float,
|
||
attn_sink: torch.Tensor | None,
|
||
nope_head_dim: int,
|
||
rope_head_dim: int,
|
||
extra_cache: torch.Tensor | None = None,
|
||
extra_indices: torch.Tensor | None = None,
|
||
extra_indptr: torch.Tensor | None = None,
|
||
out: torch.Tensor | None = None,
|
||
extra_cache_nan_free: bool = False,
|
||
adaptive_splits: bool = False,
|
||
inv_rope_positions: torch.Tensor | None = None,
|
||
inv_rope_cos_sin_cache: torch.Tensor | None = None,
|
||
out_mxfp8: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||
) -> torch.Tensor:
|
||
"""Split-K sparse decode; returns the attention output.
|
||
|
||
With ``out_mxfp8 = (data, scale)`` the reduce writes MXFP8 instead of
|
||
bf16: ``data`` is [b, h * d] e4m3 and ``scale`` [b, h * d // 32] E8M0, and
|
||
``data`` viewed as [b, h, d] is returned.
|
||
"""
|
||
assert q.ndim == 3, f"expected q=[b,h,d], got {q.shape}"
|
||
assert main_cache.ndim == 3, (
|
||
f"expected main_cache=[blocks,block,bytes], got {main_cache.shape}"
|
||
)
|
||
assert main_indices.ndim == 1, (
|
||
f"expected main_indices=[nnz], got {main_indices.shape}"
|
||
)
|
||
assert main_indptr.ndim == 1, f"expected main_indptr=[b+1], got {main_indptr.shape}"
|
||
assert (
|
||
not q.is_cpu
|
||
and not main_cache.is_cpu
|
||
and not main_indices.is_cpu
|
||
and not main_indptr.is_cpu
|
||
)
|
||
|
||
main_indices = _as_int32_contiguous_1d(main_indices)
|
||
main_indptr = _as_int32_contiguous_1d(main_indptr)
|
||
has_attn_sink = attn_sink is not None
|
||
if attn_sink is None:
|
||
attn_sink = torch.empty(1, device=q.device, dtype=torch.float32)
|
||
else:
|
||
attn_sink = attn_sink.contiguous()
|
||
|
||
num_queries, num_heads, head_dim = q.shape
|
||
assert main_indptr.numel() == num_queries + 1, (
|
||
f"expected main_indptr shape [{num_queries + 1}], got {main_indptr.shape}"
|
||
)
|
||
_validate_dsv4_sparse_dims(
|
||
head_dim,
|
||
nope_head_dim,
|
||
rope_head_dim,
|
||
"_rocm_sparse_attn_decode_ragged_triton",
|
||
)
|
||
|
||
has_extra = (
|
||
extra_cache is not None
|
||
and extra_indices is not None
|
||
and extra_indptr is not None
|
||
)
|
||
assert not extra_cache_nan_free or (_ON_GFX950 and has_extra), (
|
||
"extra_cache_nan_free requires a gfx950 compressed cache with trusted "
|
||
"canonical-writer provenance"
|
||
)
|
||
if has_extra:
|
||
assert extra_cache is not None
|
||
assert extra_indices is not None
|
||
assert extra_indptr is not None
|
||
assert extra_indices.ndim == 1, (
|
||
f"expected extra_indices=[nnz], got {extra_indices.shape}"
|
||
)
|
||
assert extra_indptr.ndim == 1, (
|
||
f"expected extra_indptr=[b+1], got {extra_indptr.shape}"
|
||
)
|
||
extra_indices = _as_int32_contiguous_1d(extra_indices)
|
||
extra_indptr = _as_int32_contiguous_1d(extra_indptr)
|
||
assert extra_indptr.numel() == num_queries + 1, (
|
||
f"expected extra_indptr shape [{num_queries + 1}], got {extra_indptr.shape}"
|
||
)
|
||
else:
|
||
extra_cache = main_cache
|
||
extra_indices = torch.empty(0, device=q.device, dtype=torch.int32)
|
||
extra_indptr = torch.zeros(num_queries + 1, device=q.device, dtype=torch.int32)
|
||
|
||
block_h = 16
|
||
out_scale = None
|
||
if out_mxfp8 is not None:
|
||
assert out is None, "out and out_mxfp8 are mutually exclusive"
|
||
assert _ON_GFX950, "the MXFP8 reduce epilogue is gfx950-only"
|
||
assert inv_rope_positions is not None, (
|
||
"the MXFP8 output feeds wo_a, so it must be inverse-RoPE'd first"
|
||
)
|
||
out_data, out_scale = out_mxfp8
|
||
assert out_data.dtype == torch.float8_e4m3fn and out_scale.dtype == (
|
||
torch.uint8
|
||
), f"expected e4m3/uint8 MXFP8 buffers, got {out_data.dtype}/{out_scale.dtype}"
|
||
assert out_data.shape == (num_queries, num_heads * head_dim), (
|
||
f"expected MXFP8 data [{num_queries}, {num_heads * head_dim}], "
|
||
f"got {tuple(out_data.shape)}"
|
||
)
|
||
assert out_scale.shape == (num_queries, num_heads * head_dim // 32), (
|
||
f"expected MXFP8 scale [{num_queries}, {num_heads * head_dim // 32}], "
|
||
f"got {tuple(out_scale.shape)}"
|
||
)
|
||
assert out_data.stride(-1) == 1 and out_scale.stride(-1) == 1
|
||
out = out_data.view(num_queries, num_heads, head_dim)
|
||
elif out is None:
|
||
out = torch.empty_like(q, dtype=torch.bfloat16)
|
||
else:
|
||
assert out.shape == q.shape, f"expected out shape {q.shape}, got {out.shape}"
|
||
assert out.device == q.device, (
|
||
f"expected out on device {q.device}, got {out.device}"
|
||
)
|
||
assert out.dtype == torch.bfloat16, (
|
||
f"expected out dtype {torch.bfloat16}, got {out.dtype}"
|
||
)
|
||
heads_blocks = triton.cdiv(num_heads, block_h)
|
||
nope_block = triton.next_power_of_2(nope_head_dim)
|
||
comb_dim = nope_head_dim + rope_head_dim
|
||
is_fnuz = current_platform.is_fp8_fnuz()
|
||
|
||
if not (_ON_GFX942 or _ON_GFX950): # Fallback path for un-tuned architectures.
|
||
block_k = 16 if head_dim >= 256 else 32
|
||
_sparse_attn_decode_ragged_kernel[(num_queries, heads_blocks)](
|
||
q,
|
||
main_cache,
|
||
main_indices,
|
||
main_indptr,
|
||
extra_cache,
|
||
extra_indices,
|
||
extra_indptr,
|
||
attn_sink,
|
||
out,
|
||
q.stride(0),
|
||
q.stride(1),
|
||
out.stride(0),
|
||
out.stride(1),
|
||
main_cache.stride(0),
|
||
extra_cache.stride(0),
|
||
main_cache.shape[0] * main_cache.shape[1],
|
||
extra_cache.shape[0] * extra_cache.shape[1],
|
||
main_cache.shape[1],
|
||
extra_cache.shape[1],
|
||
scale,
|
||
num_heads,
|
||
HAS_ATTN_SINK=has_attn_sink,
|
||
HAS_EXTRA=has_extra,
|
||
NOPE_DIM=nope_head_dim,
|
||
NOPE_BLOCK=nope_block,
|
||
ROPE_DIM=rope_head_dim,
|
||
IS_FNUZ_MAIN=is_fnuz,
|
||
IS_FNUZ_EXTRA=False,
|
||
BLOCK_H=block_h,
|
||
BLOCK_K=block_k,
|
||
num_warps=8,
|
||
)
|
||
return out
|
||
|
||
block_k = 32 # KV tokens walked per split-K iteration. Tuned on gfx950.
|
||
if _ON_GFX950:
|
||
inv_q = 1.0 / max(1, num_queries)
|
||
avg_main_len = main_indices.numel() * inv_q
|
||
avg_extra_len = (extra_indices.numel() * inv_q) if has_extra else 0.0
|
||
num_splits = _decode_gfx950_num_splits(
|
||
num_queries,
|
||
heads_blocks,
|
||
avg_main_len,
|
||
avg_extra_len,
|
||
block_k,
|
||
)
|
||
else:
|
||
# Average per-query segment lengths, read sync-free from the ragged
|
||
# index sizes, let the split heuristic avoid over-splitting.
|
||
inv_q = 1.0 / max(1, num_queries)
|
||
avg_main_len = main_indices.numel() * inv_q
|
||
avg_extra_len = (extra_indices.numel() * inv_q) if has_extra else 0.0
|
||
num_splits = _decode_num_splits(
|
||
num_queries, heads_blocks, avg_main_len, avg_extra_len, block_k
|
||
)
|
||
|
||
base_workgroups = num_queries * heads_blocks
|
||
adaptive_splits = (
|
||
_ON_GFX950 and adaptive_splits and base_workgroups >= 16 and num_splits > 4
|
||
)
|
||
one_wave_splits = (
|
||
max(1, _decode_cu_count() // base_workgroups)
|
||
if adaptive_splits and 16 <= base_workgroups < 64
|
||
else num_splits
|
||
)
|
||
|
||
part_m = torch.empty(
|
||
(num_queries, num_splits, num_heads), dtype=torch.float32, device=q.device
|
||
)
|
||
part_l = torch.empty_like(part_m)
|
||
part_acc = torch.empty(
|
||
(num_queries, num_splits, num_heads, comb_dim),
|
||
dtype=torch.float32,
|
||
device=q.device,
|
||
)
|
||
|
||
if _ON_GFX950:
|
||
_sparse_attn_decode_gfx950_partial_kernel[
|
||
(num_queries, num_splits, heads_blocks)
|
||
](
|
||
q,
|
||
main_cache,
|
||
main_indices,
|
||
main_indptr,
|
||
extra_cache,
|
||
extra_indices,
|
||
extra_indptr,
|
||
part_m,
|
||
part_l,
|
||
part_acc,
|
||
q.stride(0),
|
||
q.stride(1),
|
||
main_cache.stride(0),
|
||
extra_cache.stride(0),
|
||
main_cache.shape[0] * main_cache.shape[1],
|
||
extra_cache.shape[0] * extra_cache.shape[1],
|
||
main_cache.shape[1],
|
||
extra_cache.shape[1],
|
||
scale,
|
||
num_heads,
|
||
HAS_EXTRA=has_extra,
|
||
NOPE_DIM=nope_head_dim,
|
||
ROPE_DIM=rope_head_dim,
|
||
IS_FNUZ_MAIN=is_fnuz,
|
||
IS_FNUZ_EXTRA=False,
|
||
TRUST_EXTRA_CACHE_NAN_FREE=extra_cache_nan_free,
|
||
ADAPTIVE_SPLITS=adaptive_splits,
|
||
ONE_WAVE_SPLITS=one_wave_splits,
|
||
BLOCK_H=block_h,
|
||
BLOCK_K=block_k,
|
||
NUM_SPLITS=num_splits,
|
||
NUM_STAGES=1,
|
||
num_warps=4,
|
||
waves_per_eu=0,
|
||
)
|
||
else:
|
||
_sparse_attn_decode_partial_kernel[(num_queries, num_splits, heads_blocks)](
|
||
q,
|
||
main_cache,
|
||
main_indices,
|
||
main_indptr,
|
||
extra_cache,
|
||
extra_indices,
|
||
extra_indptr,
|
||
part_m,
|
||
part_l,
|
||
part_acc,
|
||
q.stride(0),
|
||
q.stride(1),
|
||
main_cache.stride(0),
|
||
extra_cache.stride(0),
|
||
part_m.stride(0),
|
||
part_m.stride(1),
|
||
part_acc.stride(0),
|
||
part_acc.stride(1),
|
||
part_acc.stride(2),
|
||
main_cache.shape[0] * main_cache.shape[1],
|
||
extra_cache.shape[0] * extra_cache.shape[1],
|
||
main_cache.shape[1],
|
||
extra_cache.shape[1],
|
||
scale,
|
||
num_heads,
|
||
HAS_EXTRA=has_extra,
|
||
NOPE_DIM=nope_head_dim,
|
||
NOPE_BLOCK=nope_block,
|
||
ROPE_DIM=rope_head_dim,
|
||
# main_cache = swa_k_cache (C++ encoder, FNUZ on gfx942 / OCP on gfx950).
|
||
# extra_cache = compressed kv_cache (Triton encoder, OCP everywhere).
|
||
# Reading both with a single IS_FNUZ would decode one of them with the
|
||
# wrong FNUZ/OCP scale ratio (~1.87×).
|
||
IS_FNUZ_MAIN=is_fnuz,
|
||
IS_FNUZ_EXTRA=False,
|
||
BLOCK_H=block_h,
|
||
BLOCK_K=block_k,
|
||
NUM_SPLITS=num_splits,
|
||
NUM_STAGES=1,
|
||
num_warps=4,
|
||
)
|
||
|
||
fuse_inv_rope = inv_rope_positions is not None
|
||
if fuse_inv_rope:
|
||
assert inv_rope_cos_sin_cache is not None
|
||
assert inv_rope_cos_sin_cache.shape[-1] == rope_head_dim, (
|
||
"fused inverse RoPE expects cos_sin_cache laid out as "
|
||
f"[P, {rope_head_dim}] = cos | sin, got "
|
||
f"{tuple(inv_rope_cos_sin_cache.shape)}"
|
||
)
|
||
assert nope_head_dim % 2 == 0 and rope_head_dim % 2 == 0, (
|
||
"fused inverse RoPE pairs adjacent lanes, so both head dims must "
|
||
f"be even, got nope={nope_head_dim} rope={rope_head_dim}"
|
||
)
|
||
|
||
_sparse_attn_decode_reduce_kernel[(num_queries, num_heads)](
|
||
part_m,
|
||
part_l,
|
||
part_acc,
|
||
attn_sink,
|
||
out,
|
||
inv_rope_positions,
|
||
inv_rope_cos_sin_cache,
|
||
out_scale,
|
||
out.stride(0),
|
||
out.stride(1),
|
||
out_scale.stride(0) if out_scale is not None else 0,
|
||
head_dim // 32,
|
||
part_m.stride(0),
|
||
part_m.stride(1),
|
||
part_acc.stride(0),
|
||
part_acc.stride(1),
|
||
part_acc.stride(2),
|
||
inv_rope_cos_sin_cache.stride(0) if inv_rope_cos_sin_cache is not None else 0,
|
||
num_heads,
|
||
HAS_ATTN_SINK=has_attn_sink,
|
||
ADAPTIVE_SPLITS=adaptive_splits,
|
||
COMB_DIM=comb_dim,
|
||
BLOCK_H=1,
|
||
NUM_SPLITS=num_splits,
|
||
SPLITS_PAD=triton.next_power_of_2(num_splits),
|
||
FUSE_INV_ROPE=fuse_inv_rope,
|
||
NOPE=nope_head_dim,
|
||
HALF=rope_head_dim // 2,
|
||
QUANT_OUT=out_scale is not None,
|
||
num_warps=4,
|
||
)
|
||
return out
|
||
|
||
|
||
def _rocm_sparse_attn_decode_triton(
|
||
q: torch.Tensor,
|
||
main_cache: torch.Tensor,
|
||
main_indices: torch.Tensor,
|
||
scale: float,
|
||
attn_sink: torch.Tensor | None,
|
||
nope_head_dim: int,
|
||
rope_head_dim: int,
|
||
extra_cache: torch.Tensor | None = None,
|
||
extra_indices: torch.Tensor | None = None,
|
||
main_lengths: torch.Tensor | None = None,
|
||
extra_lengths: torch.Tensor | None = None,
|
||
main_ragged_indices: torch.Tensor | None = None,
|
||
main_ragged_indptr: torch.Tensor | None = None,
|
||
extra_ragged_indices: torch.Tensor | None = None,
|
||
extra_ragged_indptr: torch.Tensor | None = None,
|
||
out: torch.Tensor | None = None,
|
||
extra_cache_nan_free: bool = False,
|
||
adaptive_splits: bool = False,
|
||
inv_rope_positions: torch.Tensor | None = None,
|
||
inv_rope_cos_sin_cache: torch.Tensor | None = None,
|
||
out_mxfp8: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||
) -> torch.Tensor:
|
||
if main_ragged_indices is None or main_ragged_indptr is None:
|
||
main_ragged_indices, main_ragged_indptr = build_ragged_indices_from_dense(
|
||
main_indices,
|
||
main_lengths
|
||
if main_lengths is not None
|
||
else (main_indices >= 0).sum(dim=-1, dtype=torch.int32),
|
||
num_rows=main_cache.shape[0] * main_cache.shape[1],
|
||
)
|
||
|
||
if (
|
||
(extra_ragged_indices is None or extra_ragged_indptr is None)
|
||
and extra_cache is not None
|
||
and extra_indices is not None
|
||
):
|
||
extra_ragged_indices, extra_ragged_indptr = build_ragged_indices_from_dense(
|
||
extra_indices,
|
||
extra_lengths
|
||
if extra_lengths is not None
|
||
else (extra_indices >= 0).sum(dim=-1, dtype=torch.int32),
|
||
num_rows=extra_cache.shape[0] * extra_cache.shape[1],
|
||
)
|
||
|
||
return _rocm_sparse_attn_decode_ragged_triton(
|
||
q=q,
|
||
main_cache=main_cache,
|
||
main_indices=main_ragged_indices,
|
||
main_indptr=main_ragged_indptr,
|
||
scale=scale,
|
||
attn_sink=attn_sink,
|
||
nope_head_dim=nope_head_dim,
|
||
rope_head_dim=rope_head_dim,
|
||
extra_cache=extra_cache,
|
||
extra_indices=extra_ragged_indices,
|
||
extra_indptr=extra_ragged_indptr,
|
||
out=out,
|
||
extra_cache_nan_free=extra_cache_nan_free,
|
||
adaptive_splits=adaptive_splits,
|
||
inv_rope_positions=inv_rope_positions,
|
||
inv_rope_cos_sin_cache=inv_rope_cos_sin_cache,
|
||
out_mxfp8=out_mxfp8,
|
||
)
|
||
|
||
|
||
def rocm_sparse_attn_prefill(
|
||
q: torch.Tensor,
|
||
kv: torch.Tensor,
|
||
indices: torch.Tensor | None,
|
||
topk_length: torch.Tensor | None,
|
||
scale: float,
|
||
head_dim: int,
|
||
nope_head_dim: int,
|
||
rope_head_dim: int,
|
||
attn_sink: torch.Tensor | None,
|
||
output: torch.Tensor,
|
||
ragged_indices: torch.Tensor | None = None,
|
||
ragged_indptr: torch.Tensor | None = None,
|
||
) -> None:
|
||
assert kv.ndim == 3 and kv.shape[1] == 1, (
|
||
f"ROCm Triton sparse prefill expects kv=[skv,1,d], got {kv.shape}"
|
||
)
|
||
_validate_sparse_dims(
|
||
head_dim,
|
||
nope_head_dim,
|
||
rope_head_dim,
|
||
"rocm_sparse_attn_prefill",
|
||
)
|
||
opus_attn_sink = None if attn_sink is None else attn_sink[: q.shape[1]]
|
||
if (
|
||
_can_use_aiter_sparse_prefill_opus(q, kv.squeeze(1), opus_attn_sink, output)
|
||
and _get_aiter_sparse_prefill_opus() is not None
|
||
):
|
||
if ragged_indices is None or ragged_indptr is None:
|
||
assert indices is not None
|
||
indices_2d = indices.reshape(indices.shape[0], -1)
|
||
ragged_indices, ragged_indptr = build_ragged_indices_from_dense(
|
||
indices_2d,
|
||
topk_length
|
||
if topk_length is not None
|
||
else (indices_2d >= 0).sum(dim=-1, dtype=torch.int32),
|
||
num_rows=kv.shape[0],
|
||
)
|
||
assert opus_attn_sink is not None
|
||
if _rocm_sparse_attn_prefill_ragged_aiter_opus(
|
||
q=q,
|
||
kv=kv.squeeze(1),
|
||
indices=ragged_indices,
|
||
indptr=ragged_indptr,
|
||
scale=scale,
|
||
attn_sink=opus_attn_sink,
|
||
output=output,
|
||
):
|
||
return
|
||
|
||
if ragged_indices is not None and ragged_indptr is not None:
|
||
_rocm_sparse_attn_prefill_ragged_triton(
|
||
q=q,
|
||
kv=kv.squeeze(1),
|
||
indices=ragged_indices,
|
||
indptr=ragged_indptr,
|
||
scale=scale,
|
||
attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]],
|
||
nope_head_dim=nope_head_dim,
|
||
rope_head_dim=rope_head_dim,
|
||
out=output,
|
||
)
|
||
else:
|
||
assert indices is not None
|
||
indices_2d = indices.reshape(indices.shape[0], -1)
|
||
_rocm_sparse_attn_prefill_triton(
|
||
q=q,
|
||
kv=kv.squeeze(1),
|
||
indices=indices_2d,
|
||
scale=scale,
|
||
attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]],
|
||
nope_head_dim=nope_head_dim,
|
||
rope_head_dim=rope_head_dim,
|
||
topk_length=topk_length,
|
||
out=output,
|
||
)
|
||
|
||
|
||
def rocm_sparse_attn_decode_bf16(
|
||
q: torch.Tensor,
|
||
kv: torch.Tensor,
|
||
scale: float,
|
||
head_dim: int,
|
||
nope_head_dim: int,
|
||
rope_head_dim: int,
|
||
attn_sink: torch.Tensor | None,
|
||
output: torch.Tensor,
|
||
ragged_indices: torch.Tensor,
|
||
ragged_indptr: torch.Tensor,
|
||
num_splits: int,
|
||
) -> None:
|
||
"""Run split-K sparse attention over decode rows using an unquantized KV cache.
|
||
|
||
Args:
|
||
q: Decode queries laid out as ``[sq, h, d]``.
|
||
kv: KV cache laid out as ``[skv, 1, d]``.
|
||
scale: Softmax scale.
|
||
head_dim: Post-absorption head width.
|
||
nope_head_dim: NoPE width of ``head_dim``.
|
||
rope_head_dim: RoPE width of ``head_dim``.
|
||
attn_sink: Optional per-head sink logits.
|
||
output: Destination, written in place.
|
||
ragged_indices: Flattened per-query KV slots.
|
||
ragged_indptr: Segment offsets into ``ragged_indices``, ``[sq + 1]``.
|
||
num_splits: KV splits per query, from
|
||
:func:`rocm_sparse_decode_bf16_num_splits`.
|
||
|
||
"""
|
||
assert kv.ndim == 3 and kv.shape[1] == 1, (
|
||
f"ROCm Triton sparse decode expects kv=[skv,1,d], got {kv.shape}"
|
||
)
|
||
_validate_sparse_dims(
|
||
head_dim,
|
||
nope_head_dim,
|
||
rope_head_dim,
|
||
"rocm_sparse_attn_decode_bf16",
|
||
)
|
||
num_queries, num_heads = q.shape[0], q.shape[1]
|
||
direct = output.shape[-1] == head_dim
|
||
out = (
|
||
output
|
||
if direct
|
||
else torch.empty(
|
||
(num_queries, num_heads, head_dim),
|
||
dtype=output.dtype,
|
||
device=output.device,
|
||
)
|
||
)
|
||
_rocm_sparse_attn_decode_ragged_bf16_triton(
|
||
q=q,
|
||
kv=kv.squeeze(1),
|
||
indices=ragged_indices,
|
||
indptr=ragged_indptr,
|
||
scale=scale,
|
||
attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]],
|
||
nope_head_dim=nope_head_dim,
|
||
rope_head_dim=rope_head_dim,
|
||
num_splits=num_splits,
|
||
out=out,
|
||
)
|
||
if not direct:
|
||
output.copy_(out[..., : output.shape[-1]])
|
||
|
||
|
||
def rocm_sparse_attn_decode(
|
||
q: torch.Tensor,
|
||
kv_cache: torch.Tensor | None,
|
||
swa_k_cache: torch.Tensor,
|
||
swa_only: bool,
|
||
topk_indices: torch.Tensor | None,
|
||
topk_lens: torch.Tensor | None,
|
||
swa_indices: torch.Tensor,
|
||
swa_lens: torch.Tensor,
|
||
swa_ragged_indices: torch.Tensor | None,
|
||
swa_ragged_indptr: torch.Tensor | None,
|
||
topk_ragged_indices: torch.Tensor | None,
|
||
topk_ragged_indptr: torch.Tensor | None,
|
||
attn_sink: torch.Tensor | None,
|
||
scale: float,
|
||
head_dim: int,
|
||
nope_head_dim: int,
|
||
rope_head_dim: int,
|
||
output: torch.Tensor | None,
|
||
extra_cache_nan_free: bool = False,
|
||
adaptive_splits: bool = False,
|
||
inv_rope_positions: torch.Tensor | None = None,
|
||
inv_rope_cos_sin_cache: torch.Tensor | None = None,
|
||
output_mxfp8: tuple[torch.Tensor, torch.Tensor] | None = None,
|
||
) -> int:
|
||
"""Run sparse MLA decode into ``output``.
|
||
|
||
Passing ``inv_rope_positions`` folds the inverse RoPE into the reduce
|
||
epilogue. Returns how many leading rows of ``output`` came back rotated,
|
||
so a caller mixing in a decode path that does not fuse still knows what it
|
||
owes the standalone pass. Read it from the eager attention segment only.
|
||
|
||
``output_mxfp8 = (data, scale)`` replaces ``output``: the reduce also
|
||
MXFP8-quantizes the rotated rows for the FP8 wo_a (see
|
||
``_rocm_sparse_attn_decode_ragged_triton``). It needs gfx950 and the fused
|
||
inverse RoPE, and always covers every row.
|
||
"""
|
||
assert swa_k_cache.dtype == torch.uint8, (
|
||
"ROCm Triton sparse decode expects uint8 fp8_ds_mla SWA cache, "
|
||
f"got {swa_k_cache.dtype}"
|
||
)
|
||
_validate_dsv4_sparse_dims(
|
||
head_dim,
|
||
nope_head_dim,
|
||
rope_head_dim,
|
||
"rocm_sparse_attn_decode",
|
||
)
|
||
|
||
main_indices = swa_indices.reshape(swa_indices.shape[0], -1)
|
||
|
||
extra_cache = None
|
||
extra_indices = None
|
||
if not swa_only:
|
||
assert kv_cache is not None
|
||
assert topk_indices is not None or (
|
||
topk_ragged_indices is not None and topk_ragged_indptr is not None
|
||
)
|
||
assert kv_cache.dtype == torch.uint8, (
|
||
"ROCm Triton sparse decode expects uint8 fp8_ds_mla extra cache, "
|
||
f"got {kv_cache.dtype}"
|
||
)
|
||
extra_cache = kv_cache
|
||
if topk_indices is not None:
|
||
extra_indices = topk_indices.reshape(topk_indices.shape[0], -1)
|
||
|
||
if output_mxfp8 is not None:
|
||
assert output is None, "output and output_mxfp8 are mutually exclusive"
|
||
direct_out = None
|
||
else:
|
||
assert output is not None
|
||
direct_out = output if _ON_GFX950 and output.dtype == torch.bfloat16 else None
|
||
attn_out = _rocm_sparse_attn_decode_triton(
|
||
q=q,
|
||
main_cache=swa_k_cache,
|
||
main_indices=main_indices,
|
||
scale=scale,
|
||
attn_sink=None if attn_sink is None else attn_sink[: q.shape[1]],
|
||
nope_head_dim=nope_head_dim,
|
||
rope_head_dim=rope_head_dim,
|
||
extra_cache=extra_cache,
|
||
extra_indices=extra_indices,
|
||
main_lengths=swa_lens,
|
||
extra_lengths=topk_lens,
|
||
main_ragged_indices=swa_ragged_indices,
|
||
main_ragged_indptr=swa_ragged_indptr,
|
||
extra_ragged_indices=topk_ragged_indices,
|
||
extra_ragged_indptr=topk_ragged_indptr,
|
||
out=direct_out,
|
||
extra_cache_nan_free=extra_cache_nan_free,
|
||
adaptive_splits=adaptive_splits,
|
||
inv_rope_positions=inv_rope_positions,
|
||
inv_rope_cos_sin_cache=inv_rope_cos_sin_cache,
|
||
out_mxfp8=output_mxfp8,
|
||
)
|
||
if output_mxfp8 is not None:
|
||
return q.shape[0]
|
||
assert output is not None
|
||
if direct_out is None:
|
||
output.copy_(attn_out.to(output.dtype))
|
||
return output.shape[0] if inv_rope_positions is not None else 0
|
||
|
||
|
||
@triton.jit(do_not_specialize=["token_offset"])
|
||
def _prefill_topk_global_slots_kernel(
|
||
out_ptr,
|
||
lens_ptr,
|
||
topk_ptr,
|
||
topk_stride,
|
||
token_to_req_ptr,
|
||
query_start_loc_ptr,
|
||
seq_lens_ptr,
|
||
is_valid_token_ptr,
|
||
block_table_ptr,
|
||
block_table_stride,
|
||
token_offset,
|
||
num_compressed,
|
||
TOPK: tl.constexpr,
|
||
COMPRESS_RATIO: tl.constexpr,
|
||
BLOCK_SIZE: tl.constexpr,
|
||
BLOCK_W: tl.constexpr,
|
||
):
|
||
row = tl.program_id(0)
|
||
token = row + token_offset
|
||
req = tl.load(token_to_req_ptr + token)
|
||
query_start = tl.load(query_start_loc_ptr + req)
|
||
query_len = tl.load(query_start_loc_ptr + req + 1) - query_start
|
||
pos = tl.load(seq_lens_ptr + req) - query_len + token - query_start
|
||
# Only (pos + 1) // ratio compressed tokens exist yet; the rest of the row
|
||
# can hold stale indices.
|
||
row_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOPK)
|
||
row_len = tl.where(tl.load(is_valid_token_ptr + token), row_len, 0)
|
||
|
||
for start in range(0, TOPK, BLOCK_W):
|
||
cols = start + tl.arange(0, BLOCK_W)
|
||
live = cols < row_len
|
||
local = tl.load(topk_ptr + row * topk_stride + cols, mask=live, other=-1)
|
||
valid = live & (local >= 0) & (local < num_compressed)
|
||
local = tl.where(valid, local, 0)
|
||
block = tl.load(
|
||
block_table_ptr + req * block_table_stride + local // BLOCK_SIZE,
|
||
mask=valid,
|
||
other=0,
|
||
)
|
||
slot = tl.where(valid, block * BLOCK_SIZE + local % BLOCK_SIZE, -1)
|
||
tl.store(out_ptr + row * TOPK + cols, slot, mask=cols < TOPK)
|
||
tl.store(lens_ptr + row, row_len)
|
||
|
||
|
||
def build_prefill_topk_ragged_indices(
|
||
topk_indices: torch.Tensor,
|
||
token_to_req_indices: torch.Tensor,
|
||
query_start_loc: torch.Tensor,
|
||
seq_lens: torch.Tensor,
|
||
is_valid_token: torch.Tensor,
|
||
block_table: torch.Tensor,
|
||
block_size: int,
|
||
compress_ratio: int,
|
||
num_compressed: int,
|
||
token_offset: int,
|
||
num_rows: int = -1,
|
||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||
"""Map prefill top-k rows to a ragged stream of compressed-cache slots.
|
||
|
||
``topk_indices`` holds local compressed positions for the prefill tokens,
|
||
which sit at ``token_offset`` in the batch; ``token_to_req_indices``,
|
||
``query_start_loc``, ``seq_lens`` and ``block_table`` are batch-wide.
|
||
``block_size`` is the compressed cache's, i.e. already divided by the ratio.
|
||
"""
|
||
topk_indices = topk_indices.reshape(topk_indices.shape[0], -1)
|
||
num_tokens, width = topk_indices.shape
|
||
dense = torch.empty(
|
||
(num_tokens, width), dtype=torch.int32, device=topk_indices.device
|
||
)
|
||
lens = torch.empty(num_tokens, dtype=torch.int32, device=topk_indices.device)
|
||
if num_tokens > 0 and width > 0:
|
||
_prefill_topk_global_slots_kernel[(num_tokens,)](
|
||
dense,
|
||
lens,
|
||
topk_indices,
|
||
topk_indices.stride(0),
|
||
token_to_req_indices,
|
||
query_start_loc,
|
||
seq_lens,
|
||
is_valid_token,
|
||
block_table,
|
||
block_table.stride(0),
|
||
token_offset,
|
||
num_compressed,
|
||
TOPK=width,
|
||
COMPRESS_RATIO=compress_ratio,
|
||
BLOCK_SIZE=block_size,
|
||
BLOCK_W=min(triton.next_power_of_2(width), 1024),
|
||
)
|
||
else:
|
||
lens.zero_()
|
||
return build_ragged_indices_from_dense(dense, lens, num_rows=num_rows)
|