[ROCm][DSv4] Fix sparse-indexer logits collapse on gfx950/gfx942 (#50455)
Signed-off-by: Frida Andersson <fanderss@amd.com> Signed-off-by: frida-andersson <fanderss@amd.com>
This commit is contained in:
@@ -66,6 +66,11 @@ _GFX950_DSV4_NATIVE_MAX_COLUMNS = 1024 * 1024
|
||||
_GFX950_AITER_SPARSE_PREFILL_OPUS_MIN_QUERIES = 1024
|
||||
|
||||
|
||||
def _indexer_k_is_c4a_block_flat(compress_ratio: int) -> bool:
|
||||
"""V4.0 C4A is block-flat (NORMAL). Ratio 1 and 2 are 16×16 SHUFFLE."""
|
||||
return compress_ratio == 4
|
||||
|
||||
|
||||
def _get_aiter_top_k_kernel(
|
||||
*,
|
||||
is_prefill: bool,
|
||||
@@ -458,6 +463,7 @@ def cp_gather_indexer_k_quant_cache_triton(
|
||||
token_to_seq: torch.Tensor,
|
||||
block_tile_size: int = 16,
|
||||
head_tile_size: int = 16,
|
||||
cache_layout: str | None = None,
|
||||
):
|
||||
num_tokens = k_fp8.size(0)
|
||||
block_size = k_cache.size(1)
|
||||
@@ -470,7 +476,10 @@ def cp_gather_indexer_k_quant_cache_triton(
|
||||
k_cache_scale = k_cache[:, block_size * head_dim :].view(torch.float32)
|
||||
grid = (num_tokens,)
|
||||
k_fp8_scale = k_fp8_scale.view(torch.float32)
|
||||
layout = "NORMAL" if block_size == 1 else "SHUFFLE"
|
||||
if cache_layout is None:
|
||||
layout = "NORMAL" if block_size == 1 else "SHUFFLE"
|
||||
else:
|
||||
layout = cache_layout
|
||||
kernel_args = (
|
||||
k_cache_value,
|
||||
k_cache_scale,
|
||||
@@ -607,6 +616,158 @@ def fp8_paged_mqa_logits_torch(
|
||||
return logits
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fp8_paged_mqa_logits_decode_kernel(
|
||||
q_ptr, # fp8 [B, NEXT_N, H, D]
|
||||
kv_val_ptr, # fp8, block-flat: [num_blocks, block_size*(D+4)]
|
||||
kv_scale_ptr, # fp32, block-flat: [num_blocks, block_size*(D+4)//4]
|
||||
weights_ptr, # fp32 [B*NEXT_N, H]
|
||||
ctx_lens_ptr, # int32 [B*NEXT_N] if CTX_PER_ROW else [B]
|
||||
block_tables_ptr, # int32 [B, max_blocks]
|
||||
logits_ptr, # fp32 [B*NEXT_N, max_model_len]
|
||||
stride_q_b,
|
||||
stride_q_n,
|
||||
stride_q_h,
|
||||
stride_w_row,
|
||||
stride_kvblk_fp8,
|
||||
stride_kvblk_f32,
|
||||
scale_region_off, # block_size*D // 4 (fp32 offset of scale region within a block)
|
||||
stride_bt_b,
|
||||
stride_logits_row,
|
||||
max_blocks, # block_tables width; guards the block-table gather
|
||||
max_model_len,
|
||||
NUM_HEADS: tl.constexpr,
|
||||
HEAD_SIZE: tl.constexpr,
|
||||
BLOCK_SIZE: tl.constexpr, # paged-cache page size
|
||||
BLOCK_KV: tl.constexpr, # positions per tile; must divide BLOCK_SIZE
|
||||
N_SPLITS: tl.constexpr, # KV-tile parallelism factor (grid dim 1)
|
||||
NEXT_N: tl.constexpr, # query positions per batch (1 = decode; >1 = MTP verify)
|
||||
CTX_PER_ROW: tl.constexpr, # ctx_lens is already per-(b, n)
|
||||
):
|
||||
# scale_region_off = block_size*D//4 needs D % 4 == 0 (D=128).
|
||||
tl.static_assert(HEAD_SIZE % 4 == 0)
|
||||
# Grid (B*NEXT_N, N_SPLITS): disjoint KV tiles per program, no host sync.
|
||||
row = tl.program_id(0)
|
||||
split = tl.program_id(1)
|
||||
b = row // NEXT_N
|
||||
n = row % NEXT_N
|
||||
if CTX_PER_ROW:
|
||||
seq_len = tl.load(ctx_lens_ptr + row)
|
||||
else:
|
||||
seq_len = tl.load(ctx_lens_ptr + b) - NEXT_N + n + 1
|
||||
seq_len = tl.minimum(tl.maximum(seq_len, 0), max_model_len)
|
||||
|
||||
h = tl.arange(0, NUM_HEADS)
|
||||
d = tl.arange(0, HEAD_SIZE)
|
||||
# keep q/kv fp8 -> fp8 MFMA (f32 accum), not the slow f32 path
|
||||
q = tl.load(
|
||||
q_ptr + b * stride_q_b + n * stride_q_n + h[:, None] * stride_q_h + d[None, :]
|
||||
)
|
||||
w = tl.load(weights_ptr + row * stride_w_row + h).to(tl.float32) # [H]
|
||||
|
||||
kv_col = tl.arange(0, BLOCK_KV)
|
||||
for kv_start in tl.range(split * BLOCK_KV, seq_len, N_SPLITS * BLOCK_KV):
|
||||
pos = kv_start + kv_col
|
||||
# Tiles are page-aligned (BLOCK_KV | BLOCK_SIZE), so a tile is one page.
|
||||
logical_blk = kv_start // BLOCK_SIZE
|
||||
blk_ok = logical_blk < max_blocks # guard against a wild page -> fault
|
||||
mask_pos = (pos < seq_len) & blk_ok
|
||||
page = tl.load(
|
||||
block_tables_ptr + b * stride_bt_b + logical_blk, mask=blk_ok, other=0
|
||||
).to(tl.int64)
|
||||
pos_in_blk = pos - logical_blk * BLOCK_SIZE # [BLOCK_KV] == pos % BLOCK_SIZE
|
||||
|
||||
# block-flat layout: values region, then scales region.
|
||||
val_off = page * stride_kvblk_fp8 + pos_in_blk[:, None] * HEAD_SIZE + d[None, :]
|
||||
kv = tl.load(
|
||||
kv_val_ptr + val_off, mask=mask_pos[:, None], other=0.0
|
||||
) # [BLOCK_KV, D] fp8
|
||||
sc = tl.load(
|
||||
kv_scale_ptr + page * stride_kvblk_f32 + scale_region_off + pos_in_blk,
|
||||
mask=mask_pos,
|
||||
other=0.0,
|
||||
) # [BLOCK_KV]
|
||||
|
||||
# fp8 upcasts exactly; bit-identical on gfx942, ~1e-5 rel on gfx950
|
||||
# (accum order), top-k unchanged.
|
||||
s = tl.dot(kv, tl.trans(q)) # [BLOCK_KV, H]
|
||||
s = tl.maximum(s, 0.0)
|
||||
s = s * w[None, :]
|
||||
s = tl.sum(s, axis=1) # [BLOCK_KV]
|
||||
s = s * sc
|
||||
tl.store(logits_ptr + row * stride_logits_row + pos, s, mask=mask_pos)
|
||||
|
||||
|
||||
def rocm_fp8_paged_mqa_logits_triton(
|
||||
q_fp8: torch.Tensor,
|
||||
kv_cache_fp8: torch.Tensor,
|
||||
weights: torch.Tensor,
|
||||
context_lens: torch.Tensor,
|
||||
block_tables: torch.Tensor,
|
||||
max_model_len: int,
|
||||
) -> torch.Tensor:
|
||||
"""Triton paged MQA-logits for decode and MTP; matches the torch ref but
|
||||
has no host sync, so it is safe to capture under a full CUDA graph."""
|
||||
batch_size, next_n, num_heads, head_size = q_fp8.shape
|
||||
block_size = kv_cache_fp8.shape[1]
|
||||
BLOCK_KV = 64
|
||||
assert block_size % BLOCK_KV == 0
|
||||
|
||||
fp8_dtype = current_platform.fp8_dtype()
|
||||
num_blocks = kv_cache_fp8.shape[0]
|
||||
kv_flat = kv_cache_fp8.reshape(
|
||||
num_blocks, -1
|
||||
) # uint8 [num_blocks, block_size*(D+4)]
|
||||
kv_val = kv_flat.view(fp8_dtype) # [num_blocks, block_size*(D+4)] fp8
|
||||
kv_scale = kv_flat.view(torch.float32) # [num_blocks, block_size*(D+4)//4] fp32
|
||||
|
||||
cl = context_lens.reshape(-1)
|
||||
ctx_per_row = not (next_n > 1 and cl.numel() == batch_size)
|
||||
|
||||
max_blocks = block_tables.shape[1]
|
||||
(out_logits,) = current_workspace_manager().get_simultaneous(
|
||||
((batch_size * next_n, max_model_len), torch.float32),
|
||||
)
|
||||
|
||||
# Memory-bound over the KV range: split each row's keys across programs so
|
||||
# few-row / long-context launches still fill the GPU. Cap splits at the
|
||||
# device CU count (304 on gfx942, 256 on gfx950) rather than a gfx950-sized
|
||||
# constant. All terms are static at launch, so the grid stays CUDA-graph-safe.
|
||||
rows = batch_size * next_n
|
||||
tiles_cap = (max_model_len + BLOCK_KV - 1) // BLOCK_KV
|
||||
N_SPLITS = max(1, min(max(1, _decode_cu_count()), tiles_cap, 1024 // rows))
|
||||
_fp8_paged_mqa_logits_decode_kernel[(rows, N_SPLITS)](
|
||||
q_fp8,
|
||||
kv_val,
|
||||
kv_scale,
|
||||
weights,
|
||||
cl,
|
||||
block_tables,
|
||||
out_logits,
|
||||
q_fp8.stride(0),
|
||||
q_fp8.stride(1),
|
||||
q_fp8.stride(2),
|
||||
weights.stride(0),
|
||||
kv_val.stride(0),
|
||||
kv_scale.stride(0),
|
||||
(block_size * head_size) // 4,
|
||||
block_tables.stride(0),
|
||||
out_logits.stride(0),
|
||||
max_blocks,
|
||||
max_model_len,
|
||||
NUM_HEADS=num_heads,
|
||||
HEAD_SIZE=head_size,
|
||||
BLOCK_SIZE=block_size,
|
||||
BLOCK_KV=BLOCK_KV,
|
||||
N_SPLITS=N_SPLITS,
|
||||
NEXT_N=next_n,
|
||||
CTX_PER_ROW=ctx_per_row,
|
||||
num_warps=4,
|
||||
num_stages=2,
|
||||
)
|
||||
return out_logits
|
||||
|
||||
|
||||
@functools.lru_cache
|
||||
def paged_mqa_logits_module():
|
||||
paged_mqa_logits_module_path = None
|
||||
@@ -632,6 +793,8 @@ def rocm_fp8_paged_mqa_logits(
|
||||
block_tables: torch.Tensor,
|
||||
schedule_metadata: torch.Tensor,
|
||||
max_model_len: int,
|
||||
*,
|
||||
compress_ratio: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""Compute FP8 MQA logits using paged KV-cache.
|
||||
|
||||
@@ -639,8 +802,7 @@ def rocm_fp8_paged_mqa_logits(
|
||||
q_fp8: Query tensor of shape [B, next_n, H, D]. Casted to
|
||||
`torch.float8_e4m3fn` by caller.
|
||||
kv_cache_fp8: Paged KV-cache in packed FP8+scale layout with shape
|
||||
[num_blocks, block_size, 1, D+4], dtype `torch.uint8`. The last
|
||||
4 bytes per (block,pos) store the `float` dequant scale.
|
||||
[num_blocks, block_size, 1, D+4], dtype `torch.uint8`.
|
||||
weights: Tensor of shape [B * next_n, H], dtype `torch.float32`.
|
||||
context_lens: Tensor of shape [B], dtype int32; effective context length
|
||||
for each batch element.
|
||||
@@ -649,6 +811,7 @@ def rocm_fp8_paged_mqa_logits(
|
||||
schedule_metadata: Returned by `get_paged_mqa_logits_metadata`;
|
||||
used to distribute work across SMs.
|
||||
max_model_len: Maximum sequence length used to size the logits output.
|
||||
compress_ratio: C4A (4) takes block-flat Triton; 1 and 2 stay on AITER.
|
||||
|
||||
Returns:
|
||||
Logits tensor of shape [B * next_n, max_model_len], dtype
|
||||
@@ -657,11 +820,26 @@ def rocm_fp8_paged_mqa_logits(
|
||||
"""
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
|
||||
aiter_paged_mqa_logits_module = None
|
||||
# if rocm_aiter_ops.is_enabled():
|
||||
batch_size, next_n = q_fp8.shape[:2]
|
||||
block_size = kv_cache_fp8.shape[1]
|
||||
|
||||
# C4A only: Flash/DSv3.2 also skip insert but still write SHUFFLE.
|
||||
if (
|
||||
(_ON_GFX950 or _ON_GFX942)
|
||||
and _indexer_k_is_c4a_block_flat(compress_ratio)
|
||||
and block_size > 1
|
||||
):
|
||||
if block_size % 64 == 0:
|
||||
return rocm_fp8_paged_mqa_logits_triton(
|
||||
q_fp8, kv_cache_fp8, weights, context_lens, block_tables, max_model_len
|
||||
)
|
||||
# Non 64-aligned page size (not used in prod): eager torch ref.
|
||||
return fp8_paged_mqa_logits_torch(
|
||||
q_fp8, kv_cache_fp8, weights, context_lens, block_tables, max_model_len
|
||||
)
|
||||
|
||||
aiter_paged_mqa_logits_module = None
|
||||
|
||||
if rocm_aiter_ops.is_enabled() or rocm_aiter_ops.is_rdna_aiter_enabled():
|
||||
aiter_paged_mqa_logits_module = paged_mqa_logits_module()
|
||||
|
||||
@@ -1128,6 +1306,9 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
chunk.block_table,
|
||||
chunk.cu_seq_lens,
|
||||
token_to_seq=chunk.token_to_seq,
|
||||
cache_layout=(
|
||||
"NORMAL" if _indexer_k_is_c4a_block_flat(compress_ratio) else None
|
||||
),
|
||||
)
|
||||
logits = rocm_fp8_mqa_logits(
|
||||
q_fp8[chunk.token_start : chunk.token_end],
|
||||
@@ -1225,6 +1406,7 @@ def rocm_aiter_sparse_attn_indexer(
|
||||
decode_metadata.block_table,
|
||||
decode_metadata.schedule_metadata,
|
||||
max_model_len=max_model_len,
|
||||
compress_ratio=compress_ratio,
|
||||
)
|
||||
|
||||
if candidate_blocks is not None:
|
||||
|
||||
Reference in New Issue
Block a user