[ROCm][DSv4.1][Perf] One-pass dequantize and a gather-sized grid for the compressed K cache (#56720)

Signed-off-by: JohnQinAMD <yanyuan.qin@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
JohnQinAMD
2026-10-05 17:57:46 -07:00
committed by GitHub
co-authored by Claude Opus 5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b2863c0de9
commit 97bd8c74eb
3 changed files with 195 additions and 2 deletions
+5
View File
@@ -1387,6 +1387,9 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
gather_lens = swa_metadata.prefill_gather_lens
assert seq_lens is not None
assert gather_lens is not None
# CPU copy, so sizing the gather grid from it does not sync.
seq_lens_cpu = swa_metadata.prefill_seq_lens_cpu
assert seq_lens_cpu is not None
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
query_start_loc = swa_metadata.query_start_loc
@@ -1438,6 +1441,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
block_size=attn_metadata.block_size // self.compress_ratio,
offset=0,
use_fnuz=False,
max_gather_len=int(seq_lens_cpu[chunk_start:chunk_end].max())
// self.compress_ratio,
)
swa_block_table = swa_metadata.block_table[num_decodes:]
@@ -33,7 +33,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.math_utils import cdiv, next_power_of_2
# Per-token byte width of the two paged fp8 records. Both put a page's whole
# data region ahead of its whole scale region, so ``k_cache.shape[-1]`` -- the
@@ -436,6 +436,95 @@ def _dequantize_and_gather_k_kernel(
tl.store(output_row_ptr + bf16_output_offset + chunk_offsets, bf16_vals)
@triton.jit
def _dequantize_and_gather_k_one_pass_kernel(
out_ptr,
out_stride0,
out_stride1,
k_cache_ptr,
seq_lens_ptr,
block_table_ptr,
offset,
gather_lens_ptr,
# Constants
max_blocks_per_seq: tl.constexpr,
fp8_dim: tl.constexpr, # 448
bf16_dim: tl.constexpr, # 64
scale_dim: tl.constexpr, # 8
quant_block: tl.constexpr, # 64 (quantization block size)
cache_block_size: tl.constexpr, # 64 or 128 (paged cache block size)
token_data_size: tl.constexpr, # 576 bytes per token data
block_stride: tl.constexpr, # total bytes per block (padded) int32
output_dim: tl.constexpr, # 512
fp8_max: tl.constexpr,
n_quant_blocks: tl.constexpr, # 7 real blocks; the per-block form only
use_fnuz: tl.constexpr = False,
):
"""``_dequantize_and_gather_k_kernel`` with the token read in one tile.
The seven quantization blocks tile the fp8 region exactly, so the per-block
form's bounds never bite and its twenty-two memory operations per token
reduce to four. Selected on ROCm; CUDA keeps the per-block kernel.
"""
batch_idx = tl.program_id(0)
worker_id = tl.program_id(1)
num_workers = tl.num_programs(1)
seq_len = tl.load(seq_lens_ptr + batch_idx)
if gather_lens_ptr is not None: # noqa: SIM108
gather_len = tl.load(gather_lens_ptr + batch_idx)
else:
gather_len = seq_len
start_pos = seq_len - gather_len
for i in range(worker_id, gather_len, num_workers):
pos = start_pos + i
block_in_seq = pos // cache_block_size
pos_in_block = pos % cache_block_size
block_table_row_ptr = block_table_ptr + batch_idx * max_blocks_per_seq
physical_block_idx = tl.load(block_table_row_ptr + block_in_seq)
cache_block_ptr = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride
token_fp8_ptr = cache_block_ptr + pos_in_block * token_data_size
token_bf16_ptr = token_fp8_ptr + fp8_dim
token_scale_ptr = (
cache_block_ptr
+ cache_block_size * token_data_size
+ pos_in_block * scale_dim
)
output_row_ptr = out_ptr + batch_idx * out_stride0 + (offset + i) * out_stride1
# The tile is scale_dim * quant_block = 512 against an fp8 region of
# 448: the tail reads the token's own bf16 bytes, is scaled by a scale
# nothing reads, and is dropped by the store mask.
tile_dim: tl.constexpr = scale_dim * quant_block
tl.static_assert(tile_dim <= token_data_size, "tile outruns the token")
fp8_offsets = tl.arange(0, tile_dim)
x_uint8 = tl.load(token_fp8_ptr + fp8_offsets)
if use_fnuz:
x_fp8 = x_uint8.to(tl.float8e4b8, bitcast=True)
else:
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
encoded_scale = tl.load(token_scale_ptr + tl.arange(0, scale_dim))
scale = tl.exp2(encoded_scale.to(tl.float32) - 127.0) # UE8M0
tiles = tl.reshape(x_fp8.to(tl.float32), (scale_dim, quant_block))
x_dequant = tl.reshape(tiles * tl.reshape(scale, (scale_dim, 1)), (tile_dim,))
tl.store(
output_row_ptr + fp8_offsets,
x_dequant.to(tl.bfloat16),
mask=fp8_offsets < fp8_dim,
)
bf16_cache_ptr = token_bf16_ptr.to(tl.pointer_type(tl.bfloat16))
bf16_offsets = tl.arange(0, bf16_dim)
tl.store(
output_row_ptr + fp8_dim + bf16_offsets,
tl.load(bf16_cache_ptr + bf16_offsets),
)
@triton.jit
def _dequantize_and_gather_k_mxfp8_kernel(
out_ptr,
@@ -566,6 +655,22 @@ def _dequantize_and_gather_k_nvfp4_kernel(
tl.store(output_row_ptr + d, dequant.to(tl.bfloat16))
# The per-token loop is a chain of dependent loads, so the grid has to hold
# enough chains in flight to cover the latency. The cap binds above 64k tokens,
# where a worker takes more than the eight steps this aims for.
_GATHER_STEPS_PER_WORKER = 8
_GATHER_MIN_WORKERS = 128
_GATHER_MAX_WORKERS = 8192
def gather_num_workers(max_gather_len: int | None) -> int:
"""Grid width for a gather of at most ``max_gather_len`` tokens."""
if max_gather_len is None:
return _GATHER_MIN_WORKERS
workers = cdiv(max_gather_len, _GATHER_STEPS_PER_WORKER)
return max(_GATHER_MIN_WORKERS, min(_GATHER_MAX_WORKERS, workers))
def dequantize_and_gather_k_cache_triton(
# [num_reqs, max_num_tokens, head_size]
out: torch.Tensor,
@@ -580,6 +685,7 @@ def dequantize_and_gather_k_cache_triton(
block_size: int,
offset: int,
use_fnuz: bool = False,
max_gather_len: int | None = None,
) -> None:
num_reqs = seq_lens.shape[0]
NUM_WORKERS = 128
@@ -633,7 +739,13 @@ def dequantize_and_gather_k_cache_triton(
FP8_MAX = 448.0
TOKEN_DATA_SIZE = TOKEN_FP8_DIM + TOKEN_BF16_DIM * 2
_dequantize_and_gather_k_kernel[(num_reqs, NUM_WORKERS)](
gather_workers = gather_num_workers(max_gather_len)
kernel = (
_dequantize_and_gather_k_one_pass_kernel
if current_platform.is_rocm()
else _dequantize_and_gather_k_kernel
)
kernel[(num_reqs, gather_workers)](
out,
out.stride(0),
out.stride(1),
@@ -671,6 +783,7 @@ def dequantize_and_gather_k_cache(
block_size: int,
offset: int,
use_fnuz: bool = False,
max_gather_len: int | None = None,
) -> None:
"""Dequantize and gather a paged DSv4 K cache.
@@ -711,6 +824,7 @@ def dequantize_and_gather_k_cache(
block_table,
block_size,
offset,
max_gather_len=max_gather_len,
use_fnuz=use_fnuz,
)