[ROCm][DSv4.1][Perf] One-pass dequantize and a gather-sized grid for the compressed K cache (#56720)

Signed-off-by: JohnQinAMD <yanyuan.qin@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
JohnQinAMD
2026-10-05 17:57:46 -07:00
committed by GitHub
co-authored by Claude Opus 5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent b2863c0de9
commit 97bd8c74eb
3 changed files with 195 additions and 2 deletions
+74
View File
@@ -405,6 +405,80 @@ def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
torch.testing.assert_close(cache_backing, expected, rtol=0, atol=0)
@pytest.mark.skipif(
not current_platform.is_cuda_alike(), reason="needs a CUDA or ROCm device"
)
@pytest.mark.parametrize("gather_len", [None, 70, 200])
@pytest.mark.parametrize("one_pass", [False, True])
def test_v41_fp8_gather_is_grid_width_invariant(gather_len, one_pass, monkeypatch):
"""The gather grid only partitions tokens, so its width cannot move the
result, and the rows it writes must match a host-side dequantization.
Covers the fp8_ds_mla record (584 B); the MXFP8 one is covered below.
Both dequantization forms are checked against the same host reference on
whatever device is present: the wrapper picks the one-pass form on ROCm
alone, so otherwise each platform would only ever see one of them.
"""
monkeypatch.setattr(current_platform, "is_rocm", lambda: one_pass)
from vllm.models.deepseek_v41.common.ops import quantize_and_insert_k_cache
from vllm.models.deepseek_v41.common.ops.cache_utils import (
dequantize_and_gather_k_cache_triton,
)
torch.manual_seed(7)
device = "cuda"
block_size, num_blocks = 64, 6
num_tokens = block_size * num_blocks
k = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
# insert takes [num_blocks, block_bytes]; gather takes the record view.
cache = torch.zeros(num_blocks, block_size * 584, dtype=torch.uint8, device=device)
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
quantize_and_insert_k_cache(k, cache, slot_mapping, block_size=block_size)
cache_view = cache.view(num_blocks, block_size, 584)
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
gls = (
None
if gather_len is None
else torch.tensor([gather_len], dtype=torch.int32, device=device)
)
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).view(1, -1)
def gather(bound):
out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device)
dequantize_and_gather_k_cache_triton(
out,
cache_view,
seq_lens,
gls,
block_table,
block_size,
0,
max_gather_len=bound,
)
return out
# None keeps the historical 128; the rest make the wrapper pick other widths
# for the same work, and the output must not notice.
baseline = gather(None)
for bound in (8, num_tokens, 1 << 20):
torch.testing.assert_close(gather(bound), baseline, rtol=0, atol=0)
n = num_tokens if gather_len is None else gather_len
start = num_tokens - n
for i in (0, 1, n // 2, n - 1):
values, scales = _ue8m0_reference(k[start + i, :448], 64, 448.0)
nope = (values.to(torch.float32).view(7, 64) * scales.view(7, 1)).reshape(448)
torch.testing.assert_close(
baseline[0, i, :448], nope.to(torch.bfloat16), rtol=4e-3, atol=1e-6
)
# The RoPE dims are stored verbatim, so they come back exactly.
torch.testing.assert_close(
baseline[0, i, 448:], k[start + i, 448:], rtol=0, atol=0
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
def test_v41_mxfp8_cache_round_trip():
"""quantize_and_insert -> dequantize_and_gather recovers the V4.1 record.
+5
View File
@@ -1387,6 +1387,9 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
gather_lens = swa_metadata.prefill_gather_lens
assert seq_lens is not None
assert gather_lens is not None
# CPU copy, so sizing the gather grid from it does not sync.
seq_lens_cpu = swa_metadata.prefill_seq_lens_cpu
assert seq_lens_cpu is not None
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
query_start_loc = swa_metadata.query_start_loc
@@ -1438,6 +1441,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
block_size=attn_metadata.block_size // self.compress_ratio,
offset=0,
use_fnuz=False,
max_gather_len=int(seq_lens_cpu[chunk_start:chunk_end].max())
// self.compress_ratio,
)
swa_block_table = swa_metadata.block_table[num_decodes:]
@@ -33,7 +33,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.import_utils import has_cutedsl
from vllm.utils.math_utils import next_power_of_2
from vllm.utils.math_utils import cdiv, next_power_of_2
# Per-token byte width of the two paged fp8 records. Both put a page's whole
# data region ahead of its whole scale region, so ``k_cache.shape[-1]`` -- the
@@ -436,6 +436,95 @@ def _dequantize_and_gather_k_kernel(
tl.store(output_row_ptr + bf16_output_offset + chunk_offsets, bf16_vals)
@triton.jit
def _dequantize_and_gather_k_one_pass_kernel(
out_ptr,
out_stride0,
out_stride1,
k_cache_ptr,
seq_lens_ptr,
block_table_ptr,
offset,
gather_lens_ptr,
# Constants
max_blocks_per_seq: tl.constexpr,
fp8_dim: tl.constexpr, # 448
bf16_dim: tl.constexpr, # 64
scale_dim: tl.constexpr, # 8
quant_block: tl.constexpr, # 64 (quantization block size)
cache_block_size: tl.constexpr, # 64 or 128 (paged cache block size)
token_data_size: tl.constexpr, # 576 bytes per token data
block_stride: tl.constexpr, # total bytes per block (padded) int32
output_dim: tl.constexpr, # 512
fp8_max: tl.constexpr,
n_quant_blocks: tl.constexpr, # 7 real blocks; the per-block form only
use_fnuz: tl.constexpr = False,
):
"""``_dequantize_and_gather_k_kernel`` with the token read in one tile.
The seven quantization blocks tile the fp8 region exactly, so the per-block
form's bounds never bite and its twenty-two memory operations per token
reduce to four. Selected on ROCm; CUDA keeps the per-block kernel.
"""
batch_idx = tl.program_id(0)
worker_id = tl.program_id(1)
num_workers = tl.num_programs(1)
seq_len = tl.load(seq_lens_ptr + batch_idx)
if gather_lens_ptr is not None: # noqa: SIM108
gather_len = tl.load(gather_lens_ptr + batch_idx)
else:
gather_len = seq_len
start_pos = seq_len - gather_len
for i in range(worker_id, gather_len, num_workers):
pos = start_pos + i
block_in_seq = pos // cache_block_size
pos_in_block = pos % cache_block_size
block_table_row_ptr = block_table_ptr + batch_idx * max_blocks_per_seq
physical_block_idx = tl.load(block_table_row_ptr + block_in_seq)
cache_block_ptr = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride
token_fp8_ptr = cache_block_ptr + pos_in_block * token_data_size
token_bf16_ptr = token_fp8_ptr + fp8_dim
token_scale_ptr = (
cache_block_ptr
+ cache_block_size * token_data_size
+ pos_in_block * scale_dim
)
output_row_ptr = out_ptr + batch_idx * out_stride0 + (offset + i) * out_stride1
# The tile is scale_dim * quant_block = 512 against an fp8 region of
# 448: the tail reads the token's own bf16 bytes, is scaled by a scale
# nothing reads, and is dropped by the store mask.
tile_dim: tl.constexpr = scale_dim * quant_block
tl.static_assert(tile_dim <= token_data_size, "tile outruns the token")
fp8_offsets = tl.arange(0, tile_dim)
x_uint8 = tl.load(token_fp8_ptr + fp8_offsets)
if use_fnuz:
x_fp8 = x_uint8.to(tl.float8e4b8, bitcast=True)
else:
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
encoded_scale = tl.load(token_scale_ptr + tl.arange(0, scale_dim))
scale = tl.exp2(encoded_scale.to(tl.float32) - 127.0) # UE8M0
tiles = tl.reshape(x_fp8.to(tl.float32), (scale_dim, quant_block))
x_dequant = tl.reshape(tiles * tl.reshape(scale, (scale_dim, 1)), (tile_dim,))
tl.store(
output_row_ptr + fp8_offsets,
x_dequant.to(tl.bfloat16),
mask=fp8_offsets < fp8_dim,
)
bf16_cache_ptr = token_bf16_ptr.to(tl.pointer_type(tl.bfloat16))
bf16_offsets = tl.arange(0, bf16_dim)
tl.store(
output_row_ptr + fp8_dim + bf16_offsets,
tl.load(bf16_cache_ptr + bf16_offsets),
)
@triton.jit
def _dequantize_and_gather_k_mxfp8_kernel(
out_ptr,
@@ -566,6 +655,22 @@ def _dequantize_and_gather_k_nvfp4_kernel(
tl.store(output_row_ptr + d, dequant.to(tl.bfloat16))
# The per-token loop is a chain of dependent loads, so the grid has to hold
# enough chains in flight to cover the latency. The cap binds above 64k tokens,
# where a worker takes more than the eight steps this aims for.
_GATHER_STEPS_PER_WORKER = 8
_GATHER_MIN_WORKERS = 128
_GATHER_MAX_WORKERS = 8192
def gather_num_workers(max_gather_len: int | None) -> int:
"""Grid width for a gather of at most ``max_gather_len`` tokens."""
if max_gather_len is None:
return _GATHER_MIN_WORKERS
workers = cdiv(max_gather_len, _GATHER_STEPS_PER_WORKER)
return max(_GATHER_MIN_WORKERS, min(_GATHER_MAX_WORKERS, workers))
def dequantize_and_gather_k_cache_triton(
# [num_reqs, max_num_tokens, head_size]
out: torch.Tensor,
@@ -580,6 +685,7 @@ def dequantize_and_gather_k_cache_triton(
block_size: int,
offset: int,
use_fnuz: bool = False,
max_gather_len: int | None = None,
) -> None:
num_reqs = seq_lens.shape[0]
NUM_WORKERS = 128
@@ -633,7 +739,13 @@ def dequantize_and_gather_k_cache_triton(
FP8_MAX = 448.0
TOKEN_DATA_SIZE = TOKEN_FP8_DIM + TOKEN_BF16_DIM * 2
_dequantize_and_gather_k_kernel[(num_reqs, NUM_WORKERS)](
gather_workers = gather_num_workers(max_gather_len)
kernel = (
_dequantize_and_gather_k_one_pass_kernel
if current_platform.is_rocm()
else _dequantize_and_gather_k_kernel
)
kernel[(num_reqs, gather_workers)](
out,
out.stride(0),
out.stride(1),
@@ -671,6 +783,7 @@ def dequantize_and_gather_k_cache(
block_size: int,
offset: int,
use_fnuz: bool = False,
max_gather_len: int | None = None,
) -> None:
"""Dequantize and gather a paged DSv4 K cache.
@@ -711,6 +824,7 @@ def dequantize_and_gather_k_cache(
block_table,
block_size,
offset,
max_gather_len=max_gather_len,
use_fnuz=use_fnuz,
)