From 97bd8c74ebf12c3d847fb59f24966a83bf6f6920 Mon Sep 17 00:00:00 2001 From: JohnQinAMD Date: Mon, 5 Oct 2026 17:57:46 -0700 Subject: [PATCH] [ROCm][DSv4.1][Perf] One-pass dequantize and a gather-sized grid for the compressed K cache (#56720) Signed-off-by: JohnQinAMD Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/kernels/test_compressor_kv_cache.py | 74 +++++++++++ vllm/models/deepseek_v41/amd/rocm.py | 5 + .../deepseek_v41/common/ops/cache_utils.py | 118 +++++++++++++++++- 3 files changed, 195 insertions(+), 2 deletions(-) diff --git a/tests/kernels/test_compressor_kv_cache.py b/tests/kernels/test_compressor_kv_cache.py index 020dd607b6..cc4f4c72a9 100644 --- a/tests/kernels/test_compressor_kv_cache.py +++ b/tests/kernels/test_compressor_kv_cache.py @@ -405,6 +405,80 @@ def test_v41_rope_insert_mxfp8_record(compress_ratio: int): torch.testing.assert_close(cache_backing, expected, rtol=0, atol=0) +@pytest.mark.skipif( + not current_platform.is_cuda_alike(), reason="needs a CUDA or ROCm device" +) +@pytest.mark.parametrize("gather_len", [None, 70, 200]) +@pytest.mark.parametrize("one_pass", [False, True]) +def test_v41_fp8_gather_is_grid_width_invariant(gather_len, one_pass, monkeypatch): + """The gather grid only partitions tokens, so its width cannot move the + result, and the rows it writes must match a host-side dequantization. + + Covers the fp8_ds_mla record (584 B); the MXFP8 one is covered below. + Both dequantization forms are checked against the same host reference on + whatever device is present: the wrapper picks the one-pass form on ROCm + alone, so otherwise each platform would only ever see one of them. + """ + monkeypatch.setattr(current_platform, "is_rocm", lambda: one_pass) + from vllm.models.deepseek_v41.common.ops import quantize_and_insert_k_cache + from vllm.models.deepseek_v41.common.ops.cache_utils import ( + dequantize_and_gather_k_cache_triton, + ) + + torch.manual_seed(7) + device = "cuda" + block_size, num_blocks = 64, 6 + num_tokens = block_size * num_blocks + + k = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device) + # insert takes [num_blocks, block_bytes]; gather takes the record view. + cache = torch.zeros(num_blocks, block_size * 584, dtype=torch.uint8, device=device) + slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device) + quantize_and_insert_k_cache(k, cache, slot_mapping, block_size=block_size) + cache_view = cache.view(num_blocks, block_size, 584) + + seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device) + gls = ( + None + if gather_len is None + else torch.tensor([gather_len], dtype=torch.int32, device=device) + ) + block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).view(1, -1) + + def gather(bound): + out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device) + dequantize_and_gather_k_cache_triton( + out, + cache_view, + seq_lens, + gls, + block_table, + block_size, + 0, + max_gather_len=bound, + ) + return out + + # None keeps the historical 128; the rest make the wrapper pick other widths + # for the same work, and the output must not notice. + baseline = gather(None) + for bound in (8, num_tokens, 1 << 20): + torch.testing.assert_close(gather(bound), baseline, rtol=0, atol=0) + + n = num_tokens if gather_len is None else gather_len + start = num_tokens - n + for i in (0, 1, n // 2, n - 1): + values, scales = _ue8m0_reference(k[start + i, :448], 64, 448.0) + nope = (values.to(torch.float32).view(7, 64) * scales.view(7, 1)).reshape(448) + torch.testing.assert_close( + baseline[0, i, :448], nope.to(torch.bfloat16), rtol=4e-3, atol=1e-6 + ) + # The RoPE dims are stored verbatim, so they come back exactly. + torch.testing.assert_close( + baseline[0, i, 448:], k[start + i, 448:], rtol=0, atol=0 + ) + + @pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only") def test_v41_mxfp8_cache_round_trip(): """quantize_and_insert -> dequantize_and_gather recovers the V4.1 record. diff --git a/vllm/models/deepseek_v41/amd/rocm.py b/vllm/models/deepseek_v41/amd/rocm.py index 99fcfa6702..64b4a9d3d1 100644 --- a/vllm/models/deepseek_v41/amd/rocm.py +++ b/vllm/models/deepseek_v41/amd/rocm.py @@ -1387,6 +1387,9 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): gather_lens = swa_metadata.prefill_gather_lens assert seq_lens is not None assert gather_lens is not None + # CPU copy, so sizing the gather grid from it does not sync. + seq_lens_cpu = swa_metadata.prefill_seq_lens_cpu + assert seq_lens_cpu is not None query_start_loc_cpu = swa_metadata.query_start_loc_cpu query_start_loc = swa_metadata.query_start_loc @@ -1438,6 +1441,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention): block_size=attn_metadata.block_size // self.compress_ratio, offset=0, use_fnuz=False, + max_gather_len=int(seq_lens_cpu[chunk_start:chunk_end].max()) + // self.compress_ratio, ) swa_block_table = swa_metadata.block_table[num_decodes:] diff --git a/vllm/models/deepseek_v41/common/ops/cache_utils.py b/vllm/models/deepseek_v41/common/ops/cache_utils.py index ad03c32945..b7f07d8d84 100644 --- a/vllm/models/deepseek_v41/common/ops/cache_utils.py +++ b/vllm/models/deepseek_v41/common/ops/cache_utils.py @@ -33,7 +33,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import ( from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.import_utils import has_cutedsl -from vllm.utils.math_utils import next_power_of_2 +from vllm.utils.math_utils import cdiv, next_power_of_2 # Per-token byte width of the two paged fp8 records. Both put a page's whole # data region ahead of its whole scale region, so ``k_cache.shape[-1]`` -- the @@ -436,6 +436,95 @@ def _dequantize_and_gather_k_kernel( tl.store(output_row_ptr + bf16_output_offset + chunk_offsets, bf16_vals) +@triton.jit +def _dequantize_and_gather_k_one_pass_kernel( + out_ptr, + out_stride0, + out_stride1, + k_cache_ptr, + seq_lens_ptr, + block_table_ptr, + offset, + gather_lens_ptr, + # Constants + max_blocks_per_seq: tl.constexpr, + fp8_dim: tl.constexpr, # 448 + bf16_dim: tl.constexpr, # 64 + scale_dim: tl.constexpr, # 8 + quant_block: tl.constexpr, # 64 (quantization block size) + cache_block_size: tl.constexpr, # 64 or 128 (paged cache block size) + token_data_size: tl.constexpr, # 576 bytes per token data + block_stride: tl.constexpr, # total bytes per block (padded) int32 + output_dim: tl.constexpr, # 512 + fp8_max: tl.constexpr, + n_quant_blocks: tl.constexpr, # 7 real blocks; the per-block form only + use_fnuz: tl.constexpr = False, +): + """``_dequantize_and_gather_k_kernel`` with the token read in one tile. + + The seven quantization blocks tile the fp8 region exactly, so the per-block + form's bounds never bite and its twenty-two memory operations per token + reduce to four. Selected on ROCm; CUDA keeps the per-block kernel. + """ + batch_idx = tl.program_id(0) + worker_id = tl.program_id(1) + num_workers = tl.num_programs(1) + + seq_len = tl.load(seq_lens_ptr + batch_idx) + if gather_lens_ptr is not None: # noqa: SIM108 + gather_len = tl.load(gather_lens_ptr + batch_idx) + else: + gather_len = seq_len + start_pos = seq_len - gather_len + + for i in range(worker_id, gather_len, num_workers): + pos = start_pos + i + block_in_seq = pos // cache_block_size + pos_in_block = pos % cache_block_size + + block_table_row_ptr = block_table_ptr + batch_idx * max_blocks_per_seq + physical_block_idx = tl.load(block_table_row_ptr + block_in_seq) + cache_block_ptr = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride + + token_fp8_ptr = cache_block_ptr + pos_in_block * token_data_size + token_bf16_ptr = token_fp8_ptr + fp8_dim + token_scale_ptr = ( + cache_block_ptr + + cache_block_size * token_data_size + + pos_in_block * scale_dim + ) + output_row_ptr = out_ptr + batch_idx * out_stride0 + (offset + i) * out_stride1 + + # The tile is scale_dim * quant_block = 512 against an fp8 region of + # 448: the tail reads the token's own bf16 bytes, is scaled by a scale + # nothing reads, and is dropped by the store mask. + tile_dim: tl.constexpr = scale_dim * quant_block + tl.static_assert(tile_dim <= token_data_size, "tile outruns the token") + fp8_offsets = tl.arange(0, tile_dim) + x_uint8 = tl.load(token_fp8_ptr + fp8_offsets) + if use_fnuz: + x_fp8 = x_uint8.to(tl.float8e4b8, bitcast=True) + else: + x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True) + + encoded_scale = tl.load(token_scale_ptr + tl.arange(0, scale_dim)) + scale = tl.exp2(encoded_scale.to(tl.float32) - 127.0) # UE8M0 + tiles = tl.reshape(x_fp8.to(tl.float32), (scale_dim, quant_block)) + x_dequant = tl.reshape(tiles * tl.reshape(scale, (scale_dim, 1)), (tile_dim,)) + tl.store( + output_row_ptr + fp8_offsets, + x_dequant.to(tl.bfloat16), + mask=fp8_offsets < fp8_dim, + ) + + bf16_cache_ptr = token_bf16_ptr.to(tl.pointer_type(tl.bfloat16)) + bf16_offsets = tl.arange(0, bf16_dim) + tl.store( + output_row_ptr + fp8_dim + bf16_offsets, + tl.load(bf16_cache_ptr + bf16_offsets), + ) + + @triton.jit def _dequantize_and_gather_k_mxfp8_kernel( out_ptr, @@ -566,6 +655,22 @@ def _dequantize_and_gather_k_nvfp4_kernel( tl.store(output_row_ptr + d, dequant.to(tl.bfloat16)) +# The per-token loop is a chain of dependent loads, so the grid has to hold +# enough chains in flight to cover the latency. The cap binds above 64k tokens, +# where a worker takes more than the eight steps this aims for. +_GATHER_STEPS_PER_WORKER = 8 +_GATHER_MIN_WORKERS = 128 +_GATHER_MAX_WORKERS = 8192 + + +def gather_num_workers(max_gather_len: int | None) -> int: + """Grid width for a gather of at most ``max_gather_len`` tokens.""" + if max_gather_len is None: + return _GATHER_MIN_WORKERS + workers = cdiv(max_gather_len, _GATHER_STEPS_PER_WORKER) + return max(_GATHER_MIN_WORKERS, min(_GATHER_MAX_WORKERS, workers)) + + def dequantize_and_gather_k_cache_triton( # [num_reqs, max_num_tokens, head_size] out: torch.Tensor, @@ -580,6 +685,7 @@ def dequantize_and_gather_k_cache_triton( block_size: int, offset: int, use_fnuz: bool = False, + max_gather_len: int | None = None, ) -> None: num_reqs = seq_lens.shape[0] NUM_WORKERS = 128 @@ -633,7 +739,13 @@ def dequantize_and_gather_k_cache_triton( FP8_MAX = 448.0 TOKEN_DATA_SIZE = TOKEN_FP8_DIM + TOKEN_BF16_DIM * 2 - _dequantize_and_gather_k_kernel[(num_reqs, NUM_WORKERS)]( + gather_workers = gather_num_workers(max_gather_len) + kernel = ( + _dequantize_and_gather_k_one_pass_kernel + if current_platform.is_rocm() + else _dequantize_and_gather_k_kernel + ) + kernel[(num_reqs, gather_workers)]( out, out.stride(0), out.stride(1), @@ -671,6 +783,7 @@ def dequantize_and_gather_k_cache( block_size: int, offset: int, use_fnuz: bool = False, + max_gather_len: int | None = None, ) -> None: """Dequantize and gather a paged DSv4 K cache. @@ -711,6 +824,7 @@ def dequantize_and_gather_k_cache( block_table, block_size, offset, + max_gather_len=max_gather_len, use_fnuz=use_fnuz, )