[ROCm][DSv4.1][Perf] One-pass dequantize and a gather-sized grid for the compressed K cache (#56720)
Signed-off-by: JohnQinAMD <yanyuan.qin@amd.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
b2863c0de9
commit
97bd8c74eb
@@ -405,6 +405,80 @@ def test_v41_rope_insert_mxfp8_record(compress_ratio: int):
|
||||
torch.testing.assert_close(cache_backing, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_cuda_alike(), reason="needs a CUDA or ROCm device"
|
||||
)
|
||||
@pytest.mark.parametrize("gather_len", [None, 70, 200])
|
||||
@pytest.mark.parametrize("one_pass", [False, True])
|
||||
def test_v41_fp8_gather_is_grid_width_invariant(gather_len, one_pass, monkeypatch):
|
||||
"""The gather grid only partitions tokens, so its width cannot move the
|
||||
result, and the rows it writes must match a host-side dequantization.
|
||||
|
||||
Covers the fp8_ds_mla record (584 B); the MXFP8 one is covered below.
|
||||
Both dequantization forms are checked against the same host reference on
|
||||
whatever device is present: the wrapper picks the one-pass form on ROCm
|
||||
alone, so otherwise each platform would only ever see one of them.
|
||||
"""
|
||||
monkeypatch.setattr(current_platform, "is_rocm", lambda: one_pass)
|
||||
from vllm.models.deepseek_v41.common.ops import quantize_and_insert_k_cache
|
||||
from vllm.models.deepseek_v41.common.ops.cache_utils import (
|
||||
dequantize_and_gather_k_cache_triton,
|
||||
)
|
||||
|
||||
torch.manual_seed(7)
|
||||
device = "cuda"
|
||||
block_size, num_blocks = 64, 6
|
||||
num_tokens = block_size * num_blocks
|
||||
|
||||
k = torch.randn(num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
# insert takes [num_blocks, block_bytes]; gather takes the record view.
|
||||
cache = torch.zeros(num_blocks, block_size * 584, dtype=torch.uint8, device=device)
|
||||
slot_mapping = torch.arange(num_tokens, dtype=torch.int64, device=device)
|
||||
quantize_and_insert_k_cache(k, cache, slot_mapping, block_size=block_size)
|
||||
cache_view = cache.view(num_blocks, block_size, 584)
|
||||
|
||||
seq_lens = torch.tensor([num_tokens], dtype=torch.int32, device=device)
|
||||
gls = (
|
||||
None
|
||||
if gather_len is None
|
||||
else torch.tensor([gather_len], dtype=torch.int32, device=device)
|
||||
)
|
||||
block_table = torch.arange(num_blocks, dtype=torch.int32, device=device).view(1, -1)
|
||||
|
||||
def gather(bound):
|
||||
out = torch.zeros(1, num_tokens, 512, dtype=torch.bfloat16, device=device)
|
||||
dequantize_and_gather_k_cache_triton(
|
||||
out,
|
||||
cache_view,
|
||||
seq_lens,
|
||||
gls,
|
||||
block_table,
|
||||
block_size,
|
||||
0,
|
||||
max_gather_len=bound,
|
||||
)
|
||||
return out
|
||||
|
||||
# None keeps the historical 128; the rest make the wrapper pick other widths
|
||||
# for the same work, and the output must not notice.
|
||||
baseline = gather(None)
|
||||
for bound in (8, num_tokens, 1 << 20):
|
||||
torch.testing.assert_close(gather(bound), baseline, rtol=0, atol=0)
|
||||
|
||||
n = num_tokens if gather_len is None else gather_len
|
||||
start = num_tokens - n
|
||||
for i in (0, 1, n // 2, n - 1):
|
||||
values, scales = _ue8m0_reference(k[start + i, :448], 64, 448.0)
|
||||
nope = (values.to(torch.float32).view(7, 64) * scales.view(7, 1)).reshape(448)
|
||||
torch.testing.assert_close(
|
||||
baseline[0, i, :448], nope.to(torch.bfloat16), rtol=4e-3, atol=1e-6
|
||||
)
|
||||
# The RoPE dims are stored verbatim, so they come back exactly.
|
||||
torch.testing.assert_close(
|
||||
baseline[0, i, 448:], k[start + i, 448:], rtol=0, atol=0
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
|
||||
def test_v41_mxfp8_cache_round_trip():
|
||||
"""quantize_and_insert -> dequantize_and_gather recovers the V4.1 record.
|
||||
|
||||
@@ -1387,6 +1387,9 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
gather_lens = swa_metadata.prefill_gather_lens
|
||||
assert seq_lens is not None
|
||||
assert gather_lens is not None
|
||||
# CPU copy, so sizing the gather grid from it does not sync.
|
||||
seq_lens_cpu = swa_metadata.prefill_seq_lens_cpu
|
||||
assert seq_lens_cpu is not None
|
||||
|
||||
query_start_loc_cpu = swa_metadata.query_start_loc_cpu
|
||||
query_start_loc = swa_metadata.query_start_loc
|
||||
@@ -1438,6 +1441,8 @@ class DeepseekV41ROCMAiterMLAAttention(DeepseekV4Attention):
|
||||
block_size=attn_metadata.block_size // self.compress_ratio,
|
||||
offset=0,
|
||||
use_fnuz=False,
|
||||
max_gather_len=int(seq_lens_cpu[chunk_start:chunk_end].max())
|
||||
// self.compress_ratio,
|
||||
)
|
||||
|
||||
swa_block_table = swa_metadata.block_table[num_decodes:]
|
||||
|
||||
@@ -33,7 +33,7 @@ from vllm.model_executor.warmup.jit_warmup_triton_helper import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.import_utils import has_cutedsl
|
||||
from vllm.utils.math_utils import next_power_of_2
|
||||
from vllm.utils.math_utils import cdiv, next_power_of_2
|
||||
|
||||
# Per-token byte width of the two paged fp8 records. Both put a page's whole
|
||||
# data region ahead of its whole scale region, so ``k_cache.shape[-1]`` -- the
|
||||
@@ -436,6 +436,95 @@ def _dequantize_and_gather_k_kernel(
|
||||
tl.store(output_row_ptr + bf16_output_offset + chunk_offsets, bf16_vals)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dequantize_and_gather_k_one_pass_kernel(
|
||||
out_ptr,
|
||||
out_stride0,
|
||||
out_stride1,
|
||||
k_cache_ptr,
|
||||
seq_lens_ptr,
|
||||
block_table_ptr,
|
||||
offset,
|
||||
gather_lens_ptr,
|
||||
# Constants
|
||||
max_blocks_per_seq: tl.constexpr,
|
||||
fp8_dim: tl.constexpr, # 448
|
||||
bf16_dim: tl.constexpr, # 64
|
||||
scale_dim: tl.constexpr, # 8
|
||||
quant_block: tl.constexpr, # 64 (quantization block size)
|
||||
cache_block_size: tl.constexpr, # 64 or 128 (paged cache block size)
|
||||
token_data_size: tl.constexpr, # 576 bytes per token data
|
||||
block_stride: tl.constexpr, # total bytes per block (padded) int32
|
||||
output_dim: tl.constexpr, # 512
|
||||
fp8_max: tl.constexpr,
|
||||
n_quant_blocks: tl.constexpr, # 7 real blocks; the per-block form only
|
||||
use_fnuz: tl.constexpr = False,
|
||||
):
|
||||
"""``_dequantize_and_gather_k_kernel`` with the token read in one tile.
|
||||
|
||||
The seven quantization blocks tile the fp8 region exactly, so the per-block
|
||||
form's bounds never bite and its twenty-two memory operations per token
|
||||
reduce to four. Selected on ROCm; CUDA keeps the per-block kernel.
|
||||
"""
|
||||
batch_idx = tl.program_id(0)
|
||||
worker_id = tl.program_id(1)
|
||||
num_workers = tl.num_programs(1)
|
||||
|
||||
seq_len = tl.load(seq_lens_ptr + batch_idx)
|
||||
if gather_lens_ptr is not None: # noqa: SIM108
|
||||
gather_len = tl.load(gather_lens_ptr + batch_idx)
|
||||
else:
|
||||
gather_len = seq_len
|
||||
start_pos = seq_len - gather_len
|
||||
|
||||
for i in range(worker_id, gather_len, num_workers):
|
||||
pos = start_pos + i
|
||||
block_in_seq = pos // cache_block_size
|
||||
pos_in_block = pos % cache_block_size
|
||||
|
||||
block_table_row_ptr = block_table_ptr + batch_idx * max_blocks_per_seq
|
||||
physical_block_idx = tl.load(block_table_row_ptr + block_in_seq)
|
||||
cache_block_ptr = k_cache_ptr + physical_block_idx.to(tl.int64) * block_stride
|
||||
|
||||
token_fp8_ptr = cache_block_ptr + pos_in_block * token_data_size
|
||||
token_bf16_ptr = token_fp8_ptr + fp8_dim
|
||||
token_scale_ptr = (
|
||||
cache_block_ptr
|
||||
+ cache_block_size * token_data_size
|
||||
+ pos_in_block * scale_dim
|
||||
)
|
||||
output_row_ptr = out_ptr + batch_idx * out_stride0 + (offset + i) * out_stride1
|
||||
|
||||
# The tile is scale_dim * quant_block = 512 against an fp8 region of
|
||||
# 448: the tail reads the token's own bf16 bytes, is scaled by a scale
|
||||
# nothing reads, and is dropped by the store mask.
|
||||
tile_dim: tl.constexpr = scale_dim * quant_block
|
||||
tl.static_assert(tile_dim <= token_data_size, "tile outruns the token")
|
||||
fp8_offsets = tl.arange(0, tile_dim)
|
||||
x_uint8 = tl.load(token_fp8_ptr + fp8_offsets)
|
||||
if use_fnuz:
|
||||
x_fp8 = x_uint8.to(tl.float8e4b8, bitcast=True)
|
||||
else:
|
||||
x_fp8 = x_uint8.to(tl.float8e4nv, bitcast=True)
|
||||
|
||||
encoded_scale = tl.load(token_scale_ptr + tl.arange(0, scale_dim))
|
||||
scale = tl.exp2(encoded_scale.to(tl.float32) - 127.0) # UE8M0
|
||||
tiles = tl.reshape(x_fp8.to(tl.float32), (scale_dim, quant_block))
|
||||
x_dequant = tl.reshape(tiles * tl.reshape(scale, (scale_dim, 1)), (tile_dim,))
|
||||
tl.store(
|
||||
output_row_ptr + fp8_offsets,
|
||||
x_dequant.to(tl.bfloat16),
|
||||
mask=fp8_offsets < fp8_dim,
|
||||
)
|
||||
|
||||
bf16_cache_ptr = token_bf16_ptr.to(tl.pointer_type(tl.bfloat16))
|
||||
bf16_offsets = tl.arange(0, bf16_dim)
|
||||
tl.store(
|
||||
output_row_ptr + fp8_dim + bf16_offsets,
|
||||
tl.load(bf16_cache_ptr + bf16_offsets),
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _dequantize_and_gather_k_mxfp8_kernel(
|
||||
out_ptr,
|
||||
@@ -566,6 +655,22 @@ def _dequantize_and_gather_k_nvfp4_kernel(
|
||||
tl.store(output_row_ptr + d, dequant.to(tl.bfloat16))
|
||||
|
||||
|
||||
# The per-token loop is a chain of dependent loads, so the grid has to hold
|
||||
# enough chains in flight to cover the latency. The cap binds above 64k tokens,
|
||||
# where a worker takes more than the eight steps this aims for.
|
||||
_GATHER_STEPS_PER_WORKER = 8
|
||||
_GATHER_MIN_WORKERS = 128
|
||||
_GATHER_MAX_WORKERS = 8192
|
||||
|
||||
|
||||
def gather_num_workers(max_gather_len: int | None) -> int:
|
||||
"""Grid width for a gather of at most ``max_gather_len`` tokens."""
|
||||
if max_gather_len is None:
|
||||
return _GATHER_MIN_WORKERS
|
||||
workers = cdiv(max_gather_len, _GATHER_STEPS_PER_WORKER)
|
||||
return max(_GATHER_MIN_WORKERS, min(_GATHER_MAX_WORKERS, workers))
|
||||
|
||||
|
||||
def dequantize_and_gather_k_cache_triton(
|
||||
# [num_reqs, max_num_tokens, head_size]
|
||||
out: torch.Tensor,
|
||||
@@ -580,6 +685,7 @@ def dequantize_and_gather_k_cache_triton(
|
||||
block_size: int,
|
||||
offset: int,
|
||||
use_fnuz: bool = False,
|
||||
max_gather_len: int | None = None,
|
||||
) -> None:
|
||||
num_reqs = seq_lens.shape[0]
|
||||
NUM_WORKERS = 128
|
||||
@@ -633,7 +739,13 @@ def dequantize_and_gather_k_cache_triton(
|
||||
FP8_MAX = 448.0
|
||||
TOKEN_DATA_SIZE = TOKEN_FP8_DIM + TOKEN_BF16_DIM * 2
|
||||
|
||||
_dequantize_and_gather_k_kernel[(num_reqs, NUM_WORKERS)](
|
||||
gather_workers = gather_num_workers(max_gather_len)
|
||||
kernel = (
|
||||
_dequantize_and_gather_k_one_pass_kernel
|
||||
if current_platform.is_rocm()
|
||||
else _dequantize_and_gather_k_kernel
|
||||
)
|
||||
kernel[(num_reqs, gather_workers)](
|
||||
out,
|
||||
out.stride(0),
|
||||
out.stride(1),
|
||||
@@ -671,6 +783,7 @@ def dequantize_and_gather_k_cache(
|
||||
block_size: int,
|
||||
offset: int,
|
||||
use_fnuz: bool = False,
|
||||
max_gather_len: int | None = None,
|
||||
) -> None:
|
||||
"""Dequantize and gather a paged DSv4 K cache.
|
||||
|
||||
@@ -711,6 +824,7 @@ def dequantize_and_gather_k_cache(
|
||||
block_table,
|
||||
block_size,
|
||||
offset,
|
||||
max_gather_len=max_gather_len,
|
||||
use_fnuz=use_fnuz,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user