[ROCm][DSv4.1][Perf] Replace the Torch prefill combine with the shared combine_topk_swa_indices kernel (#56638)

Signed-off-by: JohnQinAMD <yanyuan.qin@amd.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Ren Yuzhou <54501155+yuzhouo7@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
JohnQinAMD
2026-10-08 14:15:18 +08:00
committed by GitHub
co-authored by Claude Opus 5 Ren Yuzhou mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Douglas Lehr
parent 95307ac6d4
commit fb2ac82458
4 changed files with 185 additions and 170 deletions
@@ -41,7 +41,7 @@ def _on_gfx950() -> bool:
# architectures take the fallback decode kernel, so its tests are skipped there.
requires_split_decode_arch = pytest.mark.skipif(
not _on_split_decode_arch(),
reason="split-K decode kernel is only tuned for AMD gfx942/gfx950",
reason="split-K decode and the V4.1 combine tests need an AMD gfx942/gfx950",
)
requires_gfx950 = pytest.mark.skipif(
not _on_gfx950(),
@@ -2407,3 +2407,159 @@ def test_paged_mqa_logits_gate_v41_shuffle_vs_c4a_flat() -> None:
torch.testing.assert_close(
gated_c4a[:, :seq_len], ref[:, :seq_len], atol=2e-3, rtol=2e-3
)
def _combine_topk_swa_indices_torch(
topk_indices,
query_start_loc,
seq_lens,
gather_lens,
window_size,
compress_ratio,
topk,
M,
N,
):
"""The Torch path this kernel replaces, copied from before the switch.
Kept verbatim so the reference cannot drift toward the kernel.
"""
from vllm.models.deepseek_v41.common.ops.cache_utils import (
_SPARSE_PREFILL_TOPK_ALIGNMENT,
)
topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous()
num_tokens = topk_indices.shape[0]
combined_topk = (
(topk + window_size + _SPARSE_PREFILL_TOPK_ALIGNMENT - 1)
// _SPARSE_PREFILL_TOPK_ALIGNMENT
* _SPARSE_PREFILL_TOPK_ALIGNMENT
)
combined_indices = torch.full(
(num_tokens, combined_topk),
fill_value=-1,
dtype=torch.int32,
device=topk_indices.device,
)
combined_lens = torch.empty(
num_tokens, dtype=torch.int32, device=topk_indices.device
)
# query_start_loc may have a non-zero base for a narrowed mixed batch.
query_lens = query_start_loc[1:] - query_start_loc[:-1]
req_ids = torch.repeat_interleave(
torch.arange(seq_lens.shape[0], device=seq_lens.device), query_lens
)
query_starts = query_start_loc[:-1] - query_start_loc[0]
token_offsets = torch.arange(num_tokens, device=seq_lens.device) - (
torch.repeat_interleave(query_starts, query_lens)
)
positions = seq_lens[req_ids] - query_lens[req_ids] + token_offsets
logical_topk_width = min(topk, topk_indices.shape[1])
topk_lens = torch.minimum(
(positions + 1) // compress_ratio,
torch.full_like(positions, logical_topk_width),
).clamp_min(0)
topk_offsets = torch.arange(logical_topk_width, device=seq_lens.device)
topk_mask = topk_offsets[None, :] < topk_lens[:, None]
topk_values = topk_indices[:, :logical_topk_width].to(torch.int32)
topk_valid = topk_mask & (topk_values >= 0) & (topk_values < N)
combined_indices[:, :logical_topk_width] = torch.where(
topk_valid,
topk_values + (M * req_ids).to(torch.int32)[:, None],
-1,
)
swa_lens = torch.minimum(
positions + 1, torch.full_like(positions, window_size)
).clamp_min(0)
swa_offsets = torch.arange(window_size, device=seq_lens.device)
swa_mask = swa_offsets[None, :] < swa_lens[:, None]
swa_columns = topk_lens[:, None] + swa_offsets[None, :]
gather_starts = seq_lens - gather_lens
swa_values = (
M * req_ids[:, None]
+ N
+ swa_offsets[None, :]
+ positions[:, None]
- swa_lens[:, None]
+ 1
- gather_starts[req_ids, None]
).to(torch.int32)
rows = torch.arange(num_tokens, device=seq_lens.device)[:, None].expand_as(
swa_columns
)
flat_dst = rows[swa_mask] * combined_topk + swa_columns[swa_mask]
combined_indices.view(-1).index_copy_(0, flat_dst, swa_values[swa_mask])
combined_lens.copy_((topk_lens + swa_lens).to(torch.int32))
return combined_indices, combined_lens
def _v41_combine_case(case, window):
dev = "cuda"
ratio, topk = 2, 512
if case == "seq_len_below_query_len":
qsl = torch.tensor([0, 64], dtype=torch.int32, device=dev)
sl = torch.tensor([32], dtype=torch.int32, device=dev)
rows = 64
elif case == "seq_len_zero":
# The warmup batch: every pos is negative, so both lengths clamp to 0.
qsl = torch.tensor([0, 32, 64], dtype=torch.int32, device=dev)
sl = torch.zeros(2, dtype=torch.int32, device=dev)
rows = 64
elif case == "compress_ratio_zero":
qsl = torch.tensor([0, 4, 8], dtype=torch.int32, device=dev)
sl = torch.tensor([4, 4], dtype=torch.int32, device=dev)
rows, ratio, topk = 8, 0, 0
elif case == "nonzero_base":
# Prefill rows of a mixed batch: query_start_loc starts past the decodes.
qsl = torch.tensor([32, 64, 128], dtype=torch.int32, device=dev)
sl = torch.tensor([48, 80], dtype=torch.int32, device=dev)
rows = 96
else:
qsl = torch.tensor([0, 32, 96], dtype=torch.int32, device=dev)
sl = torch.tensor([48, 80], dtype=torch.int32, device=dev)
rows = 96
# What the SWA builder gathers: the query plus up to window - 1 rows before it.
gl = torch.minimum(sl, qsl[1:] - qsl[:-1] + window - 1)
set_random_seed(0)
ti = torch.randint(0, 4096, (rows, max(topk, 1)), dtype=torch.int32)
if case == "invalid_topk":
# Failed candidates stay -1 instead of pointing into another request.
ti[:, ::7] = -1
return ti.to(dev), qsl, sl, gl, window, ratio, topk, 100000, 4096
@requires_split_decode_arch
@pytest.mark.parametrize(
"case,window",
[
("seq_len_below_query_len", 128),
("seq_len_zero", 128),
("compress_ratio_zero", 128),
("nonzero_base", 128),
("invalid_topk", 128),
("plain", 64),
("plain", 128),
("plain", 256),
],
)
def test_v41_combine_topk_swa_indices_matches_torch(case, window) -> None:
from vllm.models.deepseek_v41.amd.rocm import combine_topk_swa_indices
args = _v41_combine_case(case, window)
got_indices, got_lens = combine_topk_swa_indices(*args)
ref_indices, ref_lens = _combine_topk_swa_indices_torch(*args)
assert torch.equal(got_lens, ref_lens), (
f"lens differ: ref={ref_lens.tolist()} got={got_lens.tolist()}"
)
assert torch.equal(got_indices, ref_indices)
def test_v41_rocm_combine_uses_shared_kernel() -> None:
"""ROCm prefill takes the shared combine, so it inherits the replay bound."""
from vllm.models.deepseek_v41.amd import rocm
from vllm.models.deepseek_v41.common.ops import cache_utils
assert rocm.combine_topk_swa_indices is cache_utils.combine_topk_swa_indices
@@ -302,8 +302,13 @@ def combine_case(
with_image: bool,
replay_starts: list[int] | None = None,
combine_fn=combine_topk_swa_indices,
in_range_topk: bool = False,
):
"""Run combine_topk_swa_indices and return (indices, lens, expected)."""
"""Run combine_topk_swa_indices and return (indices, lens, expected).
``in_range_topk`` draws top-k indices below N, for combiners that drop
out-of-range ones.
"""
device = torch.device("cuda")
num_reqs = len(seq_lens)
replay_starts = replay_starts or [0] * num_reqs
@@ -326,7 +331,11 @@ def combine_case(
M = N + int(gather_lens.max()) + 8
gen = torch.Generator(device="cpu").manual_seed(0)
topk_indices = torch.randint(
0, 4096, (num_tokens, max(topk, 1)), generator=gen, dtype=torch.int32
0,
N if in_range_topk else 4096,
(num_tokens, max(topk, 1)),
generator=gen,
dtype=torch.int32,
).to(device)
topk_indices = topk_indices[:, : max(topk, 1)]
@@ -415,24 +424,26 @@ def test_combine_topk_swa_with_image_spans(cfg):
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize("cfg", COMBINE_CASES)
def test_v41_combine_topk_swa_stops_at_replay_start(cfg):
@pytest.mark.parametrize("query_len", [24, 20])
def test_v41_combine_topk_swa_stops_at_replay_start(cfg, query_len):
"""SWA bounded replay: the gathered buffer starts at replay_start, so the
window never indexes below it."""
from vllm.models.deepseek_v41.common.ops.cache_utils import (
combine_topk_swa_indices as combine_v41,
)
# Request 0 replays [16, 40): the windows of its first rows would
# otherwise reach below 16.
# Request 0 replays from 16: a 24-token chunk starts there, a 20-token
# chunk leaves only 4 context tokens above it (fewer than the window).
indices, lens, rows, exp_lens = combine_case(
cfg["compress_ratio"],
cfg["topk"],
seq_lens=[40, 12],
query_lens=[24, 12],
query_lens=[query_len, 12],
spans=[[], []],
with_image=False,
replay_starts=[16, 0],
combine_fn=combine_v41,
in_range_topk=True,
)
assert lens.cpu().tolist() == exp_lens
assert indices.cpu().tolist() == rows
+4 -160
View File
@@ -26,7 +26,10 @@ from vllm.models.deepseek_v41.attention import (
DeepseekV4IndexerCache,
_replace_layer_index,
)
from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache
from vllm.models.deepseek_v41.common.ops import (
combine_topk_swa_indices,
dequantize_and_gather_k_cache,
)
from vllm.models.deepseek_v41.sparse_mla import (
DeepseekV4FlashMLAMetadata,
DeepseekV4SparseMLABackend,
@@ -129,165 +132,6 @@ def apply_pre_quantized_block_scaled_mm(
return out.to(dtype=kernel.config.out_dtype)
# ROCm sparse prefill keeps this dense combine local so AMD-specific SWA changes
# do not touch the shared DeepSeek V4 cache utilities.
_SPARSE_PREFILL_TOPK_ALIGNMENT = 128
@triton.jit
def _combine_topk_swa_indices_kernel(
combined_indices_ptr,
combined_indices_stride,
combined_lens_ptr,
topk_indices_ptr,
topk_indices_stride,
query_start_loc_ptr,
seq_lens_ptr,
gather_lens_ptr,
M,
N,
TOP_K: tl.constexpr,
COMPRESS_RATIO: tl.constexpr,
WINDOW_SIZE: tl.constexpr,
TOPK_WIDTH: tl.constexpr,
PADDED_TOP_K: tl.constexpr,
):
batch_idx = tl.program_id(0)
worker_id = tl.program_id(1)
num_workers = tl.num_programs(1)
base = tl.load(query_start_loc_ptr)
query_start = tl.load(query_start_loc_ptr + batch_idx) - base
query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base
query_len = query_end - query_start
seq_len = tl.load(seq_lens_ptr + batch_idx)
gather_len = tl.load(gather_lens_ptr + batch_idx)
start_pos = seq_len - query_len
gather_start = seq_len - gather_len
for token_idx in range(query_start + worker_id, query_end, num_workers):
token_idx_in_query = token_idx - query_start
pos = start_pos + token_idx_in_query
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
swa_len = tl.minimum(pos + 1, WINDOW_SIZE)
topk_offset = tl.arange(0, PADDED_TOP_K)
topk_mask = topk_offset < topk_len
safe_topk_offset = tl.where(topk_offset < TOPK_WIDTH, topk_offset, 0)
topk_indices = tl.load(
topk_indices_ptr + token_idx * topk_indices_stride + safe_topk_offset,
mask=topk_mask,
other=-1,
)
valid_topk = (topk_indices >= 0) & (topk_indices < N)
topk_indices = tl.where(valid_topk, topk_indices + M * batch_idx, -1)
tl.store(
combined_indices_ptr + token_idx * combined_indices_stride + topk_offset,
topk_indices,
mask=topk_mask,
)
swa_offset = tl.arange(0, WINDOW_SIZE)
tl.store(
combined_indices_ptr
+ token_idx * combined_indices_stride
+ topk_len
+ swa_offset,
M * batch_idx + N + swa_offset + pos - swa_len + 1 - gather_start,
mask=swa_offset < swa_len,
)
tl.store(combined_lens_ptr + token_idx, topk_len + swa_len)
def combine_topk_swa_indices(
topk_indices: torch.Tensor,
query_start_loc: torch.Tensor,
seq_lens: torch.Tensor,
gather_lens: torch.Tensor,
window_size: int,
compress_ratio: int,
topk: int,
M: int,
N: int,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Combine compressed-attention and sliding-window indices with Torch.
The Triton implementation inherited from DeepSeek V4 launches a
two-dimensional grid with 128 workers per request. On gfx950 it can issue
an out-of-bounds access for V4.1's mixed prefill metadata (including the
synthetic mixed-token warmup). This path is prefill-only and the tensors
are small, so use ordinary Torch indexing until a gfx950-safe fused kernel
is available.
"""
topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous()
num_tokens = topk_indices.shape[0]
combined_topk = (
(topk + window_size + _SPARSE_PREFILL_TOPK_ALIGNMENT - 1)
// _SPARSE_PREFILL_TOPK_ALIGNMENT
* _SPARSE_PREFILL_TOPK_ALIGNMENT
)
combined_indices = torch.full(
(num_tokens, combined_topk),
fill_value=-1,
dtype=torch.int32,
device=topk_indices.device,
)
combined_lens = torch.empty(
num_tokens, dtype=torch.int32, device=topk_indices.device
)
# query_start_loc may have a non-zero base for a narrowed mixed batch.
query_lens = query_start_loc[1:] - query_start_loc[:-1]
req_ids = torch.repeat_interleave(
torch.arange(seq_lens.shape[0], device=seq_lens.device), query_lens
)
query_starts = query_start_loc[:-1] - query_start_loc[0]
token_offsets = torch.arange(num_tokens, device=seq_lens.device) - (
torch.repeat_interleave(query_starts, query_lens)
)
positions = seq_lens[req_ids] - query_lens[req_ids] + token_offsets
logical_topk_width = min(topk, topk_indices.shape[1])
topk_lens = torch.minimum(
(positions + 1) // compress_ratio,
torch.full_like(positions, logical_topk_width),
).clamp_min(0)
topk_offsets = torch.arange(logical_topk_width, device=seq_lens.device)
topk_mask = topk_offsets[None, :] < topk_lens[:, None]
topk_values = topk_indices[:, :logical_topk_width].to(torch.int32)
topk_valid = topk_mask & (topk_values >= 0) & (topk_values < N)
combined_indices[:, :logical_topk_width] = torch.where(
topk_valid,
topk_values + (M * req_ids).to(torch.int32)[:, None],
-1,
)
swa_lens = torch.minimum(
positions + 1, torch.full_like(positions, window_size)
).clamp_min(0)
swa_offsets = torch.arange(window_size, device=seq_lens.device)
swa_mask = swa_offsets[None, :] < swa_lens[:, None]
swa_columns = topk_lens[:, None] + swa_offsets[None, :]
gather_starts = seq_lens - gather_lens
swa_values = (
M * req_ids[:, None]
+ N
+ swa_offsets[None, :]
+ positions[:, None]
- swa_lens[:, None]
+ 1
- gather_starts[req_ids, None]
).to(torch.int32)
rows = torch.arange(num_tokens, device=seq_lens.device)[:, None].expand_as(
swa_columns
)
flat_dst = rows[swa_mask] * combined_topk + swa_columns[swa_mask]
combined_indices.view(-1).index_copy_(0, flat_dst, swa_values[swa_mask])
combined_lens.copy_((topk_lens + swa_lens).to(torch.int32))
return combined_indices, combined_lens
@triton.jit
def _compute_topk_lens_kernel(
topk_lens_ptr,
@@ -1089,7 +1089,8 @@ class CombineTopkSwaIndicesKernel(
token_idx_in_query = token_idx - query_start
pos = start_pos + token_idx_in_query
if COMPRESS_RATIO > 0:
topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K)
# Warmup batches can put pos below zero; keep the length >= 0.
topk_len = tl.maximum(tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K), 0)
else:
# SWA-only layers pass TOP_K=0; skip the division entirely
# (integer div by constexpr 0 is UB and yields garbage, which
@@ -1100,7 +1101,7 @@ class CombineTopkSwaIndicesKernel(
# start (SWA bounded replay), so the window cannot start before
# the gathered buffer does.
swa_start = tl.maximum(swa_start, gather_start)
swa_len = pos - swa_start + 1
swa_len = tl.maximum(pos - swa_start + 1, 0)
offset = tl.arange(0, PADDED_TOP_K)
mask = offset < topk_len
@@ -1108,9 +1109,12 @@ class CombineTopkSwaIndicesKernel(
topk_indices_ptr + token_idx * topk_indices_stride + offset,
mask=mask,
)
# A failed (-1) or out-of-pool candidate stays -1 instead of landing
# in the previous request's rows or this request's SWA rows.
valid = (topk_indices >= 0) & (topk_indices < N)
tl.store(
combined_indices_ptr + token_idx * combined_indices_stride + offset,
topk_indices + M * batch_idx,
tl.where(valid, topk_indices + M * batch_idx, -1),
mask=mask,
)
# Index into gathered buffer: N + (position - gather_start)