diff --git a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py index ca2cf2eb70..362ac6b7ce 100644 --- a/tests/kernels/attention/test_rocm_triton_attn_dsv4.py +++ b/tests/kernels/attention/test_rocm_triton_attn_dsv4.py @@ -41,7 +41,7 @@ def _on_gfx950() -> bool: # architectures take the fallback decode kernel, so its tests are skipped there. requires_split_decode_arch = pytest.mark.skipif( not _on_split_decode_arch(), - reason="split-K decode kernel is only tuned for AMD gfx942/gfx950", + reason="split-K decode and the V4.1 combine tests need an AMD gfx942/gfx950", ) requires_gfx950 = pytest.mark.skipif( not _on_gfx950(), @@ -2407,3 +2407,159 @@ def test_paged_mqa_logits_gate_v41_shuffle_vs_c4a_flat() -> None: torch.testing.assert_close( gated_c4a[:, :seq_len], ref[:, :seq_len], atol=2e-3, rtol=2e-3 ) + + +def _combine_topk_swa_indices_torch( + topk_indices, + query_start_loc, + seq_lens, + gather_lens, + window_size, + compress_ratio, + topk, + M, + N, +): + """The Torch path this kernel replaces, copied from before the switch. + + Kept verbatim so the reference cannot drift toward the kernel. + """ + from vllm.models.deepseek_v41.common.ops.cache_utils import ( + _SPARSE_PREFILL_TOPK_ALIGNMENT, + ) + + topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous() + num_tokens = topk_indices.shape[0] + combined_topk = ( + (topk + window_size + _SPARSE_PREFILL_TOPK_ALIGNMENT - 1) + // _SPARSE_PREFILL_TOPK_ALIGNMENT + * _SPARSE_PREFILL_TOPK_ALIGNMENT + ) + combined_indices = torch.full( + (num_tokens, combined_topk), + fill_value=-1, + dtype=torch.int32, + device=topk_indices.device, + ) + combined_lens = torch.empty( + num_tokens, dtype=torch.int32, device=topk_indices.device + ) + + # query_start_loc may have a non-zero base for a narrowed mixed batch. + query_lens = query_start_loc[1:] - query_start_loc[:-1] + req_ids = torch.repeat_interleave( + torch.arange(seq_lens.shape[0], device=seq_lens.device), query_lens + ) + query_starts = query_start_loc[:-1] - query_start_loc[0] + token_offsets = torch.arange(num_tokens, device=seq_lens.device) - ( + torch.repeat_interleave(query_starts, query_lens) + ) + positions = seq_lens[req_ids] - query_lens[req_ids] + token_offsets + + logical_topk_width = min(topk, topk_indices.shape[1]) + topk_lens = torch.minimum( + (positions + 1) // compress_ratio, + torch.full_like(positions, logical_topk_width), + ).clamp_min(0) + topk_offsets = torch.arange(logical_topk_width, device=seq_lens.device) + topk_mask = topk_offsets[None, :] < topk_lens[:, None] + topk_values = topk_indices[:, :logical_topk_width].to(torch.int32) + topk_valid = topk_mask & (topk_values >= 0) & (topk_values < N) + combined_indices[:, :logical_topk_width] = torch.where( + topk_valid, + topk_values + (M * req_ids).to(torch.int32)[:, None], + -1, + ) + + swa_lens = torch.minimum( + positions + 1, torch.full_like(positions, window_size) + ).clamp_min(0) + swa_offsets = torch.arange(window_size, device=seq_lens.device) + swa_mask = swa_offsets[None, :] < swa_lens[:, None] + swa_columns = topk_lens[:, None] + swa_offsets[None, :] + gather_starts = seq_lens - gather_lens + swa_values = ( + M * req_ids[:, None] + + N + + swa_offsets[None, :] + + positions[:, None] + - swa_lens[:, None] + + 1 + - gather_starts[req_ids, None] + ).to(torch.int32) + rows = torch.arange(num_tokens, device=seq_lens.device)[:, None].expand_as( + swa_columns + ) + flat_dst = rows[swa_mask] * combined_topk + swa_columns[swa_mask] + combined_indices.view(-1).index_copy_(0, flat_dst, swa_values[swa_mask]) + combined_lens.copy_((topk_lens + swa_lens).to(torch.int32)) + return combined_indices, combined_lens + + +def _v41_combine_case(case, window): + dev = "cuda" + ratio, topk = 2, 512 + if case == "seq_len_below_query_len": + qsl = torch.tensor([0, 64], dtype=torch.int32, device=dev) + sl = torch.tensor([32], dtype=torch.int32, device=dev) + rows = 64 + elif case == "seq_len_zero": + # The warmup batch: every pos is negative, so both lengths clamp to 0. + qsl = torch.tensor([0, 32, 64], dtype=torch.int32, device=dev) + sl = torch.zeros(2, dtype=torch.int32, device=dev) + rows = 64 + elif case == "compress_ratio_zero": + qsl = torch.tensor([0, 4, 8], dtype=torch.int32, device=dev) + sl = torch.tensor([4, 4], dtype=torch.int32, device=dev) + rows, ratio, topk = 8, 0, 0 + elif case == "nonzero_base": + # Prefill rows of a mixed batch: query_start_loc starts past the decodes. + qsl = torch.tensor([32, 64, 128], dtype=torch.int32, device=dev) + sl = torch.tensor([48, 80], dtype=torch.int32, device=dev) + rows = 96 + else: + qsl = torch.tensor([0, 32, 96], dtype=torch.int32, device=dev) + sl = torch.tensor([48, 80], dtype=torch.int32, device=dev) + rows = 96 + # What the SWA builder gathers: the query plus up to window - 1 rows before it. + gl = torch.minimum(sl, qsl[1:] - qsl[:-1] + window - 1) + set_random_seed(0) + ti = torch.randint(0, 4096, (rows, max(topk, 1)), dtype=torch.int32) + if case == "invalid_topk": + # Failed candidates stay -1 instead of pointing into another request. + ti[:, ::7] = -1 + return ti.to(dev), qsl, sl, gl, window, ratio, topk, 100000, 4096 + + +@requires_split_decode_arch +@pytest.mark.parametrize( + "case,window", + [ + ("seq_len_below_query_len", 128), + ("seq_len_zero", 128), + ("compress_ratio_zero", 128), + ("nonzero_base", 128), + ("invalid_topk", 128), + ("plain", 64), + ("plain", 128), + ("plain", 256), + ], +) +def test_v41_combine_topk_swa_indices_matches_torch(case, window) -> None: + from vllm.models.deepseek_v41.amd.rocm import combine_topk_swa_indices + + args = _v41_combine_case(case, window) + got_indices, got_lens = combine_topk_swa_indices(*args) + ref_indices, ref_lens = _combine_topk_swa_indices_torch(*args) + assert torch.equal(got_lens, ref_lens), ( + f"lens differ: ref={ref_lens.tolist()} got={got_lens.tolist()}" + ) + assert torch.equal(got_indices, ref_indices) + + +def test_v41_rocm_combine_uses_shared_kernel() -> None: + """ROCm prefill takes the shared combine, so it inherits the replay bound.""" + from vllm.models.deepseek_v41.amd import rocm + from vllm.models.deepseek_v41.common.ops import cache_utils + + assert rocm.combine_topk_swa_indices is cache_utils.combine_topk_swa_indices diff --git a/tests/v1/attention/test_deepseek_v4_swa_visible.py b/tests/v1/attention/test_deepseek_v4_swa_visible.py index 7a74ce8c81..8f08929549 100644 --- a/tests/v1/attention/test_deepseek_v4_swa_visible.py +++ b/tests/v1/attention/test_deepseek_v4_swa_visible.py @@ -302,8 +302,13 @@ def combine_case( with_image: bool, replay_starts: list[int] | None = None, combine_fn=combine_topk_swa_indices, + in_range_topk: bool = False, ): - """Run combine_topk_swa_indices and return (indices, lens, expected).""" + """Run combine_topk_swa_indices and return (indices, lens, expected). + + ``in_range_topk`` draws top-k indices below N, for combiners that drop + out-of-range ones. + """ device = torch.device("cuda") num_reqs = len(seq_lens) replay_starts = replay_starts or [0] * num_reqs @@ -326,7 +331,11 @@ def combine_case( M = N + int(gather_lens.max()) + 8 gen = torch.Generator(device="cpu").manual_seed(0) topk_indices = torch.randint( - 0, 4096, (num_tokens, max(topk, 1)), generator=gen, dtype=torch.int32 + 0, + N if in_range_topk else 4096, + (num_tokens, max(topk, 1)), + generator=gen, + dtype=torch.int32, ).to(device) topk_indices = topk_indices[:, : max(topk, 1)] @@ -415,24 +424,26 @@ def test_combine_topk_swa_with_image_spans(cfg): @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") @pytest.mark.parametrize("cfg", COMBINE_CASES) -def test_v41_combine_topk_swa_stops_at_replay_start(cfg): +@pytest.mark.parametrize("query_len", [24, 20]) +def test_v41_combine_topk_swa_stops_at_replay_start(cfg, query_len): """SWA bounded replay: the gathered buffer starts at replay_start, so the window never indexes below it.""" from vllm.models.deepseek_v41.common.ops.cache_utils import ( combine_topk_swa_indices as combine_v41, ) - # Request 0 replays [16, 40): the windows of its first rows would - # otherwise reach below 16. + # Request 0 replays from 16: a 24-token chunk starts there, a 20-token + # chunk leaves only 4 context tokens above it (fewer than the window). indices, lens, rows, exp_lens = combine_case( cfg["compress_ratio"], cfg["topk"], seq_lens=[40, 12], - query_lens=[24, 12], + query_lens=[query_len, 12], spans=[[], []], with_image=False, replay_starts=[16, 0], combine_fn=combine_v41, + in_range_topk=True, ) assert lens.cpu().tolist() == exp_lens assert indices.cpu().tolist() == rows diff --git a/vllm/models/deepseek_v41/amd/rocm.py b/vllm/models/deepseek_v41/amd/rocm.py index 64b4a9d3d1..3aa54f9b0e 100644 --- a/vllm/models/deepseek_v41/amd/rocm.py +++ b/vllm/models/deepseek_v41/amd/rocm.py @@ -26,7 +26,10 @@ from vllm.models.deepseek_v41.attention import ( DeepseekV4IndexerCache, _replace_layer_index, ) -from vllm.models.deepseek_v41.common.ops import dequantize_and_gather_k_cache +from vllm.models.deepseek_v41.common.ops import ( + combine_topk_swa_indices, + dequantize_and_gather_k_cache, +) from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4FlashMLAMetadata, DeepseekV4SparseMLABackend, @@ -129,165 +132,6 @@ def apply_pre_quantized_block_scaled_mm( return out.to(dtype=kernel.config.out_dtype) -# ROCm sparse prefill keeps this dense combine local so AMD-specific SWA changes -# do not touch the shared DeepSeek V4 cache utilities. -_SPARSE_PREFILL_TOPK_ALIGNMENT = 128 - - -@triton.jit -def _combine_topk_swa_indices_kernel( - combined_indices_ptr, - combined_indices_stride, - combined_lens_ptr, - topk_indices_ptr, - topk_indices_stride, - query_start_loc_ptr, - seq_lens_ptr, - gather_lens_ptr, - M, - N, - TOP_K: tl.constexpr, - COMPRESS_RATIO: tl.constexpr, - WINDOW_SIZE: tl.constexpr, - TOPK_WIDTH: tl.constexpr, - PADDED_TOP_K: tl.constexpr, -): - batch_idx = tl.program_id(0) - worker_id = tl.program_id(1) - num_workers = tl.num_programs(1) - - base = tl.load(query_start_loc_ptr) - query_start = tl.load(query_start_loc_ptr + batch_idx) - base - query_end = tl.load(query_start_loc_ptr + batch_idx + 1) - base - query_len = query_end - query_start - seq_len = tl.load(seq_lens_ptr + batch_idx) - gather_len = tl.load(gather_lens_ptr + batch_idx) - start_pos = seq_len - query_len - gather_start = seq_len - gather_len - - for token_idx in range(query_start + worker_id, query_end, num_workers): - token_idx_in_query = token_idx - query_start - pos = start_pos + token_idx_in_query - topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K) - swa_len = tl.minimum(pos + 1, WINDOW_SIZE) - - topk_offset = tl.arange(0, PADDED_TOP_K) - topk_mask = topk_offset < topk_len - safe_topk_offset = tl.where(topk_offset < TOPK_WIDTH, topk_offset, 0) - topk_indices = tl.load( - topk_indices_ptr + token_idx * topk_indices_stride + safe_topk_offset, - mask=topk_mask, - other=-1, - ) - valid_topk = (topk_indices >= 0) & (topk_indices < N) - topk_indices = tl.where(valid_topk, topk_indices + M * batch_idx, -1) - tl.store( - combined_indices_ptr + token_idx * combined_indices_stride + topk_offset, - topk_indices, - mask=topk_mask, - ) - - swa_offset = tl.arange(0, WINDOW_SIZE) - tl.store( - combined_indices_ptr - + token_idx * combined_indices_stride - + topk_len - + swa_offset, - M * batch_idx + N + swa_offset + pos - swa_len + 1 - gather_start, - mask=swa_offset < swa_len, - ) - - tl.store(combined_lens_ptr + token_idx, topk_len + swa_len) - - -def combine_topk_swa_indices( - topk_indices: torch.Tensor, - query_start_loc: torch.Tensor, - seq_lens: torch.Tensor, - gather_lens: torch.Tensor, - window_size: int, - compress_ratio: int, - topk: int, - M: int, - N: int, -) -> tuple[torch.Tensor, torch.Tensor]: - """Combine compressed-attention and sliding-window indices with Torch. - - The Triton implementation inherited from DeepSeek V4 launches a - two-dimensional grid with 128 workers per request. On gfx950 it can issue - an out-of-bounds access for V4.1's mixed prefill metadata (including the - synthetic mixed-token warmup). This path is prefill-only and the tensors - are small, so use ordinary Torch indexing until a gfx950-safe fused kernel - is available. - """ - topk_indices = topk_indices.reshape(topk_indices.shape[0], -1).contiguous() - num_tokens = topk_indices.shape[0] - combined_topk = ( - (topk + window_size + _SPARSE_PREFILL_TOPK_ALIGNMENT - 1) - // _SPARSE_PREFILL_TOPK_ALIGNMENT - * _SPARSE_PREFILL_TOPK_ALIGNMENT - ) - combined_indices = torch.full( - (num_tokens, combined_topk), - fill_value=-1, - dtype=torch.int32, - device=topk_indices.device, - ) - combined_lens = torch.empty( - num_tokens, dtype=torch.int32, device=topk_indices.device - ) - - # query_start_loc may have a non-zero base for a narrowed mixed batch. - query_lens = query_start_loc[1:] - query_start_loc[:-1] - req_ids = torch.repeat_interleave( - torch.arange(seq_lens.shape[0], device=seq_lens.device), query_lens - ) - query_starts = query_start_loc[:-1] - query_start_loc[0] - token_offsets = torch.arange(num_tokens, device=seq_lens.device) - ( - torch.repeat_interleave(query_starts, query_lens) - ) - positions = seq_lens[req_ids] - query_lens[req_ids] + token_offsets - - logical_topk_width = min(topk, topk_indices.shape[1]) - topk_lens = torch.minimum( - (positions + 1) // compress_ratio, - torch.full_like(positions, logical_topk_width), - ).clamp_min(0) - topk_offsets = torch.arange(logical_topk_width, device=seq_lens.device) - topk_mask = topk_offsets[None, :] < topk_lens[:, None] - topk_values = topk_indices[:, :logical_topk_width].to(torch.int32) - topk_valid = topk_mask & (topk_values >= 0) & (topk_values < N) - combined_indices[:, :logical_topk_width] = torch.where( - topk_valid, - topk_values + (M * req_ids).to(torch.int32)[:, None], - -1, - ) - - swa_lens = torch.minimum( - positions + 1, torch.full_like(positions, window_size) - ).clamp_min(0) - swa_offsets = torch.arange(window_size, device=seq_lens.device) - swa_mask = swa_offsets[None, :] < swa_lens[:, None] - swa_columns = topk_lens[:, None] + swa_offsets[None, :] - gather_starts = seq_lens - gather_lens - swa_values = ( - M * req_ids[:, None] - + N - + swa_offsets[None, :] - + positions[:, None] - - swa_lens[:, None] - + 1 - - gather_starts[req_ids, None] - ).to(torch.int32) - rows = torch.arange(num_tokens, device=seq_lens.device)[:, None].expand_as( - swa_columns - ) - flat_dst = rows[swa_mask] * combined_topk + swa_columns[swa_mask] - combined_indices.view(-1).index_copy_(0, flat_dst, swa_values[swa_mask]) - combined_lens.copy_((topk_lens + swa_lens).to(torch.int32)) - return combined_indices, combined_lens - - @triton.jit def _compute_topk_lens_kernel( topk_lens_ptr, diff --git a/vllm/models/deepseek_v41/common/ops/cache_utils.py b/vllm/models/deepseek_v41/common/ops/cache_utils.py index b7f07d8d84..a6139c71f8 100644 --- a/vllm/models/deepseek_v41/common/ops/cache_utils.py +++ b/vllm/models/deepseek_v41/common/ops/cache_utils.py @@ -1089,7 +1089,8 @@ class CombineTopkSwaIndicesKernel( token_idx_in_query = token_idx - query_start pos = start_pos + token_idx_in_query if COMPRESS_RATIO > 0: - topk_len = tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K) + # Warmup batches can put pos below zero; keep the length >= 0. + topk_len = tl.maximum(tl.minimum((pos + 1) // COMPRESS_RATIO, TOP_K), 0) else: # SWA-only layers pass TOP_K=0; skip the division entirely # (integer div by constexpr 0 is UB and yields garbage, which @@ -1100,7 +1101,7 @@ class CombineTopkSwaIndicesKernel( # start (SWA bounded replay), so the window cannot start before # the gathered buffer does. swa_start = tl.maximum(swa_start, gather_start) - swa_len = pos - swa_start + 1 + swa_len = tl.maximum(pos - swa_start + 1, 0) offset = tl.arange(0, PADDED_TOP_K) mask = offset < topk_len @@ -1108,9 +1109,12 @@ class CombineTopkSwaIndicesKernel( topk_indices_ptr + token_idx * topk_indices_stride + offset, mask=mask, ) + # A failed (-1) or out-of-pool candidate stays -1 instead of landing + # in the previous request's rows or this request's SWA rows. + valid = (topk_indices >= 0) & (topk_indices < N) tl.store( combined_indices_ptr + token_idx * combined_indices_stride + offset, - topk_indices + M * batch_idx, + tl.where(valid, topk_indices + M * batch_idx, -1), mask=mask, ) # Index into gathered buffer: N + (position - gather_start)