From d19db10974587335ec3a37e0a424abb57430574e Mon Sep 17 00:00:00 2001 From: Xiaogang Zhou Date: Sat, 23 May 2026 13:56:16 +0800 Subject: [PATCH] =?UTF-8?q?[Bugfix]=20Fix=20native=20Triton=20top-k/top-p?= =?UTF-8?q?=20kernel=20assumes=20contiguous=20logi=E2=80=A6=20(#42739)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: xiaogang.zhou Co-authored-by: xiaogang.zhou --- tests/v1/sample/test_topk_topp_sampler.py | 50 +++++++++++++++++++++++ vllm/v1/sample/ops/topk_topp_triton.py | 17 +++++--- 2 files changed, 62 insertions(+), 5 deletions(-) diff --git a/tests/v1/sample/test_topk_topp_sampler.py b/tests/v1/sample/test_topk_topp_sampler.py index 659577b754..7d488aaabf 100644 --- a/tests/v1/sample/test_topk_topp_sampler.py +++ b/tests/v1/sample/test_topk_topp_sampler.py @@ -320,6 +320,56 @@ class TestTritonTopkTopp: self._compare_results(logits, k, p) + @pytest.mark.parametrize( + "mode", + ["topk_only", "topp_only", "topk_and_topp"], + ) + def test_noncontiguous_logits_match_contiguous(self, mode: str): + """Non-contiguous logits views should behave like contiguous inputs.""" + from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton + + device = torch.device(DEVICE_TYPE) + batch_size, vocab_size, pad = 16, 4096, 8 + backing = torch.full( + (batch_size, vocab_size + pad), + -1000.0, + device=device, + dtype=torch.float32, + ) + base = torch.linspace( + 10.0, -10.0, vocab_size, device=device, dtype=torch.float32 + ) + source = base[None, :] + ( + torch.arange(batch_size, device=device, dtype=torch.float32)[:, None] + / 1000.0 + ) + + logits = backing[:, :vocab_size] + logits.copy_(source) + contig_logits = source.clone() + pytorch_logits = source.clone() + + assert logits.shape == (batch_size, vocab_size) + assert logits.stride() == (vocab_size + pad, 1) + assert not logits.is_contiguous() + + k: torch.Tensor | None = None + p: torch.Tensor | None = None + if mode in ("topk_only", "topk_and_topp"): + k = torch.full((batch_size,), 154, device=device, dtype=torch.int32) + if mode in ("topp_only", "topk_and_topp"): + p = torch.full((batch_size,), 0.95, device=device, dtype=torch.float32) + + noncontig_out = apply_top_k_top_p_triton(logits, k, p) + contig_out = apply_top_k_top_p_triton(contig_logits, k, p) + pytorch_out = apply_top_k_top_p_pytorch(pytorch_logits, k, p) + + assert noncontig_out.data_ptr() == logits.data_ptr() + assert not noncontig_out.is_contiguous() + assert torch.equal(logits, noncontig_out) + assert torch.equal(torch.isfinite(noncontig_out), torch.isfinite(contig_out)) + assert torch.equal(torch.isfinite(noncontig_out), torch.isfinite(pytorch_out)) + # ----------------------------------------------------------------- # Tests for -inf logits (e.g. from grammar / structured output masks) # ----------------------------------------------------------------- diff --git a/vllm/v1/sample/ops/topk_topp_triton.py b/vllm/v1/sample/ops/topk_topp_triton.py index 4c7c3e99d4..fee87883c9 100644 --- a/vllm/v1/sample/ops/topk_topp_triton.py +++ b/vllm/v1/sample/ops/topk_topp_triton.py @@ -93,6 +93,7 @@ def _update_min_larger_stats(data, above_mask, min_larger, num_min_larger, senti @triton.jit def _topk_topp_kernel( LOGITS, + LOGITS_STRIDE_0, BUFFER, PERCENTILE_TO_STD_TABLE, NORMAL_CDF_TO_SIGMA_TABLE, @@ -110,7 +111,7 @@ def _topk_topp_kernel( pid = tl.program_id(0) num_programs = tl.num_programs(0) for row_id in tl.range(pid, BATCH_SIZE, num_programs): - LOGITS_ROW = LOGITS + row_id * VOCAB_SIZE + LOGITS_ROW = LOGITS + row_id * LOGITS_STRIDE_0 BUFFER_ROW = BUFFER + pid * VOCAB_SIZE final_pivot = -float("inf") @@ -975,26 +976,31 @@ def apply_top_k_top_p_triton( to the remaining k values (by probability). Args: - logits: [batch_size, vocab_size] float32 tensor, modified in-place + logits: [batch_size, vocab_size] float32 tensor. The returned tensor + may alias this input or be a new contiguous tensor for unsupported + layouts. k: [batch_size] int32 tensor of top-k values per row, or None to disable top-k p: [batch_size] float32 tensor of top-p values per row (0 to 1), or None to disable top-p mask_value: Value for masked positions (default: -inf) Returns: - The logits tensor (modified in-place) + The masked logits tensor. It may or may not be modified in-place. """ assert logits.ndim == 2 assert logits.dtype == torch.float32 - batch_size, vocab_size = logits.shape - topk_enabled = k is not None topp_enabled = p is not None if batch_size == 0 or not (topk_enabled or topp_enabled): return logits + # The Triton kernel supports arbitrary row strides, but it still assumes + # the vocab dimension is laid out contiguously within each row. + if logits.stride(1) != 1: + logits = logits.contiguous() + if k is not None: assert k.ndim == 1 and k.shape[0] == batch_size k_ptr = k.to(torch.int32) @@ -1034,6 +1040,7 @@ def apply_top_k_top_p_triton( _topk_topp_kernel[(NUM_PROGRAMS,)]( logits, + logits.stride(0), buffer, percentile_to_std_table, normal_cdf_to_sigma_table,