[Bugfix] Fix native Triton top-k/top-p kernel assumes contiguous logi… (#42739)

Signed-off-by: xiaogang.zhou <xiaogang.zhou@bytedance.com>
Co-authored-by: xiaogang.zhou <xiaogang.zhou@bytedance.com>
This commit is contained in:
Xiaogang Zhou
2026-05-22 22:56:16 -07:00
committed by GitHub
co-authored by xiaogang.zhou
parent 3a1c062151
commit d19db10974
2 changed files with 62 additions and 5 deletions
+50
View File
@@ -320,6 +320,56 @@ class TestTritonTopkTopp:
self._compare_results(logits, k, p)
@pytest.mark.parametrize(
"mode",
["topk_only", "topp_only", "topk_and_topp"],
)
def test_noncontiguous_logits_match_contiguous(self, mode: str):
"""Non-contiguous logits views should behave like contiguous inputs."""
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
device = torch.device(DEVICE_TYPE)
batch_size, vocab_size, pad = 16, 4096, 8
backing = torch.full(
(batch_size, vocab_size + pad),
-1000.0,
device=device,
dtype=torch.float32,
)
base = torch.linspace(
10.0, -10.0, vocab_size, device=device, dtype=torch.float32
)
source = base[None, :] + (
torch.arange(batch_size, device=device, dtype=torch.float32)[:, None]
/ 1000.0
)
logits = backing[:, :vocab_size]
logits.copy_(source)
contig_logits = source.clone()
pytorch_logits = source.clone()
assert logits.shape == (batch_size, vocab_size)
assert logits.stride() == (vocab_size + pad, 1)
assert not logits.is_contiguous()
k: torch.Tensor | None = None
p: torch.Tensor | None = None
if mode in ("topk_only", "topk_and_topp"):
k = torch.full((batch_size,), 154, device=device, dtype=torch.int32)
if mode in ("topp_only", "topk_and_topp"):
p = torch.full((batch_size,), 0.95, device=device, dtype=torch.float32)
noncontig_out = apply_top_k_top_p_triton(logits, k, p)
contig_out = apply_top_k_top_p_triton(contig_logits, k, p)
pytorch_out = apply_top_k_top_p_pytorch(pytorch_logits, k, p)
assert noncontig_out.data_ptr() == logits.data_ptr()
assert not noncontig_out.is_contiguous()
assert torch.equal(logits, noncontig_out)
assert torch.equal(torch.isfinite(noncontig_out), torch.isfinite(contig_out))
assert torch.equal(torch.isfinite(noncontig_out), torch.isfinite(pytorch_out))
# -----------------------------------------------------------------
# Tests for -inf logits (e.g. from grammar / structured output masks)
# -----------------------------------------------------------------
+12 -5
View File
@@ -93,6 +93,7 @@ def _update_min_larger_stats(data, above_mask, min_larger, num_min_larger, senti
@triton.jit
def _topk_topp_kernel(
LOGITS,
LOGITS_STRIDE_0,
BUFFER,
PERCENTILE_TO_STD_TABLE,
NORMAL_CDF_TO_SIGMA_TABLE,
@@ -110,7 +111,7 @@ def _topk_topp_kernel(
pid = tl.program_id(0)
num_programs = tl.num_programs(0)
for row_id in tl.range(pid, BATCH_SIZE, num_programs):
LOGITS_ROW = LOGITS + row_id * VOCAB_SIZE
LOGITS_ROW = LOGITS + row_id * LOGITS_STRIDE_0
BUFFER_ROW = BUFFER + pid * VOCAB_SIZE
final_pivot = -float("inf")
@@ -975,26 +976,31 @@ def apply_top_k_top_p_triton(
to the remaining k values (by probability).
Args:
logits: [batch_size, vocab_size] float32 tensor, modified in-place
logits: [batch_size, vocab_size] float32 tensor. The returned tensor
may alias this input or be a new contiguous tensor for unsupported
layouts.
k: [batch_size] int32 tensor of top-k values per row, or None to disable top-k
p: [batch_size] float32 tensor of top-p values per row (0 to 1),
or None to disable top-p
mask_value: Value for masked positions (default: -inf)
Returns:
The logits tensor (modified in-place)
The masked logits tensor. It may or may not be modified in-place.
"""
assert logits.ndim == 2
assert logits.dtype == torch.float32
batch_size, vocab_size = logits.shape
topk_enabled = k is not None
topp_enabled = p is not None
if batch_size == 0 or not (topk_enabled or topp_enabled):
return logits
# The Triton kernel supports arbitrary row strides, but it still assumes
# the vocab dimension is laid out contiguously within each row.
if logits.stride(1) != 1:
logits = logits.contiguous()
if k is not None:
assert k.ndim == 1 and k.shape[0] == batch_size
k_ptr = k.to(torch.int32)
@@ -1034,6 +1040,7 @@ def apply_top_k_top_p_triton(
_topk_topp_kernel[(NUM_PROGRAMS,)](
logits,
logits.stride(0),
buffer,
percentile_to_std_table,
normal_cdf_to_sigma_table,