[Bugfix] Fix native Triton top-k/top-p kernel assumes contiguous logi… (#42739)
Signed-off-by: xiaogang.zhou <xiaogang.zhou@bytedance.com> Co-authored-by: xiaogang.zhou <xiaogang.zhou@bytedance.com>
This commit is contained in:
co-authored by
xiaogang.zhou
parent
3a1c062151
commit
d19db10974
@@ -320,6 +320,56 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k, p)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode",
|
||||
["topk_only", "topp_only", "topk_and_topp"],
|
||||
)
|
||||
def test_noncontiguous_logits_match_contiguous(self, mode: str):
|
||||
"""Non-contiguous logits views should behave like contiguous inputs."""
|
||||
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
|
||||
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
batch_size, vocab_size, pad = 16, 4096, 8
|
||||
backing = torch.full(
|
||||
(batch_size, vocab_size + pad),
|
||||
-1000.0,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
base = torch.linspace(
|
||||
10.0, -10.0, vocab_size, device=device, dtype=torch.float32
|
||||
)
|
||||
source = base[None, :] + (
|
||||
torch.arange(batch_size, device=device, dtype=torch.float32)[:, None]
|
||||
/ 1000.0
|
||||
)
|
||||
|
||||
logits = backing[:, :vocab_size]
|
||||
logits.copy_(source)
|
||||
contig_logits = source.clone()
|
||||
pytorch_logits = source.clone()
|
||||
|
||||
assert logits.shape == (batch_size, vocab_size)
|
||||
assert logits.stride() == (vocab_size + pad, 1)
|
||||
assert not logits.is_contiguous()
|
||||
|
||||
k: torch.Tensor | None = None
|
||||
p: torch.Tensor | None = None
|
||||
if mode in ("topk_only", "topk_and_topp"):
|
||||
k = torch.full((batch_size,), 154, device=device, dtype=torch.int32)
|
||||
if mode in ("topp_only", "topk_and_topp"):
|
||||
p = torch.full((batch_size,), 0.95, device=device, dtype=torch.float32)
|
||||
|
||||
noncontig_out = apply_top_k_top_p_triton(logits, k, p)
|
||||
contig_out = apply_top_k_top_p_triton(contig_logits, k, p)
|
||||
pytorch_out = apply_top_k_top_p_pytorch(pytorch_logits, k, p)
|
||||
|
||||
assert noncontig_out.data_ptr() == logits.data_ptr()
|
||||
assert not noncontig_out.is_contiguous()
|
||||
assert torch.equal(logits, noncontig_out)
|
||||
assert torch.equal(torch.isfinite(noncontig_out), torch.isfinite(contig_out))
|
||||
assert torch.equal(torch.isfinite(noncontig_out), torch.isfinite(pytorch_out))
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Tests for -inf logits (e.g. from grammar / structured output masks)
|
||||
# -----------------------------------------------------------------
|
||||
|
||||
@@ -93,6 +93,7 @@ def _update_min_larger_stats(data, above_mask, min_larger, num_min_larger, senti
|
||||
@triton.jit
|
||||
def _topk_topp_kernel(
|
||||
LOGITS,
|
||||
LOGITS_STRIDE_0,
|
||||
BUFFER,
|
||||
PERCENTILE_TO_STD_TABLE,
|
||||
NORMAL_CDF_TO_SIGMA_TABLE,
|
||||
@@ -110,7 +111,7 @@ def _topk_topp_kernel(
|
||||
pid = tl.program_id(0)
|
||||
num_programs = tl.num_programs(0)
|
||||
for row_id in tl.range(pid, BATCH_SIZE, num_programs):
|
||||
LOGITS_ROW = LOGITS + row_id * VOCAB_SIZE
|
||||
LOGITS_ROW = LOGITS + row_id * LOGITS_STRIDE_0
|
||||
BUFFER_ROW = BUFFER + pid * VOCAB_SIZE
|
||||
|
||||
final_pivot = -float("inf")
|
||||
@@ -975,26 +976,31 @@ def apply_top_k_top_p_triton(
|
||||
to the remaining k values (by probability).
|
||||
|
||||
Args:
|
||||
logits: [batch_size, vocab_size] float32 tensor, modified in-place
|
||||
logits: [batch_size, vocab_size] float32 tensor. The returned tensor
|
||||
may alias this input or be a new contiguous tensor for unsupported
|
||||
layouts.
|
||||
k: [batch_size] int32 tensor of top-k values per row, or None to disable top-k
|
||||
p: [batch_size] float32 tensor of top-p values per row (0 to 1),
|
||||
or None to disable top-p
|
||||
mask_value: Value for masked positions (default: -inf)
|
||||
|
||||
Returns:
|
||||
The logits tensor (modified in-place)
|
||||
The masked logits tensor. It may or may not be modified in-place.
|
||||
"""
|
||||
assert logits.ndim == 2
|
||||
assert logits.dtype == torch.float32
|
||||
|
||||
batch_size, vocab_size = logits.shape
|
||||
|
||||
topk_enabled = k is not None
|
||||
topp_enabled = p is not None
|
||||
|
||||
if batch_size == 0 or not (topk_enabled or topp_enabled):
|
||||
return logits
|
||||
|
||||
# The Triton kernel supports arbitrary row strides, but it still assumes
|
||||
# the vocab dimension is laid out contiguously within each row.
|
||||
if logits.stride(1) != 1:
|
||||
logits = logits.contiguous()
|
||||
|
||||
if k is not None:
|
||||
assert k.ndim == 1 and k.shape[0] == batch_size
|
||||
k_ptr = k.to(torch.int32)
|
||||
@@ -1034,6 +1040,7 @@ def apply_top_k_top_p_triton(
|
||||
|
||||
_topk_topp_kernel[(NUM_PROGRAMS,)](
|
||||
logits,
|
||||
logits.stride(0),
|
||||
buffer,
|
||||
percentile_to_std_table,
|
||||
normal_cdf_to_sigma_table,
|
||||
|
||||
Reference in New Issue
Block a user