[Bugfix] Fix native Triton top-k/top-p kernel assumes contiguous logi… (#42739)
Signed-off-by: xiaogang.zhou <xiaogang.zhou@bytedance.com> Co-authored-by: xiaogang.zhou <xiaogang.zhou@bytedance.com>
This commit is contained in:
co-authored by
xiaogang.zhou
parent
3a1c062151
commit
d19db10974
@@ -320,6 +320,56 @@ class TestTritonTopkTopp:
|
||||
|
||||
self._compare_results(logits, k, p)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"mode",
|
||||
["topk_only", "topp_only", "topk_and_topp"],
|
||||
)
|
||||
def test_noncontiguous_logits_match_contiguous(self, mode: str):
|
||||
"""Non-contiguous logits views should behave like contiguous inputs."""
|
||||
from vllm.v1.sample.ops.topk_topp_triton import apply_top_k_top_p_triton
|
||||
|
||||
device = torch.device(DEVICE_TYPE)
|
||||
batch_size, vocab_size, pad = 16, 4096, 8
|
||||
backing = torch.full(
|
||||
(batch_size, vocab_size + pad),
|
||||
-1000.0,
|
||||
device=device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
base = torch.linspace(
|
||||
10.0, -10.0, vocab_size, device=device, dtype=torch.float32
|
||||
)
|
||||
source = base[None, :] + (
|
||||
torch.arange(batch_size, device=device, dtype=torch.float32)[:, None]
|
||||
/ 1000.0
|
||||
)
|
||||
|
||||
logits = backing[:, :vocab_size]
|
||||
logits.copy_(source)
|
||||
contig_logits = source.clone()
|
||||
pytorch_logits = source.clone()
|
||||
|
||||
assert logits.shape == (batch_size, vocab_size)
|
||||
assert logits.stride() == (vocab_size + pad, 1)
|
||||
assert not logits.is_contiguous()
|
||||
|
||||
k: torch.Tensor | None = None
|
||||
p: torch.Tensor | None = None
|
||||
if mode in ("topk_only", "topk_and_topp"):
|
||||
k = torch.full((batch_size,), 154, device=device, dtype=torch.int32)
|
||||
if mode in ("topp_only", "topk_and_topp"):
|
||||
p = torch.full((batch_size,), 0.95, device=device, dtype=torch.float32)
|
||||
|
||||
noncontig_out = apply_top_k_top_p_triton(logits, k, p)
|
||||
contig_out = apply_top_k_top_p_triton(contig_logits, k, p)
|
||||
pytorch_out = apply_top_k_top_p_pytorch(pytorch_logits, k, p)
|
||||
|
||||
assert noncontig_out.data_ptr() == logits.data_ptr()
|
||||
assert not noncontig_out.is_contiguous()
|
||||
assert torch.equal(logits, noncontig_out)
|
||||
assert torch.equal(torch.isfinite(noncontig_out), torch.isfinite(contig_out))
|
||||
assert torch.equal(torch.isfinite(noncontig_out), torch.isfinite(pytorch_out))
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Tests for -inf logits (e.g. from grammar / structured output masks)
|
||||
# -----------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user