[Sampler] Stop upcasting logits to fp32 in apply_sampling_params (#48641)

Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Michael Goin
2026-07-17 18:15:20 -07:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Nick Hill
parent 02c01f442b
commit 425c4eafb0
5 changed files with 85 additions and 25 deletions
+38
View File
@@ -5,11 +5,13 @@ import torch
from torch import Generator
from tests.utils import large_gpu_mark
from vllm.model_executor.layers.vocab_parallel_embedding import pad_vocab_size
from vllm.platforms import current_platform
from vllm.triton_utils import HAS_TRITON
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.sample.ops.topk_topp_sampler import (
apply_top_k_top_p_pytorch,
flashinfer_sample,
random_sample,
)
from vllm.v1.sample.sampler import Sampler
@@ -1043,3 +1045,39 @@ class TestFlashInferDistributionMatch:
f"{label}: distribution differs from theoretical: "
f"chi2={chi2:.2f} p_value={p_value:.2e} alpha={self.ALPHA}"
)
@pytest.mark.skipif(
not FLASHINFER_TOPK_TOPP_SUPPORTED,
reason="FlashInfer top-k/top-p sampler is not available on this platform.",
)
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32])
@pytest.mark.parametrize("k, p", [(20, 0.95), (20, None), (None, 0.95)])
def test_flashinfer_sample_padded_vocab(
dtype: torch.dtype, k: int | None, p: float | None
):
"""flashinfer_sample must accept the logits the sampler actually hands it.
compute_logits slices the padding off the vocab, so for a vocab that isn't a
multiple of 64 (e.g. opt's 50272) the logits are a strided view in the model
dtype, while FlashInfer requires contiguous fp32.
"""
torch.set_default_device(DEVICE_TYPE)
batch_size = 8
org_vocab_size = 50272
padded_vocab_size = pad_vocab_size(org_vocab_size)
assert padded_vocab_size != org_vocab_size
logits = torch.randn(batch_size, padded_vocab_size, dtype=dtype)[
..., :org_vocab_size
]
# A single row stays contiguous despite the padded stride, hence batch_size > 1.
assert not logits.is_contiguous()
token_ids = flashinfer_sample(
logits,
torch.full((batch_size,), k, dtype=torch.int32) if k is not None else None,
torch.full((batch_size,), p, dtype=torch.float32) if p is not None else None,
)
assert token_ids.shape == (batch_size,)
assert torch.all((token_ids >= 0) & (token_ids < org_vocab_size))