Co-authored-by: vllm-agent <vllm-agent@users.noreply.github.com>
This commit is contained in:
@@ -5,13 +5,11 @@ import torch
|
||||
from torch import Generator
|
||||
|
||||
from tests.utils import large_gpu_mark
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import pad_vocab_size
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.sample.ops.topk_topp_sampler import (
|
||||
apply_top_k_top_p_pytorch,
|
||||
flashinfer_sample,
|
||||
random_sample,
|
||||
)
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
@@ -1045,39 +1043,3 @@ class TestFlashInferDistributionMatch:
|
||||
f"{label}: distribution differs from theoretical: "
|
||||
f"chi2={chi2:.2f} p_value={p_value:.2e} alpha={self.ALPHA}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not FLASHINFER_TOPK_TOPP_SUPPORTED,
|
||||
reason="FlashInfer top-k/top-p sampler is not available on this platform.",
|
||||
)
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("k, p", [(20, 0.95), (20, None), (None, 0.95)])
|
||||
def test_flashinfer_sample_padded_vocab(
|
||||
dtype: torch.dtype, k: int | None, p: float | None
|
||||
):
|
||||
"""flashinfer_sample must accept the logits the sampler actually hands it.
|
||||
|
||||
compute_logits slices the padding off the vocab, so for a vocab that isn't a
|
||||
multiple of 64 (e.g. opt's 50272) the logits are a strided view in the model
|
||||
dtype, while FlashInfer requires contiguous fp32.
|
||||
"""
|
||||
torch.set_default_device(DEVICE_TYPE)
|
||||
batch_size = 8
|
||||
org_vocab_size = 50272
|
||||
padded_vocab_size = pad_vocab_size(org_vocab_size)
|
||||
assert padded_vocab_size != org_vocab_size
|
||||
|
||||
logits = torch.randn(batch_size, padded_vocab_size, dtype=dtype)[
|
||||
..., :org_vocab_size
|
||||
]
|
||||
# A single row stays contiguous despite the padded stride, hence batch_size > 1.
|
||||
assert not logits.is_contiguous()
|
||||
|
||||
token_ids = flashinfer_sample(
|
||||
logits,
|
||||
torch.full((batch_size,), k, dtype=torch.int32) if k is not None else None,
|
||||
torch.full((batch_size,), p, dtype=torch.float32) if p is not None else None,
|
||||
)
|
||||
assert token_ids.shape == (batch_size,)
|
||||
assert torch.all((token_ids >= 0) & (token_ids < org_vocab_size))
|
||||
|
||||
Reference in New Issue
Block a user