[Sampler] Adapt to FlashInfer 0.2.3 sampler API (#15777)
Signed-off-by: Bowen Wang <abmfy@icloud.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -123,12 +123,13 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
|
||||
# for rejection sampling
|
||||
if self.use_flashinfer and chain_speculative_sampling is not None:
|
||||
batch_size, k, _ = draft_probs.shape
|
||||
uniform_samples = self._create_uniform_samples(
|
||||
seeded_seqs, batch_size, k, draft_probs.device)
|
||||
output_token_ids, accepted_token_num, emitted_token_num \
|
||||
= chain_speculative_sampling(
|
||||
draft_probs, draft_token_ids, uniform_samples,
|
||||
target_with_bonus_probs)
|
||||
|
||||
(output_token_ids, accepted_token_num,
|
||||
emitted_token_num) = chain_speculative_sampling(
|
||||
draft_probs,
|
||||
draft_token_ids,
|
||||
target_with_bonus_probs,
|
||||
)
|
||||
|
||||
# num_emitted_tokens returned by flashinfer
|
||||
# does not include the bonus token
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""A layer that samples the next tokens from the model's outputs."""
|
||||
import itertools
|
||||
import warnings
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from importlib.util import find_spec
|
||||
@@ -24,7 +23,6 @@ from vllm.sequence import (VLLM_INVALID_TOKEN_ID,
|
||||
from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
|
||||
|
||||
if envs.VLLM_USE_FLASHINFER_SAMPLER and find_spec("flashinfer"):
|
||||
import flashinfer.sampling
|
||||
# yapf: disable
|
||||
from flashinfer.sampling import (
|
||||
top_k_top_p_sampling_from_probs as flashinfer_top_k_top_p_sampling)
|
||||
@@ -33,6 +31,10 @@ if envs.VLLM_USE_FLASHINFER_SAMPLER and find_spec("flashinfer"):
|
||||
else:
|
||||
flashinfer_top_k_top_p_sampling = None
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_sampler() -> torch.nn.Module:
|
||||
if envs.VLLM_USE_V1:
|
||||
@@ -545,38 +547,15 @@ def _multinomial(
|
||||
def _top_k_top_p_multinomial_with_flashinfer(
|
||||
probs: torch.Tensor, top_ks: torch.Tensor, top_ps: torch.Tensor,
|
||||
num_samples: int, seq_groups: Optional[list[SequenceGroupToSample]]):
|
||||
max_top_k_round = 32
|
||||
if num_samples > 1:
|
||||
probs = probs.repeat_interleave(num_samples, dim=0)
|
||||
top_ks = top_ks.repeat_interleave(num_samples)
|
||||
top_ps = top_ps.repeat_interleave(num_samples)
|
||||
batch_size = probs.shape[0]
|
||||
uniform_samples = torch.empty((max_top_k_round, batch_size),
|
||||
device=probs.device)
|
||||
if seq_groups is None:
|
||||
uniform_samples.uniform_()
|
||||
else:
|
||||
sample_idx = 0
|
||||
for seq_group in seq_groups:
|
||||
seq_ids = seq_group.seq_ids
|
||||
stride = len(seq_ids) * num_samples
|
||||
assert seq_group.generator is not None
|
||||
uniform_samples[:, sample_idx:sample_idx +
|
||||
stride].uniform_(generator=seq_group.generator)
|
||||
sample_idx += stride
|
||||
batch_next_token_ids, success = flashinfer_top_k_top_p_sampling(
|
||||
batch_next_token_ids = flashinfer_top_k_top_p_sampling(
|
||||
probs,
|
||||
uniform_samples,
|
||||
top_ks,
|
||||
top_ps,
|
||||
)
|
||||
if not success.all():
|
||||
warnings.warn("FlashInfer rejection sampling failed, fallback.",
|
||||
stacklevel=1)
|
||||
probs = flashinfer.sampling.top_k_renorm_prob(probs, top_ks)
|
||||
probs = flashinfer.sampling.top_p_renorm_prob(probs, top_ps)
|
||||
batch_next_token_ids = flashinfer.sampling.sampling_from_probs(
|
||||
probs, uniform_samples[0])
|
||||
return batch_next_token_ids.view(-1, num_samples)
|
||||
|
||||
|
||||
@@ -712,19 +691,14 @@ def _sample_with_torch(
|
||||
seq_groups)
|
||||
|
||||
if flashinfer_top_k_top_p_sampling is not None:
|
||||
multinomial_samples[
|
||||
sampling_type] = _top_k_top_p_multinomial_with_flashinfer(
|
||||
probs[long_sample_indices],
|
||||
sampling_tensors.top_ks[long_sample_indices],
|
||||
sampling_tensors.top_ps[long_sample_indices],
|
||||
max_n_in_batch,
|
||||
seq_groups_arg,
|
||||
)
|
||||
else:
|
||||
multinomial_samples[sampling_type] = _multinomial(
|
||||
probs[long_sample_indices],
|
||||
max_n_in_batch,
|
||||
seq_groups=seq_groups_arg)
|
||||
logger.warning("FlashInfer 0.2.3+ does not support "
|
||||
"per-request generators. Falling back to "
|
||||
"PyTorch-native implementation.")
|
||||
|
||||
multinomial_samples[sampling_type] = _multinomial(
|
||||
probs[long_sample_indices],
|
||||
max_n_in_batch,
|
||||
seq_groups=seq_groups_arg)
|
||||
|
||||
if sampled_token_ids_tensor is not None:
|
||||
# Store sampled tokens in output tensor.
|
||||
|
||||
@@ -31,21 +31,10 @@ class TopKTopPSampler(nn.Module):
|
||||
if current_platform.is_cuda():
|
||||
if is_flashinfer_available:
|
||||
flashinfer_version = flashinfer.__version__
|
||||
if flashinfer_version >= "0.2.3":
|
||||
# FIXME(DefTruth): Currently, we have errors when using
|
||||
# FlashInfer>=v0.2.3 for top-p & top-k sampling. As a
|
||||
# workaround, we disable FlashInfer for top-p & top-k
|
||||
# sampling by default while FlashInfer>=v0.2.3.
|
||||
# The sampling API removes the success return value
|
||||
# of all sampling API, which is not compatible with
|
||||
# earlier design.
|
||||
# https://github.com/flashinfer-ai/flashinfer/releases/
|
||||
# tag/v0.2.3
|
||||
logger.info(
|
||||
"Currently, FlashInfer top-p & top-k sampling sampler "
|
||||
"is disabled because FlashInfer>=v0.2.3 is not "
|
||||
"backward compatible. Falling back to the PyTorch-"
|
||||
"native implementation of top-p & top-k sampling.")
|
||||
if flashinfer_version < "0.2.3":
|
||||
logger.warning(
|
||||
"FlashInfer version >= 0.2.3 required. "
|
||||
"Falling back to default sampling implementation.")
|
||||
self.forward = self.forward_native
|
||||
elif envs.VLLM_USE_FLASHINFER_SAMPLER is not False:
|
||||
# NOTE(woosuk): The V0 sampler doesn't use FlashInfer for
|
||||
@@ -106,6 +95,11 @@ class TopKTopPSampler(nn.Module):
|
||||
# not needed. This is because `random_sample` does not require
|
||||
# CPU-GPU synchronization while `flashinfer_sample` does.
|
||||
return random_sample(probs, generators)
|
||||
if generators:
|
||||
logger.warning("FlashInfer 0.2.3+ does not support "
|
||||
"per-request generators. Falling back to "
|
||||
"PyTorch-native implementation.")
|
||||
return self.forward_native(logits, generators, k, p)
|
||||
return flashinfer_sample(probs, k, p, generators)
|
||||
|
||||
def forward_tpu(
|
||||
@@ -280,36 +274,18 @@ def flashinfer_sample(
|
||||
the synchronization overhead.
|
||||
"""
|
||||
assert not (k is None and p is None)
|
||||
max_top_k_round = 32
|
||||
batch_size = probs.shape[0]
|
||||
uniform_samples = torch.empty((max_top_k_round, batch_size),
|
||||
device=probs.device)
|
||||
if len(generators) != batch_size:
|
||||
uniform_samples.uniform_()
|
||||
if generators:
|
||||
for i, generator in generators.items():
|
||||
uniform_samples[:, i].uniform_(generator=generator)
|
||||
|
||||
if k is None:
|
||||
# Top-p only.
|
||||
next_token_ids, success = flashinfer.sampling.top_p_sampling_from_probs(
|
||||
probs, uniform_samples, p, deterministic=True)
|
||||
next_token_ids = flashinfer.sampling.top_p_sampling_from_probs(
|
||||
probs, p, deterministic=True)
|
||||
elif p is None:
|
||||
# Top-k only.
|
||||
next_token_ids, success = flashinfer.sampling.top_k_sampling_from_probs(
|
||||
probs, uniform_samples, k, deterministic=True)
|
||||
next_token_ids = flashinfer.sampling.top_k_sampling_from_probs(
|
||||
probs, k, deterministic=True)
|
||||
else:
|
||||
# Both top-k and top-p.
|
||||
next_token_ids, success = (
|
||||
flashinfer.sampling.top_k_top_p_sampling_from_probs(
|
||||
probs, uniform_samples, k, p, deterministic=True))
|
||||
next_token_ids = (flashinfer.sampling.top_k_top_p_sampling_from_probs(
|
||||
probs, k, p, deterministic=True))
|
||||
|
||||
# NOTE: CPU-GPU synchronization happens here.
|
||||
if not success.all():
|
||||
if k is not None:
|
||||
probs = flashinfer.sampling.top_k_renorm_prob(probs, k)
|
||||
if p is not None:
|
||||
probs = flashinfer.sampling.top_p_renorm_prob(probs, p)
|
||||
next_token_ids = flashinfer.sampling.sampling_from_probs(
|
||||
probs, uniform_samples[0], deterministic=True)
|
||||
return next_token_ids.view(-1)
|
||||
|
||||
Reference in New Issue
Block a user