[Sampler] Adapt to FlashInfer 0.2.3 sampler API (#15777)

Signed-off-by: Bowen Wang <abmfy@icloud.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Bowen Wang
2025-05-16 15:14:03 -07:00
committed by GitHub
co-authored by mgoin
parent aef94c6d07
commit 7fdfa01530
7 changed files with 122 additions and 88 deletions
@@ -123,12 +123,13 @@ class RejectionSampler(SpecDecodeStochasticBaseSampler):
# for rejection sampling
if self.use_flashinfer and chain_speculative_sampling is not None:
batch_size, k, _ = draft_probs.shape
uniform_samples = self._create_uniform_samples(
seeded_seqs, batch_size, k, draft_probs.device)
output_token_ids, accepted_token_num, emitted_token_num \
= chain_speculative_sampling(
draft_probs, draft_token_ids, uniform_samples,
target_with_bonus_probs)
(output_token_ids, accepted_token_num,
emitted_token_num) = chain_speculative_sampling(
draft_probs,
draft_token_ids,
target_with_bonus_probs,
)
# num_emitted_tokens returned by flashinfer
# does not include the bonus token