[Core] Support min_tokens with speculative decoding (#32642)
Signed-off-by: qianlihuang <yiliu.dong@qq.com> Co-authored-by: qianlihuang <yiliu.dong@qq.com>
This commit is contained in:
co-authored by
qianlihuang
parent
99c7892c5b
commit
d940607629
@@ -10,6 +10,7 @@ import torch.nn as nn
|
||||
from vllm.logger import init_logger
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.outputs import LogprobsLists, LogprobsTensors, SamplerOutput
|
||||
from vllm.v1.sample.logits_processor.builtin import MinTokensLogitsProcessor
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.ops.bad_words import apply_bad_words_with_drafts
|
||||
from vllm.v1.sample.ops.penalties import apply_all_penalties
|
||||
@@ -292,6 +293,12 @@ class RejectionSampler(nn.Module):
|
||||
logits, bad_words_token_ids, output_token_ids, metadata.num_draft_tokens
|
||||
)
|
||||
|
||||
for processor in sampling_metadata.logitsprocs.non_argmax_invariant:
|
||||
if isinstance(processor, MinTokensLogitsProcessor):
|
||||
logits = processor.apply_with_spec_decode(
|
||||
logits, metadata.num_draft_tokens
|
||||
)
|
||||
|
||||
return logits
|
||||
|
||||
@staticmethod
|
||||
|
||||
Reference in New Issue
Block a user