[Speculative Decoding] Add ProposerWorkerBase abstract class (#5252)
This commit is contained in:
@@ -14,6 +14,7 @@ from vllm.spec_decode.interfaces import (SpeculativeProposals,
|
||||
from vllm.spec_decode.metrics import AsyncMetricsCollector
|
||||
from vllm.spec_decode.multi_step_worker import MultiStepWorker
|
||||
from vllm.spec_decode.ngram_worker import NGramWorker
|
||||
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
|
||||
from vllm.spec_decode.util import (create_sequence_group_output,
|
||||
get_all_num_logprobs, get_all_seq_ids,
|
||||
get_sampled_token_logprobs, nvtx_range,
|
||||
@@ -117,7 +118,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
proposer_worker: WorkerBase,
|
||||
proposer_worker: ProposerWorkerBase,
|
||||
scorer_worker: WorkerBase,
|
||||
rejection_sampler: RejectionSampler,
|
||||
metrics_collector: Optional[AsyncMetricsCollector] = None,
|
||||
@@ -260,7 +261,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
|
||||
# This is required as if the number of draft model runs changes
|
||||
# dynamically, the non-driver workers won't know unless we perform a
|
||||
# communication to inform then.
|
||||
# communication to inform them.
|
||||
broadcast_dict = dict(
|
||||
num_lookahead_slots=num_lookahead_slots,
|
||||
disable_all_speculation=disable_all_speculation,
|
||||
|
||||
Reference in New Issue
Block a user