Remove hard-dependencies of Speculative decode to CUDA workers (#10587)
Signed-off-by: Chendi Xue <chendi.xue@intel.com>
This commit is contained in:
@@ -14,12 +14,16 @@ from vllm.model_executor.layers.spec_decode_base_sampler import (
|
||||
SpecDecodeBaseSampler, SpecDecodeStochasticBaseSampler)
|
||||
from vllm.model_executor.layers.typical_acceptance_sampler import (
|
||||
TypicalAcceptanceSampler)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import (VLLM_INVALID_TOKEN_ID,
|
||||
CompletionSequenceGroupOutput, ExecuteModelRequest,
|
||||
HiddenStates, SequenceGroupMetadata,
|
||||
get_all_seq_ids_and_request_ids)
|
||||
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
|
||||
from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
|
||||
|
||||
if current_platform.is_cuda_alike():
|
||||
from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
|
||||
|
||||
from vllm.spec_decode.interfaces import (SpeculativeProposals,
|
||||
SpeculativeScorer, SpeculativeScores)
|
||||
from vllm.spec_decode.medusa_worker import MedusaWorker
|
||||
@@ -36,8 +40,8 @@ from vllm.spec_decode.util import (Timer, create_logprobs_output,
|
||||
get_all_num_logprobs,
|
||||
get_sampled_token_logprobs, nvtx_range,
|
||||
split_batch_by_proposal_len)
|
||||
from vllm.worker.worker import Worker
|
||||
from vllm.worker.worker_base import LoraNotSupportedWorkerBase, WorkerBase
|
||||
from vllm.worker.worker_base import (LoraNotSupportedWorkerBase, WorkerBase,
|
||||
WorkerWrapperBase)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -53,7 +57,11 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
|
||||
draft_worker_kwargs = kwargs.copy()
|
||||
|
||||
kwargs["model_runner_cls"] = TargetModelRunner
|
||||
target_worker = Worker(*args, **kwargs)
|
||||
target_worker_config = copy.deepcopy(vllm_config)
|
||||
target_worker_config.parallel_config.worker_cls =\
|
||||
target_worker_config.parallel_config.sd_worker_cls
|
||||
target_worker = WorkerWrapperBase(vllm_config=target_worker_config)
|
||||
target_worker.init_worker(*args, **kwargs)
|
||||
# Set the disable_logprobs variable in the TargetModelRunner instance
|
||||
# as per its value specified in the SpeculativeConfig.
|
||||
target_worker.model_runner.disable_logprobs =\
|
||||
@@ -65,6 +73,8 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
|
||||
draft_worker_config.model_config,
|
||||
vllm_config.load_config,
|
||||
)
|
||||
speculative_config.draft_parallel_config.worker_cls =\
|
||||
draft_worker_config.parallel_config.sd_worker_cls
|
||||
draft_worker_config.parallel_config = speculative_config.draft_parallel_config # noqa
|
||||
# TODO allow draft-model specific load config.
|
||||
|
||||
@@ -125,7 +135,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
@classmethod
|
||||
def create_worker(
|
||||
cls,
|
||||
scorer_worker: Worker,
|
||||
scorer_worker: WorkerBase,
|
||||
draft_worker_kwargs: Dict[str, Any],
|
||||
disable_mqa_scorer: bool,
|
||||
disable_by_batch_size: Optional[int],
|
||||
@@ -145,6 +155,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
draft_parallel_config: ParallelConfig = draft_worker_kwargs[
|
||||
'vllm_config'].parallel_config
|
||||
if ngram_prompt_lookup_max > 0:
|
||||
draft_worker_kwargs[
|
||||
"device_type"] = scorer_worker.device_config.device.type
|
||||
proposer_worker = NGramWorker(**draft_worker_kwargs)
|
||||
proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
|
||||
ngram_prompt_lookup_max)
|
||||
@@ -158,8 +170,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
proposer_worker = MedusaWorker(**draft_worker_kwargs)
|
||||
else:
|
||||
if draft_tp == 1:
|
||||
draft_worker_kwargs[
|
||||
"model_runner_cls"] = TP1DraftModelRunner
|
||||
if current_platform.is_cuda_alike():
|
||||
draft_worker_kwargs[
|
||||
"model_runner_cls"] = TP1DraftModelRunner
|
||||
else:
|
||||
if draft_model_config.hf_config.model_type == "eagle":
|
||||
raise NotImplementedError(
|
||||
@@ -306,8 +319,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
self.scorer_worker.load_model()
|
||||
self.proposer_worker.load_model()
|
||||
|
||||
self._metrics.init_gpu_tensors(self.rank)
|
||||
self.spec_decode_sampler.init_gpu_tensors(self.rank)
|
||||
self._metrics.init_tensors(self.rank, device_type=self.device)
|
||||
self.spec_decode_sampler.init_tensors(self.rank,
|
||||
device_type=self.device)
|
||||
|
||||
scorer_cls: Type[SpeculativeScorer]
|
||||
if self.disable_mqa_scorer:
|
||||
@@ -1111,11 +1125,11 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
|
||||
raise NotImplementedError
|
||||
|
||||
def start_profile(self):
|
||||
if isinstance(self.scorer_worker, Worker):
|
||||
if isinstance(self.scorer_worker, WorkerBase):
|
||||
self.scorer_worker.start_profile()
|
||||
|
||||
def stop_profile(self):
|
||||
if isinstance(self.scorer_worker, Worker):
|
||||
if isinstance(self.scorer_worker, WorkerBase):
|
||||
self.scorer_worker.stop_profile()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user