Remove hard-dependencies of Speculative decode to CUDA workers (#10587)
Signed-off-by: Chendi Xue <chendi.xue@intel.com>
This commit is contained in:
@@ -22,6 +22,7 @@ class NGramWorker(NonLLMProposerWorkerBase):
|
||||
# Get local_rank/vocab_size from kwargs attribute
|
||||
self.local_rank = kwargs["local_rank"]
|
||||
self.vocab_size = kwargs["vllm_config"].model_config.get_vocab_size()
|
||||
self.device_type = kwargs.get("device_type", "cuda")
|
||||
|
||||
# Lazy initialization list.
|
||||
self._proposer: Top1Proposer
|
||||
@@ -34,7 +35,7 @@ class NGramWorker(NonLLMProposerWorkerBase):
|
||||
self.ngram_prompt_lookup_min = ngram_prompt_lookup_min
|
||||
|
||||
def init_device(self):
|
||||
self.device = torch.device(f"cuda:{self.local_rank}")
|
||||
self.device = torch.device(f"{self.device_type}:{self.local_rank}")
|
||||
self.load_model = lambda *args, **kwargs: None
|
||||
|
||||
# Current NGramWorker only supports Top1Proposer
|
||||
|
||||
Reference in New Issue
Block a user