Remove hard-dependencies of Speculative decode to CUDA workers (#10587)
Signed-off-by: Chendi Xue <chendi.xue@intel.com>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Set
|
||||
from typing import Optional, Set, Union
|
||||
|
||||
import torch
|
||||
|
||||
@@ -75,9 +75,11 @@ class SpeculativeProposer(ABC):
|
||||
|
||||
class SpeculativeScorer(ABC):
|
||||
|
||||
def __init__(self, scorer_worker: WorkerBase, device: str,
|
||||
vocab_size: int):
|
||||
def __init__(self, scorer_worker: WorkerBase,
|
||||
device: Union[torch.device, str], vocab_size: int):
|
||||
self._scorer_worker = scorer_worker
|
||||
if isinstance(device, torch.device):
|
||||
device = device.type
|
||||
self._device = device
|
||||
self._vocab_size = vocab_size
|
||||
|
||||
|
||||
Reference in New Issue
Block a user