[Spec Decode] (1/2) Remove batch expansion (#8839)
This commit is contained in:
@@ -5,6 +5,7 @@ from typing import Optional, Set
|
||||
import torch
|
||||
|
||||
from vllm.sequence import ExecuteModelRequest
|
||||
from vllm.worker.worker_base import WorkerBase
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -74,6 +75,12 @@ class SpeculativeProposer(ABC):
|
||||
|
||||
class SpeculativeScorer(ABC):
|
||||
|
||||
def __init__(self, scorer_worker: WorkerBase, device: str,
|
||||
vocab_size: int):
|
||||
self._scorer_worker = scorer_worker
|
||||
self._device = device
|
||||
self._vocab_size = vocab_size
|
||||
|
||||
@abstractmethod
|
||||
def score_proposals(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user