[Spec Decode] (1/2) Remove batch expansion (#8839)

This commit is contained in:
Lily Liu
2024-10-01 16:04:42 -07:00
committed by GitHub
parent 22f5851b80
commit 1570203864
29 changed files with 531 additions and 99 deletions
+7
View File
@@ -5,6 +5,7 @@ from typing import Optional, Set
import torch
from vllm.sequence import ExecuteModelRequest
from vllm.worker.worker_base import WorkerBase
@dataclass
@@ -74,6 +75,12 @@ class SpeculativeProposer(ABC):
class SpeculativeScorer(ABC):
def __init__(self, scorer_worker: WorkerBase, device: str,
vocab_size: int):
self._scorer_worker = scorer_worker
self._device = device
self._vocab_size = vocab_size
@abstractmethod
def score_proposals(
self,