[Speculative Decoding] Support draft model on different tensor-parallel size than target model (#5414)

This commit is contained in:
Woo-Yeon Lee
2024-06-25 09:56:06 +00:00
committed by GitHub
parent f23871e9ee
commit 2ce5d6688b
11 changed files with 388 additions and 59 deletions
+6 -5
View File
@@ -6,7 +6,8 @@ import torch
from vllm.sequence import (ExecuteModelRequest, SamplerOutput, SequenceData,
SequenceGroupMetadata)
from vllm.spec_decode.interfaces import SpeculativeProposals
from vllm.spec_decode.interfaces import (SpeculativeProposals,
SpeculativeProposer)
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
from vllm.spec_decode.top1_proposer import Top1Proposer
from vllm.worker.worker import Worker
@@ -28,9 +29,9 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
super().__init__(*args, **kwargs)
# Lazy initialization list.
self._proposer: Top1Proposer
self._proposer: SpeculativeProposer
def init_device(self):
def init_device(self) -> None:
super().init_device()
self._proposer = Top1Proposer(
@@ -40,7 +41,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
max_proposal_len=self.max_model_len,
)
def set_include_gpu_probs_tensor(self):
def set_include_gpu_probs_tensor(self) -> None:
# Need include_gpu_probs_tensor for multi_step_worker
self.model_runner.model.sampler.include_gpu_probs_tensor = True
@@ -73,7 +74,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
# Run model sample_len times.
model_outputs: List[SamplerOutput] = []
for _ in range(sample_len):
model_output = super().execute_model(
model_output: List[SamplerOutput] = super().execute_model(
execute_model_req=copied_execute_model_req)
assert (len(model_output) == 1
), "composing multistep workers not supported"