[Speculative decoding 7/9] Speculative decoding end-to-end correctness tests. (#3951)
This commit is contained in:
@@ -6,8 +6,7 @@ import torch
|
||||
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
|
||||
from vllm.spec_decode.interfaces import (SpeculativeProposals,
|
||||
SpeculativeProposer)
|
||||
from vllm.spec_decode.util import (maybe_mock_device_tensors,
|
||||
sampler_output_to_torch)
|
||||
from vllm.spec_decode.util import sampler_output_to_torch
|
||||
from vllm.worker.worker import Worker
|
||||
|
||||
|
||||
@@ -329,12 +328,15 @@ class DraftModelTop1Proposer(SpeculativeProposer):
|
||||
"""
|
||||
if maybe_sampler_output is None:
|
||||
# If no speculative tokens, the sampler output will be None.
|
||||
# In this case we return empty tensors.
|
||||
proposal_tokens = torch.zeros(0,
|
||||
max_proposal_len,
|
||||
dtype=torch.long,
|
||||
device=self._device)
|
||||
proposal_probs = torch.zeros(0,
|
||||
# In this case we return empty proposals.
|
||||
proposal_tokens = torch.full(size=(
|
||||
batch_size,
|
||||
max_proposal_len,
|
||||
),
|
||||
fill_value=-1,
|
||||
dtype=torch.long,
|
||||
device=self._device)
|
||||
proposal_probs = torch.zeros(batch_size,
|
||||
max_proposal_len,
|
||||
self._vocab_size,
|
||||
dtype=torch.float32,
|
||||
@@ -345,17 +347,6 @@ class DraftModelTop1Proposer(SpeculativeProposer):
|
||||
return proposal_tokens, proposal_probs, proposal_lens_tensor
|
||||
|
||||
sampler_output = maybe_sampler_output
|
||||
|
||||
# We mock the device tensors until PR 7/9 is merged (e2e correctness).
|
||||
# https://docs.google.com/document/d/1rE4pr3IdspRw97XbImY4fS9IWYuJJ3HGtL7AdIKGrw8/edit#heading=h.qijw1sdidrer
|
||||
for step_output in sampler_output:
|
||||
maybe_mock_device_tensors(
|
||||
sampler_output=step_output,
|
||||
batch_size=len(proposal_lens),
|
||||
vocab_size=self._vocab_size,
|
||||
device=self._device,
|
||||
)
|
||||
|
||||
proposal_tokens, proposal_probs = sampler_output_to_torch(
|
||||
sampler_output)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user