[Spec][V2] Support MTP speculative decoding under pipeline parallelism (#46994)
Signed-off-by: Chris Eastwood <chris.eastwood@pwn4g3.dev> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Nick Hill
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
9a35c081e8
commit
6fe67cbbf3
@@ -2030,7 +2030,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
)
|
||||
|
||||
mm_inputs: tuple[list[torch.Tensor], torch.Tensor] | None = None
|
||||
if self.speculator is not None and self.speculator.supports_mm_inputs:
|
||||
# The encoder runner exists only on the first PP rank, so later ranks
|
||||
# have no cached embeddings to gather.
|
||||
if (
|
||||
self.speculator is not None
|
||||
and self.speculator.supports_mm_inputs
|
||||
and self.model_state.supports_mm_inputs
|
||||
):
|
||||
# Get cached multimodal embeddings for draft forward.
|
||||
# NOTE: This is done here because postprocess updates
|
||||
# num_computed_prefill_tokens.
|
||||
|
||||
Reference in New Issue
Block a user