[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)

Signed-off-by: Yongqin Wang <yongqinwang@roblox.com>
Co-authored-by: Cursor <noreply@cursor.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
yongqinwang-cmd
2026-09-05 11:45:34 +08:00
committed by GitHub
co-authored by Cursor Wentao Ye
parent d4ef5e53fc
commit d87a440f88
27 changed files with 582 additions and 152 deletions
+76 -2
View File
@@ -10,6 +10,7 @@ import torch
from vllm.distributed.parallel_state import get_pp_group
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
from vllm.v1.worker.gpu.input_batch import InputBatch
@@ -30,6 +31,7 @@ class PendingRecv:
# Snapshot of slot generation counters at receive time, used to
# detect requests aborted since then.
gen_at_receive_np: np.ndarray # [num_reqs]
draft_tokens: torch.Tensor | None = None # [num_reqs, num_speculative_steps]
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
@@ -61,6 +63,7 @@ class PPHandler:
self.is_last_rank = get_pp_group().is_last_rank
self.last_rank = get_pp_group().last_rank
self.max_sample_len = num_speculative_steps + 1
self.num_speculative_steps = num_speculative_steps
self.device = device
self.main_stream = torch.cuda.current_stream(device)
self.broadcast_stream = torch.cuda.Stream(device)
@@ -82,11 +85,37 @@ class PPHandler:
self.broadcast_group = get_pp_group().make_sibling_device_group(
group_desc="pp_broadcast"
)
self.aux_hidden_state_relay_keys: tuple[str, ...] = ()
def on_req_idx_freed(self, req_idx: int) -> None:
self.req_idx_gen_np[req_idx] += 1
def get_prev_sampled_outputs(self) -> dict[str, torch.Tensor] | None:
def configure_aux_hidden_state_relay(self, model: torch.nn.Module) -> None:
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
aux_hidden_state_relay_keys,
)
self.aux_hidden_state_relay_keys = aux_hidden_state_relay_keys(model)
def relay_aux_hidden_states(
self,
intermediate_tensors: IntermediateTensors | None,
output_intermediate_tensors: IntermediateTensors,
) -> IntermediateTensors:
if not self.aux_hidden_state_relay_keys:
return output_intermediate_tensors
assert intermediate_tensors is not None
return IntermediateTensors(
output_intermediate_tensors.tensors
| {
key: intermediate_tensors[key]
for key in self.aux_hidden_state_relay_keys
}
)
def get_prev_sampled_outputs(
self, draft_tokens_to_update: torch.Tensor | None = None
) -> dict[str, torch.Tensor] | None:
"""Consume the entry from pp_size steps ago and wait for its recv event,
then filter out entries whose request was freed since `receive`.
"""
@@ -112,6 +141,18 @@ class PPHandler:
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
self.main_stream.wait_event(slot.event)
if slot.draft_tokens is not None and draft_tokens_to_update is not None:
draft_tokens = slot.draft_tokens
draft_idx_mapping = slot.idx_mapping
if exclude_mask.any():
keep = ~exclude_mask
keep_t = torch.as_tensor(keep, device=self.device)
draft_tokens = draft_tokens[keep_t]
draft_idx_mapping = async_copy_to_gpu(
slot.idx_mapping_np[keep], device=self.device
)
draft_tokens_to_update[draft_idx_mapping] = draft_tokens
return dict(
sampled_tokens=slot.sampled_tokens,
num_sampled=slot.num_sampled,
@@ -119,6 +160,21 @@ class PPHandler:
idx_mapping=idx_mapping,
)
def broadcast_drafts(
self, draft_tokens: torch.Tensor, input_batch: InputBatch
) -> None:
"""Broadcast draft proposals so non-last ranks can embed real token ids."""
assert self.is_last_rank
if compute_need_sampled_mask(input_batch) is None:
return
with torch.cuda.stream(self.broadcast_stream):
self.broadcast_stream.wait_stream(self.main_stream)
send = draft_tokens[input_batch.idx_mapping].contiguous()
torch.distributed.broadcast(
send, src=self.last_rank, group=self.broadcast_group
)
send.record_stream(self.broadcast_stream)
def receive(self, input_batch: InputBatch) -> bool:
"""Returns True iff sampled tokens need to be gathered from *all*
requests in the batch."""
@@ -145,12 +201,25 @@ class PPHandler:
torch.distributed.broadcast(
combined, src=self.last_rank, group=self.broadcast_group
)
draft_tokens = None
if self.num_speculative_steps > 0:
draft_tokens = torch.empty(
num_reqs,
self.num_speculative_steps,
dtype=torch.int64,
device=self.device,
)
torch.distributed.broadcast(
draft_tokens, src=self.last_rank, group=self.broadcast_group
)
event = self.broadcast_stream.record_event()
num_sampled, num_rejected = combined.unbind(dim=0)
# Must record_stream since these were allocated on broadcast stream but
# later used on the main stream.
sampled_tokens.record_stream(self.main_stream)
combined.record_stream(self.main_stream)
if draft_tokens is not None:
draft_tokens.record_stream(self.main_stream)
self.queue[-1] = PendingRecv(
event,
sampled_tokens,
@@ -160,6 +229,7 @@ class PPHandler:
input_batch.idx_mapping_np,
need_sampled_mask,
gen_at_receive_np,
draft_tokens,
)
return bool(need_sampled_mask.all())
@@ -182,8 +252,12 @@ class PPHandler:
with torch.cuda.stream(self.broadcast_stream):
self.broadcast_stream.wait_stream(self.main_stream)
send_tokens = torch.nn.functional.pad(
sampled_token_ids,
(0, self.max_sample_len - sampled_token_ids.shape[-1]),
)
torch.distributed.broadcast(
sampled_token_ids.contiguous(),
send_tokens.contiguous(),
src=self.last_rank,
group=self.broadcast_group,
)