[Core][MRV2] Support eagle3 spec decode with pipeline parallel (#50514)
Signed-off-by: Yongqin Wang <yongqinwang@roblox.com> Co-authored-by: Cursor <noreply@cursor.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
Wentao Ye
parent
d4ef5e53fc
commit
d87a440f88
@@ -10,6 +10,7 @@ import torch
|
||||
|
||||
from vllm.distributed.parallel_state import get_pp_group
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.worker.gpu.buffer_utils import async_copy_to_gpu
|
||||
from vllm.v1.worker.gpu.input_batch import InputBatch
|
||||
|
||||
@@ -30,6 +31,7 @@ class PendingRecv:
|
||||
# Snapshot of slot generation counters at receive time, used to
|
||||
# detect requests aborted since then.
|
||||
gen_at_receive_np: np.ndarray # [num_reqs]
|
||||
draft_tokens: torch.Tensor | None = None # [num_reqs, num_speculative_steps]
|
||||
|
||||
|
||||
def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
|
||||
@@ -61,6 +63,7 @@ class PPHandler:
|
||||
self.is_last_rank = get_pp_group().is_last_rank
|
||||
self.last_rank = get_pp_group().last_rank
|
||||
self.max_sample_len = num_speculative_steps + 1
|
||||
self.num_speculative_steps = num_speculative_steps
|
||||
self.device = device
|
||||
self.main_stream = torch.cuda.current_stream(device)
|
||||
self.broadcast_stream = torch.cuda.Stream(device)
|
||||
@@ -82,11 +85,37 @@ class PPHandler:
|
||||
self.broadcast_group = get_pp_group().make_sibling_device_group(
|
||||
group_desc="pp_broadcast"
|
||||
)
|
||||
self.aux_hidden_state_relay_keys: tuple[str, ...] = ()
|
||||
|
||||
def on_req_idx_freed(self, req_idx: int) -> None:
|
||||
self.req_idx_gen_np[req_idx] += 1
|
||||
|
||||
def get_prev_sampled_outputs(self) -> dict[str, torch.Tensor] | None:
|
||||
def configure_aux_hidden_state_relay(self, model: torch.nn.Module) -> None:
|
||||
from vllm.v1.worker.gpu.spec_decode.eagle.eagle3_utils import (
|
||||
aux_hidden_state_relay_keys,
|
||||
)
|
||||
|
||||
self.aux_hidden_state_relay_keys = aux_hidden_state_relay_keys(model)
|
||||
|
||||
def relay_aux_hidden_states(
|
||||
self,
|
||||
intermediate_tensors: IntermediateTensors | None,
|
||||
output_intermediate_tensors: IntermediateTensors,
|
||||
) -> IntermediateTensors:
|
||||
if not self.aux_hidden_state_relay_keys:
|
||||
return output_intermediate_tensors
|
||||
assert intermediate_tensors is not None
|
||||
return IntermediateTensors(
|
||||
output_intermediate_tensors.tensors
|
||||
| {
|
||||
key: intermediate_tensors[key]
|
||||
for key in self.aux_hidden_state_relay_keys
|
||||
}
|
||||
)
|
||||
|
||||
def get_prev_sampled_outputs(
|
||||
self, draft_tokens_to_update: torch.Tensor | None = None
|
||||
) -> dict[str, torch.Tensor] | None:
|
||||
"""Consume the entry from pp_size steps ago and wait for its recv event,
|
||||
then filter out entries whose request was freed since `receive`.
|
||||
"""
|
||||
@@ -112,6 +141,18 @@ class PPHandler:
|
||||
idx_mapping = async_copy_to_gpu(idx_mapping_np, device=self.device)
|
||||
|
||||
self.main_stream.wait_event(slot.event)
|
||||
if slot.draft_tokens is not None and draft_tokens_to_update is not None:
|
||||
draft_tokens = slot.draft_tokens
|
||||
draft_idx_mapping = slot.idx_mapping
|
||||
if exclude_mask.any():
|
||||
keep = ~exclude_mask
|
||||
keep_t = torch.as_tensor(keep, device=self.device)
|
||||
draft_tokens = draft_tokens[keep_t]
|
||||
draft_idx_mapping = async_copy_to_gpu(
|
||||
slot.idx_mapping_np[keep], device=self.device
|
||||
)
|
||||
draft_tokens_to_update[draft_idx_mapping] = draft_tokens
|
||||
|
||||
return dict(
|
||||
sampled_tokens=slot.sampled_tokens,
|
||||
num_sampled=slot.num_sampled,
|
||||
@@ -119,6 +160,21 @@ class PPHandler:
|
||||
idx_mapping=idx_mapping,
|
||||
)
|
||||
|
||||
def broadcast_drafts(
|
||||
self, draft_tokens: torch.Tensor, input_batch: InputBatch
|
||||
) -> None:
|
||||
"""Broadcast draft proposals so non-last ranks can embed real token ids."""
|
||||
assert self.is_last_rank
|
||||
if compute_need_sampled_mask(input_batch) is None:
|
||||
return
|
||||
with torch.cuda.stream(self.broadcast_stream):
|
||||
self.broadcast_stream.wait_stream(self.main_stream)
|
||||
send = draft_tokens[input_batch.idx_mapping].contiguous()
|
||||
torch.distributed.broadcast(
|
||||
send, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
send.record_stream(self.broadcast_stream)
|
||||
|
||||
def receive(self, input_batch: InputBatch) -> bool:
|
||||
"""Returns True iff sampled tokens need to be gathered from *all*
|
||||
requests in the batch."""
|
||||
@@ -145,12 +201,25 @@ class PPHandler:
|
||||
torch.distributed.broadcast(
|
||||
combined, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
draft_tokens = None
|
||||
if self.num_speculative_steps > 0:
|
||||
draft_tokens = torch.empty(
|
||||
num_reqs,
|
||||
self.num_speculative_steps,
|
||||
dtype=torch.int64,
|
||||
device=self.device,
|
||||
)
|
||||
torch.distributed.broadcast(
|
||||
draft_tokens, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
event = self.broadcast_stream.record_event()
|
||||
num_sampled, num_rejected = combined.unbind(dim=0)
|
||||
# Must record_stream since these were allocated on broadcast stream but
|
||||
# later used on the main stream.
|
||||
sampled_tokens.record_stream(self.main_stream)
|
||||
combined.record_stream(self.main_stream)
|
||||
if draft_tokens is not None:
|
||||
draft_tokens.record_stream(self.main_stream)
|
||||
self.queue[-1] = PendingRecv(
|
||||
event,
|
||||
sampled_tokens,
|
||||
@@ -160,6 +229,7 @@ class PPHandler:
|
||||
input_batch.idx_mapping_np,
|
||||
need_sampled_mask,
|
||||
gen_at_receive_np,
|
||||
draft_tokens,
|
||||
)
|
||||
return bool(need_sampled_mask.all())
|
||||
|
||||
@@ -182,8 +252,12 @@ class PPHandler:
|
||||
|
||||
with torch.cuda.stream(self.broadcast_stream):
|
||||
self.broadcast_stream.wait_stream(self.main_stream)
|
||||
send_tokens = torch.nn.functional.pad(
|
||||
sampled_token_ids,
|
||||
(0, self.max_sample_len - sampled_token_ids.shape[-1]),
|
||||
)
|
||||
torch.distributed.broadcast(
|
||||
sampled_token_ids.contiguous(),
|
||||
send_tokens.contiguous(),
|
||||
src=self.last_rank,
|
||||
group=self.broadcast_group,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user