Revert "[DSpark] Support pipeline-parallel targets in aggregated serving (#56956)" (#58484)

This commit is contained in:
Nick Hill
2026-09-24 08:48:36 -07:00
committed by GitHub
parent d636456b1f
commit 09fe178dba
11 changed files with 28 additions and 629 deletions
+20 -65
View File
@@ -46,18 +46,6 @@ def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
return produces_sample if produces_sample.any() else None
def _alloc_combined(num_reqs: int, device: torch.device) -> torch.Tensor:
"""Allocate the (2, N) int32 buffer broadcast alongside sampled tokens.
The inner dim is padded to a multiple of 4 so that both `unbind` views
stay 16-byte aligned for any `num_reqs`: triton specializes on pointer
alignment, and a misaligned `num_rejected` would JIT-compile a second
`_post_update_kernel` variant at serving time. The padding is broadcast
but never read. Sender and receiver must both use this allocation.
"""
return torch.empty(2, -(-num_reqs // 4) * 4, dtype=torch.int32, device=device)
class PPHandler:
"""Runs the PP sampled-token broadcast/recv on a side stream so the
default stream isn't gated by the matching peer call. Step T's recv is
@@ -98,14 +86,6 @@ class PPHandler:
)
self.aux_hidden_state_relay_keys: tuple[str, ...] = ()
# Warmup steps run the pipeline with synthetic batches whose outputs are
# discarded; the sampled-token broadcast is disabled there so its
# side-stream NCCL ops cannot overlap the next step's activation p2p.
self.disabled = False
def set_disabled(self, disabled: bool) -> None:
self.disabled = disabled
def on_req_idx_freed(self, req_idx: int) -> None:
self.req_idx_gen_np[req_idx] += 1
@@ -177,15 +157,29 @@ class PPHandler:
num_sampled=slot.num_sampled,
num_rejected=slot.num_rejected,
idx_mapping=idx_mapping,
broadcast_drafts=slot.draft_tokens,
)
def broadcast_drafts(
self, draft_tokens: torch.Tensor, input_batch: InputBatch
) -> None:
"""Broadcast draft proposals so non-last ranks can embed real token ids."""
assert self.is_last_rank
if compute_need_sampled_mask(input_batch) is None:
return
with torch.cuda.stream(self.broadcast_stream):
self.broadcast_stream.wait_stream(self.main_stream)
send = draft_tokens[input_batch.idx_mapping].contiguous()
# Must record the idx_mapping tensor since it was allocated
# on the main stream.
input_batch.idx_mapping.record_stream(self.broadcast_stream)
torch.distributed.broadcast(
send, src=self.last_rank, group=self.broadcast_group
)
def receive(self, input_batch: InputBatch) -> bool:
"""Returns True iff sampled tokens need to be gathered from *all*
requests in the batch."""
assert not self.is_last_rank
if self.disabled:
return False
need_sampled_mask = compute_need_sampled_mask(input_batch)
if need_sampled_mask is None:
# Leave this step's reserved slot as None.
@@ -201,7 +195,7 @@ class PPHandler:
sampled_tokens = torch.empty(
num_reqs, self.max_sample_len, dtype=torch.int64, device=self.device
)
combined = _alloc_combined(num_reqs, self.device)
combined = torch.empty(2, num_reqs, dtype=torch.int32, device=self.device)
torch.distributed.broadcast(
sampled_tokens, src=self.last_rank, group=self.broadcast_group
)
@@ -219,7 +213,6 @@ class PPHandler:
torch.distributed.broadcast(
draft_tokens, src=self.last_rank, group=self.broadcast_group
)
event = self.broadcast_stream.record_event()
num_sampled, num_rejected = combined.unbind(dim=0)
# Must record_stream since these were allocated on broadcast stream but
@@ -249,16 +242,11 @@ class PPHandler:
input_batch: InputBatch,
) -> None:
assert self.is_last_rank
if self.disabled:
return
mask = compute_need_sampled_mask(input_batch)
if mask is None:
if compute_need_sampled_mask(input_batch) is None:
# No request needs sampled outputs for a subsequent decode step.
return
assert sampled_token_ids.dtype == torch.int64
assert num_sampled.dtype == torch.int32
assert num_rejected.dtype == torch.int32
if current_platform.is_xpu():
self.main_stream.synchronize()
@@ -274,42 +262,9 @@ class PPHandler:
src=self.last_rank,
group=self.broadcast_group,
)
combined = _alloc_combined(num_sampled.shape[0], self.device)
combined[0, : num_sampled.shape[0]] = num_sampled
combined[1, : num_sampled.shape[0]] = num_rejected
combined = torch.stack((num_sampled, num_rejected), dim=0)
torch.distributed.broadcast(
combined, src=self.last_rank, group=self.broadcast_group
)
for tensor in (sampled_token_ids, num_sampled, num_rejected):
tensor.record_stream(self.broadcast_stream)
def broadcast_drafts(
self, draft_token_table: torch.Tensor, input_batch: InputBatch
) -> None:
"""Broadcast the speculator's freshly proposed draft tokens.
Runs after propose() on the last rank; the send is stream-ordered
after broadcast()'s sends, matching receive()'s enqueue order. The
payload is gathered from the runner's draft table (just updated from
propose()'s output) into a fresh compact tensor: the speculator
overwrites its own persistent buffer on the next step, possibly
before this async send completes.
"""
assert self.is_last_rank
if self.disabled or self.max_sample_len == 1:
return
if compute_need_sampled_mask(input_batch) is None:
return
assert draft_token_table.dtype == torch.int64
assert draft_token_table.shape[1] == self.max_sample_len - 1
# Gather on the main stream so the payload is ordered after this
# step's table update and before any later one.
drafts = draft_token_table[input_batch.idx_mapping]
assert drafts.shape[0] == input_batch.num_reqs
with torch.cuda.stream(self.broadcast_stream):
self.broadcast_stream.wait_stream(self.main_stream)
torch.distributed.broadcast(
drafts, src=self.last_rank, group=self.broadcast_group
)
drafts.record_stream(self.broadcast_stream)