This commit is contained in:
@@ -46,18 +46,6 @@ def compute_need_sampled_mask(input_batch: InputBatch) -> np.ndarray | None:
|
||||
return produces_sample if produces_sample.any() else None
|
||||
|
||||
|
||||
def _alloc_combined(num_reqs: int, device: torch.device) -> torch.Tensor:
|
||||
"""Allocate the (2, N) int32 buffer broadcast alongside sampled tokens.
|
||||
|
||||
The inner dim is padded to a multiple of 4 so that both `unbind` views
|
||||
stay 16-byte aligned for any `num_reqs`: triton specializes on pointer
|
||||
alignment, and a misaligned `num_rejected` would JIT-compile a second
|
||||
`_post_update_kernel` variant at serving time. The padding is broadcast
|
||||
but never read. Sender and receiver must both use this allocation.
|
||||
"""
|
||||
return torch.empty(2, -(-num_reqs // 4) * 4, dtype=torch.int32, device=device)
|
||||
|
||||
|
||||
class PPHandler:
|
||||
"""Runs the PP sampled-token broadcast/recv on a side stream so the
|
||||
default stream isn't gated by the matching peer call. Step T's recv is
|
||||
@@ -98,14 +86,6 @@ class PPHandler:
|
||||
)
|
||||
self.aux_hidden_state_relay_keys: tuple[str, ...] = ()
|
||||
|
||||
# Warmup steps run the pipeline with synthetic batches whose outputs are
|
||||
# discarded; the sampled-token broadcast is disabled there so its
|
||||
# side-stream NCCL ops cannot overlap the next step's activation p2p.
|
||||
self.disabled = False
|
||||
|
||||
def set_disabled(self, disabled: bool) -> None:
|
||||
self.disabled = disabled
|
||||
|
||||
def on_req_idx_freed(self, req_idx: int) -> None:
|
||||
self.req_idx_gen_np[req_idx] += 1
|
||||
|
||||
@@ -177,15 +157,29 @@ class PPHandler:
|
||||
num_sampled=slot.num_sampled,
|
||||
num_rejected=slot.num_rejected,
|
||||
idx_mapping=idx_mapping,
|
||||
broadcast_drafts=slot.draft_tokens,
|
||||
)
|
||||
|
||||
def broadcast_drafts(
|
||||
self, draft_tokens: torch.Tensor, input_batch: InputBatch
|
||||
) -> None:
|
||||
"""Broadcast draft proposals so non-last ranks can embed real token ids."""
|
||||
assert self.is_last_rank
|
||||
if compute_need_sampled_mask(input_batch) is None:
|
||||
return
|
||||
with torch.cuda.stream(self.broadcast_stream):
|
||||
self.broadcast_stream.wait_stream(self.main_stream)
|
||||
send = draft_tokens[input_batch.idx_mapping].contiguous()
|
||||
# Must record the idx_mapping tensor since it was allocated
|
||||
# on the main stream.
|
||||
input_batch.idx_mapping.record_stream(self.broadcast_stream)
|
||||
torch.distributed.broadcast(
|
||||
send, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
|
||||
def receive(self, input_batch: InputBatch) -> bool:
|
||||
"""Returns True iff sampled tokens need to be gathered from *all*
|
||||
requests in the batch."""
|
||||
assert not self.is_last_rank
|
||||
if self.disabled:
|
||||
return False
|
||||
need_sampled_mask = compute_need_sampled_mask(input_batch)
|
||||
if need_sampled_mask is None:
|
||||
# Leave this step's reserved slot as None.
|
||||
@@ -201,7 +195,7 @@ class PPHandler:
|
||||
sampled_tokens = torch.empty(
|
||||
num_reqs, self.max_sample_len, dtype=torch.int64, device=self.device
|
||||
)
|
||||
combined = _alloc_combined(num_reqs, self.device)
|
||||
combined = torch.empty(2, num_reqs, dtype=torch.int32, device=self.device)
|
||||
torch.distributed.broadcast(
|
||||
sampled_tokens, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
@@ -219,7 +213,6 @@ class PPHandler:
|
||||
torch.distributed.broadcast(
|
||||
draft_tokens, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
|
||||
event = self.broadcast_stream.record_event()
|
||||
num_sampled, num_rejected = combined.unbind(dim=0)
|
||||
# Must record_stream since these were allocated on broadcast stream but
|
||||
@@ -249,16 +242,11 @@ class PPHandler:
|
||||
input_batch: InputBatch,
|
||||
) -> None:
|
||||
assert self.is_last_rank
|
||||
if self.disabled:
|
||||
return
|
||||
mask = compute_need_sampled_mask(input_batch)
|
||||
if mask is None:
|
||||
if compute_need_sampled_mask(input_batch) is None:
|
||||
# No request needs sampled outputs for a subsequent decode step.
|
||||
return
|
||||
|
||||
assert sampled_token_ids.dtype == torch.int64
|
||||
assert num_sampled.dtype == torch.int32
|
||||
assert num_rejected.dtype == torch.int32
|
||||
|
||||
if current_platform.is_xpu():
|
||||
self.main_stream.synchronize()
|
||||
@@ -274,42 +262,9 @@ class PPHandler:
|
||||
src=self.last_rank,
|
||||
group=self.broadcast_group,
|
||||
)
|
||||
combined = _alloc_combined(num_sampled.shape[0], self.device)
|
||||
combined[0, : num_sampled.shape[0]] = num_sampled
|
||||
combined[1, : num_sampled.shape[0]] = num_rejected
|
||||
combined = torch.stack((num_sampled, num_rejected), dim=0)
|
||||
torch.distributed.broadcast(
|
||||
combined, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
for tensor in (sampled_token_ids, num_sampled, num_rejected):
|
||||
tensor.record_stream(self.broadcast_stream)
|
||||
|
||||
def broadcast_drafts(
|
||||
self, draft_token_table: torch.Tensor, input_batch: InputBatch
|
||||
) -> None:
|
||||
"""Broadcast the speculator's freshly proposed draft tokens.
|
||||
|
||||
Runs after propose() on the last rank; the send is stream-ordered
|
||||
after broadcast()'s sends, matching receive()'s enqueue order. The
|
||||
payload is gathered from the runner's draft table (just updated from
|
||||
propose()'s output) into a fresh compact tensor: the speculator
|
||||
overwrites its own persistent buffer on the next step, possibly
|
||||
before this async send completes.
|
||||
"""
|
||||
assert self.is_last_rank
|
||||
if self.disabled or self.max_sample_len == 1:
|
||||
return
|
||||
if compute_need_sampled_mask(input_batch) is None:
|
||||
return
|
||||
assert draft_token_table.dtype == torch.int64
|
||||
assert draft_token_table.shape[1] == self.max_sample_len - 1
|
||||
|
||||
# Gather on the main stream so the payload is ordered after this
|
||||
# step's table update and before any later one.
|
||||
drafts = draft_token_table[input_batch.idx_mapping]
|
||||
assert drafts.shape[0] == input_batch.num_reqs
|
||||
with torch.cuda.stream(self.broadcast_stream):
|
||||
self.broadcast_stream.wait_stream(self.main_stream)
|
||||
torch.distributed.broadcast(
|
||||
drafts, src=self.last_rank, group=self.broadcast_group
|
||||
)
|
||||
drafts.record_stream(self.broadcast_stream)
|
||||
|
||||
Reference in New Issue
Block a user