[PP][XPU]Overlap async-scheduling PP sampled-token broadcast with compute (#51650)
Signed-off-by: yisheng <yi.sheng@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -586,6 +586,9 @@ class GPUModelRunner(
|
|||||||
# Async scheduling
|
# Async scheduling
|
||||||
self.use_async_scheduling = self.scheduler_config.async_scheduling
|
self.use_async_scheduling = self.scheduler_config.async_scheduling
|
||||||
|
|
||||||
|
# Async PP broadcast of sampled token ids, waited on in _prepare_input_ids.
|
||||||
|
self._pp_recv_work: torch.distributed.Work | None = None
|
||||||
|
|
||||||
# Sampler
|
# Sampler
|
||||||
self.sampler = Sampler(
|
self.sampler = Sampler(
|
||||||
logprobs_mode=self.model_config.logprobs_mode,
|
logprobs_mode=self.model_config.logprobs_mode,
|
||||||
@@ -1843,6 +1846,11 @@ class GPUModelRunner(
|
|||||||
(-1 for new requests).
|
(-1 for new requests).
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
# Sync the async PP broadcast before reading sampled tokens.
|
||||||
|
if self._pp_recv_work is not None:
|
||||||
|
self._pp_recv_work.wait()
|
||||||
|
self._pp_recv_work = None
|
||||||
|
|
||||||
if self.input_batch.prev_sampled_token_ids is None:
|
if self.input_batch.prev_sampled_token_ids is None:
|
||||||
# Normal scheduling case
|
# Normal scheduling case
|
||||||
self.input_ids.copy_to_gpu(total_num_scheduled_tokens)
|
self.input_ids.copy_to_gpu(total_num_scheduled_tokens)
|
||||||
@@ -4963,7 +4971,9 @@ class GPUModelRunner(
|
|||||||
recv = torch.empty((num_reqs, 1), dtype=torch.int32, device=self.device)
|
recv = torch.empty((num_reqs, 1), dtype=torch.int32, device=self.device)
|
||||||
# skip for chunked prefill.
|
# skip for chunked prefill.
|
||||||
if not self._is_all_reqs_chunked_prefill():
|
if not self._is_all_reqs_chunked_prefill():
|
||||||
torch.distributed.broadcast(recv, src=pp.last_rank, group=pp.device_group)
|
self._pp_recv_work = torch.distributed.broadcast(
|
||||||
|
recv, src=pp.last_rank, group=pp.device_group, async_op=True
|
||||||
|
)
|
||||||
self.input_batch.prev_sampled_token_ids = recv
|
self.input_batch.prev_sampled_token_ids = recv
|
||||||
|
|
||||||
# construct `prev_req_id_to_index` here so `_prepare_input_ids`
|
# construct `prev_req_id_to_index` here so `_prepare_input_ids`
|
||||||
|
|||||||
Reference in New Issue
Block a user