[AsyncScheduling] Make async overlap work with logprobs (#27615)

Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Nick Hill
2025-10-28 22:35:54 +00:00
committed by GitHub
parent 111faf1118
commit 4fe5895361
4 changed files with 65 additions and 10 deletions
+15 -4
View File
@@ -164,6 +164,7 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
self,
model_runner_output: ModelRunnerOutput,
sampled_token_ids: torch.Tensor,
logprobs_tensors: torch.Tensor | None,
invalid_req_indices: list[int],
async_output_copy_stream: torch.cuda.Stream,
):
@@ -176,6 +177,7 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
# Keep a reference to the device tensor to avoid it being
# deallocated until we finish copying it to the host.
self._sampled_token_ids = sampled_token_ids
self._logprobs_tensors = logprobs_tensors
# Initiate the copy on a separate stream, but do not synchronize it.
default_stream = torch.cuda.current_stream()
@@ -184,6 +186,11 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
self.sampled_token_ids_cpu = self._sampled_token_ids.to(
"cpu", non_blocking=True
)
self._logprobs_tensors_cpu = (
self._logprobs_tensors.to_cpu_nonblocking()
if self._logprobs_tensors
else None
)
self.async_copy_ready_event.record()
def get_output(self) -> ModelRunnerOutput:
@@ -193,7 +200,8 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
"""
self.async_copy_ready_event.synchronize()
# Release the device tensor once the copy has completed
# Release the device tensors once the copy has completed.
del self._logprobs_tensors
del self._sampled_token_ids
valid_sampled_token_ids = self.sampled_token_ids_cpu.tolist()
@@ -202,6 +210,10 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
output = self._model_runner_output
output.sampled_token_ids = valid_sampled_token_ids
if self._logprobs_tensors_cpu:
# NOTE(nick): this will need to be updated to use cu_num_accepted_tokens
# for async sched + spec decode + logprobs compatibility.
output.logprobs = self._logprobs_tensors_cpu.tolists()
return output
@@ -2334,11 +2346,9 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
cu_num_accepted_tokens[-1] + len(sampled_ids)
)
# NOTE: GPU -> CPU Sync happens here.
# Move as many CPU operations as possible before this sync point.
logprobs_lists = (
logprobs_tensors.tolists(cu_num_accepted_tokens)
if logprobs_tensors is not None
if not self.use_async_scheduling and logprobs_tensors is not None
else None
)
@@ -2664,6 +2674,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
async_output = AsyncGPUModelRunnerOutput(
model_runner_output=output,
sampled_token_ids=sampler_output.sampled_token_ids,
logprobs_tensors=sampler_output.logprobs_tensors,
invalid_req_indices=invalid_req_indices,
async_output_copy_stream=self.async_output_copy_stream,
)