[AsyncScheduling] Make async overlap work with logprobs (#27615)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -164,6 +164,7 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
self,
|
||||
model_runner_output: ModelRunnerOutput,
|
||||
sampled_token_ids: torch.Tensor,
|
||||
logprobs_tensors: torch.Tensor | None,
|
||||
invalid_req_indices: list[int],
|
||||
async_output_copy_stream: torch.cuda.Stream,
|
||||
):
|
||||
@@ -176,6 +177,7 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
# Keep a reference to the device tensor to avoid it being
|
||||
# deallocated until we finish copying it to the host.
|
||||
self._sampled_token_ids = sampled_token_ids
|
||||
self._logprobs_tensors = logprobs_tensors
|
||||
|
||||
# Initiate the copy on a separate stream, but do not synchronize it.
|
||||
default_stream = torch.cuda.current_stream()
|
||||
@@ -184,6 +186,11 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
self.sampled_token_ids_cpu = self._sampled_token_ids.to(
|
||||
"cpu", non_blocking=True
|
||||
)
|
||||
self._logprobs_tensors_cpu = (
|
||||
self._logprobs_tensors.to_cpu_nonblocking()
|
||||
if self._logprobs_tensors
|
||||
else None
|
||||
)
|
||||
self.async_copy_ready_event.record()
|
||||
|
||||
def get_output(self) -> ModelRunnerOutput:
|
||||
@@ -193,7 +200,8 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
"""
|
||||
self.async_copy_ready_event.synchronize()
|
||||
|
||||
# Release the device tensor once the copy has completed
|
||||
# Release the device tensors once the copy has completed.
|
||||
del self._logprobs_tensors
|
||||
del self._sampled_token_ids
|
||||
|
||||
valid_sampled_token_ids = self.sampled_token_ids_cpu.tolist()
|
||||
@@ -202,6 +210,10 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
|
||||
|
||||
output = self._model_runner_output
|
||||
output.sampled_token_ids = valid_sampled_token_ids
|
||||
if self._logprobs_tensors_cpu:
|
||||
# NOTE(nick): this will need to be updated to use cu_num_accepted_tokens
|
||||
# for async sched + spec decode + logprobs compatibility.
|
||||
output.logprobs = self._logprobs_tensors_cpu.tolists()
|
||||
return output
|
||||
|
||||
|
||||
@@ -2334,11 +2346,9 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
cu_num_accepted_tokens[-1] + len(sampled_ids)
|
||||
)
|
||||
|
||||
# NOTE: GPU -> CPU Sync happens here.
|
||||
# Move as many CPU operations as possible before this sync point.
|
||||
logprobs_lists = (
|
||||
logprobs_tensors.tolists(cu_num_accepted_tokens)
|
||||
if logprobs_tensors is not None
|
||||
if not self.use_async_scheduling and logprobs_tensors is not None
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -2664,6 +2674,7 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
|
||||
async_output = AsyncGPUModelRunnerOutput(
|
||||
model_runner_output=output,
|
||||
sampled_token_ids=sampler_output.sampled_token_ids,
|
||||
logprobs_tensors=sampler_output.logprobs_tensors,
|
||||
invalid_req_indices=invalid_req_indices,
|
||||
async_output_copy_stream=self.async_output_copy_stream,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user