[BugFix] Fix async scheduling for pooling models (#31584)

Signed-off-by: njhill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2025-12-31 14:48:51 -08:00
committed by GitHub
parent d8da76f3b7
commit 6c2cfb62ff
7 changed files with 132 additions and 93 deletions
+92 -35
View File
@@ -254,6 +254,50 @@ class AsyncGPUModelRunnerOutput(AsyncModelRunnerOutput):
return output
class AsyncGPUPoolingModelRunnerOutput(AsyncModelRunnerOutput):
def __init__(
self,
model_runner_output: ModelRunnerOutput,
raw_pooler_output: PoolerOutput,
finished_mask: list[bool],
async_output_copy_stream: torch.cuda.Stream,
):
self._model_runner_output = model_runner_output
self._finished_mask = finished_mask
# Event on the copy stream so we can synchronize the non-blocking copy.
self.async_copy_ready_event = torch.Event()
# Keep a reference to the device tensors to avoid them being
# deallocated until we finish copying it to the host.
self._raw_pooler_output = raw_pooler_output
# Initiate the copy on a separate stream, but do not synchronize it.
default_stream = torch.cuda.current_stream()
with torch.cuda.stream(async_output_copy_stream):
async_output_copy_stream.wait_stream(default_stream)
self._raw_pooler_output_cpu = json_map_leaves(
lambda x: None if x is None else x.to("cpu", non_blocking=True),
self._raw_pooler_output,
)
self.async_copy_ready_event.record()
def get_output(self) -> ModelRunnerOutput:
"""Copy the device tensors to the host and return a ModelRunnerOutput.
This function blocks until the copy is finished.
"""
self.async_copy_ready_event.synchronize()
# Release the device tensors once the copy has completed.
del self._raw_pooler_output
self._model_runner_output.pooler_output = [
out if include else None
for out, include in zip(self._raw_pooler_output_cpu, self._finished_mask)
]
return self._model_runner_output
class ExecuteModelState(NamedTuple):
"""Ephemeral cached state transferred between execute_model() and
sample_tokens(), after execute_model() returns None."""
@@ -2476,17 +2520,19 @@ class GPUModelRunner(
hidden_states: torch.Tensor,
num_scheduled_tokens: int,
num_scheduled_tokens_np: np.ndarray,
) -> ModelRunnerOutput:
assert self.input_batch.num_reqs == len(self.input_batch.pooling_params), (
kv_connector_output: KVConnectorOutput | None,
) -> ModelRunnerOutput | AsyncModelRunnerOutput:
num_reqs = self.input_batch.num_reqs
assert num_reqs == len(self.input_batch.pooling_params), (
"Either all or none of the requests in a batch must be pooling request"
)
hidden_states = hidden_states[:num_scheduled_tokens]
seq_lens_cpu = self.seq_lens.cpu[: self.input_batch.num_reqs]
seq_lens_cpu = self.seq_lens.cpu[:num_reqs]
pooling_metadata = self.input_batch.get_pooling_metadata()
pooling_metadata.build_pooling_cursor(
num_scheduled_tokens_np.tolist(), seq_lens_cpu, device=hidden_states.device
num_scheduled_tokens_np, seq_lens_cpu, device=hidden_states.device
)
model = cast(VllmModelForPooling, self.model)
@@ -2494,27 +2540,41 @@ class GPUModelRunner(
hidden_states=hidden_states,
pooling_metadata=pooling_metadata,
)
finished_mask = [
seq_len == prompt_len
for seq_len, prompt_len in zip(seq_lens_cpu, pooling_metadata.prompt_lens)
]
model_runner_output = ModelRunnerOutput(
req_ids=self.input_batch.req_ids.copy(),
req_id_to_index=self.input_batch.req_id_to_index.copy(),
kv_connector_output=kv_connector_output,
)
if raw_pooler_output is None or not any(finished_mask):
model_runner_output.pooler_output = [None] * num_reqs
return model_runner_output
if self.use_async_scheduling:
return AsyncGPUPoolingModelRunnerOutput(
model_runner_output=model_runner_output,
raw_pooler_output=raw_pooler_output,
finished_mask=finished_mask,
async_output_copy_stream=self.async_output_copy_stream,
)
raw_pooler_output = json_map_leaves(
lambda x: x.to("cpu", non_blocking=True) if x is not None else x,
lambda x: None if x is None else x.to("cpu", non_blocking=True),
raw_pooler_output,
)
self._sync_device()
pooler_output: list[torch.Tensor | None] = []
for raw_output, seq_len, prompt_len in zip(
raw_pooler_output, seq_lens_cpu, pooling_metadata.prompt_lens
):
output = raw_output if seq_len == prompt_len else None
pooler_output.append(output)
return ModelRunnerOutput(
req_ids=self.input_batch.req_ids,
req_id_to_index=self.input_batch.req_id_to_index,
sampled_token_ids=[],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=pooler_output,
)
model_runner_output.pooler_output = [
out if include else None
for out, include in zip(raw_pooler_output, finished_mask)
]
return model_runner_output
def _pad_for_sequence_parallelism(self, num_scheduled_tokens: int) -> int:
# Pad tokens to multiple of tensor_parallel_size when
@@ -3036,7 +3096,7 @@ class GPUModelRunner(
self,
scheduler_output: "SchedulerOutput",
intermediate_tensors: IntermediateTensors | None = None,
) -> ModelRunnerOutput | IntermediateTensors | None:
) -> ModelRunnerOutput | AsyncModelRunnerOutput | IntermediateTensors | None:
if self.execute_model_state is not None:
raise RuntimeError(
"State error: sample_tokens() must be called "
@@ -3244,11 +3304,12 @@ class GPUModelRunner(
if self.is_pooling_model:
# Return the pooling output.
output = self._pool(
hidden_states, num_scheduled_tokens, num_scheduled_tokens_np
return self._pool(
hidden_states,
num_scheduled_tokens,
num_scheduled_tokens_np,
kv_connector_output,
)
output.kv_connector_output = kv_connector_output
return output
sample_hidden_states = hidden_states[logits_indices]
logits = self.model.compute_logits(sample_hidden_states)
@@ -3437,7 +3498,6 @@ class GPUModelRunner(
sampled_token_ids=valid_sampled_token_ids,
logprobs=logprobs_lists,
prompt_logprobs_dict=prompt_logprobs_dict,
pooler_output=[],
kv_connector_output=kv_connector_output,
ec_connector_output=ec_connector_output
if self.supports_mm_inputs
@@ -4508,17 +4568,14 @@ class GPUModelRunner(
max_num_reqs = self.scheduler_config.max_num_seqs
num_reqs = min(num_tokens, max_num_reqs)
min_tokens_per_req = num_tokens // num_reqs
num_scheduled_tokens_list = [min_tokens_per_req] * num_reqs
num_scheduled_tokens_list[-1] += num_tokens % num_reqs
assert sum(num_scheduled_tokens_list) == num_tokens
assert len(num_scheduled_tokens_list) == num_reqs
num_scheduled_tokens_np = np.full(num_reqs, min_tokens_per_req)
num_scheduled_tokens_np[-1] += num_tokens % num_reqs
assert np.sum(num_scheduled_tokens_np) == num_tokens
assert len(num_scheduled_tokens_np) == num_reqs
req_num_tokens = num_tokens // num_reqs
dummy_prompt_lens = torch.tensor(
num_scheduled_tokens_list,
device="cpu",
)
dummy_prompt_lens = torch.from_numpy(num_scheduled_tokens_np)
dummy_token_ids = torch.zeros(
(num_reqs, req_num_tokens), dtype=torch.int32, device=self.device
)
@@ -4537,7 +4594,7 @@ class GPUModelRunner(
)
dummy_metadata.build_pooling_cursor(
num_scheduled_tokens_list,
num_scheduled_tokens_np,
seq_lens_cpu=dummy_prompt_lens,
device=hidden_states.device,
)