[BugFix] Fix async scheduling for pooling models (#31584)

Signed-off-by: njhill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2025-12-31 14:48:51 -08:00
committed by GitHub
parent d8da76f3b7
commit 6c2cfb62ff
7 changed files with 132 additions and 93 deletions
+2 -2
View File
@@ -124,7 +124,7 @@ class WorkerBase:
def execute_model(
self, scheduler_output: SchedulerOutput
) -> ModelRunnerOutput | None:
) -> ModelRunnerOutput | AsyncModelRunnerOutput | None:
"""If this method returns None, sample_tokens should be called immediately after
to obtain the ModelRunnerOutput.
@@ -362,7 +362,7 @@ class WorkerWrapperBase:
scheduler_output: SchedulerOutput,
*args,
**kwargs,
) -> ModelRunnerOutput | None:
) -> ModelRunnerOutput | AsyncModelRunnerOutput | None:
self._apply_mm_cache(scheduler_output)
assert self.worker is not None