[Core] Async scheduling + structured outputs compatibility (#26866)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -20,10 +20,12 @@ from vllm.v1.kv_cache_interface import KVCacheSpec
|
||||
from vllm.v1.serial_utils import run_method
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.outputs import AsyncModelRunnerOutput, ModelRunnerOutput
|
||||
else:
|
||||
SchedulerOutput = object
|
||||
GrammarOutput = object
|
||||
AsyncModelRunnerOutput = object
|
||||
ModelRunnerOutput = object
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -122,7 +124,21 @@ class WorkerBase:
|
||||
"""Load model onto target device."""
|
||||
raise NotImplementedError
|
||||
|
||||
def execute_model(self, scheduler_output: SchedulerOutput) -> ModelRunnerOutput:
|
||||
def execute_model(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> ModelRunnerOutput | None:
|
||||
"""If this method returns None, sample_tokens should be called immediately after
|
||||
to obtain the ModelRunnerOutput.
|
||||
|
||||
Note that this design may be changed in future if/when structured outputs
|
||||
parallelism is re-architected.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def sample_tokens(
|
||||
self, grammar_output: GrammarOutput
|
||||
) -> ModelRunnerOutput | AsyncModelRunnerOutput:
|
||||
"""Should be called immediately after execute_model iff it returned None."""
|
||||
raise NotImplementedError
|
||||
|
||||
def get_cache_block_size_bytes(self) -> int:
|
||||
@@ -344,7 +360,7 @@ class WorkerWrapperBase:
|
||||
scheduler_output: SchedulerOutput,
|
||||
*args,
|
||||
**kwargs,
|
||||
) -> ModelRunnerOutput:
|
||||
) -> ModelRunnerOutput | None:
|
||||
self._apply_mm_cache(scheduler_output)
|
||||
|
||||
assert self.worker is not None
|
||||
|
||||
Reference in New Issue
Block a user