[Core] Async scheduling + structured outputs compatibility (#26866)
Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
@@ -19,7 +19,7 @@ from vllm.v1.outputs import AsyncModelRunnerOutput
|
||||
from vllm.v1.worker.worker_base import WorkerWrapperBase
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.core.sched.output import GrammarOutput, SchedulerOutput
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -82,36 +82,41 @@ try:
|
||||
|
||||
def execute_model_ray(
|
||||
self,
|
||||
scheduler_output: Union[
|
||||
"SchedulerOutput", tuple["SchedulerOutput", "IntermediateTensors"]
|
||||
],
|
||||
execute_model_input: tuple["SchedulerOutput", "GrammarOutput"]
|
||||
| tuple["SchedulerOutput", "GrammarOutput", "IntermediateTensors"],
|
||||
) -> Union[
|
||||
"ModelRunnerOutput", tuple["SchedulerOutput", "IntermediateTensors"]
|
||||
"ModelRunnerOutput",
|
||||
tuple["SchedulerOutput", "GrammarOutput", "IntermediateTensors"],
|
||||
]:
|
||||
# This method is used by Ray Compiled Graph to execute the model,
|
||||
# and it needs a special logic of self.setup_device_if_necessary()
|
||||
self.setup_device_if_necessary()
|
||||
assert self.worker is not None, "Worker is not initialized"
|
||||
if isinstance(scheduler_output, tuple):
|
||||
scheduler_output, intermediate_tensors = scheduler_output
|
||||
if len(execute_model_input) == 3:
|
||||
scheduler_output, grammar_output, intermediate_tensors = (
|
||||
execute_model_input
|
||||
)
|
||||
else:
|
||||
scheduler_output, intermediate_tensors = scheduler_output, None
|
||||
scheduler_output, grammar_output = execute_model_input
|
||||
intermediate_tensors = None
|
||||
assert self.worker.model_runner is not None
|
||||
output = self.worker.model_runner.execute_model(
|
||||
scheduler_output, intermediate_tensors
|
||||
)
|
||||
if isinstance(output, IntermediateTensors):
|
||||
output = scheduler_output, output
|
||||
output = scheduler_output, grammar_output, output
|
||||
elif not get_pp_group().is_last_rank:
|
||||
# Case where there are no scheduled requests
|
||||
# but may still be finished requests.
|
||||
assert not output or not output.req_ids
|
||||
output = scheduler_output, None
|
||||
# Ensure outputs crossing Ray compiled DAG are serializable.
|
||||
# AsyncModelRunnerOutput holds CUDA events and cannot be
|
||||
# pickled.
|
||||
if isinstance(output, AsyncModelRunnerOutput):
|
||||
output = output.get_output()
|
||||
output = scheduler_output, grammar_output, None
|
||||
elif output is None:
|
||||
output = self.worker.model_runner.sample_tokens(grammar_output)
|
||||
# Ensure outputs crossing Ray compiled DAG are serializable.
|
||||
# AsyncModelRunnerOutput holds CUDA events and cannot be
|
||||
# pickled.
|
||||
if isinstance(output, AsyncModelRunnerOutput):
|
||||
output = output.get_output()
|
||||
return output
|
||||
|
||||
def override_env_vars(self, vars: dict[str, str]):
|
||||
|
||||
Reference in New Issue
Block a user