[core] Multi Step Scheduling (#7000)
Co-authored-by: afeldman-nm <156691304+afeldman-nm@users.noreply.github.com>
This commit is contained in:
co-authored by
afeldman-nm
parent
dad961ef5c
commit
47b65a5508
@@ -16,7 +16,9 @@ from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
|
||||
SamplerOutput)
|
||||
from vllm.utils import (enable_trace_function_call_for_thread,
|
||||
update_environment_variables)
|
||||
from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
|
||||
from vllm.worker.model_runner_base import (BroadcastableModelInput,
|
||||
ModelRunnerBase,
|
||||
ModelRunnerInputBase)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -220,7 +222,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
|
||||
raise NotImplementedError
|
||||
|
||||
def _get_worker_input_from_broadcast(
|
||||
self) -> Optional[Tuple[ModelRunnerInputBase, WorkerInput]]:
|
||||
self) -> Optional[Tuple[BroadcastableModelInput, WorkerInput]]:
|
||||
""" Get the worker input from the broadcasted tensor dict. """
|
||||
assert self.do_metadata_broadcast
|
||||
assert not self.is_driver_worker
|
||||
@@ -237,7 +239,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
|
||||
|
||||
def _get_driver_input_and_broadcast(
|
||||
self, execute_model_req: ExecuteModelRequest
|
||||
) -> Tuple[ModelRunnerInputBase, WorkerInput]:
|
||||
) -> Tuple[BroadcastableModelInput, WorkerInput]:
|
||||
""" Get the driver input and broadcast it to other workers. """
|
||||
assert self.is_driver_worker
|
||||
|
||||
@@ -259,7 +261,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
|
||||
def prepare_input(
|
||||
self,
|
||||
execute_model_req: Optional[ExecuteModelRequest] = None
|
||||
) -> Optional[Tuple[ModelRunnerInputBase, WorkerInput]]:
|
||||
) -> Optional[Tuple[BroadcastableModelInput, WorkerInput]]:
|
||||
"""
|
||||
Prepare the inputs to ModelRunner and workers.
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user