[core] Multi Step Scheduling (#7000)

Co-authored-by: afeldman-nm <156691304+afeldman-nm@users.noreply.github.com>
This commit is contained in:
William Lin
2024-08-19 13:52:13 -07:00
committed by GitHub
co-authored by afeldman-nm
parent dad961ef5c
commit 47b65a5508
13 changed files with 1004 additions and 34 deletions
+6 -4
View File
@@ -16,7 +16,9 @@ from vllm.sequence import (ExecuteModelRequest, IntermediateTensors,
SamplerOutput)
from vllm.utils import (enable_trace_function_call_for_thread,
update_environment_variables)
from vllm.worker.model_runner_base import ModelRunnerBase, ModelRunnerInputBase
from vllm.worker.model_runner_base import (BroadcastableModelInput,
ModelRunnerBase,
ModelRunnerInputBase)
logger = init_logger(__name__)
@@ -220,7 +222,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
raise NotImplementedError
def _get_worker_input_from_broadcast(
self) -> Optional[Tuple[ModelRunnerInputBase, WorkerInput]]:
self) -> Optional[Tuple[BroadcastableModelInput, WorkerInput]]:
""" Get the worker input from the broadcasted tensor dict. """
assert self.do_metadata_broadcast
assert not self.is_driver_worker
@@ -237,7 +239,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
def _get_driver_input_and_broadcast(
self, execute_model_req: ExecuteModelRequest
) -> Tuple[ModelRunnerInputBase, WorkerInput]:
) -> Tuple[BroadcastableModelInput, WorkerInput]:
""" Get the driver input and broadcast it to other workers. """
assert self.is_driver_worker
@@ -259,7 +261,7 @@ class LocalOrDistributedWorkerBase(WorkerBase):
def prepare_input(
self,
execute_model_req: Optional[ExecuteModelRequest] = None
) -> Optional[Tuple[ModelRunnerInputBase, WorkerInput]]:
) -> Optional[Tuple[BroadcastableModelInput, WorkerInput]]:
"""
Prepare the inputs to ModelRunner and workers.
"""