[Core] Pipeline parallel with Ray ADAG (#6837)
Support pipeline-parallelism with Ray accelerated DAG. Signed-off-by: Rui Qiao <ruisearch42@gmail.com>
This commit is contained in:
@@ -1,8 +1,8 @@
|
||||
from typing import List, Optional, Tuple
|
||||
from typing import List, Optional, Tuple, Union
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sequence import ExecuteModelRequest
|
||||
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
|
||||
from vllm.utils import get_ip, is_hip, is_tpu, is_xpu
|
||||
from vllm.worker.worker_base import WorkerWrapperBase
|
||||
|
||||
@@ -31,9 +31,17 @@ try:
|
||||
gpu_ids = ray.get_gpu_ids()
|
||||
return node_id, gpu_ids
|
||||
|
||||
def execute_model_spmd(self, execute_model_req: ExecuteModelRequest):
|
||||
"""Used only when SPMD worker and compiled DAG are both
|
||||
enabled."""
|
||||
def execute_model_spmd(
|
||||
self, req_or_tuple: Union[ExecuteModelRequest,
|
||||
Tuple[ExecuteModelRequest,
|
||||
IntermediateTensors]]):
|
||||
"""Execute model in SPMD fashion: used only when SPMD worker and
|
||||
compiled DAG are both enabled.
|
||||
|
||||
Args:
|
||||
req_or_tuple: The request to execute the model, or a tuple
|
||||
containing the request and intermediate tensors.
|
||||
"""
|
||||
# TODO(swang): This is needed right now because Ray aDAG executes
|
||||
# on a background thread, so we need to reset torch's current
|
||||
# device.
|
||||
@@ -42,7 +50,17 @@ try:
|
||||
torch.cuda.set_device(self.worker.device)
|
||||
self.compiled_dag_cuda_device_set = True
|
||||
|
||||
return self.worker._execute_model_spmd(execute_model_req)
|
||||
if isinstance(req_or_tuple, tuple):
|
||||
execute_model_req, intermediate_tensors = req_or_tuple
|
||||
else:
|
||||
execute_model_req = req_or_tuple
|
||||
intermediate_tensors = None
|
||||
|
||||
output = self.worker._execute_model_spmd(execute_model_req,
|
||||
intermediate_tensors)
|
||||
if isinstance(output, IntermediateTensors):
|
||||
return execute_model_req, output
|
||||
return output
|
||||
|
||||
ray_import_err = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user