[Core] Introduce SPMD worker execution using Ray accelerated DAG (#6032)
Signed-off-by: Rui Qiao <ruisearch42@gmail.com> Co-authored-by: Stephanie Wang <swang@cs.berkeley.edu>
This commit is contained in:
co-authored by
Stephanie Wang
parent
d25877dd9b
commit
61e592747c
@@ -1,8 +1,8 @@
|
||||
import pickle
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sequence import ExecuteModelRequest
|
||||
from vllm.utils import get_ip, is_hip, is_xpu
|
||||
from vllm.worker.worker_base import WorkerWrapperBase
|
||||
|
||||
@@ -31,16 +31,18 @@ try:
|
||||
gpu_ids = ray.get_gpu_ids()
|
||||
return node_id, gpu_ids
|
||||
|
||||
def execute_model_compiled_dag_remote(self, ignored):
|
||||
"""Used only when compiled DAG is enabled."""
|
||||
def execute_model_spmd(self, execute_model_req: ExecuteModelRequest):
|
||||
"""Used only when SPMD worker and compiled DAG are both
|
||||
enabled."""
|
||||
# TODO(swang): This is needed right now because Ray aDAG executes
|
||||
# on a background thread, so we need to reset torch's current
|
||||
# device.
|
||||
import torch
|
||||
if not self.compiled_dag_cuda_device_set:
|
||||
torch.cuda.set_device(self.worker.device)
|
||||
self.compiled_dag_cuda_device_set = True
|
||||
|
||||
output = self.worker.execute_model()
|
||||
output = pickle.dumps(output)
|
||||
return output
|
||||
return self.worker._execute_model_spmd(execute_model_req)
|
||||
|
||||
ray_import_err = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user