Support torchrun and SPMD-style offline inference (#12071)

Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
youkaichao
2025-01-16 19:58:53 +08:00
committed by GitHub
parent dd7c9ad870
commit bf53e0c70b
14 changed files with 248 additions and 30 deletions
+3 -3
View File
@@ -172,7 +172,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
scheduling_strategy=scheduling_strategy,
**ray_remote_kwargs,
)(RayWorkerWrapper).remote(vllm_config=self.vllm_config,
rank=rank)
rpc_rank=rank)
else:
worker = ray.remote(
num_cpus=0,
@@ -181,7 +181,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
scheduling_strategy=scheduling_strategy,
**ray_remote_kwargs,
)(RayWorkerWrapper).remote(vllm_config=self.vllm_config,
rank=rank)
rpc_rank=rank)
worker_metadata.append(
RayWorkerMetaData(worker=worker, created_rank=rank))
rank += 1
@@ -204,7 +204,7 @@ class RayDistributedExecutor(DistributedExecutorBase):
# as the resource holder for the driver process.
self.driver_dummy_worker = worker
self.driver_worker = RayWorkerWrapper(
vllm_config=self.vllm_config, rank=0)
vllm_config=self.vllm_config, rpc_rank=0)
worker_metadata.pop(i)
break