[Hardware][TPU] Implement tensor parallelism with Ray (#5871)

This commit is contained in:
Woosuk Kwon
2024-07-26 20:54:27 -07:00
committed by GitHub
parent 14dbd5a767
commit 52f07e3dec
6 changed files with 365 additions and 21 deletions
+8 -2
View File
@@ -394,8 +394,14 @@ class LLMEngine:
from vllm.executor.neuron_executor import NeuronExecutor
executor_class = NeuronExecutor
elif engine_config.device_config.device_type == "tpu":
from vllm.executor.tpu_executor import TPUExecutor
executor_class = TPUExecutor
if distributed_executor_backend == "ray":
initialize_ray_cluster(engine_config.parallel_config)
from vllm.executor.ray_tpu_executor import RayTPUExecutor
executor_class = RayTPUExecutor
else:
assert distributed_executor_backend is None
from vllm.executor.tpu_executor import TPUExecutor
executor_class = TPUExecutor
elif engine_config.device_config.device_type == "cpu":
from vllm.executor.cpu_executor import CPUExecutor
executor_class = CPUExecutor