[V1] TPU - Add tensor parallel support via Ray (#13618)
Signed-off-by: Alexander Matveev <amatveev@redhat.com>
This commit is contained in:
@@ -11,6 +11,7 @@ import vllm.platforms
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.executor.msgspec_utils import decode_hook, encode_hook
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import ExecuteModelRequest, IntermediateTensors
|
||||
from vllm.utils import get_ip
|
||||
from vllm.worker.worker_base import WorkerWrapperBase
|
||||
@@ -106,10 +107,15 @@ try:
|
||||
# on a background thread, so we need to reset torch's current
|
||||
# device.
|
||||
# We can remove this API after it is fixed in compiled graph.
|
||||
import torch
|
||||
assert self.worker is not None, "Worker is not initialized"
|
||||
if not self.compiled_dag_cuda_device_set:
|
||||
torch.cuda.set_device(self.worker.device)
|
||||
if current_platform.is_tpu():
|
||||
# Not needed
|
||||
pass
|
||||
else:
|
||||
import torch
|
||||
torch.cuda.set_device(self.worker.device)
|
||||
|
||||
self.compiled_dag_cuda_device_set = True
|
||||
|
||||
def execute_model_ray(
|
||||
|
||||
Reference in New Issue
Block a user