[Hardware][TPU] Implement tensor parallelism with Ray (#5871)
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -45,6 +46,7 @@ class ModelInputForTPU(ModelRunnerInputBase):
|
||||
num_samples: int
|
||||
best_of: List[int]
|
||||
seq_groups: List[List[int]]
|
||||
virtual_engine: int = 0
|
||||
|
||||
def as_broadcastable_tensor_dict(
|
||||
self) -> Dict[str, Union[int, torch.Tensor]]:
|
||||
@@ -55,6 +57,9 @@ class ModelInputForTPU(ModelRunnerInputBase):
|
||||
"t": self.t,
|
||||
"p": self.p,
|
||||
"num_samples": self.num_samples,
|
||||
"best_of": self.best_of,
|
||||
"seq_groups": self.seq_groups,
|
||||
"virtual_engine": self.virtual_engine,
|
||||
}
|
||||
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
|
||||
return tensor_dict
|
||||
@@ -113,16 +118,30 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
def load_model(self) -> None:
|
||||
self.device = self.device_config.device
|
||||
|
||||
model = get_model(
|
||||
model_config=self.model_config,
|
||||
load_config=self.load_config,
|
||||
device_config=self.device_config,
|
||||
parallel_config=self.parallel_config,
|
||||
cache_config=self.cache_config,
|
||||
scheduler_config=self.scheduler_config,
|
||||
multimodal_config=self.multimodal_config,
|
||||
lora_config=None,
|
||||
)
|
||||
# NOTE(woosuk): While the executor assigns the TP ranks to the worker
|
||||
# process, the ranks can be different from the ranks internally assigned
|
||||
# by the xm runtime. Therefore, there is a mismatch in the rank
|
||||
# assignment between the gloo (cpu) runtime and the xm (tpu) runtime.
|
||||
# This is not a problem in linear layers because all-reduce is
|
||||
# rank-agnostic. However, it matters for all-gather as the ranks
|
||||
# determine the order of concatenating the output tensors.
|
||||
# As a workaround, we use the xm's rank assignment only when loading
|
||||
# the embedding weights.
|
||||
xm_tp_rank = xm.get_ordinal()
|
||||
with patch(
|
||||
"vllm.model_executor.layers.vocab_parallel_embedding."
|
||||
"get_tensor_model_parallel_rank",
|
||||
return_value=xm_tp_rank):
|
||||
model = get_model(
|
||||
model_config=self.model_config,
|
||||
load_config=self.load_config,
|
||||
device_config=self.device_config,
|
||||
parallel_config=self.parallel_config,
|
||||
cache_config=self.cache_config,
|
||||
scheduler_config=self.scheduler_config,
|
||||
multimodal_config=self.multimodal_config,
|
||||
lora_config=None,
|
||||
)
|
||||
model = model.eval()
|
||||
xm.wait_device_ops()
|
||||
|
||||
@@ -463,10 +482,11 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
|
||||
tensor_dict, attn_backend=self.attn_backend)
|
||||
return model_input
|
||||
|
||||
@torch.no_grad()
|
||||
def execute_model(
|
||||
self,
|
||||
model_input: ModelInputForTPU,
|
||||
kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
|
||||
kv_caches: Optional[List[Any]],
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
num_steps: int = 1,
|
||||
) -> List[SamplerOutput]:
|
||||
|
||||
@@ -70,13 +70,13 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
|
||||
|
||||
def init_device(self) -> None:
|
||||
os.environ["PJRT_DEVICE"] = "TPU"
|
||||
self.device = xm.xla_device()
|
||||
self.device_config.device = self.device
|
||||
torch.set_grad_enabled(False)
|
||||
torch.set_default_dtype(self.model_config.dtype)
|
||||
|
||||
# NOTE(woosuk): This is just a hack to initialize the TP group.
|
||||
# This cannot perform the actual communication ops.
|
||||
# NOTE(woosuk): This is just to initialize the TP group and broadcast
|
||||
# the input objects on CPU. The all-reduce and all-gather ops on TPU
|
||||
# are invoked by `xm.all_reduce` and `xm.all_gather` which use their
|
||||
# own context.
|
||||
init_distributed_environment(
|
||||
world_size=self.parallel_config.world_size,
|
||||
rank=self.rank,
|
||||
@@ -88,6 +88,11 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
|
||||
self.parallel_config.tensor_parallel_size,
|
||||
self.parallel_config.pipeline_parallel_size)
|
||||
|
||||
# Device initialization should happen after initializing the distributed
|
||||
# runtime.
|
||||
self.device = xm.xla_device()
|
||||
self.device_config.device = self.device
|
||||
|
||||
# Set random seed.
|
||||
set_random_seed(self.model_config.seed)
|
||||
xm.set_rng_state(self.model_config.seed, self.device)
|
||||
@@ -200,8 +205,7 @@ class TPUWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
|
||||
|
||||
@property
|
||||
def do_metadata_broadcast(self) -> bool:
|
||||
# TODO(woosuk): Support TP.
|
||||
return False
|
||||
return self.parallel_config.tensor_parallel_size > 1
|
||||
|
||||
@property
|
||||
def kv_cache(self) -> Optional[List[List[torch.Tensor]]]:
|
||||
|
||||
Reference in New Issue
Block a user