[V1] Multiprocessing Tensor Parallel Support for v1 (#9856)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
This commit is contained in:
@@ -34,6 +34,7 @@ class GPUModelRunner:
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
device: torch.device,
|
||||
input_registry: InputRegistry = INPUT_REGISTRY,
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
@@ -43,7 +44,6 @@ class GPUModelRunner:
|
||||
self.load_config = vllm_config.load_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
self.scheduler_config = vllm_config.scheduler_config
|
||||
self.device_config = vllm_config.device_config
|
||||
self.speculative_config = vllm_config.speculative_config
|
||||
self.prompt_adapter_config = vllm_config.prompt_adapter_config
|
||||
self.observability_config = vllm_config.observability_config
|
||||
@@ -52,7 +52,7 @@ class GPUModelRunner:
|
||||
cache_config = self.cache_config
|
||||
scheduler_config = self.scheduler_config
|
||||
parallel_config = self.parallel_config
|
||||
self.device = self.device_config.device
|
||||
self.device = device
|
||||
self.pin_memory = is_pin_memory_available()
|
||||
self.dtype = self.model_config.dtype
|
||||
if cache_config.cache_dtype == "auto":
|
||||
@@ -477,9 +477,7 @@ class GPUModelRunner:
|
||||
sampling_metadata=sampling_metadata,
|
||||
)
|
||||
|
||||
# NOTE: CPU-GPU synchronization happens here.
|
||||
sampled_token_ids = sampler_output.sampled_token_ids.cpu()
|
||||
sampled_token_ids_list = sampled_token_ids.tolist()
|
||||
sampled_token_ids = sampler_output.sampled_token_ids
|
||||
# TODO(woosuk): The following loop can be slow since it iterates over
|
||||
# the requests one by one. Optimize.
|
||||
num_reqs = self.input_batch.num_reqs
|
||||
@@ -490,7 +488,7 @@ class GPUModelRunner:
|
||||
assert seq_len <= req_state.num_tokens
|
||||
if seq_len == req_state.num_tokens:
|
||||
# Append the sampled token to the output token ids.
|
||||
token_id = sampled_token_ids_list[i]
|
||||
token_id = sampled_token_ids[i]
|
||||
self.input_batch.token_ids_cpu[i, seq_len] = token_id
|
||||
req_state.output_token_ids.append(token_id)
|
||||
else:
|
||||
@@ -512,7 +510,7 @@ class GPUModelRunner:
|
||||
model_runner_output = ModelRunnerOutput(
|
||||
req_ids=self.input_batch.req_ids[:num_reqs],
|
||||
req_id_to_index=self.input_batch.req_id_to_index,
|
||||
sampled_token_ids_cpu=sampled_token_ids,
|
||||
sampled_token_ids=sampled_token_ids,
|
||||
logprob_token_ids_cpu=logprob_token_ids,
|
||||
logprobs_cpu=logprobs,
|
||||
)
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor import set_random_seed
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
|
||||
from vllm.v1.core.scheduler import SchedulerOutput
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
|
||||
@@ -56,7 +57,6 @@ class Worker:
|
||||
from vllm.utils import init_cached_hf_modules
|
||||
init_cached_hf_modules()
|
||||
|
||||
self.model_runner = GPUModelRunner(vllm_config)
|
||||
# Torch profiler. Enabled and configured through env vars:
|
||||
# VLLM_TORCH_PROFILER_DIR=/path/to/save/trace
|
||||
if envs.VLLM_TORCH_PROFILER_DIR:
|
||||
@@ -103,6 +103,9 @@ class Worker:
|
||||
# Set random seed.
|
||||
set_random_seed(self.model_config.seed)
|
||||
|
||||
# Construct the model runner
|
||||
self.model_runner = GPUModelRunner(self.vllm_config, self.device)
|
||||
|
||||
def load_model(self) -> None:
|
||||
self.model_runner.load_model()
|
||||
|
||||
@@ -198,7 +201,7 @@ class Worker:
|
||||
scheduler_output: "SchedulerOutput",
|
||||
) -> ModelRunnerOutput:
|
||||
output = self.model_runner.execute_model(scheduler_output)
|
||||
# TODO(woosuk): Send the output to the engine process.
|
||||
return output if self.rank == 0 else None
|
||||
return output
|
||||
|
||||
def profile(self, is_start=True):
|
||||
@@ -209,6 +212,10 @@ class Worker:
|
||||
else:
|
||||
self.profiler.stop()
|
||||
|
||||
def check_health(self) -> None:
|
||||
# worker will always be healthy as long as it's running.
|
||||
return
|
||||
|
||||
|
||||
def init_worker_distributed_environment(
|
||||
parallel_config: ParallelConfig,
|
||||
|
||||
Reference in New Issue
Block a user