Remove hardcoded device="cuda" to support more devices (#2503)
Co-authored-by: Jiang Li <jiang1.li@intel.com> Co-authored-by: Kunshang Ji <kunshang.ji@intel.com>
This commit is contained in:
@@ -104,11 +104,13 @@ class CacheEngine:
|
||||
size=(self.num_cpu_blocks, *key_block_shape),
|
||||
dtype=self.dtype,
|
||||
pin_memory=pin_memory,
|
||||
device="cpu",
|
||||
)
|
||||
value_blocks = torch.empty(
|
||||
size=(self.num_cpu_blocks, *value_block_shape),
|
||||
dtype=self.dtype,
|
||||
pin_memory=pin_memory,
|
||||
device="cpu",
|
||||
)
|
||||
cpu_cache.append((key_blocks, value_blocks))
|
||||
return cpu_cache
|
||||
|
||||
+46
-29
@@ -5,7 +5,7 @@ import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig, LoRAConfig, ParallelConfig, SchedulerConfig
|
||||
from vllm.config import DeviceConfig, ModelConfig, LoRAConfig, ParallelConfig, SchedulerConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor import get_model, InputMetadata, SamplingMetadata
|
||||
from vllm.model_executor.parallel_utils.communication_op import (
|
||||
@@ -35,6 +35,7 @@ class ModelRunner:
|
||||
model_config: ModelConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
scheduler_config: SchedulerConfig,
|
||||
device_config: DeviceConfig,
|
||||
lora_config: Optional[LoRAConfig],
|
||||
kv_cache_dtype: Optional[str] = "auto",
|
||||
is_driver_worker: bool = False,
|
||||
@@ -49,7 +50,10 @@ class ModelRunner:
|
||||
# FIXME(woosuk): This is a hack to make the tests work. Refactor this.
|
||||
self.sliding_window = (model_config.get_sliding_window()
|
||||
if model_config is not None else None)
|
||||
self.device = torch.device(torch.cuda.current_device())
|
||||
self.device_config = (device_config
|
||||
if device_config is not None else DeviceConfig())
|
||||
self.device = self.device_config.device
|
||||
|
||||
self.model = None
|
||||
self.block_size = None # Set after initial profiling.
|
||||
self.lora_manager = None
|
||||
@@ -72,7 +76,8 @@ class ModelRunner:
|
||||
self.kv_cache_dtype = kv_cache_dtype
|
||||
|
||||
def load_model(self) -> None:
|
||||
self.model = get_model(self.model_config, self.lora_config)
|
||||
self.model = get_model(self.model_config, self.device_config,
|
||||
self.lora_config)
|
||||
|
||||
vocab_size = self.model.config.vocab_size
|
||||
|
||||
@@ -182,22 +187,25 @@ class ModelRunner:
|
||||
input_tokens = _make_tensor_with_pad(input_tokens,
|
||||
max_prompt_len,
|
||||
pad=0,
|
||||
dtype=torch.long)
|
||||
dtype=torch.long,
|
||||
device=self.device)
|
||||
input_positions = _make_tensor_with_pad(input_positions,
|
||||
max_prompt_len,
|
||||
pad=0,
|
||||
dtype=torch.long)
|
||||
dtype=torch.long,
|
||||
device=self.device)
|
||||
slot_mapping = _make_tensor_with_pad(slot_mapping,
|
||||
max_prompt_len,
|
||||
pad=_PAD_SLOT_ID,
|
||||
dtype=torch.long)
|
||||
dtype=torch.long,
|
||||
device=self.device)
|
||||
lora_index_mapping = [
|
||||
_pad_to_max(mapping, max_prompt_len, pad=0)
|
||||
for mapping in lora_index_mapping
|
||||
]
|
||||
context_lens_tensor = torch.tensor(context_lens,
|
||||
dtype=torch.int,
|
||||
device='cuda')
|
||||
device=self.device)
|
||||
# Prepare prefix block tables
|
||||
max_prompt_block_table_len = max(len(t) for t in prefix_block_tables)
|
||||
block_tables = _make_tensor_with_pad(
|
||||
@@ -205,15 +213,16 @@ class ModelRunner:
|
||||
max_len=max_prompt_block_table_len,
|
||||
pad=0,
|
||||
dtype=torch.int,
|
||||
device=self.device,
|
||||
)
|
||||
start_loc_tensor = torch.arange(0,
|
||||
len(prompt_lens) * max_prompt_len,
|
||||
max_prompt_len,
|
||||
dtype=torch.long,
|
||||
device='cuda')
|
||||
device=self.device)
|
||||
prompt_lens_tensor = torch.tensor(prompt_lens,
|
||||
dtype=torch.long,
|
||||
device='cuda')
|
||||
device=self.device)
|
||||
|
||||
input_metadata = InputMetadata(
|
||||
is_prompt=True,
|
||||
@@ -305,20 +314,20 @@ class ModelRunner:
|
||||
max_len=1,
|
||||
pad=0,
|
||||
dtype=torch.long,
|
||||
device="cuda")
|
||||
device=self.device)
|
||||
input_positions = _make_tensor_with_pad(input_positions,
|
||||
max_len=1,
|
||||
pad=0,
|
||||
dtype=torch.long,
|
||||
device="cuda")
|
||||
device=self.device)
|
||||
slot_mapping = _make_tensor_with_pad(slot_mapping,
|
||||
max_len=1,
|
||||
pad=_PAD_SLOT_ID,
|
||||
dtype=torch.long,
|
||||
device="cuda")
|
||||
device=self.device)
|
||||
context_lens = torch.tensor(context_lens,
|
||||
dtype=torch.int,
|
||||
device="cuda")
|
||||
device=self.device)
|
||||
|
||||
if use_captured_graph:
|
||||
# The shape of graph_block_tables is
|
||||
@@ -327,7 +336,7 @@ class ModelRunner:
|
||||
for i, block_table in enumerate(block_tables):
|
||||
if block_table:
|
||||
input_block_tables[i, :len(block_table)] = block_table
|
||||
block_tables = torch.tensor(input_block_tables, device="cuda")
|
||||
block_tables = torch.tensor(input_block_tables, device=self.device)
|
||||
else:
|
||||
max_block_table_len = max(
|
||||
len(block_table) for block_table in block_tables)
|
||||
@@ -336,7 +345,7 @@ class ModelRunner:
|
||||
max_len=max_block_table_len,
|
||||
pad=0,
|
||||
dtype=torch.int,
|
||||
device="cuda",
|
||||
device=self.device,
|
||||
)
|
||||
|
||||
lora_index_mapping = [
|
||||
@@ -355,7 +364,8 @@ class ModelRunner:
|
||||
use_cuda_graph=use_captured_graph,
|
||||
kv_cache_dtype=self.kv_cache_dtype,
|
||||
)
|
||||
return input_tokens, input_positions, input_metadata, lora_index_mapping, lora_prompt_mapping, lora_requests
|
||||
return (input_tokens, input_positions, input_metadata,
|
||||
lora_index_mapping, lora_prompt_mapping, lora_requests)
|
||||
|
||||
def _prepare_sample(
|
||||
self,
|
||||
@@ -410,9 +420,13 @@ class ModelRunner:
|
||||
|
||||
selected_token_indices = _async_h2d(selected_token_indices,
|
||||
dtype=torch.long,
|
||||
target_device=self.device,
|
||||
pin_memory=not self.in_wsl)
|
||||
categorized_sample_indices = {
|
||||
t: _async_h2d(seq_ids, dtype=torch.int, pin_memory=not self.in_wsl)
|
||||
t: _async_h2d(seq_ids,
|
||||
dtype=torch.int,
|
||||
target_device=self.device,
|
||||
pin_memory=not self.in_wsl)
|
||||
for t, seq_ids in categorized_sample_indices.items()
|
||||
}
|
||||
|
||||
@@ -511,7 +525,8 @@ class ModelRunner:
|
||||
perform_sampling=False,
|
||||
)
|
||||
|
||||
return input_tokens, input_positions, input_metadata, sampling_metadata, lora_requests, lora_mapping
|
||||
return (input_tokens, input_positions, input_metadata,
|
||||
sampling_metadata, lora_requests, lora_mapping)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
@@ -519,8 +534,9 @@ class ModelRunner:
|
||||
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
|
||||
kv_caches: List[Tuple[torch.Tensor, torch.Tensor]],
|
||||
) -> Optional[SamplerOutput]:
|
||||
input_tokens, input_positions, input_metadata, sampling_metadata, lora_requests, lora_mapping = (
|
||||
self.prepare_input_tensors(seq_group_metadata_list))
|
||||
(input_tokens, input_positions, input_metadata, sampling_metadata,
|
||||
lora_requests,
|
||||
lora_mapping) = self.prepare_input_tensors(seq_group_metadata_list)
|
||||
|
||||
if self.lora_config:
|
||||
self.set_active_loras(lora_requests, lora_mapping)
|
||||
@@ -789,14 +805,10 @@ def _make_tensor_with_pad(
|
||||
max_len: int,
|
||||
pad: int,
|
||||
dtype: torch.dtype,
|
||||
device: Union[str, torch.device] = "cuda",
|
||||
pin_memory: bool = False,
|
||||
device: Optional[Union[str, torch.device]],
|
||||
) -> torch.Tensor:
|
||||
padded_x = [_pad_to_max(x_i, max_len, pad) for x_i in x]
|
||||
return torch.tensor(padded_x,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
pin_memory=pin_memory and str(device) == "cpu")
|
||||
return torch.tensor(padded_x, dtype=dtype, device=device)
|
||||
|
||||
|
||||
def _get_graph_batch_size(batch_size: int) -> int:
|
||||
@@ -808,6 +820,11 @@ def _get_graph_batch_size(batch_size: int) -> int:
|
||||
return (batch_size + 7) // 8 * 8
|
||||
|
||||
|
||||
def _async_h2d(data: list, dtype, pin_memory):
|
||||
t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory)
|
||||
return t.to(device="cuda", non_blocking=True)
|
||||
def _async_h2d(
|
||||
data: list,
|
||||
dtype: torch.dtype,
|
||||
target_device: Union[str, torch.device],
|
||||
pin_memory: bool,
|
||||
) -> torch.Tensor:
|
||||
t = torch.tensor(data, dtype=dtype, pin_memory=pin_memory, device="cpu")
|
||||
return t.to(device=target_device, non_blocking=True)
|
||||
|
||||
+21
-15
@@ -6,8 +6,8 @@ from typing import Dict, List, Tuple, Set, Optional
|
||||
import torch
|
||||
import torch.distributed
|
||||
|
||||
from vllm.config import (CacheConfig, ModelConfig, ParallelConfig,
|
||||
SchedulerConfig, LoRAConfig)
|
||||
from vllm.config import (CacheConfig, DeviceConfig, ModelConfig,
|
||||
ParallelConfig, SchedulerConfig, LoRAConfig)
|
||||
from vllm.model_executor import set_random_seed
|
||||
from vllm.model_executor.parallel_utils.communication_op import (
|
||||
broadcast_tensor_dict)
|
||||
@@ -33,6 +33,7 @@ class Worker:
|
||||
model_config: ModelConfig,
|
||||
parallel_config: ParallelConfig,
|
||||
scheduler_config: SchedulerConfig,
|
||||
device_config: DeviceConfig,
|
||||
local_rank: int,
|
||||
rank: int,
|
||||
distributed_init_method: str,
|
||||
@@ -43,6 +44,7 @@ class Worker:
|
||||
self.model_config = model_config
|
||||
self.parallel_config = parallel_config
|
||||
self.scheduler_config = scheduler_config
|
||||
self.device_config = device_config
|
||||
self.local_rank = local_rank
|
||||
self.rank = rank
|
||||
self.distributed_init_method = distributed_init_method
|
||||
@@ -54,6 +56,7 @@ class Worker:
|
||||
self.model_runner = ModelRunner(model_config,
|
||||
parallel_config,
|
||||
scheduler_config,
|
||||
device_config,
|
||||
lora_config=self.lora_config,
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
is_driver_worker=is_driver_worker)
|
||||
@@ -65,21 +68,24 @@ class Worker:
|
||||
self.gpu_cache = None
|
||||
|
||||
def init_model(self) -> None:
|
||||
# torch.distributed.all_reduce does not free the input tensor until
|
||||
# the synchronization point. This causes the memory usage to grow
|
||||
# as the number of all_reduce calls increases. This env var disables
|
||||
# this behavior.
|
||||
# Related issue:
|
||||
# https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573
|
||||
os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
|
||||
if self.device_config.device.type == "cuda":
|
||||
# torch.distributed.all_reduce does not free the input tensor until
|
||||
# the synchronization point. This causes the memory usage to grow
|
||||
# as the number of all_reduce calls increases. This env var disables
|
||||
# this behavior.
|
||||
# Related issue:
|
||||
# https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573
|
||||
os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
|
||||
|
||||
# This env var set by Ray causes exceptions with graph building.
|
||||
os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
|
||||
self.device = torch.device(f"cuda:{self.local_rank}")
|
||||
torch.cuda.set_device(self.device)
|
||||
|
||||
_check_if_gpu_supports_dtype(self.model_config.dtype)
|
||||
# This env var set by Ray causes exceptions with graph building.
|
||||
os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
|
||||
self.device = torch.device(f"cuda:{self.local_rank}")
|
||||
torch.cuda.set_device(self.device)
|
||||
|
||||
_check_if_gpu_supports_dtype(self.model_config.dtype)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Not support device type: {self.device_config.device}")
|
||||
# Initialize the distributed environment.
|
||||
init_distributed_environment(self.parallel_config, self.rank,
|
||||
self.distributed_init_method)
|
||||
|
||||
Reference in New Issue
Block a user