[Neuron] Support inference with transformers-neuronx (#2569)
This commit is contained in:
@@ -3,10 +3,9 @@ from typing import Dict, List, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
from vllm._C import cache_ops
|
||||
from vllm.config import CacheConfig, ModelConfig, ParallelConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils import in_wsl, STR_DTYPE_TO_TORCH_DTYPE
|
||||
from vllm.utils import in_wsl, is_neuron, STR_DTYPE_TO_TORCH_DTYPE
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -39,6 +38,10 @@ class CacheEngine:
|
||||
self.num_gpu_blocks = cache_config.num_gpu_blocks
|
||||
self.num_cpu_blocks = cache_config.num_cpu_blocks
|
||||
|
||||
# Skip initializing CUDA stream and buffer for Neuron backend.
|
||||
if is_neuron():
|
||||
return
|
||||
|
||||
if cache_config.cache_dtype == "auto":
|
||||
self.dtype = model_config.dtype
|
||||
else:
|
||||
@@ -121,6 +124,8 @@ class CacheEngine:
|
||||
dst: List[KVCache],
|
||||
src_to_dst: Dict[int, int],
|
||||
) -> None:
|
||||
from vllm._C import cache_ops
|
||||
|
||||
with torch.cuda.stream(self.cache_stream):
|
||||
for i in range(self.num_layers):
|
||||
src_key_cache, src_value_cache = src[i]
|
||||
@@ -140,6 +145,8 @@ class CacheEngine:
|
||||
self._swap(self.gpu_cache, self.cpu_cache, src_to_dst)
|
||||
|
||||
def copy(self, src_to_dsts: Dict[int, List[int]]) -> None:
|
||||
from vllm._C import cache_ops
|
||||
|
||||
key_caches = [key_cache for key_cache, _ in self.gpu_cache]
|
||||
value_caches = [value_cache for _, value_cache in self.gpu_cache]
|
||||
# NOTE(woosuk): This operation implicitly synchronizes the CPU and GPU.
|
||||
|
||||
Reference in New Issue
Block a user