Allocate more shared memory to attention kernel (#1154)

This commit is contained in:
Antoni Baum
2023-09-26 22:27:13 -07:00
committed by GitHub
parent 03ffd0a022
commit cf5cb1e33e
7 changed files with 87 additions and 3 deletions
+12 -1
View File
@@ -1,10 +1,12 @@
import enum
from platform import uname
import uuid
from platform import uname
import psutil
import torch
from vllm import cuda_utils
class Device(enum.Enum):
GPU = enum.auto()
@@ -25,6 +27,15 @@ class Counter:
self.counter = 0
def get_max_shared_memory_bytes(gpu: int = 0) -> int:
"""Returns the maximum shared memory per thread block in bytes."""
# https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 # pylint: disable=invalid-name
max_shared_mem = cuda_utils.get_device_attribute(
cudaDevAttrMaxSharedMemoryPerBlockOptin, gpu)
return int(max_shared_mem)
def get_gpu_memory(gpu: int = 0) -> int:
"""Returns the total memory of the GPU in bytes."""
return torch.cuda.get_device_properties(gpu).total_memory