Allocate more shared memory to attention kernel (#1154)
This commit is contained in:
+12
-1
@@ -1,10 +1,12 @@
|
||||
import enum
|
||||
from platform import uname
|
||||
import uuid
|
||||
from platform import uname
|
||||
|
||||
import psutil
|
||||
import torch
|
||||
|
||||
from vllm import cuda_utils
|
||||
|
||||
|
||||
class Device(enum.Enum):
|
||||
GPU = enum.auto()
|
||||
@@ -25,6 +27,15 @@ class Counter:
|
||||
self.counter = 0
|
||||
|
||||
|
||||
def get_max_shared_memory_bytes(gpu: int = 0) -> int:
|
||||
"""Returns the maximum shared memory per thread block in bytes."""
|
||||
# https://docs.nvidia.com/cuda/cuda-runtime-api/group__CUDART__TYPES.html
|
||||
cudaDevAttrMaxSharedMemoryPerBlockOptin = 97 # pylint: disable=invalid-name
|
||||
max_shared_mem = cuda_utils.get_device_attribute(
|
||||
cudaDevAttrMaxSharedMemoryPerBlockOptin, gpu)
|
||||
return int(max_shared_mem)
|
||||
|
||||
|
||||
def get_gpu_memory(gpu: int = 0) -> int:
|
||||
"""Returns the total memory of the GPU in bytes."""
|
||||
return torch.cuda.get_device_properties(gpu).total_memory
|
||||
|
||||
Reference in New Issue
Block a user