[CPU] Add device-memory-utilization CLI alias (#56547)
Signed-off-by: louie-tsai <louie.tsai@intel.com> Signed-off-by: Louie Tsai <louie.tsai@intel.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
91d7324cb1
commit
6dc34b6334
+13
-1
@@ -107,7 +107,19 @@ class CacheConfig:
|
||||
per-instance limit, and only applies to the current vLLM instance. It does
|
||||
not matter if you have another vLLM instance running on the same GPU. For
|
||||
example, if you have two vLLM instances running on the same GPU, you can
|
||||
set the GPU memory utilization to 0.5 for each instance."""
|
||||
set the GPU memory utilization to 0.5 for each instance. On non-GPU
|
||||
installs, this value controls the corresponding device memory utilization.
|
||||
"""
|
||||
|
||||
@property
|
||||
def device_memory_utilization(self) -> float:
|
||||
"""Device-neutral alias for ``gpu_memory_utilization``."""
|
||||
return self.gpu_memory_utilization
|
||||
|
||||
@device_memory_utilization.setter
|
||||
def device_memory_utilization(self, value: float) -> None:
|
||||
self.gpu_memory_utilization = value
|
||||
|
||||
cache_dtype: CacheDType = "auto"
|
||||
"""Data type for kv cache storage. If "auto", will use model data type.
|
||||
CUDA 11.8+ supports fp8 (=fp8_e4m3) and fp8_e5m2. ROCm (AMD GPU) supports
|
||||
|
||||
Reference in New Issue
Block a user