[ROCm] Cpu offload for ROCm 7.13+ to align the hipMemcpyBatchAsync params and perf in 7.14x (#43018)
Signed-off-by: Hongxia Yang <hongxia.yang@amd.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Yifan Qiao
parent
c1d7a3808d
commit
cd7b7c265a
@@ -299,6 +299,7 @@ if TYPE_CHECKING:
|
||||
VLLM_DEBUG_MFU_METRICS: bool = False
|
||||
VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False
|
||||
VLLM_WEIGHT_OFFLOADING_DISABLE_UVA: bool = False
|
||||
VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS: int = 0
|
||||
VLLM_WSL2_ENABLE_PIN_MEMORY: bool = False
|
||||
VLLM_DISABLE_LOG_LOGO: bool = False
|
||||
VLLM_LORA_DISABLE_PDL: bool = False
|
||||
@@ -2059,6 +2060,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_WEIGHT_OFFLOADING_DISABLE_UVA": lambda: bool(
|
||||
int(os.getenv("VLLM_WEIGHT_OFFLOADING_DISABLE_UVA", "0"))
|
||||
),
|
||||
# Max descriptors per CPU-KV-offload batch-memcpy call. 0 = platform default
|
||||
# (ROCm chunks at 8192, since hipMemcpyBatchAsync faults above that on
|
||||
# rocm 7.14/7.15 when batch copy is optimized; CUDA uncapped). Set >0 to override.
|
||||
"VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS": lambda: int(
|
||||
os.getenv("VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS", "0")
|
||||
),
|
||||
# On WSL2 with a compatible kernel (>= 4.19.121), pinned memory is
|
||||
# supported but disabled by default due to a small performance regression.
|
||||
# Set to 1 when pinned memory or UVA is required (e.g. CPU offloading
|
||||
|
||||
Reference in New Issue
Block a user