[ROCm] Cpu offload for ROCm 7.13+ to align the hipMemcpyBatchAsync params and perf in 7.14x (#43018)

Signed-off-by: Hongxia Yang <hongxia.yang@amd.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Hongxia Yang
2026-08-21 09:07:03 -07:00
committed by GitHub
co-authored by Yifan Qiao
parent c1d7a3808d
commit cd7b7c265a
4 changed files with 338 additions and 59 deletions
+7
View File
@@ -299,6 +299,7 @@ if TYPE_CHECKING:
VLLM_DEBUG_MFU_METRICS: bool = False
VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False
VLLM_WEIGHT_OFFLOADING_DISABLE_UVA: bool = False
VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS: int = 0
VLLM_WSL2_ENABLE_PIN_MEMORY: bool = False
VLLM_DISABLE_LOG_LOGO: bool = False
VLLM_LORA_DISABLE_PDL: bool = False
@@ -2059,6 +2060,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_WEIGHT_OFFLOADING_DISABLE_UVA": lambda: bool(
int(os.getenv("VLLM_WEIGHT_OFFLOADING_DISABLE_UVA", "0"))
),
# Max descriptors per CPU-KV-offload batch-memcpy call. 0 = platform default
# (ROCm chunks at 8192, since hipMemcpyBatchAsync faults above that on
# rocm 7.14/7.15 when batch copy is optimized; CUDA uncapped). Set >0 to override.
"VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS": lambda: int(
os.getenv("VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS", "0")
),
# On WSL2 with a compatible kernel (>= 4.19.121), pinned memory is
# supported but disabled by default due to a small performance regression.
# Set to 1 when pinned memory or UVA is required (e.g. CPU offloading