From 7075df79b3094bb6f6d28021c4df8631af10b2b8 Mon Sep 17 00:00:00 2001 From: raviguptaamd Date: Fri, 1 May 2026 02:18:30 -0700 Subject: [PATCH] [ROCm] Enable DBO (Dynamic Batch Optimization) on ROCm (#34726) Signed-off-by: raviguptaamd --- vllm/envs.py | 15 ++++++++++++--- vllm/v1/worker/gpu_ubatch_wrapper.py | 4 ++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/vllm/envs.py b/vllm/envs.py index d55732cb6a..4191cd6a97 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -1627,9 +1627,18 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL": lambda: bool( int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL", "0")) ), - # The number of SMs to allocate for communication kernels when running DBO - # the rest of the SMs on the device will be allocated to compute - "VLLM_DBO_COMM_SMS": lambda: int(os.getenv("VLLM_DBO_COMM_SMS", "20")), + # The number of SMs/CUs to allocate for communication kernels when + # running DBO; the rest will be allocated to compute. + # Default: 20 on CUDA (SMs), 64 on ROCm (CUs). + "VLLM_DBO_COMM_SMS": lambda: int( + os.getenv( + "VLLM_DBO_COMM_SMS", + "64" + if hasattr(__import__("torch").version, "hip") + and __import__("torch").version.hip is not None + else "20", + ) + ), # Enable max_autotune & coordinate_descent_tuning in inductor_config # to compile static shapes passed from compile_sizes in compilation_config # If set to 1, enable max_autotune; By default, this is enabled (1) diff --git a/vllm/v1/worker/gpu_ubatch_wrapper.py b/vllm/v1/worker/gpu_ubatch_wrapper.py index 01f18a1194..a8c54c4f06 100644 --- a/vllm/v1/worker/gpu_ubatch_wrapper.py +++ b/vllm/v1/worker/gpu_ubatch_wrapper.py @@ -71,8 +71,8 @@ class SMControlContextManager: A function that sets the number of SMs for computation. """ - assert current_platform.is_cuda(), ( - "SM control is currently only supported on CUDA" + assert current_platform.is_cuda() or current_platform.is_rocm(), ( + "SM/CU control is supported on CUDA and ROCm platforms" ) device = torch.accelerator.current_device_index() total_sms = num_compute_units(device)