diff --git a/vllm/envs.py b/vllm/envs.py index d55732cb6a..4191cd6a97 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -1627,9 +1627,18 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL": lambda: bool( int(os.getenv("VLLM_DEEPEP_LOW_LATENCY_USE_MNNVL", "0")) ), - # The number of SMs to allocate for communication kernels when running DBO - # the rest of the SMs on the device will be allocated to compute - "VLLM_DBO_COMM_SMS": lambda: int(os.getenv("VLLM_DBO_COMM_SMS", "20")), + # The number of SMs/CUs to allocate for communication kernels when + # running DBO; the rest will be allocated to compute. + # Default: 20 on CUDA (SMs), 64 on ROCm (CUs). + "VLLM_DBO_COMM_SMS": lambda: int( + os.getenv( + "VLLM_DBO_COMM_SMS", + "64" + if hasattr(__import__("torch").version, "hip") + and __import__("torch").version.hip is not None + else "20", + ) + ), # Enable max_autotune & coordinate_descent_tuning in inductor_config # to compile static shapes passed from compile_sizes in compilation_config # If set to 1, enable max_autotune; By default, this is enabled (1) diff --git a/vllm/v1/worker/gpu_ubatch_wrapper.py b/vllm/v1/worker/gpu_ubatch_wrapper.py index 01f18a1194..a8c54c4f06 100644 --- a/vllm/v1/worker/gpu_ubatch_wrapper.py +++ b/vllm/v1/worker/gpu_ubatch_wrapper.py @@ -71,8 +71,8 @@ class SMControlContextManager: A function that sets the number of SMs for computation. """ - assert current_platform.is_cuda(), ( - "SM control is currently only supported on CUDA" + assert current_platform.is_cuda() or current_platform.is_rocm(), ( + "SM/CU control is supported on CUDA and ROCm platforms" ) device = torch.accelerator.current_device_index() total_sms = num_compute_units(device)