diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index ed89f526dc..b880fadf35 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -210,7 +210,7 @@ steps: - TARGET_TEST_SUITE=A100 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' - pytest -v -s -x lora/test_mixtral.py -- label: Distributed Tests (2 GPUs)(H100) +- label: Distributed Tests (2xH100-2xMI300) key: distributed-tests-2-gpus-h100 timeout_in_minutes: 15 device: h100 @@ -224,6 +224,18 @@ steps: - pytest -v -s tests/v1/distributed/test_dbo.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py - pytest -v -s tests/distributed/test_packed_tensor.py + mirror: + amd: + device: mi300_2 + timeout_in_minutes: 180 + depends_on: + - image-build-amd + commands: + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py + - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 + - pytest -v -s tests/v1/distributed/test_dbo.py + - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py + - pytest -v -s tests/distributed/test_packed_tensor.py - label: Distributed Tests (2 GPUs)(B200) key: distributed-tests-2-gpus-b200 diff --git a/vllm/triton_utils/importing.py b/vllm/triton_utils/importing.py index 8dea20fd3e..e17450f78d 100644 --- a/vllm/triton_utils/importing.py +++ b/vllm/triton_utils/importing.py @@ -7,6 +7,7 @@ from importlib.metadata import version from importlib.util import find_spec from vllm.logger import init_logger +from vllm.platforms import current_platform from vllm.utils.math_utils import cdiv logger = init_logger(__name__) @@ -27,10 +28,16 @@ if HAS_TRITON: ] # Check if we're in a distributed environment where CUDA_VISIBLE_DEVICES - # might be temporarily empty (e.g., Ray sets it to "" during actor init) - cuda_visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES") + # or HIP_VISIBLE_DEVICES might be temporarily empty (e.g., Ray sets it to "" + # during actor init) + visible_devices_env = ( + "HIP_VISIBLE_DEVICES" + if current_platform.is_rocm() + else "CUDA_VISIBLE_DEVICES" + ) + visible_devices = os.environ.get(visible_devices_env) is_distributed_env = ( - cuda_visible_devices is not None and len(cuda_visible_devices.strip()) == 0 + visible_devices is not None and len(visible_devices.strip()) == 0 ) # Apply lenient driver check for distributed environments