diff --git a/.buildkite/scripts/ci-bake-rocm.sh b/.buildkite/scripts/ci-bake-rocm.sh index 51cffb8e20..4ccbbb352d 100644 --- a/.buildkite/scripts/ci-bake-rocm.sh +++ b/.buildkite/scripts/ci-bake-rocm.sh @@ -15,7 +15,7 @@ set -euo pipefail DEFAULT_REPO_SLUG="vllm-project/vllm" DEFAULT_CI_HCL_SOURCE="docker/ci-rocm.hcl" -DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base tools/install_torchcodec_rocm.sh tests/vllm_test_utils" +DEFAULT_CI_BASE_CONTENT_FILES="requirements/common.txt requirements/rocm.txt requirements/test/rocm.txt docker/Dockerfile.rocm_base docker/ci-rocm.hcl docker/docker-bake-rocm.hcl tools/install_torchcodec_rocm.sh tests/vllm_test_utils .buildkite/scripts/ci-bake-rocm.sh" DEFAULT_CI_BASE_DOCKERFILE="docker/Dockerfile.rocm" DEFAULT_CI_BASE_DOCKERFILE_STAGES="base build_rixl build_rocshmem build_deepep mori_base ci_base" DEFAULT_CI_BASE_METADATA_VERSION="1" diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index a9608cc332..7521901c9a 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -438,7 +438,7 @@ steps: #----------------------------------------------------- mi300 · basic_correctness -----------------------------------------------------# - label: Basic Correctness # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 fast_check: true @@ -456,7 +456,7 @@ steps: - pytest -v -s basic_correctness/test_cpu_offload.py - label: Distributed Model Tests (2 GPUs) # TBD - timeout_in_minutes: 180 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_2 num_gpus: 2 @@ -678,7 +678,7 @@ steps: - pytest -v -s distributed/test_eplb_spec_decode.py - label: Distributed Tests (2xH100-2xMI300) # TBD - timeout_in_minutes: 180 + timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_2 num_gpus: 2 @@ -1199,7 +1199,7 @@ steps: #--------------------------------------------------------- mi300 · examples ----------------------------------------------------------# - label: Examples # TBD - timeout_in_minutes: 180 + timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 optional: true @@ -1212,7 +1212,7 @@ steps: commands: - pip install tensorizer # Basic - - python3 basic/offline_inference/chat.py --attention-backend TRITON_ATTN + - python3 basic/offline_inference/chat.py - python3 basic/offline_inference/generate.py --model facebook/opt-125m - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 - python3 basic/offline_inference/classify.py @@ -1220,11 +1220,8 @@ steps: - python3 basic/offline_inference/score.py # Multi-modal models - python3 generate/multimodal/audio_language_offline.py --seed 0 - # These two examples import transformers before vllm, so on ROCm the HIP context - # is initialized in the parent before vllm sets this guard, poisoning fork. Set it - # inline to keep torch.cuda.is_available() fork-safe (see vllm/env_override.py). - - PYTORCH_NVML_BASED_CUDA_CHECK=1 python3 generate/multimodal/vision_language_offline.py --seed 0 - - PYTORCH_NVML_BASED_CUDA_CHECK=1 python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 + - python3 generate/multimodal/vision_language_offline.py --seed 0 + - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 # Pooling models - python3 pooling/embed/vision_embedding_offline.py --seed 0 @@ -1666,10 +1663,7 @@ steps: - pytest -v -s tests/models/test_transformers.py - pytest -v -s tests/models/multimodal/test_mapping.py - python3 examples/basic/offline_inference/chat.py - # This example imports transformers before vllm, so on ROCm the HIP context is - # initialized in the parent before vllm sets this guard, poisoning fork. Set it - # inline to keep torch.cuda.is_available() fork-safe (see vllm/env_override.py). - - PYTORCH_NVML_BASED_CUDA_CHECK=1 python3 examples/generate/multimodal/vision_language_offline.py --model-type qwen2_5_vl + - python3 examples/generate/multimodal/vision_language_offline.py --model-type qwen2_5_vl - VLLM_WORKER_MULTIPROC_METHOD=spawn python3 examples/generate/multimodal/audio_language_offline.py --model-type whisper #---------------------------------------------------------- mi300 · plugins ----------------------------------------------------------# @@ -2773,7 +2767,7 @@ steps: #--------------------------------------------------------- mi355 · examples ----------------------------------------------------------# - label: Examples # TBD - timeout_in_minutes: 180 + timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_1 working_dir: "/vllm-workspace/examples" @@ -2785,7 +2779,7 @@ steps: commands: - pip install tensorizer # Basic - - python3 basic/offline_inference/chat.py --attention-backend TRITON_ATTN + - python3 basic/offline_inference/chat.py - python3 basic/offline_inference/generate.py --model facebook/opt-125m - python3 basic/offline_inference/generate.py --model meta-llama/Llama-2-13b-chat-hf --cpu-offload-gb 10 - python3 basic/offline_inference/classify.py @@ -2793,11 +2787,8 @@ steps: - python3 basic/offline_inference/score.py # Multi-modal models - python3 generate/multimodal/audio_language_offline.py --seed 0 - # These two examples import transformers before vllm, so on ROCm the HIP context - # is initialized in the parent before vllm sets this guard, poisoning fork. Set it - # inline to keep torch.cuda.is_available() fork-safe (see vllm/env_override.py). - - PYTORCH_NVML_BASED_CUDA_CHECK=1 python3 generate/multimodal/vision_language_offline.py --seed 0 - - PYTORCH_NVML_BASED_CUDA_CHECK=1 python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 + - python3 generate/multimodal/vision_language_offline.py --seed 0 + - python3 generate/multimodal/vision_language_multi_image_offline.py --seed 0 - python3 generate/multimodal/encoder_decoder_multimodal_offline.py --model-type whisper --seed 0 # Pooling models - python3 pooling/embed/vision_embedding_offline.py --seed 0 diff --git a/.buildkite/test_areas/basic_correctness.yaml b/.buildkite/test_areas/basic_correctness.yaml index 7e166a8a28..d7173b6438 100644 --- a/.buildkite/test_areas/basic_correctness.yaml +++ b/.buildkite/test_areas/basic_correctness.yaml @@ -19,6 +19,6 @@ steps: mirror: amd: device: mi325_1 - timeout_in_minutes: 40 + timeout_in_minutes: 50 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index 5ff4b24b74..2cc52603c2 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -224,18 +224,6 @@ steps: - pytest -v -s tests/v1/distributed/test_dbo.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py - pytest -v -s tests/distributed/test_packed_tensor.py - mirror: - amd: - device: mi300_2 - timeout_in_minutes: 180 - depends_on: - - image-build-amd - commands: - - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py - - VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 - - pytest -v -s tests/v1/distributed/test_dbo.py - - VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py - - pytest -v -s tests/distributed/test_packed_tensor.py - label: Distributed Tests (2xB200) key: distributed-tests-2xb200 diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index f5db2e956b..450cfcbc26 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -224,6 +224,16 @@ steps: - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 2048 # https://github.com/vllm-project/vllm/pull/26682 uses slightly more memory in PyTorch 2.9+ causing this test to OOM in 1xL4 GPU - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 + mirror: + amd: + device: mi325_1 + source_file_dependencies: + - vllm/entrypoints + - vllm/multimodal + - examples/ + - vllm/platforms/rocm.py + depends_on: + - image-build-amd - label: Metrics, Tracing (2 GPUs) key: metrics-tracing-2-gpus diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index dcae40c524..c17444217f 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -575,6 +575,9 @@ RUN --mount=type=cache,target=/root/.cache/uv \ ENV HF_XET_HIGH_PERFORMANCE=1 ENV HF_HUB_DOWNLOAD_TIMEOUT=60 +# Keep torch.cuda.is_available() fork-safe (see vllm/env_override.py). +ENV PYTORCH_NVML_BASED_CUDA_CHECK=1 + # Pre-install vLLM test dependencies. COPY requirements/test/rocm.txt /tmp/rocm-test-reqs.txt RUN --mount=type=cache,target=/root/.cache/uv \ @@ -695,6 +698,9 @@ ENV SAFETENSORS_FAST_GPU=1 # Performance environment variable. ENV HIP_FORCE_DEV_KERNARG=1 +# Keep torch.cuda.is_available() fork-safe (see vllm/env_override.py). +ENV PYTORCH_NVML_BASED_CUDA_CHECK=1 + # Workaround for ROCm profiler limits RUN echo "ROCTRACER_MAX_EVENTS=10000000" > ${COMMON_WORKDIR}/libkineto.conf ENV KINETO_CONFIG="${COMMON_WORKDIR}/libkineto.conf"