From 967c5c3bc38891f4465d3f4e99917ed837bb3833 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Mon, 8 Jun 2026 01:00:59 -0500 Subject: [PATCH] [ROCm][CI] Stage C mirrors (#42793) Signed-off-by: Andreas Karatzas --- .../scripts/hardware_ci/run-amd-test.sh | 32 ++++++- .buildkite/test-amd.yaml | 86 +++++++++++++++---- .buildkite/test_areas/attention.yaml | 18 +++- .buildkite/test_areas/engine.yaml | 8 +- .buildkite/test_areas/entrypoints.yaml | 29 +++++-- .buildkite/test_areas/kernels.yaml | 2 +- .buildkite/test_areas/lm_eval.yaml | 15 ++++ .buildkite/test_areas/models_language.yaml | 1 + .buildkite/test_areas/models_multimodal.yaml | 15 +++- .buildkite/test_areas/spec_decode.yaml | 43 ++++++++++ docker/Dockerfile.rocm | 2 + 11 files changed, 213 insertions(+), 38 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 953074c388..5c994e25d0 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -28,8 +28,10 @@ ############################################################################### set -o pipefail -# Export Python path -export PYTHONPATH=".." +# Export Python path for commands that run directly on the host. Containerized +# tests set this to /vllm-workspace below so spawned Python processes do not +# depend on their current working directory. +export PYTHONPATH="${PYTHONPATH:-..}" ############################################################################### # Helper Functions @@ -377,6 +379,14 @@ HF_CACHE="$(realpath ~)/huggingface" mkdir -p "${HF_CACHE}" HF_MOUNT="/root/.cache/huggingface" +# Hugging Face Hub defaults to 10s request/download timeouts, while the ROCm +# CI image currently raises downloads to 60s. AMD model-test jobs routinely +# start from a cold or partially-populated shared cache, and the 60s read cap +# has still timed out before pytest reached the vLLM behavior under test. +# Keep the CI default explicit and overridable from the Buildkite environment. +: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}" +: "${HF_HUB_ETAG_TIMEOUT:=60}" + # ---- Command source selection ---- # Prefer VLLM_TEST_COMMANDS (preserves all inner quoting intact). # Fall back to $* for backward compatibility, but warn that inner @@ -416,7 +426,14 @@ fi echo "Final commands: $commands" -MYPYTHONPATH=".." +MYPYTHONPATH="/vllm-workspace" + +container_job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-0}}" +container_job_id="${container_job_id//[^A-Za-z0-9_.-]/_}" +container_job_id_short="${container_job_id:0:8}" +CONTAINER_TMPDIR="/tmp/vllm-${container_job_id_short}" +CONTAINER_CACHE_ROOT="/tmp/vllm-buildkite-${container_job_id}/cache" +CONTAINER_PREFLIGHT="mkdir -p \"\$TMPDIR\" \"\$TORCHINDUCTOR_CACHE_DIR\" \"\$TRITON_CACHE_DIR\" \"\$VLLM_CACHE_ROOT\" \"\$XDG_CACHE_HOME\" && python -c \"import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing\"" # Verify GPU access render_gid=$(getent group render | cut -d: -f3) @@ -493,6 +510,8 @@ else --group-add "$render_gid" \ --rm \ -e HF_TOKEN \ + -e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \ + -e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \ -e AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY \ -e BUILDKITE_PARALLEL_JOB \ @@ -500,10 +519,15 @@ else -v "${HF_CACHE}:${HF_MOUNT}" \ -e "HF_HOME=${HF_MOUNT}" \ -e "PYTHONPATH=${MYPYTHONPATH}" \ + -e "TMPDIR=${CONTAINER_TMPDIR}/tmp" \ + -e "TORCHINDUCTOR_CACHE_DIR=${CONTAINER_CACHE_ROOT}/torchinductor" \ + -e "TRITON_CACHE_DIR=${CONTAINER_CACHE_ROOT}/triton" \ + -e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \ + -e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \ -e "PYTORCH_ROCM_ARCH=" \ --name "${container_name}" \ "${image_name}" \ - /bin/bash -c "${commands}" + /bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}" exit_code=$? handle_pytest_exit "$exit_code" diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 1641300b06..17b2932e13 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -315,24 +315,6 @@ steps: - pytest -v -s distributed/test_pp_cudagraph.py - pytest -v -s distributed/test_pipeline_parallel.py -#---------------------------------------------------------- mi250 · engine -----------------------------------------------------------# - -- label: Engine # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] - agent_pool: mi250_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/engine - - tests/test_sequence - - tests/test_config - - tests/test_logger - - tests/test_vllm_port - commands: - - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py - #----------------------------------------------------------- mi250 · evals -----------------------------------------------------------# - label: Multi-Modal Accuracy Eval (Small Models) # TBD @@ -1351,6 +1333,7 @@ steps: timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - csrc/ @@ -1409,6 +1392,7 @@ steps: timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 + optional: true working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" source_file_dependencies: - csrc/ @@ -2183,10 +2167,72 @@ steps: commands: - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness" +- label: Speculators Correctness # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + agent_pool: mi300_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/config/speculative.py + - vllm/engine/arg_utils.py + - vllm/transformers_utils/config.py + - vllm/transformers_utils/configs/speculators/ + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/v1/worker/gpu_model_runner.py + - vllm/v1/sample/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/model_executor/model_loader/ + - vllm/model_executor/layers/ + - vllm/model_executor/models/llama_eagle3.py + - vllm/model_executor/models/qwen3.py + - vllm/model_executor/models/qwen3_dflash.py + - vllm/model_executor/models/registry.py + - vllm/_aiter_ops.py + - tests/evals/gsm8k/ + - tests/v1/spec_decode/test_speculators_correctness.py + - vllm/platforms/rocm.py + commands: + - export VLLM_ALLOW_INSECURE_SERIALIZATION=1 + - pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test + +- label: Extract Hidden States Integration # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + agent_pool: mi300_1 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/config/speculative.py + - vllm/distributed/kv_transfer/kv_connector/ + - vllm/model_executor/layers/attention/ + - vllm/model_executor/layers/mamba/ + - vllm/model_executor/model_loader/ + - vllm/model_executor/models/extract_hidden_states.py + - vllm/model_executor/models/llama.py + - vllm/model_executor/models/qwen3_5.py + - vllm/model_executor/models/qwen3_next.py + - vllm/model_executor/models/registry.py + - vllm/transformers_utils/configs/extract_hidden_states.py + - vllm/transformers_utils/configs/qwen3_5.py + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/v1/kv_cache_interface.py + - vllm/v1/spec_decode/extract_hidden_states.py + - vllm/v1/worker/gpu_model_runner.py + - vllm/_aiter_ops.py + - tests/v1/kv_connector/extract_hidden_states_integration/ + - vllm/platforms/rocm.py + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - pytest -v -s v1/kv_connector/extract_hidden_states_integration + - label: V1 attention (H100-MI300) # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/config/attention.py @@ -2545,6 +2591,7 @@ steps: timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] agent_pool: mi325_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -2554,7 +2601,7 @@ steps: - tests/test_logger - tests/test_vllm_port commands: - - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py + - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py #----------------------------------------------------------- mi325 · evals -----------------------------------------------------------# @@ -2636,6 +2683,7 @@ steps: timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] agent_pool: mi325_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ diff --git a/.buildkite/test_areas/attention.yaml b/.buildkite/test_areas/attention.yaml index d3947a0316..8814a3a8f0 100644 --- a/.buildkite/test_areas/attention.yaml +++ b/.buildkite/test_areas/attention.yaml @@ -2,8 +2,8 @@ group: Attention depends_on: - image-build steps: -- label: V1 attention (H100) - key: v1-attention-h100 +- label: V1 attention (H100-MI300) + key: v1-attention-h100-mi300 timeout_in_minutes: 30 device: h100 source_file_dependencies: @@ -13,6 +13,20 @@ steps: - tests/v1/attention commands: - pytest -v -s v1/attention + mirror: + amd: + device: mi300_1 + timeout_in_minutes: 70 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/config/attention.py + - vllm/model_executor/layers/attention + - vllm/v1/attention + - tests/v1/attention + - vllm/_aiter_ops.py + - vllm/envs.py + - vllm/platforms/rocm.py - label: V1 attention (B200) key: v1-attention-b200 diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index f3862789ee..ddae5e774f 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -26,6 +26,12 @@ steps: - tests/test_jit_monitor.py commands: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py + mirror: + amd: + device: mi300_1 + timeout_in_minutes: 60 + depends_on: + - image-build-amd - label: Engine (1 GPU) key: engine-1-gpu @@ -38,7 +44,7 @@ steps: - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py mirror: amd: - device: mi325_1 + device: mi300_1 timeout_in_minutes: 40 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 613cb76eb4..88b0eae739 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -28,7 +28,6 @@ steps: mirror: amd: device: mi325_1 - soft_fail: true depends_on: - image-build-amd @@ -45,8 +44,7 @@ steps: - pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_oot_registration.py mirror: amd: - device: mi325_1 - soft_fail: true + device: mi300_1 timeout_in_minutes: 80 depends_on: - image-build-amd @@ -68,8 +66,7 @@ steps: - pytest -v -s tool_use mirror: amd: - device: mi325_1 - soft_fail: true + device: mi300_1 timeout_in_minutes: 60 depends_on: - image-build-amd @@ -89,7 +86,6 @@ steps: mirror: amd: device: mi325_1 - soft_fail: true timeout_in_minutes: 60 depends_on: - image-build-amd @@ -108,8 +104,7 @@ steps: - PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc mirror: amd: - device: mi325_1 - soft_fail: true + device: mi300_1 depends_on: - image-build-amd @@ -155,3 +150,21 @@ steps: - vllm/entrypoints/openai/ commands: # LMEval - pytest -s entrypoints/openai/correctness/ + mirror: + amd: + device: mi300_1 + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/entrypoints/openai/ + - vllm/model_executor/models/whisper.py + - vllm/model_executor/layers/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - vllm/model_executor/model_loader/ + commands: + - bash ../tools/install_torchcodec_rocm.sh || exit 1 + - pytest -s entrypoints/openai/correctness/ diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 68e6a5762e..fc7fc25ff2 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -86,7 +86,7 @@ steps: parallelism: 2 mirror: amd: - device: mi325_1 + device: mi300_1 source_file_dependencies: - csrc/quantization/ - vllm/model_executor/layers/quantization diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 06f530ecc2..8219fa1915 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -12,6 +12,21 @@ steps: autorun_on_main: true commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt + mirror: + amd: + device: mi300_1 + timeout_in_minutes: 55 + depends_on: + - image-build-amd + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py # - label: LM Eval Large Models (4 GPUs)(A100) # device: a100 diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index b9f7861d11..f5e23cd95f 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -51,6 +51,7 @@ steps: torch_nightly: {} amd: device: mi325_1 + timeout_in_minutes: 90 depends_on: - image-build-amd commands: diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index 06cfe11f08..7c7e2163fe 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -15,7 +15,7 @@ steps: - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model mirror: amd: - device: mi325_1 + device: mi300_1 depends_on: - image-build-amd @@ -33,7 +33,7 @@ steps: - pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model mirror: amd: - device: mi325_1 + device: mi300_1 depends_on: - image-build-amd @@ -50,7 +50,7 @@ steps: - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model mirror: amd: - device: mi325_1 + device: mi300_1 depends_on: - image-build-amd @@ -153,3 +153,12 @@ steps: - tests/models/multimodal/pooling commands: - pytest -v -s models/multimodal/pooling -m 'not core_model' + mirror: + amd: + device: mi300_1 + timeout_in_minutes: 60 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/ + - tests/models/multimodal/pooling diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 68dc8e7ef3..27396118db 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -37,6 +37,21 @@ steps: - tests/v1/e2e/spec_decode/ commands: - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness" + mirror: + amd: + device: mi300_1 + timeout_in_minutes: 65 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - vllm/transformers_utils/configs/speculators/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py - label: Spec Decode Speculators + MTP Nightly B200 key: spec-decode-speculators-mtp-nightly-b200 @@ -61,6 +76,20 @@ steps: - tests/v1/e2e/spec_decode/ commands: - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix" + mirror: + amd: + device: mi300_1 + timeout_in_minutes: 65 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py - label: Spec Decode Draft Model key: spec-decode-draft-model @@ -72,6 +101,20 @@ steps: - tests/v1/e2e/spec_decode/ commands: - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference" + mirror: + amd: + device: mi300_1 + timeout_in_minutes: 50 + depends_on: + - image-build-amd + source_file_dependencies: + - vllm/v1/spec_decode/ + - vllm/v1/worker/gpu/spec_decode/ + - vllm/model_executor/model_loader/ + - vllm/v1/sample/ + - vllm/model_executor/layers/ + - tests/v1/e2e/spec_decode/ + - vllm/platforms/rocm.py - label: Spec Decode Draft Model Nightly B200 key: spec-decode-draft-model-nightly-b200 diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index 545765ea85..16284e999a 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -230,6 +230,7 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples +COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/pyproject.toml /pyproject.toml @@ -513,6 +514,7 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/requirements /requir COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples +COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/pyproject.toml /pyproject.toml