[ROCm][CI] Stage C mirrors (#42793)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-06-07 23:00:59 -07:00
committed by GitHub
parent 54c660c3a6
commit 967c5c3bc3
11 changed files with 213 additions and 38 deletions
+28 -4
View File
@@ -28,8 +28,10 @@
###############################################################################
set -o pipefail
# Export Python path
export PYTHONPATH=".."
# Export Python path for commands that run directly on the host. Containerized
# tests set this to /vllm-workspace below so spawned Python processes do not
# depend on their current working directory.
export PYTHONPATH="${PYTHONPATH:-..}"
###############################################################################
# Helper Functions
@@ -377,6 +379,14 @@ HF_CACHE="$(realpath ~)/huggingface"
mkdir -p "${HF_CACHE}"
HF_MOUNT="/root/.cache/huggingface"
# Hugging Face Hub defaults to 10s request/download timeouts, while the ROCm
# CI image currently raises downloads to 60s. AMD model-test jobs routinely
# start from a cold or partially-populated shared cache, and the 60s read cap
# has still timed out before pytest reached the vLLM behavior under test.
# Keep the CI default explicit and overridable from the Buildkite environment.
: "${HF_HUB_DOWNLOAD_TIMEOUT:=300}"
: "${HF_HUB_ETAG_TIMEOUT:=60}"
# ---- Command source selection ----
# Prefer VLLM_TEST_COMMANDS (preserves all inner quoting intact).
# Fall back to $* for backward compatibility, but warn that inner
@@ -416,7 +426,14 @@ fi
echo "Final commands: $commands"
MYPYTHONPATH=".."
MYPYTHONPATH="/vllm-workspace"
container_job_id="${BUILDKITE_JOB_ID:-${BUILDKITE_PARALLEL_JOB:-0}}"
container_job_id="${container_job_id//[^A-Za-z0-9_.-]/_}"
container_job_id_short="${container_job_id:0:8}"
CONTAINER_TMPDIR="/tmp/vllm-${container_job_id_short}"
CONTAINER_CACHE_ROOT="/tmp/vllm-buildkite-${container_job_id}/cache"
CONTAINER_PREFLIGHT="mkdir -p \"\$TMPDIR\" \"\$TORCHINDUCTOR_CACHE_DIR\" \"\$TRITON_CACHE_DIR\" \"\$VLLM_CACHE_ROOT\" \"\$XDG_CACHE_HOME\" && python -c \"import encodings, importlib.metadata as im, importlib.util as iu; [im.version(d) for d in ('transformers', 'torch', 'ray', 'sympy', 'markupsafe', 'vllm')]; missing=[m for m in ('torch.utils.model_zoo', 'transformers.models.nomic_bert', 'ray.dag', 'sympy.physics', 'markupsafe._speedups') if iu.find_spec(m) is None]; assert not missing, missing\""
# Verify GPU access
render_gid=$(getent group render | cut -d: -f3)
@@ -493,6 +510,8 @@ else
--group-add "$render_gid" \
--rm \
-e HF_TOKEN \
-e "HF_HUB_DOWNLOAD_TIMEOUT=${HF_HUB_DOWNLOAD_TIMEOUT}" \
-e "HF_HUB_ETAG_TIMEOUT=${HF_HUB_ETAG_TIMEOUT}" \
-e AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY \
-e BUILDKITE_PARALLEL_JOB \
@@ -500,10 +519,15 @@ else
-v "${HF_CACHE}:${HF_MOUNT}" \
-e "HF_HOME=${HF_MOUNT}" \
-e "PYTHONPATH=${MYPYTHONPATH}" \
-e "TMPDIR=${CONTAINER_TMPDIR}/tmp" \
-e "TORCHINDUCTOR_CACHE_DIR=${CONTAINER_CACHE_ROOT}/torchinductor" \
-e "TRITON_CACHE_DIR=${CONTAINER_CACHE_ROOT}/triton" \
-e "VLLM_CACHE_ROOT=${CONTAINER_CACHE_ROOT}/vllm" \
-e "XDG_CACHE_HOME=${CONTAINER_CACHE_ROOT}/xdg" \
-e "PYTORCH_ROCM_ARCH=" \
--name "${container_name}" \
"${image_name}" \
/bin/bash -c "${commands}"
/bin/bash -c "${CONTAINER_PREFLIGHT} && ${commands}"
exit_code=$?
handle_pytest_exit "$exit_code"
+67 -19
View File
@@ -315,24 +315,6 @@ steps:
- pytest -v -s distributed/test_pp_cudagraph.py
- pytest -v -s distributed/test_pipeline_parallel.py
#---------------------------------------------------------- mi250 · engine -----------------------------------------------------------#
- label: Engine # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250]
agent_pool: mi250_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
- tests/engine
- tests/test_sequence
- tests/test_config
- tests/test_logger
- tests/test_vllm_port
commands:
- pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
#----------------------------------------------------------- mi250 · evals -----------------------------------------------------------#
- label: Multi-Modal Accuracy Eval (Small Models) # TBD
@@ -1351,6 +1333,7 @@ steps:
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- csrc/
@@ -1409,6 +1392,7 @@ steps:
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/.buildkite/lm-eval-harness"
source_file_dependencies:
- csrc/
@@ -2183,10 +2167,72 @@ steps:
commands:
- pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
- label: Speculators Correctness # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/config/speculative.py
- vllm/engine/arg_utils.py
- vllm/transformers_utils/config.py
- vllm/transformers_utils/configs/speculators/
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/v1/worker/gpu_model_runner.py
- vllm/v1/sample/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/model_executor/model_loader/
- vllm/model_executor/layers/
- vllm/model_executor/models/llama_eagle3.py
- vllm/model_executor/models/qwen3.py
- vllm/model_executor/models/qwen3_dflash.py
- vllm/model_executor/models/registry.py
- vllm/_aiter_ops.py
- tests/evals/gsm8k/
- tests/v1/spec_decode/test_speculators_correctness.py
- vllm/platforms/rocm.py
commands:
- export VLLM_ALLOW_INSECURE_SERIALIZATION=1
- pytest -v -s v1/spec_decode/test_speculators_correctness.py -m slow_test
- label: Extract Hidden States Integration # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
agent_pool: mi300_1
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/config/speculative.py
- vllm/distributed/kv_transfer/kv_connector/
- vllm/model_executor/layers/attention/
- vllm/model_executor/layers/mamba/
- vllm/model_executor/model_loader/
- vllm/model_executor/models/extract_hidden_states.py
- vllm/model_executor/models/llama.py
- vllm/model_executor/models/qwen3_5.py
- vllm/model_executor/models/qwen3_next.py
- vllm/model_executor/models/registry.py
- vllm/transformers_utils/configs/extract_hidden_states.py
- vllm/transformers_utils/configs/qwen3_5.py
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/v1/kv_cache_interface.py
- vllm/v1/spec_decode/extract_hidden_states.py
- vllm/v1/worker/gpu_model_runner.py
- vllm/_aiter_ops.py
- tests/v1/kv_connector/extract_hidden_states_integration/
- vllm/platforms/rocm.py
commands:
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
- pytest -v -s v1/kv_connector/extract_hidden_states_integration
- label: V1 attention (H100-MI300) # TBD
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
agent_pool: mi300_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/config/attention.py
@@ -2545,6 +2591,7 @@ steps:
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
agent_pool: mi325_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
@@ -2554,7 +2601,7 @@ steps:
- tests/test_logger
- tests/test_vllm_port
commands:
- pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py
- pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py
#----------------------------------------------------------- mi325 · evals -----------------------------------------------------------#
@@ -2636,6 +2683,7 @@ steps:
timeout_in_minutes: 180
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325]
agent_pool: mi325_1
optional: true
working_dir: "/vllm-workspace/tests"
source_file_dependencies:
- vllm/
+16 -2
View File
@@ -2,8 +2,8 @@ group: Attention
depends_on:
- image-build
steps:
- label: V1 attention (H100)
key: v1-attention-h100
- label: V1 attention (H100-MI300)
key: v1-attention-h100-mi300
timeout_in_minutes: 30
device: h100
source_file_dependencies:
@@ -13,6 +13,20 @@ steps:
- tests/v1/attention
commands:
- pytest -v -s v1/attention
mirror:
amd:
device: mi300_1
timeout_in_minutes: 70
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/config/attention.py
- vllm/model_executor/layers/attention
- vllm/v1/attention
- tests/v1/attention
- vllm/_aiter_ops.py
- vllm/envs.py
- vllm/platforms/rocm.py
- label: V1 attention (B200)
key: v1-attention-b200
+7 -1
View File
@@ -26,6 +26,12 @@ steps:
- tests/test_jit_monitor.py
commands:
- pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py
mirror:
amd:
device: mi300_1
timeout_in_minutes: 60
depends_on:
- image-build-amd
- label: Engine (1 GPU)
key: engine-1-gpu
@@ -38,7 +44,7 @@ steps:
- pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py
mirror:
amd:
device: mi325_1
device: mi300_1
timeout_in_minutes: 40
depends_on:
- image-build-amd
+21 -8
View File
@@ -28,7 +28,6 @@ steps:
mirror:
amd:
device: mi325_1
soft_fail: true
depends_on:
- image-build-amd
@@ -45,8 +44,7 @@ steps:
- pytest -v -s entrypoints/openai/chat_completion --ignore=entrypoints/openai/chat_completion/test_oot_registration.py
mirror:
amd:
device: mi325_1
soft_fail: true
device: mi300_1
timeout_in_minutes: 80
depends_on:
- image-build-amd
@@ -68,8 +66,7 @@ steps:
- pytest -v -s tool_use
mirror:
amd:
device: mi325_1
soft_fail: true
device: mi300_1
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -89,7 +86,6 @@ steps:
mirror:
amd:
device: mi325_1
soft_fail: true
timeout_in_minutes: 60
depends_on:
- image-build-amd
@@ -108,8 +104,7 @@ steps:
- PYTHONPATH=/vllm-workspace pytest -v -s entrypoints/serve/dev/rpc
mirror:
amd:
device: mi325_1
soft_fail: true
device: mi300_1
depends_on:
- image-build-amd
@@ -155,3 +150,21 @@ steps:
- vllm/entrypoints/openai/
commands: # LMEval
- pytest -s entrypoints/openai/correctness/
mirror:
amd:
device: mi300_1
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/
- vllm/entrypoints/openai/
- vllm/model_executor/models/whisper.py
- vllm/model_executor/layers/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
- vllm/model_executor/model_loader/
commands:
- bash ../tools/install_torchcodec_rocm.sh || exit 1
- pytest -s entrypoints/openai/correctness/
+1 -1
View File
@@ -86,7 +86,7 @@ steps:
parallelism: 2
mirror:
amd:
device: mi325_1
device: mi300_1
source_file_dependencies:
- csrc/quantization/
- vllm/model_executor/layers/quantization
+15
View File
@@ -12,6 +12,21 @@ steps:
autorun_on_main: true
commands:
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt
mirror:
amd:
device: mi300_1
timeout_in_minutes: 55
depends_on:
- image-build-amd
source_file_dependencies:
- csrc/
- vllm/model_executor/layers/quantization
- vllm/model_executor/models/
- vllm/model_executor/model_loader/
- vllm/v1/attention/backends/
- vllm/v1/attention/selector.py
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
# - label: LM Eval Large Models (4 GPUs)(A100)
# device: a100
@@ -51,6 +51,7 @@ steps:
torch_nightly: {}
amd:
device: mi325_1
timeout_in_minutes: 90
depends_on:
- image-build-amd
commands:
+12 -3
View File
@@ -15,7 +15,7 @@ steps:
- pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model
mirror:
amd:
device: mi325_1
device: mi300_1
depends_on:
- image-build-amd
@@ -33,7 +33,7 @@ steps:
- pytest -v -s models/multimodal/generation/test_vit_cudagraph.py -m core_model
mirror:
amd:
device: mi325_1
device: mi300_1
depends_on:
- image-build-amd
@@ -50,7 +50,7 @@ steps:
- pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model
mirror:
amd:
device: mi325_1
device: mi300_1
depends_on:
- image-build-amd
@@ -153,3 +153,12 @@ steps:
- tests/models/multimodal/pooling
commands:
- pytest -v -s models/multimodal/pooling -m 'not core_model'
mirror:
amd:
device: mi300_1
timeout_in_minutes: 60
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/
- tests/models/multimodal/pooling
+43
View File
@@ -37,6 +37,21 @@ steps:
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness"
mirror:
amd:
device: mi300_1
timeout_in_minutes: 65
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/model_executor/model_loader/
- vllm/v1/sample/
- vllm/model_executor/layers/
- vllm/transformers_utils/configs/speculators/
- tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
- label: Spec Decode Speculators + MTP Nightly B200
key: spec-decode-speculators-mtp-nightly-b200
@@ -61,6 +76,20 @@ steps:
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode -k "ngram or suffix"
mirror:
amd:
device: mi300_1
timeout_in_minutes: 65
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/model_executor/model_loader/
- vllm/v1/sample/
- vllm/model_executor/layers/
- tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
- label: Spec Decode Draft Model
key: spec-decode-draft-model
@@ -72,6 +101,20 @@ steps:
- tests/v1/e2e/spec_decode/
commands:
- pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference"
mirror:
amd:
device: mi300_1
timeout_in_minutes: 50
depends_on:
- image-build-amd
source_file_dependencies:
- vllm/v1/spec_decode/
- vllm/v1/worker/gpu/spec_decode/
- vllm/model_executor/model_loader/
- vllm/v1/sample/
- vllm/model_executor/layers/
- tests/v1/e2e/spec_decode/
- vllm/platforms/rocm.py
- label: Spec Decode Draft Model Nightly B200
key: spec-decode-draft-model-nightly-b200
+2
View File
@@ -230,6 +230,7 @@ COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/requirements /requirements
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tests /tests
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/examples /examples
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite
COPY --from=build_vllm ${COMMON_WORKDIR}/vllm/pyproject.toml /pyproject.toml
@@ -513,6 +514,7 @@ COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/requirements /requir
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/benchmarks /benchmarks
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tests /tests
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/examples /examples
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/tools/install_torchcodec_rocm.sh /tools/install_torchcodec_rocm.sh
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/docker/Dockerfile.rocm /docker/
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/.buildkite /.buildkite
COPY --from=build_vllm_wheel_release ${COMMON_WORKDIR}/vllm/pyproject.toml /pyproject.toml