From 1479bd9e9d3e7e06a4167980d4d4662eeda0638c Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Thu, 23 Jul 2026 20:42:25 -0500 Subject: [PATCH] [ROCm][CI] Prepare AMD mirrors for regating (#49270) Signed-off-by: Andreas Karatzas Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- .../scripts/hardware_ci/run-amd-test.sh | 6 +- .buildkite/test-amd.yaml | 449 +++++++++--------- .buildkite/test_areas/attention.yaml | 5 +- .buildkite/test_areas/basic_correctness.yaml | 5 +- .buildkite/test_areas/benchmarks.yaml | 1 + .buildkite/test_areas/disaggregated.yaml | 10 +- .buildkite/test_areas/distributed.yaml | 1 + .buildkite/test_areas/engine.yaml | 18 +- .buildkite/test_areas/entrypoints.yaml | 25 +- .buildkite/test_areas/expert_parallelism.yaml | 1 + .buildkite/test_areas/kernels.yaml | 12 +- .buildkite/test_areas/lm_eval.yaml | 7 +- .buildkite/test_areas/lora.yaml | 7 +- .buildkite/test_areas/misc.yaml | 24 +- .buildkite/test_areas/model_executor.yaml | 1 + .buildkite/test_areas/models_basic.yaml | 4 +- .buildkite/test_areas/models_language.yaml | 12 +- .buildkite/test_areas/models_multimodal.yaml | 25 +- .buildkite/test_areas/pytorch.yaml | 8 +- .buildkite/test_areas/samplers.yaml | 12 +- .buildkite/test_areas/spec_decode.yaml | 20 +- .buildkite/test_areas/weight_loading.yaml | 1 + docker/Dockerfile.rocm | 2 + 23 files changed, 364 insertions(+), 292 deletions(-) diff --git a/.buildkite/scripts/hardware_ci/run-amd-test.sh b/.buildkite/scripts/hardware_ci/run-amd-test.sh index 4e8735c841..de3f443ad5 100755 --- a/.buildkite/scripts/hardware_ci/run-amd-test.sh +++ b/.buildkite/scripts/hardware_ci/run-amd-test.sh @@ -35,7 +35,7 @@ set -o pipefail : "${PY_COLORS:=1}" : "${ROCM_DOCKER_TTY:=1}" : "${PYTHONFAULTHANDLER:=1}" -: "${PYTEST_TIMEOUT:=2100}" +: "${PYTEST_TIMEOUT:=2400}" if [[ " ${PYTEST_ADDOPTS:-} " != *" --color"* ]]; then PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--color=yes" fi @@ -45,9 +45,9 @@ fi if [[ " ${PYTEST_ADDOPTS:-} " != *" --durations-min="* ]]; then PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }--durations-min=1.0" fi -# Dump stacks after 15 minutes, then stop an individual test after 35 minutes. +# Dump stacks after 25 minutes, then stop an individual test after 40 minutes. if [[ " ${PYTEST_ADDOPTS:-} " != *" faulthandler_timeout="* ]]; then - PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }-o faulthandler_timeout=900" + PYTEST_ADDOPTS="${PYTEST_ADDOPTS:+${PYTEST_ADDOPTS} }-o faulthandler_timeout=1500" fi if [[ " ${PYTEST_ADDOPTS:-} " != *" --timeout-method="* && " ${PYTEST_ADDOPTS:-} " != *" --timeout-method "* ]]; then diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index bbde36e970..d3cdbc388f 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -40,7 +40,7 @@ ##################################################################################################################################### # # # IMPORTANT: # -# * Currently AMD CI has MI250 agents, MI300 agents, MI325 agents, and MI355 agents. All upcoming feature improvements are # +# * Currently AMD CI has MI250 agents, MI300 agents, and MI355 agents. All upcoming feature improvements are # # tracked in: https://github.com/vllm-project/vllm/issues/34994 # # # #-----------------------------------------------------------------------------------------------------------------------------------# @@ -207,6 +207,7 @@ steps: timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -240,6 +241,20 @@ steps: - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "not qwen2 and not qwen3 and not gemma" - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model +- label: Multi-Modal Processor (CPU) %N # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + agent_pool: mi250_1 + parallelism: 6 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/models/multimodal + - tests/models/registry.py + commands: + - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + #------------------------------------------------------------ mi250 · v1 -------------------------------------------------------------# - label: Batch Invariance (H100-MI250) # TBD @@ -367,6 +382,22 @@ steps: commands: - pytest -v -s v1/attention +- label: V1 others (CPU) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + agent_pool: mi250_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/v1 + commands: + - pytest -v -s -m 'cpu_test' v1/core + - pytest -v -s v1/structured_output + - pytest -v -s v1/test_serial_utils.py + - pytest -v -s -m 'cpu_test' v1/kv_connector/unit + - pytest -v -s -m 'cpu_test' v1/metrics + #------------------------------------------------------------- mi250 · misc ------------------------------------------------------------# - label: Async Engine, Inputs, Utils, Worker, Config (CPU) # TBD @@ -408,6 +439,19 @@ steps: - pytest -v -s transformers_utils - pytest -v -s config +- label: Python-only Installation # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] + agent_pool: mi250_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - tests/standalone_tests/python_only_compile.sh + - setup.py + - vllm/platforms/rocm.py + commands: + - bash standalone_tests/python_only_compile.sh + #------------------------------------------------------------ mi250 · rust -----------------------------------------------------------# - label: Rust Frontend Cargo Style + Clippy # TBD @@ -445,6 +489,7 @@ steps: mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 no_gpu: true + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - .buildkite/scripts/docker-build-metadata-args.sh @@ -507,7 +552,7 @@ steps: - tests/models/ commands: - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' - - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' + - HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' - pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' - pytest models/language -v -s -m 'distributed(num_gpus=2)' - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py @@ -657,6 +702,30 @@ steps: - VLLM_TEST_CLEAN_GPU_MEMORY=1 pytest -v -s tests/compile/passes/distributed/test_async_tp.py - pytest -v -s tests/compile/fusions_e2e/test_tp2_ar_rms.py::test_tp2_ar_rms_fusions +- label: Distributed Compile + RPC Tests (2 GPUs) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_2 + num_gpus: 2 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/compilation/ + - vllm/distributed/ + - vllm/engine/ + - vllm/executor/ + - vllm/worker/worker_base.py + - vllm/v1/engine/ + - vllm/v1/worker/ + - tests/compile/fullgraph/test_basic_correctness.py + - tests/compile/test_wrapper.py + - tests/entrypoints/llm/test_collective_rpc.py + - vllm/platforms/rocm.py + commands: + - pytest -v -s entrypoints/llm/test_collective_rpc.py + - pytest -v -s ./compile/fullgraph/test_basic_correctness.py + - pytest -v -s ./compile/test_wrapper.py + #----------------------------------------------------------- mi300 · cuda ------------------------------------------------------------# - label: Platform Tests # TBD @@ -679,6 +748,7 @@ steps: mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ @@ -870,6 +940,71 @@ steps: commands: - torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep +- label: Distributed Torchrun + Shutdown Tests (2 GPUs) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_2 + num_gpus: 2 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/ + - vllm/engine/ + - vllm/executor/ + - vllm/worker/worker_base.py + - vllm/v1/engine/ + - vllm/v1/worker/ + - tests/distributed/ + - tests/v1/shutdown + - tests/v1/worker/test_worker_memory_snapshot.py + - vllm/platforms/rocm.py + commands: + - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed' + - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed' + - HIP_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown + - pytest -v -s v1/worker/test_worker_memory_snapshot.py + +- label: Distributed Compile + Comm (4 GPUs) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_4 + num_gpus: 4 + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/distributed/ + - tests/distributed/test_pynccl + - tests/distributed/test_events + - tests/compile/fullgraph/test_basic_correctness.py + - tests/distributed/test_symm_mem_allreduce.py + - tests/distributed/test_multiproc_executor.py + - vllm/platforms/rocm.py + commands: + - pytest -v -s compile/fullgraph/test_basic_correctness.py + - pytest -v -s distributed/test_pynccl.py + - pytest -v -s distributed/test_events.py + - pytest -v -s distributed/test_symm_mem_allreduce.py + - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node + +#---------------------------------------------------------- mi300 · engine -----------------------------------------------------------# + +- label: Engine # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/engine + - tests/test_sequence + - tests/test_config + - tests/test_logger + - tests/test_vllm_port + commands: + - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py + #-------------------------------------------------------- mi300 · entrypoints --------------------------------------------------------# - label: Entrypoints Unit Tests # TBD @@ -878,6 +1013,7 @@ steps: dind: false agent_pool: mi300_1 fast_check: true + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/entrypoints @@ -982,6 +1118,7 @@ steps: mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 + optional: true fast_check: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1010,6 +1147,7 @@ steps: mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 + optional: true fast_check: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1024,6 +1162,7 @@ steps: mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 + optional: true fast_check: true working_dir: "/vllm-workspace/tests" source_file_dependencies: @@ -1345,6 +1484,27 @@ steps: - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm.txt --tp-size=8 +- label: LM Eval Large Models (4xH100-4xMI300) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_4 + num_gpus: 4 + optional: true + working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + commands: + - export VLLM_USE_DEEP_GEMM=0 + - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4 + #--------------------------------------------------------- mi300 · examples ----------------------------------------------------------# - label: Examples # TBD @@ -1593,7 +1753,7 @@ steps: - set -x - export VLLM_USE_V2_MODEL_RUNNER=1 - pytest -v -s v1/engine/test_llm_engine.py -k "not test_engine_metrics" - - ENFORCE_EAGER=1 pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" + - pytest -v -s v1/e2e/general/test_async_scheduling.py -k "not ngram" - pytest -v -s v1/e2e/general/test_context_length.py - pytest -v -s v1/e2e/general/test_min_tokens.py - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0" @@ -1797,6 +1957,37 @@ steps: - pip freeze | grep -E 'torch' - pytest -v -s models/language -m 'core_model and slow_test' --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB +- label: Language Models Test (Extended Generation) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/models/language/generation + commands: + - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' + - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' + - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' + +- label: Language Models Tests (Hybrid) %N # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + parallelism: 2 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/models/language/generation + commands: + - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' + - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' + - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + #---------------------------------------------------- mi300 · models / multimodal ----------------------------------------------------# - label: Multi-Modal Models (Extended Generation 1) # TBD @@ -1899,20 +2090,32 @@ steps: commands: - pytest -v -s models/multimodal/processing/test_tensor_schema.py -- label: Multi-Modal Processor (CPU) %N # TBD +- label: Multi-Modal Models (Extended Pooling) # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + optional: true + working_dir: "/vllm-workspace/tests" + source_file_dependencies: + - vllm/ + - tests/models/multimodal/pooling + commands: + - pytest -v -s models/multimodal/pooling -m 'not core_model' + +- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 - parallelism: 4 optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/ - tests/models/multimodal - - tests/models/registry.py commands: - - pytest -v -s models/multimodal/processing --ignore models/multimodal/processing/test_tensor_schema.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma" + - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model #----------------------------------------------------- mi300 · models / quantized -----------------------------------------------------# @@ -1933,7 +2136,29 @@ steps: #-------------------------------------------------- mi300 · models / transformers ---------------------------------------------------# -- label: Transformers Nightly Models (Shardable) %N # TBD +- label: Transformers Nightly Models (Initialization) %N # TBD + timeout_in_minutes: 180 + mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] + dind: false + agent_pool: mi300_1 + parallelism: 6 + optional: true + working_dir: "/vllm-workspace/" + source_file_dependencies: + - vllm/model_executor/models/ + - vllm/model_executor/model_loader/ + - vllm/multimodal/ + - vllm/model_executor/layers/ + - vllm/v1/attention/backends/ + - vllm/v1/attention/selector.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py + - tests/models/ + commands: + - pip install --upgrade git+https://github.com/huggingface/transformers + - pytest -v -s tests/models/test_initialization.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB + +- label: Transformers Nightly Models (Processing) %N # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false @@ -1953,7 +2178,6 @@ steps: - tests/models/ commands: - pip install --upgrade git+https://github.com/huggingface/transformers - - pytest -v -s tests/models/test_initialization.py --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - pytest -v -s tests/models/multimodal/processing/ --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - label: Transformers Nightly Models (Single) # TBD @@ -2517,23 +2741,6 @@ steps: # - export HSA_NO_SCRATCH_RECLAIM=1 - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine -- label: V1 others (CPU) # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] - dind: false - agent_pool: mi300_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/v1 - commands: - - pytest -v -s -m 'cpu_test' v1/core - - pytest -v -s v1/structured_output - - pytest -v -s v1/test_serial_utils.py - - pytest -v -s -m 'cpu_test' v1/kv_connector/unit - - pytest -v -s -m 'cpu_test' v1/metrics - - label: V1 Sample + Logits # TBD timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] @@ -2830,195 +3037,6 @@ steps: commands: - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-large-amd.txt -######################################################################################################################################### -# # -# MI325 (gfx942) tests # -# # -######################################################################################################################################### - -#---------------------------------------------------------- mi325 · compile ----------------------------------------------------------# - -- label: Distributed Compile + RPC Tests (2 GPUs) # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - agent_pool: mi325_2 - num_gpus: 2 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/compilation/ - - vllm/distributed/ - - vllm/engine/ - - vllm/executor/ - - vllm/worker/worker_base.py - - vllm/v1/engine/ - - vllm/v1/worker/ - - tests/compile/fullgraph/test_basic_correctness.py - - tests/compile/test_wrapper.py - - tests/entrypoints/llm/test_collective_rpc.py - - vllm/platforms/rocm.py - commands: - - pytest -v -s entrypoints/llm/test_collective_rpc.py - - pytest -v -s ./compile/fullgraph/test_basic_correctness.py - - pytest -v -s ./compile/test_wrapper.py - -#-------------------------------------------------------- mi325 · distributed --------------------------------------------------------# - -- label: Distributed Torchrun + Shutdown Tests (2 GPUs) # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - agent_pool: mi325_2 - num_gpus: 2 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/distributed/ - - vllm/engine/ - - vllm/executor/ - - vllm/worker/worker_base.py - - vllm/v1/engine/ - - vllm/v1/worker/ - - tests/distributed/ - - tests/v1/shutdown - - tests/v1/worker/test_worker_memory_snapshot.py - - vllm/platforms/rocm.py - commands: - - VLLM_TEST_SAME_HOST=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed' - - VLLM_TEST_SAME_HOST=1 VLLM_TEST_WITH_DEFAULT_DEVICE_SET=1 torchrun --nproc-per-node=4 distributed/test_same_node.py | grep 'Same node test passed' - - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s v1/shutdown - - pytest -v -s v1/worker/test_worker_memory_snapshot.py - -- label: Distributed Compile + Comm (4 GPUs) # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - agent_pool: mi325_4 - num_gpus: 4 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/distributed/ - - tests/distributed/test_pynccl - - tests/distributed/test_events - - tests/compile/fullgraph/test_basic_correctness.py - - tests/distributed/test_symm_mem_allreduce.py - - tests/distributed/test_multiproc_executor.py - - vllm/platforms/rocm.py - commands: - - pytest -v -s compile/fullgraph/test_basic_correctness.py - - pytest -v -s distributed/test_pynccl.py - - pytest -v -s distributed/test_events.py - - pytest -v -s distributed/test_symm_mem_allreduce.py - - pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node - -#---------------------------------------------------------- mi325 · engine -----------------------------------------------------------# - -- label: Engine # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - agent_pool: mi325_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/engine - - tests/test_sequence - - tests/test_config - - tests/test_logger - - tests/test_vllm_port - commands: - - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py - -#----------------------------------------------------------- mi325 · evals -----------------------------------------------------------# - -- label: LM Eval Large Models (4xH100-4xMI325) # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - agent_pool: mi325_4 - num_gpus: 4 - optional: true - working_dir: "/vllm-workspace/.buildkite/lm-eval-harness" - source_file_dependencies: - - csrc/ - - vllm/model_executor/layers/quantization - - vllm/model_executor/models/ - - vllm/model_executor/model_loader/ - - vllm/v1/attention/backends/ - - vllm/v1/attention/selector.py - - vllm/_aiter_ops.py - - vllm/platforms/rocm.py - commands: - - export VLLM_USE_DEEP_GEMM=0 - - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-rocm-fp8.txt --tp-size=4 - -#----------------------------------------------------- mi325 · models / language -----------------------------------------------------# - -- label: Language Models Test (Extended Generation) # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - agent_pool: mi325_1 - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/language/generation - commands: - - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' - - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - - pytest -v -s models/language/generation -m '(not core_model) and (not hybrid_model)' - -- label: Language Models Tests (Hybrid) %N # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - agent_pool: mi325_1 - parallelism: 2 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/language/generation - commands: - - uv pip install --system --no-build-isolation 'git+https://github.com/AndreasKaratzas/mamba@fix-rocm-7.0-warp-size-constexpr' - - uv pip install --system --no-build-isolation 'git+https://github.com/Dao-AILab/causal-conv1d@v1.6.0' - - pytest -v -s models/language/generation -m hybrid_model --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB - -#---------------------------------------------------- mi325 · models / multimodal ----------------------------------------------------# - -- label: Multi-Modal Models (Extended Pooling) # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - agent_pool: mi325_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/multimodal/pooling - commands: - - pytest -v -s models/multimodal/pooling -m 'not core_model' - -- label: "Multi-Modal Models (Standard) 2: qwen3 + gemma" # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - agent_pool: mi325_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - vllm/ - - tests/models/multimodal - commands: - - pytest -v -s models/multimodal/generation/test_common.py -m core_model -k "qwen3 or gemma" - - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model - -#----------------------------------------------------------- mi325 · misc ------------------------------------------------------------# - -- label: Python-only Installation # TBD - timeout_in_minutes: 180 - mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi325] - agent_pool: mi325_1 - optional: true - working_dir: "/vllm-workspace/tests" - source_file_dependencies: - - tests/standalone_tests/python_only_compile.sh - - setup.py - - vllm/platforms/rocm.py - commands: - - bash standalone_tests/python_only_compile.sh - ######################################################################################################################################### # # # MI355 (gfx950) tests # @@ -3463,6 +3481,7 @@ steps: timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_1 + optional: true working_dir: "/vllm-workspace/tests" source_file_dependencies: - vllm/model_executor/models/qwen3_5.py diff --git a/.buildkite/test_areas/attention.yaml b/.buildkite/test_areas/attention.yaml index aadea2908c..71d2975a5f 100644 --- a/.buildkite/test_areas/attention.yaml +++ b/.buildkite/test_areas/attention.yaml @@ -16,8 +16,9 @@ steps: parallelism: 2 mirror: amd: - device: mi325_1 - timeout_in_minutes: 95 + dind: false + device: mi300_1 + timeout_in_minutes: 125 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/basic_correctness.yaml b/.buildkite/test_areas/basic_correctness.yaml index 88f3e05482..6f0c22cc8b 100644 --- a/.buildkite/test_areas/basic_correctness.yaml +++ b/.buildkite/test_areas/basic_correctness.yaml @@ -18,7 +18,8 @@ steps: - pytest -v -s basic_correctness/test_cpu_offload.py mirror: amd: - device: mi325_1 - timeout_in_minutes: 70 + dind: false + device: mi300_1 + timeout_in_minutes: 60 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/benchmarks.yaml b/.buildkite/test_areas/benchmarks.yaml index 8aac6545cf..034fb58840 100644 --- a/.buildkite/test_areas/benchmarks.yaml +++ b/.buildkite/test_areas/benchmarks.yaml @@ -15,6 +15,7 @@ steps: amd: dind: false device: mi300_1 + timeout_in_minutes: 40 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/disaggregated.yaml b/.buildkite/test_areas/disaggregated.yaml index 4012f1ebd5..a3342e362e 100644 --- a/.buildkite/test_areas/disaggregated.yaml +++ b/.buildkite/test_areas/disaggregated.yaml @@ -17,7 +17,7 @@ steps: amd: dind: false device: mi300_4 - timeout_in_minutes: 85 + timeout_in_minutes: 60 depends_on: - image-build-amd source_file_dependencies: @@ -68,7 +68,7 @@ steps: amd: dind: false device: mi300_4 - timeout_in_minutes: 60 + timeout_in_minutes: 40 depends_on: - image-build-amd source_file_dependencies: @@ -94,7 +94,7 @@ steps: amd: dind: false device: mi300_4 - timeout_in_minutes: 85 + timeout_in_minutes: 60 depends_on: - image-build-amd source_file_dependencies: @@ -120,7 +120,7 @@ steps: amd: dind: false device: mi300_4 - timeout_in_minutes: 80 + timeout_in_minutes: 55 depends_on: - image-build-amd source_file_dependencies: @@ -177,7 +177,7 @@ steps: amd: dind: false device: mi300_2 - timeout_in_minutes: 70 + timeout_in_minutes: 45 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index d9fcff19f0..258a60451e 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -41,6 +41,7 @@ steps: amd: dind: false device: mi300_2 + timeout_in_minutes: 45 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/engine.yaml b/.buildkite/test_areas/engine.yaml index 6e88451af8..ed593c4aba 100644 --- a/.buildkite/test_areas/engine.yaml +++ b/.buildkite/test_areas/engine.yaml @@ -28,8 +28,9 @@ steps: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py test_jit_monitor.py mirror: amd: - device: mi325_1 - timeout_in_minutes: 50 + dind: false + device: mi300_1 + timeout_in_minutes: 40 depends_on: - image-build-amd @@ -44,8 +45,8 @@ steps: - pytest -v -s v1/engine --ignore v1/engine/test_preprocess_error_handling.py mirror: amd: - device: mi325_1 - timeout_in_minutes: 55 + device: mi250_1 + timeout_in_minutes: 45 depends_on: - image-build-amd @@ -60,8 +61,8 @@ steps: - pytest -v -s v1/e2e/general/test_async_scheduling.py mirror: amd: - device: mi325_1 - timeout_in_minutes: 70 + device: mi250_1 + timeout_in_minutes: 55 depends_on: - image-build-amd @@ -76,8 +77,8 @@ steps: - pytest -v -s v1/e2e/general --ignore v1/e2e/general/test_async_scheduling.py mirror: amd: - device: mi325_1 - timeout_in_minutes: 60 + device: mi250_1 + timeout_in_minutes: 50 depends_on: - image-build-amd source_file_dependencies: @@ -116,6 +117,7 @@ steps: amd: dind: false device: mi300_2 + timeout_in_minutes: 30 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/entrypoints.yaml b/.buildkite/test_areas/entrypoints.yaml index 9d571ff0af..47c0db109c 100644 --- a/.buildkite/test_areas/entrypoints.yaml +++ b/.buildkite/test_areas/entrypoints.yaml @@ -30,9 +30,9 @@ steps: - pytest -v -s entrypoints/llm/offline_mode # Needs to avoid interference with other tests mirror: amd: - device: mi325_1 - # TODO(akaratza): Test after Torch >= 2.12 bump - soft_fail: true + dind: false + device: mi300_1 + timeout_in_minutes: 55 depends_on: - image-build-amd @@ -52,7 +52,9 @@ steps: - pytest -v -s entrypoints/scale_out mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 + timeout_in_minutes: 65 depends_on: - image-build-amd @@ -70,7 +72,8 @@ steps: - pytest -v -s entrypoints/openai --ignore=entrypoints/openai/completion --ignore=entrypoints/openai/chat_completion --ignore=entrypoints/openai/responses --ignore=entrypoints/openai/correctness mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 timeout_in_minutes: 65 depends_on: - image-build-amd @@ -90,8 +93,9 @@ steps: - pytest -v -s entrypoints/openai/completion --ignore=entrypoints/openai/completion/test_tensorizer_entrypoint.py mirror: amd: - device: mi325_1 - timeout_in_minutes: 80 + dind: false + device: mi300_1 + timeout_in_minutes: 70 depends_on: - image-build-amd @@ -113,7 +117,8 @@ steps: - pytest -v -s entrypoints/anthropic mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 timeout_in_minutes: 65 depends_on: - image-build-amd @@ -176,7 +181,9 @@ steps: - pytest -s entrypoints/openai/correctness/ mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 + timeout_in_minutes: 30 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/expert_parallelism.yaml b/.buildkite/test_areas/expert_parallelism.yaml index d2278408fa..a3b46b5828 100644 --- a/.buildkite/test_areas/expert_parallelism.yaml +++ b/.buildkite/test_areas/expert_parallelism.yaml @@ -18,6 +18,7 @@ steps: amd: dind: false device: mi300_1 + timeout_in_minutes: 30 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/kernels.yaml b/.buildkite/test_areas/kernels.yaml index 3618cb6e96..e1951685f6 100644 --- a/.buildkite/test_areas/kernels.yaml +++ b/.buildkite/test_areas/kernels.yaml @@ -80,7 +80,8 @@ steps: parallelism: 2 mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 timeout_in_minutes: 90 depends_on: - image-build-amd @@ -118,7 +119,9 @@ steps: parallelism: 2 mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 + timeout_in_minutes: 120 source_file_dependencies: - csrc/quantization/ - vllm/model_executor/layers/quantization @@ -148,8 +151,9 @@ steps: parallelism: 5 mirror: amd: - device: mi325_1 - timeout_in_minutes: 65 + dind: false + device: mi300_1 + timeout_in_minutes: 55 source_file_dependencies: - csrc/quantization/cutlass_w8a8/moe/ - csrc/moe/ diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 95a8bafdd5..7d0f987d10 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -14,8 +14,9 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-small.txt mirror: amd: - device: mi325_1 - timeout_in_minutes: 55 + dind: false + device: mi300_1 + timeout_in_minutes: 45 depends_on: - image-build-amd source_file_dependencies: @@ -141,7 +142,7 @@ steps: amd: dind: false device: mi300_8 - timeout_in_minutes: 60 + timeout_in_minutes: 40 depends_on: - image-build-amd commands: diff --git a/.buildkite/test_areas/lora.yaml b/.buildkite/test_areas/lora.yaml index 46a3710ea8..a79196ffbd 100644 --- a/.buildkite/test_areas/lora.yaml +++ b/.buildkite/test_areas/lora.yaml @@ -14,9 +14,10 @@ steps: parallelism: 4 mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 working_dir: "/vllm-workspace/tests" - timeout_in_minutes: 65 + timeout_in_minutes: 85 source_file_dependencies: - vllm/lora - tests/lora @@ -46,4 +47,4 @@ steps: - pytest -v -s -x lora/test_qwen3_with_multi_loras.py - pytest -v -s -x lora/test_olmoe_tp.py - pytest -v -s -x lora/test_gptoss_tp.py - - pytest -v -s -x lora/test_qwen35_densemodel_lora.py \ No newline at end of file + - pytest -v -s -x lora/test_qwen35_densemodel_lora.py diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index d61a6a816f..1c2000a9e5 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -25,7 +25,7 @@ steps: amd: dind: false device: mi300_1 - timeout_in_minutes: 75 + timeout_in_minutes: 50 depends_on: - image-build-amd @@ -59,7 +59,9 @@ steps: - pytest -v -s v1/test_outputs.py mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 + timeout_in_minutes: 70 depends_on: - image-build-amd @@ -113,8 +115,9 @@ steps: - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine mirror: amd: - device: mi325_1 - timeout_in_minutes: 75 + dind: false + device: mi300_1 + timeout_in_minutes: 65 depends_on: - image-build-amd @@ -233,7 +236,9 @@ steps: - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 + timeout_in_minutes: 75 source_file_dependencies: - vllm/entrypoints - vllm/multimodal @@ -269,10 +274,11 @@ steps: - pytest -v -s v1/tracing mirror: amd: - device: mi325_2 + dind: false + device: mi300_2 + timeout_in_minutes: 30 depends_on: - image-build-amd - optional: true - label: Python-only Installation key: python-only-installation @@ -287,8 +293,8 @@ steps: - bash standalone_tests/python_only_compile.sh mirror: amd: - device: mi325_1 - timeout_in_minutes: 45 + device: mi250_1 + timeout_in_minutes: 55 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/model_executor.yaml b/.buildkite/test_areas/model_executor.yaml index e2f9007929..07d4aea871 100644 --- a/.buildkite/test_areas/model_executor.yaml +++ b/.buildkite/test_areas/model_executor.yaml @@ -30,6 +30,7 @@ steps: amd: dind: false device: mi300_1 + timeout_in_minutes: 60 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/models_basic.yaml b/.buildkite/test_areas/models_basic.yaml index 2e542b6db4..af90308d6c 100644 --- a/.buildkite/test_areas/models_basic.yaml +++ b/.buildkite/test_areas/models_basic.yaml @@ -42,7 +42,9 @@ steps: - pytest -v -s models/test_terratorch.py models/transformers/test_backend.py models/test_registry.py mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 + timeout_in_minutes: 50 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/models_language.yaml b/.buildkite/test_areas/models_language.yaml index 2aea721e56..6124cb023d 100644 --- a/.buildkite/test_areas/models_language.yaml +++ b/.buildkite/test_areas/models_language.yaml @@ -17,6 +17,7 @@ steps: amd: dind: false device: mi300_1 + timeout_in_minutes: 45 depends_on: - image-build-amd @@ -39,6 +40,7 @@ steps: amd: dind: false device: mi300_1 + timeout_in_minutes: 40 depends_on: - image-build-amd source_file_dependencies: @@ -69,8 +71,9 @@ steps: parallelism: 2 mirror: amd: - device: mi325_1 - timeout_in_minutes: 70 + dind: false + device: mi300_1 + timeout_in_minutes: 60 depends_on: - image-build-amd commands: @@ -130,8 +133,9 @@ steps: - pytest -v -s models/language/pooling -m 'not core_model' mirror: amd: - device: mi325_1 - timeout_in_minutes: 120 + dind: false + device: mi300_1 + timeout_in_minutes: 95 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/models_multimodal.yaml b/.buildkite/test_areas/models_multimodal.yaml index 57f559c59f..095af12d28 100644 --- a/.buildkite/test_areas/models_multimodal.yaml +++ b/.buildkite/test_areas/models_multimodal.yaml @@ -14,7 +14,9 @@ steps: - pytest -v -s models/multimodal/generation/test_ultravox.py -m core_model mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 + timeout_in_minutes: 65 depends_on: - image-build-amd @@ -31,7 +33,9 @@ steps: - pytest -v -s models/multimodal/generation/test_qwen2_5_vl.py -m core_model mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 + timeout_in_minutes: 55 depends_on: - image-build-amd @@ -47,7 +51,8 @@ steps: - pytest -v -s models/multimodal/generation/test_qwen2_vl.py -m core_model mirror: amd: - device: mi325_1 + device: mi250_1 + timeout_in_minutes: 55 depends_on: - image-build-amd @@ -65,7 +70,9 @@ steps: - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model # Otherwise, mp_method="spawn" doesn't work mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 + timeout_in_minutes: 50 depends_on: - image-build-amd @@ -109,6 +116,7 @@ steps: amd: dind: false device: mi300_1 + timeout_in_minutes: 35 depends_on: - image-build-amd source_file_dependencies: @@ -131,7 +139,9 @@ steps: - pytest -v -s models/multimodal/test_mapping.py mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 + timeout_in_minutes: 90 depends_on: - image-build-amd @@ -166,8 +176,9 @@ steps: - pytest -v -s models/multimodal/pooling -m 'not core_model' mirror: amd: - device: mi325_1 - timeout_in_minutes: 75 + dind: false + device: mi300_1 + timeout_in_minutes: 60 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/pytorch.yaml b/.buildkite/test_areas/pytorch.yaml index 59a3632d42..f53f399946 100644 --- a/.buildkite/test_areas/pytorch.yaml +++ b/.buildkite/test_areas/pytorch.yaml @@ -107,13 +107,6 @@ steps: - tests/compile/passes commands: - pytest -s -v compile/passes --ignore compile/passes/distributed - mirror: - amd: - dind: false - device: mi300_1 - timeout_in_minutes: 65 - depends_on: - - image-build-amd - label: PyTorch Fullgraph Smoke Test device: h200_35gb @@ -236,6 +229,7 @@ steps: amd: dind: false device: mi300_1 + timeout_in_minutes: 30 depends_on: - image-build-amd source_file_dependencies: diff --git a/.buildkite/test_areas/samplers.yaml b/.buildkite/test_areas/samplers.yaml index 2e7cd4a623..929cbec2ae 100644 --- a/.buildkite/test_areas/samplers.yaml +++ b/.buildkite/test_areas/samplers.yaml @@ -19,8 +19,18 @@ steps: - VLLM_USE_FLASHINFER_SAMPLER=1 pytest -v -s samplers mirror: amd: - device: mi325_1 + device: mi250_1 + timeout_in_minutes: 40 depends_on: - image-build-amd + source_file_dependencies: + - vllm/model_executor/layers + - vllm/sampling_metadata.py + - vllm/v1/sample/ + - vllm/entrypoints/generate/beam_search/ + - tests/samplers + - tests/conftest.py + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py commands: - pytest -v -s samplers diff --git a/.buildkite/test_areas/spec_decode.yaml b/.buildkite/test_areas/spec_decode.yaml index 4efeb6526e..977562ee97 100644 --- a/.buildkite/test_areas/spec_decode.yaml +++ b/.buildkite/test_areas/spec_decode.yaml @@ -14,8 +14,9 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "eagle_correctness" mirror: amd: - device: mi325_1 - timeout_in_minutes: 60 + dind: false + device: mi300_1 + timeout_in_minutes: 55 depends_on: - image-build-amd source_file_dependencies: @@ -53,8 +54,9 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "speculators or mtp_correctness" mirror: amd: - device: mi325_1 - timeout_in_minutes: 65 + dind: false + device: mi300_1 + timeout_in_minutes: 75 depends_on: - image-build-amd source_file_dependencies: @@ -92,10 +94,9 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "ngram or suffix" mirror: amd: - device: mi325_1 - timeout_in_minutes: 55 - # TODO(akaratza): Test after Torch >= 2.12 bump - soft_fail: true + dind: false + device: mi300_1 + timeout_in_minutes: 35 depends_on: - image-build-amd source_file_dependencies: @@ -119,7 +120,8 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference" mirror: amd: - device: mi325_1 + dind: false + device: mi300_1 timeout_in_minutes: 55 depends_on: - image-build-amd diff --git a/.buildkite/test_areas/weight_loading.yaml b/.buildkite/test_areas/weight_loading.yaml index 46a874178b..2e05e5b5ba 100644 --- a/.buildkite/test_areas/weight_loading.yaml +++ b/.buildkite/test_areas/weight_loading.yaml @@ -17,6 +17,7 @@ steps: amd: dind: false device: mi300_2 + timeout_in_minutes: 35 depends_on: - image-build-amd commands: diff --git a/docker/Dockerfile.rocm b/docker/Dockerfile.rocm index 47438b06b5..27c199317d 100644 --- a/docker/Dockerfile.rocm +++ b/docker/Dockerfile.rocm @@ -745,6 +745,7 @@ ENV MIOPEN_DEBUG_CONV_GEMM=0 # See: https://github.com/ROCm/rocm-libraries/issues/6266 ENV HSA_ENABLE_IPC_MODE_LEGACY=1 ENV UCX_RMA_PPLN_ENABLE=y +ENV UCX_ROCM_COPY_SIGPOOL_MAX_ELEMS=inf # ROCm profiler limits workaround. RUN echo "ROCTRACER_MAX_EVENTS=10000000" > ${COMMON_WORKDIR}/libkineto.conf @@ -835,6 +836,7 @@ COPY --from=export_vllm /docker ${COMMON_WORKDIR}/vllm/docker # See: https://github.com/ROCm/rocm-libraries/issues/6266 ENV HSA_ENABLE_IPC_MODE_LEGACY=1 ENV UCX_RMA_PPLN_ENABLE=y +ENV UCX_ROCM_COPY_SIGPOOL_MAX_ELEMS=inf ENV TOKENIZERS_PARALLELISM=false