From 67e7ea8977bc4281d4d33dd3a81a5c5fab3df920 Mon Sep 17 00:00:00 2001 From: Charlie Fu Date: Thu, 9 Jul 2026 12:52:26 -0500 Subject: [PATCH] [ROCm][CI] Set all timeout_in_minutes to 180 (#48146) Signed-off-by: charlifu --- .buildkite/test-amd.yaml | 54 ++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 0b72c852e7..ba7823e420 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -275,7 +275,7 @@ steps: - pytest -v -s v1/cudagraph/test_cudagraph_mode.py - label: e2e Core (1 GPU) # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 optional: true @@ -439,7 +439,7 @@ steps: #----------------------------------------------------------- mi250 · docker ----------------------------------------------------------# - label: Docker Build Metadata (ROCm) # TBD - timeout_in_minutes: 20 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx90anightly, amdmi250] agent_pool: mi250_1 no_gpu: true @@ -467,7 +467,7 @@ steps: #----------------------------------------------------- mi300 · basic_correctness -----------------------------------------------------# - label: Basic Correctness # TBD - timeout_in_minutes: 95 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 fast_check: true @@ -484,7 +484,7 @@ steps: - pytest -v -s basic_correctness/test_cpu_offload.py - label: Distributed Model Tests (2 GPUs) # TBD - timeout_in_minutes: 110 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_2 num_gpus: 2 @@ -724,7 +724,7 @@ steps: - pytest -v -s distributed/test_eplb_spec_decode.py - label: Distributed Tests (2xH100-2xMI300) # TBD - timeout_in_minutes: 75 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_2 num_gpus: 2 @@ -1297,7 +1297,7 @@ steps: #--------------------------------------------------------- mi300 · examples ----------------------------------------------------------# - label: Examples # TBD - timeout_in_minutes: 90 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 optional: true @@ -1348,7 +1348,7 @@ steps: - pytest -v -s tests/kernels/ir - label: Kernels Attention Test %N # TBD - timeout_in_minutes: 100 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 optional: true @@ -1397,7 +1397,7 @@ steps: - pytest -v -s kernels/test_kda.py - label: Kernels MoE Test %N # TBD - timeout_in_minutes: 95 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 optional: true @@ -1511,7 +1511,7 @@ steps: #---------------------------------------------------- mi300 · model_runner_v2 -------------------------------------------------------# - label: Model Runner V2 Core Tests # TBD - timeout_in_minutes: 60 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 optional: true @@ -1535,7 +1535,7 @@ steps: - pytest -v -s entrypoints/llm/test_struct_output_generate.py -k "xgrammar and not speculative_config6 and not speculative_config7 and not speculative_config8 and not speculative_config0" - label: Model Runner V2 Examples # TBD - timeout_in_minutes: 60 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 optional: true @@ -1569,7 +1569,7 @@ steps: - python3 features/speculative_decoding/spec_decode_offline.py --test --method eagle3 --num_spec_tokens 3 --dataset-name hf --dataset-path philschmid/mt-bench --num-prompts 80 --temp 0 --top-p 1.0 --top-k -1 --tp 1 --enable-chunked-prefill --max-model-len 1536 - label: Model Runner V2 Distributed (2 GPUs) # TBD - timeout_in_minutes: 60 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_2 num_gpus: 2 @@ -1661,7 +1661,7 @@ steps: - pytest -v -s models/test_initialization.py::test_can_initialize_small_subset - label: Basic Models Tests (Other) # TBD - timeout_in_minutes: 90 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 optional: true @@ -1806,7 +1806,7 @@ steps: - cd .. && VLLM_WORKER_MULTIPROC_METHOD=spawn pytest -v -s tests/models/multimodal/generation/test_whisper.py -m core_model - label: Multi-Modal Processor # 1h 42m - timeout_in_minutes: 138 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 optional: true @@ -2255,7 +2255,7 @@ steps: - pytest -v -s v1/e2e/spec_decode -k "draft_model or no_sync or batch_inference" - label: Spec Decode Eagle # TBD - timeout_in_minutes: 90 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_1 optional: true @@ -2472,7 +2472,7 @@ steps: - DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py - label: Metrics, Tracing (2 GPUs) # TBD - timeout_in_minutes: 65 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_2 optional: true @@ -2678,7 +2678,7 @@ steps: #------------------------------------------------------ mi300 · weight_loading -------------------------------------------------------# - label: Weight Loading Multiple GPU # TBD - timeout_in_minutes: 75 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_2 num_gpus: 2 @@ -2690,7 +2690,7 @@ steps: - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt - label: Weight Loading Multiple GPU - Large Models # TBD - timeout_in_minutes: 75 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] agent_pool: mi300_2 num_gpus: 2 @@ -3089,7 +3089,7 @@ steps: - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt - label: LM Eval Qwen3-5 Models (B200-MI355) # TBD - timeout_in_minutes: 120 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_2 num_gpus: 2 @@ -3171,7 +3171,7 @@ steps: #--------------------------------------------------------- mi355 · examples ----------------------------------------------------------# - label: Examples # TBD - timeout_in_minutes: 100 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_1 working_dir: "/vllm-workspace/examples" @@ -3206,7 +3206,7 @@ steps: #---------------------------------------------------------- mi355 · kernels ----------------------------------------------------------# - label: Kernels (B200-MI355) # TBD - timeout_in_minutes: 15 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_1 working_dir: "/vllm-workspace/" @@ -3231,7 +3231,7 @@ steps: - pytest -v -s tests/kernels/attention/test_rocm_aiter_mla_decode_metadata.py - label: Kernels Attention Test %N # TBD - timeout_in_minutes: 100 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_1 parallelism: 2 @@ -3248,7 +3248,7 @@ steps: - pytest -v -s kernels/attention --shard-id=$$BUILDKITE_PARALLEL_JOB --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT - label: Kernels MoE Test %N # TBD - timeout_in_minutes: 50 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_1 parallelism: 5 @@ -3512,7 +3512,7 @@ steps: - pytest -v -s v1/attention - label: V1 Core + KV + Metrics # TBD - timeout_in_minutes: 60 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_1 optional: true @@ -3538,7 +3538,7 @@ steps: - pytest -v -s entrypoints/openai/correctness/test_lmeval.py::test_lm_eval_accuracy_v1_engine - label: V1 Sample + Logits # TBD - timeout_in_minutes: 60 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_1 optional: true @@ -3558,7 +3558,7 @@ steps: - pytest -v -s v1/test_outputs.py - label: V1 Spec Decode # TBD - timeout_in_minutes: 60 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_1 working_dir: "/vllm-workspace/tests" @@ -3571,7 +3571,7 @@ steps: #------------------------------------------------------ mi355 · weight_loading -------------------------------------------------------# - label: Weight Loading Multiple GPU # TBD - timeout_in_minutes: 75 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_2 num_gpus: 2 @@ -3583,7 +3583,7 @@ steps: - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models-amd.txt - label: Weight Loading Multiple GPU - Large Models # TBD - timeout_in_minutes: 75 + timeout_in_minutes: 180 mirror_hardwares: [amdexperimental, amdproduction, amdgfx950nightly, amdmi355] agent_pool: mi355_2 working_dir: "/vllm-workspace/tests"