From a7f9be9612861d6425738dd0ba1c7fd5805e6c95 Mon Sep 17 00:00:00 2001 From: Andreas Karatzas Date: Fri, 4 Sep 2026 21:50:51 -0500 Subject: [PATCH] [CI][ROCm] Raise AMD job and server readiness timeouts (#55308) Signed-off-by: Andreas Karatzas Co-authored-by: OpenAI Codex --- .../deepseek_v2_lite_ep_eplb.sh | 2 +- .../qwen3_next_mtp_async_eplb.sh | 2 +- .buildkite/test-amd.yaml | 24 +++++++++---------- tests/v1/distributed/test_internal_lb_dp.py | 3 +++ 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh b/.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh index 2ff0b2e125..9f30f7bd3c 100644 --- a/.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh +++ b/.buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh @@ -10,7 +10,7 @@ mkdir -p "${OUT_DIR}" wait_for_server() { local port=$1 - timeout 600 bash -c ' + timeout "${VLLM_ENGINE_READY_TIMEOUT_S:-600}" bash -c ' until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do sleep 1 done' diff --git a/.buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh b/.buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh index d587f26ae8..f7d4ecf4a7 100644 --- a/.buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh +++ b/.buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh @@ -10,7 +10,7 @@ mkdir -p "${OUT_DIR}" wait_for_server() { local port=$1 - timeout 600 bash -c ' + timeout "${VLLM_ENGINE_READY_TIMEOUT_S:-600}" bash -c ' until curl -sf "http://127.0.0.1:'"$port"'/health" > /dev/null; do sleep 1 done' diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index c1f429aaf1..ca790a7e7d 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -617,7 +617,7 @@ steps: - "pytest -v -s tests/compile/fusions_e2e/test_tp1_quant.py -k 'inductor_partition and +rms_norm and +quant_fp8 and qwen3'" - label: ":amd: (MI300) PyTorch Compilation Passes" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 80 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -901,7 +901,7 @@ steps: - TARGET_TEST_SUITE=L4 pytest -v -s basic_correctness/test_basic_correctness.py -k "ray" - label: ":amd: (MI300) Distributed DP + EP" # TBD - timeout_in_minutes: 30 + timeout_in_minutes: 50 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_8 @@ -1008,7 +1008,7 @@ steps: - pytest -v -s entrypoints/launchers - label: ":amd: (MI300) Entrypoints Integration (LLM)" # TBD - timeout_in_minutes: 60 + timeout_in_minutes: 90 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_1 @@ -1312,7 +1312,7 @@ steps: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx-fp8-and-mixed.txt - label: ":amd: (MI300) DeepSeek V2-Lite Sync EPLB Accuracy" # TBD - timeout_in_minutes: 30 + timeout_in_minutes: 40 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 @@ -1332,10 +1332,10 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py commands: - - bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010 + - VLLM_ENGINE_READY_TIMEOUT_S=1800 bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010 - label: ":amd: (MI300) LM Eval Large Models Harness" # TBD - timeout_in_minutes: 55 + timeout_in_minutes: 85 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 @@ -1400,7 +1400,7 @@ steps: - bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_dp4_async_eplb.sh 0.8 200 8050 - label: ":amd: (MI300) Qwen3-Next-80B-A3B-Instruct MTP Async EPLB Accuracy" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 @@ -1420,10 +1420,10 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py commands: - - bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040 + - VLLM_ENGINE_READY_TIMEOUT_S=1800 bash .buildkite/scripts/scheduled_integration_test/qwen3_next_mtp_async_eplb.sh 0.8 1319 8040 - label: ":amd: (MI300) LM Eval Large Models" # TBD - timeout_in_minutes: 35 + timeout_in_minutes: 65 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_8 @@ -2450,7 +2450,7 @@ steps: - pytest -v -s tool_use --ignore=tool_use/mistral --models llama3.2 -k "not test_response_format_with_tool_choice_required and not test_parallel_tool_calls_false and not test_tool_call_and_choice" - label: ":amd: (MI300) Rust Frontend Distributed" # TBD - timeout_in_minutes: 30 + timeout_in_minutes: 45 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_4 @@ -2474,7 +2474,7 @@ steps: - export VLLM_USE_RUST_FRONTEND=1 - export VLLM_WORKER_MULTIPROC_METHOD=spawn - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_dense_dp_world_size.py - - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info" + - VLLM_ENGINE_READY_TIMEOUT_S=1800 TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_internal_lb_dp.py -k "not 4 and not server_info" - TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py -k "not 4 and not server_info" - TP_SIZE=1 DP_SIZE=4 pytest -v -s v1/distributed/test_hybrid_lb_dp.py -k "not 4 and not server_info" @@ -3076,7 +3076,7 @@ steps: #------------------------------------------------------ mi300 ยท weight_loading -------------------------------------------------------# - label: ":amd: (MI300) Weight Loading Multi-GPU" # TBD - timeout_in_minutes: 40 + timeout_in_minutes: 70 mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300] dind: false agent_pool: mi300_2 diff --git a/tests/v1/distributed/test_internal_lb_dp.py b/tests/v1/distributed/test_internal_lb_dp.py index 70e416787c..e4f7b3d5a8 100644 --- a/tests/v1/distributed/test_internal_lb_dp.py +++ b/tests/v1/distributed/test_internal_lb_dp.py @@ -184,6 +184,9 @@ class MultinodeInternalLBServerManager: self.model_name, sargs, auto_port=False, + max_wait_seconds=int( + os.getenv("VLLM_ENGINE_READY_TIMEOUT_S", "480") + ), env_dict={ "VLLM_SERVER_DEV_MODE": "1", current_platform.device_control_env_var: ",".join(