From 96031b8623f36522f5e6e70f778d9654341a8d86 Mon Sep 17 00:00:00 2001 From: "Kevin H. Luu" Date: Tue, 1 Sep 2026 15:32:11 -0700 Subject: [PATCH] [CI] Shard distributed model jobs above the 24h P90 threshold (#54752) Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com> --- .buildkite/test_areas/models_distributed.yaml | 58 +++++++++++++------ 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/.buildkite/test_areas/models_distributed.yaml b/.buildkite/test_areas/models_distributed.yaml index 62b545cbb8..cf7623ce3e 100644 --- a/.buildkite/test_areas/models_distributed.yaml +++ b/.buildkite/test_areas/models_distributed.yaml @@ -2,9 +2,9 @@ group: Models - Distributed depends_on: - image-build steps: -- label: ":nvidia: (L4) Distributed Models" +- label: ":nvidia: (L4) Distributed Models Shard %N" key: distributed-model-tests-2-gpus - timeout_in_minutes: 60 + timeout_in_minutes: 35 working_dir: "/vllm-workspace/tests" device: l4 num_devices: 2 @@ -15,20 +15,30 @@ steps: - tests/model_executor/model_loader/test_sharded_state_loader.py - tests/models/ commands: - - TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' - - CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' - # Avoid importing model tests that cause CUDA reinitialization error - - pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' - - pytest models/language -v -s -m 'distributed(num_gpus=2)' - - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' - - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py - - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' + - | + case "$$BUILDKITE_PARALLEL_JOB" in + 0) + TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' + CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' + ;; + 1) + pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' + pytest models/language -v -s -m 'distributed(num_gpus=2)' + ;; + 2) + pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' + pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py + VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' + ;; + *) exit 2 ;; + esac + parallelism: 3 mirror: amd: - label: ":amd: (MI300) Distributed Models" + label: ":amd: (MI300) Distributed Models Shard %N" dind: false device: mi300_2 - timeout_in_minutes: 75 + timeout_in_minutes: 35 depends_on: - image-build-amd source_file_dependencies: @@ -43,10 +53,20 @@ steps: - tests/model_executor/model_loader/test_sharded_state_loader.py - tests/models/ commands: - - TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' - - HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' - - pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' - - pytest models/language -v -s -m 'distributed(num_gpus=2)' - - pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py - - pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' - - VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' + - | + case "$$BUILDKITE_PARALLEL_JOB" in + 0) + TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)' + HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)' + ;; + 1) + pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)' + pytest models/language -v -s -m 'distributed(num_gpus=2)' + ;; + 2) + pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py + pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)' + VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)' + ;; + *) exit 2 ;; + esac