[CI] Shard distributed model jobs above the 24h P90 threshold (#54752)

Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
This commit is contained in:
Kevin H. Luu
2026-09-01 15:32:11 -07:00
committed by GitHub
parent e90b608c42
commit 96031b8623
+39 -19
View File
@@ -2,9 +2,9 @@ group: Models - Distributed
depends_on:
- image-build
steps:
- label: ":nvidia: (L4) Distributed Models"
- label: ":nvidia: (L4) Distributed Models Shard %N"
key: distributed-model-tests-2-gpus
timeout_in_minutes: 60
timeout_in_minutes: 35
working_dir: "/vllm-workspace/tests"
device: l4
num_devices: 2
@@ -15,20 +15,30 @@ steps:
- tests/model_executor/model_loader/test_sharded_state_loader.py
- tests/models/
commands:
- TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
# Avoid importing model tests that cause CUDA reinitialization error
- pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
- pytest models/language -v -s -m 'distributed(num_gpus=2)'
- pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
- pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
- VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
- |
case "$$BUILDKITE_PARALLEL_JOB" in
0)
TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
;;
1)
pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
pytest models/language -v -s -m 'distributed(num_gpus=2)'
;;
2)
pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
;;
*) exit 2 ;;
esac
parallelism: 3
mirror:
amd:
label: ":amd: (MI300) Distributed Models"
label: ":amd: (MI300) Distributed Models Shard %N"
dind: false
device: mi300_2
timeout_in_minutes: 75
timeout_in_minutes: 35
depends_on:
- image-build-amd
source_file_dependencies:
@@ -43,10 +53,20 @@ steps:
- tests/model_executor/model_loader/test_sharded_state_loader.py
- tests/models/
commands:
- TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
- HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
- pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
- pytest models/language -v -s -m 'distributed(num_gpus=2)'
- pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
- pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
- VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
- |
case "$$BUILDKITE_PARALLEL_JOB" in
0)
TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
;;
1)
pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
pytest models/language -v -s -m 'distributed(num_gpus=2)'
;;
2)
pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
;;
*) exit 2 ;;
esac