[CI] Shard distributed model jobs above the 24h P90 threshold (#54752)
Signed-off-by: Kevin Luu <51931015+khluu@users.noreply.github.com>
This commit is contained in:
@@ -2,9 +2,9 @@ group: Models - Distributed
|
||||
depends_on:
|
||||
- image-build
|
||||
steps:
|
||||
- label: ":nvidia: (L4) Distributed Models"
|
||||
- label: ":nvidia: (L4) Distributed Models Shard %N"
|
||||
key: distributed-model-tests-2-gpus
|
||||
timeout_in_minutes: 60
|
||||
timeout_in_minutes: 35
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
device: l4
|
||||
num_devices: 2
|
||||
@@ -15,20 +15,30 @@ steps:
|
||||
- tests/model_executor/model_loader/test_sharded_state_loader.py
|
||||
- tests/models/
|
||||
commands:
|
||||
- TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
|
||||
- CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
|
||||
# Avoid importing model tests that cause CUDA reinitialization error
|
||||
- pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest models/language -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
|
||||
- VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
|
||||
- |
|
||||
case "$$BUILDKITE_PARALLEL_JOB" in
|
||||
0)
|
||||
TARGET_TEST_SUITE=L4 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
|
||||
CUDA_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
|
||||
;;
|
||||
1)
|
||||
pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
|
||||
pytest models/language -v -s -m 'distributed(num_gpus=2)'
|
||||
;;
|
||||
2)
|
||||
pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
|
||||
pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
|
||||
;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
parallelism: 3
|
||||
mirror:
|
||||
amd:
|
||||
label: ":amd: (MI300) Distributed Models"
|
||||
label: ":amd: (MI300) Distributed Models Shard %N"
|
||||
dind: false
|
||||
device: mi300_2
|
||||
timeout_in_minutes: 75
|
||||
timeout_in_minutes: 35
|
||||
depends_on:
|
||||
- image-build-amd
|
||||
source_file_dependencies:
|
||||
@@ -43,10 +53,20 @@ steps:
|
||||
- tests/model_executor/model_loader/test_sharded_state_loader.py
|
||||
- tests/models/
|
||||
commands:
|
||||
- TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
|
||||
- HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
|
||||
- pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest models/language -v -s -m 'distributed(num_gpus=2)'
|
||||
- pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
|
||||
- pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
|
||||
- VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
|
||||
- |
|
||||
case "$$BUILDKITE_PARALLEL_JOB" in
|
||||
0)
|
||||
TARGET_TEST_SUITE=MI300 pytest basic_correctness/ -v -s -m 'distributed(num_gpus=2)'
|
||||
HIP_VISIBLE_DEVICES=0,1 pytest -v -s model_executor/model_loader/test_sharded_state_loader.py -m '(not slow_test)'
|
||||
;;
|
||||
1)
|
||||
pytest models/transformers/test_backend.py -v -s -m 'distributed(num_gpus=2)'
|
||||
pytest models/language -v -s -m 'distributed(num_gpus=2)'
|
||||
;;
|
||||
2)
|
||||
pytest models/multimodal -v -s -m 'distributed(num_gpus=2)' --ignore models/multimodal/generation/test_whisper.py --ignore models/multimodal/generation/test_phi4siglip.py
|
||||
pytest models/multimodal/generation/test_phi4siglip.py -v -s -m 'distributed(num_gpus=2)'
|
||||
VLLM_WORKER_MULTIPROC_METHOD=spawn pytest models/multimodal/generation/test_whisper.py -v -s -m 'distributed(num_gpus=2)'
|
||||
;;
|
||||
*) exit 2 ;;
|
||||
esac
|
||||
|
||||
Reference in New Issue
Block a user