[ROCm][CI] Fine-tuning queues and test names (#39238)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -771,7 +771,7 @@ steps:
|
|||||||
|
|
||||||
#----------------------------------------------------------- mi300 · cuda ------------------------------------------------------------#
|
#----------------------------------------------------------- mi300 · cuda ------------------------------------------------------------#
|
||||||
|
|
||||||
- label: Platform Tests (CUDA) # TBD
|
- label: Platform Tests # TBD
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
||||||
agent_pool: mi300_1
|
agent_pool: mi300_1
|
||||||
@@ -831,7 +831,7 @@ steps:
|
|||||||
- pytest -v -s distributed/test_eplb_execute.py
|
- pytest -v -s distributed/test_eplb_execute.py
|
||||||
- pytest -v -s distributed/test_eplb_spec_decode.py
|
- pytest -v -s distributed/test_eplb_spec_decode.py
|
||||||
|
|
||||||
- label: Distributed Tests (2xH100-2xMI250) # TBD
|
- label: Distributed Tests (2xH100-2xMI300) # TBD
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
||||||
agent_pool: mi300_2
|
agent_pool: mi300_2
|
||||||
@@ -843,13 +843,19 @@ steps:
|
|||||||
- vllm/model_executor/layers/fused_moe/
|
- vllm/model_executor/layers/fused_moe/
|
||||||
- vllm/v1/attention/backends/
|
- vllm/v1/attention/backends/
|
||||||
- vllm/v1/attention/selector.py
|
- vllm/v1/attention/selector.py
|
||||||
|
- tests/v1/distributed/test_dbo.py
|
||||||
- tests/distributed/test_context_parallel.py
|
- tests/distributed/test_context_parallel.py
|
||||||
- examples/features/data_parallel/data_parallel_offline.py
|
- examples/features/data_parallel/data_parallel_offline.py
|
||||||
- vllm/_aiter_ops.py
|
- vllm/_aiter_ops.py
|
||||||
- vllm/platforms/rocm.py
|
- vllm/platforms/rocm.py
|
||||||
commands:
|
commands:
|
||||||
- pytest -v -s tests/distributed/test_context_parallel.py
|
- pytest -v -s tests/distributed/test_context_parallel.py
|
||||||
|
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
|
||||||
|
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
|
||||||
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
|
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=allgather_reducescatter --disable-nccl-for-dp-synchronization
|
||||||
|
- pytest -v -s tests/v1/distributed/test_dbo.py
|
||||||
|
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
|
||||||
|
- pytest -v -s tests/distributed/test_packed_tensor.py
|
||||||
|
|
||||||
- label: Distributed Tests (4xA100-4xMI300) # TBD
|
- label: Distributed Tests (4xA100-4xMI300) # TBD
|
||||||
timeout_in_minutes: 180
|
timeout_in_minutes: 180
|
||||||
@@ -2195,26 +2201,6 @@ steps:
|
|||||||
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
|
- TP_SIZE=1 DP_SIZE=2 pytest -v -s v1/distributed/test_external_lb_dp.py
|
||||||
- DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
|
- DP_SIZE=2 pytest -v -s entrypoints/openai/test_multi_api_servers.py
|
||||||
|
|
||||||
- label: Distributed Tests (2xH100-2xMI300) # TBD
|
|
||||||
timeout_in_minutes: 180
|
|
||||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
|
||||||
agent_pool: mi300_2
|
|
||||||
num_gpus: 2
|
|
||||||
working_dir: "/vllm-workspace/"
|
|
||||||
source_file_dependencies:
|
|
||||||
- vllm/distributed/
|
|
||||||
- vllm/v1/distributed/
|
|
||||||
- vllm/model_executor/layers/fused_moe/
|
|
||||||
- tests/v1/distributed/test_dbo.py
|
|
||||||
- vllm/_aiter_ops.py
|
|
||||||
- vllm/platforms/rocm.py
|
|
||||||
commands:
|
|
||||||
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_async_new_apis.py
|
|
||||||
- VLLM_LOGGING_LEVEL=DEBUG python3 examples/features/data_parallel/data_parallel_offline.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput
|
|
||||||
- pytest -v -s tests/v1/distributed/test_dbo.py
|
|
||||||
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
|
|
||||||
- pytest -v -s tests/distributed/test_packed_tensor.py
|
|
||||||
|
|
||||||
- label: Metrics, Tracing (2 GPUs) # TBD
|
- label: Metrics, Tracing (2 GPUs) # TBD
|
||||||
timeout_in_minutes: 20
|
timeout_in_minutes: 20
|
||||||
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
mirror_hardwares: [amdexperimental, amdproduction, amdgfx942nightly, amdmi300]
|
||||||
|
|||||||
@@ -2,8 +2,8 @@ group: CUDA
|
|||||||
depends_on:
|
depends_on:
|
||||||
- image-build
|
- image-build
|
||||||
steps:
|
steps:
|
||||||
- label: Platform Tests (CUDA)
|
- label: Platform Tests
|
||||||
key: platform-tests-cuda
|
key: platform-tests
|
||||||
timeout_in_minutes: 15
|
timeout_in_minutes: 15
|
||||||
device: h200_18gb
|
device: h200_18gb
|
||||||
source_file_dependencies:
|
source_file_dependencies:
|
||||||
|
|||||||
@@ -174,8 +174,8 @@ steps:
|
|||||||
# test multi-node TP with multiproc executor (simulated on single node)
|
# test multi-node TP with multiproc executor (simulated on single node)
|
||||||
- pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
|
- pytest -v -s distributed/test_multiproc_executor.py::test_multiproc_executor_multi_node
|
||||||
|
|
||||||
- label: Distributed Tests (8 GPUs)(H100)
|
- label: Distributed Tests (8xH100)
|
||||||
key: distributed-tests-8-gpus-h100
|
key: distributed-tests-8xh100
|
||||||
timeout_in_minutes: 10
|
timeout_in_minutes: 10
|
||||||
device: h100
|
device: h100
|
||||||
num_devices: 8
|
num_devices: 8
|
||||||
@@ -195,8 +195,8 @@ steps:
|
|||||||
# test with torchrun tp=2 and dp=4 with ep
|
# test with torchrun tp=2 and dp=4 with ep
|
||||||
- torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
|
- torchrun --nproc-per-node=8 ../examples/features/torchrun/torchrun_dp_example_offline.py --tp-size=2 --pp-size=1 --dp-size=4 --enable-ep
|
||||||
|
|
||||||
- label: Distributed Tests (4 GPUs)(A100)
|
- label: Distributed Tests (4xA100)
|
||||||
key: distributed-tests-4-gpus-a100
|
key: distributed-tests-4xa100
|
||||||
device: a100
|
device: a100
|
||||||
optional: true
|
optional: true
|
||||||
num_devices: 4
|
num_devices: 4
|
||||||
@@ -211,7 +211,7 @@ steps:
|
|||||||
- pytest -v -s -x lora/test_mixtral.py
|
- pytest -v -s -x lora/test_mixtral.py
|
||||||
|
|
||||||
- label: Distributed Tests (2xH100-2xMI300)
|
- label: Distributed Tests (2xH100-2xMI300)
|
||||||
key: distributed-tests-2-gpus-h100
|
key: distributed-tests-2xh100-2xmi300
|
||||||
timeout_in_minutes: 15
|
timeout_in_minutes: 15
|
||||||
device: h100
|
device: h100
|
||||||
optional: true
|
optional: true
|
||||||
@@ -237,8 +237,8 @@ steps:
|
|||||||
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
|
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 pytest -v -s tests/distributed/test_weight_transfer.py
|
||||||
- pytest -v -s tests/distributed/test_packed_tensor.py
|
- pytest -v -s tests/distributed/test_packed_tensor.py
|
||||||
|
|
||||||
- label: Distributed Tests (2 GPUs)(B200)
|
- label: Distributed Tests (2xB200)
|
||||||
key: distributed-tests-2-gpus-b200
|
key: distributed-tests-2xb200
|
||||||
device: b200-k8s
|
device: b200-k8s
|
||||||
optional: true
|
optional: true
|
||||||
working_dir: "/vllm-workspace/"
|
working_dir: "/vllm-workspace/"
|
||||||
|
|||||||
@@ -2,8 +2,8 @@ group: E2E Integration
|
|||||||
depends_on:
|
depends_on:
|
||||||
- image-build
|
- image-build
|
||||||
steps:
|
steps:
|
||||||
- label: DeepSeek V2-Lite Sync EPLB Accuracy
|
- label: DeepSeek V2-Lite Sync EPLB Accuracy (4xH100)
|
||||||
key: deepseek-v2-lite-sync-eplb-accuracy
|
key: deepseek-v2-lite-sync-eplb-accuracy-4xh100
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 60
|
||||||
device: h100
|
device: h100
|
||||||
optional: true
|
optional: true
|
||||||
@@ -12,8 +12,8 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
|
- bash .buildkite/scripts/scheduled_integration_test/deepseek_v2_lite_ep_eplb.sh 0.25 200 8010
|
||||||
|
|
||||||
- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy
|
- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (4xH100)
|
||||||
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy
|
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-4xh100
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 60
|
||||||
device: h100
|
device: h100
|
||||||
optional: true
|
optional: true
|
||||||
@@ -22,8 +22,8 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
|
- bash .buildkite/scripts/scheduled_integration_test/qwen30b_a3b_fp8_block_ep_eplb.sh 0.8 200 8020
|
||||||
|
|
||||||
- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (B200)
|
- label: Qwen3-30B-A3B-FP8-block Sync EPLB Accuracy (2xB200)
|
||||||
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-b200
|
key: qwen3-30b-a3b-fp8-block-sync-eplb-accuracy-2xb200
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 60
|
||||||
device: b200-k8s
|
device: b200-k8s
|
||||||
optional: true
|
optional: true
|
||||||
|
|||||||
@@ -274,8 +274,8 @@ steps:
|
|||||||
- pytest -v -s kernels/helion/
|
- pytest -v -s kernels/helion/
|
||||||
|
|
||||||
|
|
||||||
- label: Kernels FP8 MoE Test (1 H100)
|
- label: Kernels FP8 MoE Test (1xH100)
|
||||||
key: kernels-fp8-moe-test-1-h100
|
key: kernels-fp8-moe-test-1xh100
|
||||||
timeout_in_minutes: 90
|
timeout_in_minutes: 90
|
||||||
device: h100
|
device: h100
|
||||||
num_devices: 1
|
num_devices: 1
|
||||||
@@ -291,8 +291,8 @@ steps:
|
|||||||
- pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
|
- pytest -v -s kernels/moe/test_triton_moe_no_act_mul.py
|
||||||
- pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py
|
- pytest -v -s kernels/moe/test_triton_moe_ptpc_fp8.py
|
||||||
|
|
||||||
- label: Kernels FP8 MoE Test (2 H100s)
|
- label: Kernels FP8 MoE Test (2xH100)
|
||||||
key: kernels-fp8-moe-test-2-h100s
|
key: kernels-fp8-moe-test-2xh100
|
||||||
timeout_in_minutes: 90
|
timeout_in_minutes: 90
|
||||||
device: h100
|
device: h100
|
||||||
num_devices: 2
|
num_devices: 2
|
||||||
|
|||||||
@@ -28,7 +28,8 @@ steps:
|
|||||||
- vllm/_aiter_ops.py
|
- vllm/_aiter_ops.py
|
||||||
- vllm/platforms/rocm.py
|
- vllm/platforms/rocm.py
|
||||||
|
|
||||||
# - label: LM Eval Large Models (4 GPUs)(A100)
|
# - label: LM Eval Large Models (4xA100)
|
||||||
|
# key: lm-eval-large-models-4xa100
|
||||||
# device: a100
|
# device: a100
|
||||||
# optional: true
|
# optional: true
|
||||||
# num_devices: 4
|
# num_devices: 4
|
||||||
@@ -40,8 +41,8 @@ steps:
|
|||||||
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
# - export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||||
# - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
|
# - pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large.txt --tp-size=4
|
||||||
|
|
||||||
- label: LM Eval Large Models (4 GPUs)(H100)
|
- label: LM Eval Large Models (4xH100)
|
||||||
key: lm-eval-large-models-4-gpus-h100
|
key: lm-eval-large-models-4xh100
|
||||||
device: h100
|
device: h100
|
||||||
optional: true
|
optional: true
|
||||||
num_devices: 4
|
num_devices: 4
|
||||||
@@ -53,8 +54,8 @@ steps:
|
|||||||
- export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
|
- export VLLM_USE_DEEP_GEMM=0 # We found Triton is faster than DeepGEMM for H100
|
||||||
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
|
- pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-hopper.txt --tp-size=4
|
||||||
|
|
||||||
- label: LM Eval Small Models (B200)
|
- label: LM Eval Small Models (2xB200)
|
||||||
key: lm-eval-small-models-b200
|
key: lm-eval-small-models-2xb200
|
||||||
timeout_in_minutes: 120
|
timeout_in_minutes: 120
|
||||||
device: b200-k8s
|
device: b200-k8s
|
||||||
optional: true
|
optional: true
|
||||||
@@ -64,8 +65,8 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
|
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell.txt
|
||||||
|
|
||||||
- label: LM Eval Large Models (B200, EP)
|
- label: LM Eval Large Models EP (2xB200)
|
||||||
key: lm-eval-large-models-b200-ep
|
key: lm-eval-large-models-ep-2xb200
|
||||||
timeout_in_minutes: 120
|
timeout_in_minutes: 120
|
||||||
device: b200-k8s
|
device: b200-k8s
|
||||||
optional: true
|
optional: true
|
||||||
@@ -76,8 +77,8 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep.txt
|
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-blackwell-ep.txt
|
||||||
|
|
||||||
- label: LM Eval Qwen3.5 Models (B200)
|
- label: LM Eval Qwen3.5 Models (2xB200)
|
||||||
key: lm-eval-qwen3-5-models-b200
|
key: lm-eval-qwen3-5-models-2xb200
|
||||||
timeout_in_minutes: 120
|
timeout_in_minutes: 120
|
||||||
device: b200-k8s
|
device: b200-k8s
|
||||||
optional: true
|
optional: true
|
||||||
@@ -93,8 +94,8 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
|
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt
|
||||||
|
|
||||||
- label: LM Eval Large Models (H200)
|
- label: LM Eval Large Models (8xH200)
|
||||||
key: lm-eval-large-models-h200
|
key: lm-eval-large-models-8xh200
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 60
|
||||||
device: h200
|
device: h200
|
||||||
optional: true
|
optional: true
|
||||||
@@ -192,8 +193,8 @@ steps:
|
|||||||
commands:
|
commands:
|
||||||
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt
|
- pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=evals/gsm8k/configs/models-turboquant.txt
|
||||||
|
|
||||||
- label: GPQA Eval (GPT-OSS) (H100)
|
- label: GPQA Eval (GPT-OSS) (2xH100)
|
||||||
key: gpqa-eval-gpt-oss-h100
|
key: gpqa-eval-gpt-oss-2xh100
|
||||||
timeout_in_minutes: 120
|
timeout_in_minutes: 120
|
||||||
device: h100
|
device: h100
|
||||||
optional: true
|
optional: true
|
||||||
@@ -206,8 +207,8 @@ steps:
|
|||||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
|
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
|
||||||
|
|
||||||
- label: GPQA Eval (GPT-OSS) (B200)
|
- label: GPQA Eval (GPT-OSS) (2xB200)
|
||||||
key: gpqa-eval-gpt-oss-b200
|
key: gpqa-eval-gpt-oss-2xb200
|
||||||
timeout_in_minutes: 120
|
timeout_in_minutes: 120
|
||||||
device: b200-k8s
|
device: b200-k8s
|
||||||
optional: true
|
optional: true
|
||||||
|
|||||||
Reference in New Issue
Block a user