From 78d78365254dfa20d3a265b6da695ab8151ec8b1 Mon Sep 17 00:00:00 2001 From: Thang Nguyen <69278249+Thangnguyenvn98@users.noreply.github.com> Date: Mon, 21 Sep 2026 20:19:17 -0400 Subject: [PATCH] [CI] Split (H200) LM Eval Large Models into per-model jobs (#57965) Signed-off-by: Thang Nguyen Co-authored-by: Kimi Code Co-authored-by: Claude Opus 5 (1M context) --- .buildkite/test_areas/lm_eval.yaml | 75 ++++++++++++++----- .../configs/models-h200-deepseek-r1-dp.txt | 1 + .../configs/models-h200-deepseek-r1-tp.txt | 1 + .../configs/models-h200-deepseek-v32-dp.txt | 1 + .../configs/models-h200-deepseek-v32-tp.txt | 1 + .../configs/models-h200-nemotron-3-super.txt | 2 + tests/evals/gsm8k/configs/models-h200.txt | 6 -- 7 files changed, 62 insertions(+), 25 deletions(-) create mode 100644 tests/evals/gsm8k/configs/models-h200-deepseek-r1-dp.txt create mode 100644 tests/evals/gsm8k/configs/models-h200-deepseek-r1-tp.txt create mode 100644 tests/evals/gsm8k/configs/models-h200-deepseek-v32-dp.txt create mode 100644 tests/evals/gsm8k/configs/models-h200-deepseek-v32-tp.txt create mode 100644 tests/evals/gsm8k/configs/models-h200-nemotron-3-super.txt delete mode 100644 tests/evals/gsm8k/configs/models-h200.txt diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index 5a284d172a..3450b70c28 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -343,29 +343,66 @@ steps: commands: - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-qwen35-blackwell.txt -- label: ":nvidia: (H200) LM Eval Large Models" - key: lm-eval-large-models-8xh200 - timeout_in_minutes: 120 +- label: ":nvidia: (H200) LM Eval DeepSeek R1 TP" + key: lm-eval-deepseek-r1-tp-8xh200 + timeout_in_minutes: 35 device: h200 optional: true num_devices: 8 commands: - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt - mirror: - amd: - label: ":amd: (MI300) LM Eval Large Models" - dind: false - device: mi300_8 - timeout_in_minutes: 65 - depends_on: - - image-build-amd - commands: - - export VLLM_WORKER_MULTIPROC_METHOD=spawn - - export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time - - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt - source_file_dependencies: - - vllm/_aiter_ops.py - - vllm/platforms/rocm.py + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-r1-tp.txt + +- label: ":nvidia: (H200) LM Eval DeepSeek R1 DP" + key: lm-eval-deepseek-r1-dp-8xh200 + timeout_in_minutes: 35 + device: h200 + optional: true + num_devices: 8 + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-r1-dp.txt + +- label: ":nvidia: (H200) LM Eval DeepSeek V3.2 TP" + key: lm-eval-deepseek-v32-tp-8xh200 + timeout_in_minutes: 35 + device: h200 + optional: true + num_devices: 8 + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-v32-tp.txt + +- label: ":nvidia: (H200) LM Eval DeepSeek V3.2 DP" + key: lm-eval-deepseek-v32-dp-8xh200 + timeout_in_minutes: 35 + device: h200 + optional: true + num_devices: 8 + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-deepseek-v32-dp.txt + +- label: ":nvidia: (H200) LM Eval Nemotron 3 Super" + key: lm-eval-nemotron-3-super-8xh200 + timeout_in_minutes: 35 + device: h200 + optional: true + num_devices: 8 + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200-nemotron-3-super.txt + +- label: ":amd: (MI300) LM Eval Large Models" + key: amd-lm-eval-large-models + dind: false + device: mi300_8 + optional: true + timeout_in_minutes: 65 + depends_on: + - image-build-amd + commands: + - export VLLM_WORKER_MULTIPROC_METHOD=spawn + - export PYTORCH_ROCM_ARCH=gfx942 # Limit Quark compilation to save time + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-mi3xx.txt + source_file_dependencies: + - vllm/_aiter_ops.py + - vllm/platforms/rocm.py - label: ":nvidia: (H100) MoE Refactor Integration TEMPORARY" key: moe-refactor-integration-test-h100-temporary diff --git a/tests/evals/gsm8k/configs/models-h200-deepseek-r1-dp.txt b/tests/evals/gsm8k/configs/models-h200-deepseek-r1-dp.txt new file mode 100644 index 0000000000..9e293daf16 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-h200-deepseek-r1-dp.txt @@ -0,0 +1 @@ +DeepSeek-R1-DP.yaml diff --git a/tests/evals/gsm8k/configs/models-h200-deepseek-r1-tp.txt b/tests/evals/gsm8k/configs/models-h200-deepseek-r1-tp.txt new file mode 100644 index 0000000000..30826a6f76 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-h200-deepseek-r1-tp.txt @@ -0,0 +1 @@ +DeepSeek-R1-TP.yaml diff --git a/tests/evals/gsm8k/configs/models-h200-deepseek-v32-dp.txt b/tests/evals/gsm8k/configs/models-h200-deepseek-v32-dp.txt new file mode 100644 index 0000000000..57faba312f --- /dev/null +++ b/tests/evals/gsm8k/configs/models-h200-deepseek-v32-dp.txt @@ -0,0 +1 @@ +DeepSeek-V3.2-DP.yaml diff --git a/tests/evals/gsm8k/configs/models-h200-deepseek-v32-tp.txt b/tests/evals/gsm8k/configs/models-h200-deepseek-v32-tp.txt new file mode 100644 index 0000000000..027f168fd5 --- /dev/null +++ b/tests/evals/gsm8k/configs/models-h200-deepseek-v32-tp.txt @@ -0,0 +1 @@ +DeepSeek-V3.2-TP.yaml diff --git a/tests/evals/gsm8k/configs/models-h200-nemotron-3-super.txt b/tests/evals/gsm8k/configs/models-h200-nemotron-3-super.txt new file mode 100644 index 0000000000..ec6fc214cd --- /dev/null +++ b/tests/evals/gsm8k/configs/models-h200-nemotron-3-super.txt @@ -0,0 +1,2 @@ +Nemotron-3-Super-120B-A12B-BF16.yaml +Nemotron-3-Super-120B-A12B-FP8.yaml diff --git a/tests/evals/gsm8k/configs/models-h200.txt b/tests/evals/gsm8k/configs/models-h200.txt deleted file mode 100644 index 255e93b674..0000000000 --- a/tests/evals/gsm8k/configs/models-h200.txt +++ /dev/null @@ -1,6 +0,0 @@ -DeepSeek-R1-TP.yaml -DeepSeek-R1-DP.yaml -DeepSeek-V3.2-TP.yaml -DeepSeek-V3.2-DP.yaml -Nemotron-3-Super-120B-A12B-BF16.yaml -Nemotron-3-Super-120B-A12B-FP8.yaml