From 276e03b92c168f9457f922d8d957900dd30e5d89 Mon Sep 17 00:00:00 2001 From: Matthew Bonanni Date: Mon, 5 Jan 2026 17:17:59 -0500 Subject: [PATCH] [CI][DeepSeek] Add nightly DeepSeek R1 `lm_eval` tests on H200 (#30356) Signed-off-by: Matthew Bonanni Co-authored-by: Michael Goin --- .buildkite/test-pipeline.yaml | 8 ++++++++ tests/evals/gsm8k/configs/DeepSeek-R1-DP.yaml | 11 +++++++++++ tests/evals/gsm8k/configs/DeepSeek-R1-TP.yaml | 11 +++++++++++ tests/evals/gsm8k/configs/models-h200.txt | 2 ++ tests/evals/gsm8k/test_gsm8k_correctness.py | 2 +- 5 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 tests/evals/gsm8k/configs/DeepSeek-R1-DP.yaml create mode 100644 tests/evals/gsm8k/configs/DeepSeek-R1-TP.yaml create mode 100644 tests/evals/gsm8k/configs/models-h200.txt diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index bd607cd3a9..44165b9ba5 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -1351,6 +1351,14 @@ steps: - CUDA_VISIBLE_DEVICES=1,2 VLLM_USE_DEEP_GEMM=1 VLLM_LOGGING_LEVEL=DEBUG python3 examples/offline_inference/data_parallel.py --model=Qwen/Qwen1.5-MoE-A2.7B -tp=1 -dp=2 --max-model-len=2048 --all2all-backend=deepep_high_throughput - pytest -v -s tests/v1/distributed/test_dbo.py +- label: LM Eval Large Models (H200) # optional + timeout_in_minutes: 60 + gpu: h200 + optional: true + num_gpus: 8 + commands: + - pytest -s -v evals/gsm8k/test_gsm8k_correctness.py --config-list-file=configs/models-h200.txt + ##### B200 test ##### - label: Distributed Tests (B200) # optional gpu: b200 diff --git a/tests/evals/gsm8k/configs/DeepSeek-R1-DP.yaml b/tests/evals/gsm8k/configs/DeepSeek-R1-DP.yaml new file mode 100644 index 0000000000..f351a17220 --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-R1-DP.yaml @@ -0,0 +1,11 @@ +model_name: "deepseek-ai/DeepSeek-R1" +accuracy_threshold: 0.95 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +server_args: >- + --enforce-eager + --max-model-len 4096 + --data-parallel-size 8 + --enable-expert-parallel + --speculative-config '{"method":"mtp","num_speculative_tokens":1}' diff --git a/tests/evals/gsm8k/configs/DeepSeek-R1-TP.yaml b/tests/evals/gsm8k/configs/DeepSeek-R1-TP.yaml new file mode 100644 index 0000000000..ba3463463b --- /dev/null +++ b/tests/evals/gsm8k/configs/DeepSeek-R1-TP.yaml @@ -0,0 +1,11 @@ +model_name: "deepseek-ai/DeepSeek-R1" +accuracy_threshold: 0.95 +num_questions: 1319 +num_fewshot: 5 +startup_max_wait_seconds: 1200 +server_args: >- + --enforce-eager + --max-model-len 4096 + --tensor-parallel-size 8 + --enable-expert-parallel + --speculative-config '{"method":"mtp","num_speculative_tokens":1}' diff --git a/tests/evals/gsm8k/configs/models-h200.txt b/tests/evals/gsm8k/configs/models-h200.txt new file mode 100644 index 0000000000..ec936533bb --- /dev/null +++ b/tests/evals/gsm8k/configs/models-h200.txt @@ -0,0 +1,2 @@ +DeepSeek-R1-TP.yaml +DeepSeek-R1-DP.yaml diff --git a/tests/evals/gsm8k/test_gsm8k_correctness.py b/tests/evals/gsm8k/test_gsm8k_correctness.py index dd0d3ae0cc..991b905211 100644 --- a/tests/evals/gsm8k/test_gsm8k_correctness.py +++ b/tests/evals/gsm8k/test_gsm8k_correctness.py @@ -78,7 +78,7 @@ def test_gsm8k_correctness(config_filename): eval_config["model_name"], server_args, env_dict=env_dict, - max_wait_seconds=600, + max_wait_seconds=eval_config.get("startup_max_wait_seconds", 600), ) as remote_server: server_url = remote_server.url_for("v1") print(f"Server started at: {server_url}")