From 6cc2c9ba3a2c78714b897474e087745058570902 Mon Sep 17 00:00:00 2001 From: Michael Goin Date: Mon, 22 Jun 2026 22:52:38 +0100 Subject: [PATCH] [CI] Add DGX Spark GPQA smoke test (#39541) Signed-off-by: mgoin Signed-off-by: Michael Goin Co-authored-by: Claude Opus 4.8 --- .buildkite/image_build/image_build_arm64.sh | 5 +++-- .buildkite/test_areas/lm_eval.yaml | 14 ++++++++++++++ tests/evals/gpt_oss/configs/gpt-oss-20b-sm120.yaml | 5 +++++ tests/evals/gpt_oss/configs/models-spark.txt | 2 ++ 4 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 tests/evals/gpt_oss/configs/gpt-oss-20b-sm120.yaml create mode 100644 tests/evals/gpt_oss/configs/models-spark.txt diff --git a/.buildkite/image_build/image_build_arm64.sh b/.buildkite/image_build/image_build_arm64.sh index 5baa55a196..0280d31c0c 100755 --- a/.buildkite/image_build/image_build_arm64.sh +++ b/.buildkite/image_build/image_build_arm64.sh @@ -21,12 +21,13 @@ else exit 0 fi -# build (Grace/GH200 is the arm64 GPU target; sm_90) +# build for arm64 GPU targets: Grace/GH200 (sm_90) and DGX Spark/GB10 +# (sm_121, family-covered by 12.0 under CUDA 13) docker build --file docker/Dockerfile \ --platform linux/arm64 \ --build-arg max_jobs=16 \ --build-arg nvcc_threads=4 \ - --build-arg torch_cuda_arch_list="9.0" \ + --build-arg torch_cuda_arch_list="9.0 12.0" \ --build-arg USE_SCCACHE=1 \ --build-arg buildkite_commit="$BUILDKITE_COMMIT" \ --tag "$REGISTRY"/"$REPO":"$BUILDKITE_COMMIT"-arm64 \ diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index f1d787313c..a64edbd1c4 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -220,6 +220,20 @@ steps: - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt +- label: GPQA Eval (GPT-OSS) (DGX Spark) + key: gpqa-eval-gpt-oss-spark + timeout_in_minutes: 120 + device: dgx-spark + optional: true + num_devices: 1 + source_file_dependencies: + - csrc/ + - vllm/model_executor/layers/quantization + - tests/evals/gpt_oss/ + commands: + - uv pip install --system 'gpt-oss[eval]==0.0.5' + - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt + - label: MRCR Eval Small Models device: h200_35gb timeout_in_minutes: 30 diff --git a/tests/evals/gpt_oss/configs/gpt-oss-20b-sm120.yaml b/tests/evals/gpt_oss/configs/gpt-oss-20b-sm120.yaml new file mode 100644 index 0000000000..934f9f4994 --- /dev/null +++ b/tests/evals/gpt_oss/configs/gpt-oss-20b-sm120.yaml @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +model_name: "openai/gpt-oss-20b" +metric_threshold: 0.568 +reasoning_effort: "low" diff --git a/tests/evals/gpt_oss/configs/models-spark.txt b/tests/evals/gpt_oss/configs/models-spark.txt new file mode 100644 index 0000000000..d1efb254c8 --- /dev/null +++ b/tests/evals/gpt_oss/configs/models-spark.txt @@ -0,0 +1,2 @@ +# DGX Spark model configurations for GPQA evaluation +gpt-oss-20b-sm120.yaml