From 1fee69838eeb72622b441a593888687831c1e434 Mon Sep 17 00:00:00 2001 From: Chaojun Zhang Date: Tue, 22 Sep 2026 10:05:26 +0800 Subject: [PATCH] [XPU] Fix Nemotron FP8 LM-eval config: drop CUDA-only moe_backend and wire to new Buildkite job (#49685) Signed-off-by: Chaojun Zhang Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .buildkite/intel_jobs/lm_eval_intel.yaml | 24 +++++++++++++++++++ ...VIDIA-Nemotron-3-Nano-30B-A3B-FP8-xpu.yaml | 16 +++++++++++++ .../configs/models-large-xpu-fp8.txt | 1 + 3 files changed, 41 insertions(+) create mode 100644 .buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8-xpu.yaml create mode 100644 .buildkite/lm-eval-harness/configs/models-large-xpu-fp8.txt diff --git a/.buildkite/intel_jobs/lm_eval_intel.yaml b/.buildkite/intel_jobs/lm_eval_intel.yaml index 1ef00a4794..e1f649ad91 100644 --- a/.buildkite/intel_jobs/lm_eval_intel.yaml +++ b/.buildkite/intel_jobs/lm_eval_intel.yaml @@ -26,3 +26,27 @@ steps: 'uv pip install "gpt-oss[eval]==0.0.5" && cd tests && pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt' +- label: LM Eval Large Models(2 GPUs) + key: lm-eval-large-models-2-gpus + timeout_in_minutes: 45 + device: intel_gpu + agent_tags: + label: production + gpu: 2+ + mem: 24+ + no_plugin: true + optional: true + working_dir: "." + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - .buildkite/lm-eval-harness/ + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'cd .buildkite/lm-eval-harness && + export VLLM_USE_DEEP_GEMM=0 && + pytest -s -v test_lm_eval_correctness.py --config-list-file=configs/models-large-xpu-fp8.txt --tp-size=2' diff --git a/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8-xpu.yaml b/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8-xpu.yaml new file mode 100644 index 0000000000..72ba861195 --- /dev/null +++ b/.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8-xpu.yaml @@ -0,0 +1,16 @@ +model_name: "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8" +tasks: +- name: "gsm8k" + metrics: + - name: "exact_match,strict-match" + value: 0.7142 + - name: "exact_match,flexible-extract" + value: 0.4579 +limit: 256 +num_fewshot: 5 +max_model_len: 262144 +kv_cache_dtype: fp8 +enforce_eager: true +apply_chat_template: true +fewshot_as_multiturn: true +trust_remote_code: true diff --git a/.buildkite/lm-eval-harness/configs/models-large-xpu-fp8.txt b/.buildkite/lm-eval-harness/configs/models-large-xpu-fp8.txt new file mode 100644 index 0000000000..73b28f062c --- /dev/null +++ b/.buildkite/lm-eval-harness/configs/models-large-xpu-fp8.txt @@ -0,0 +1 @@ +NVIDIA-Nemotron-3-Nano-30B-A3B-FP8-xpu.yaml \ No newline at end of file