diff --git a/.buildkite/intel_jobs/test-intel.yaml b/.buildkite/intel_jobs/test-intel.yaml index 3fadb07f39..ec5cb2fd9e 100644 --- a/.buildkite/intel_jobs/test-intel.yaml +++ b/.buildkite/intel_jobs/test-intel.yaml @@ -147,6 +147,30 @@ steps: 'cd tests && pytest -v -s quantization/test_auto_round.py && pytest -v -s quantization/test_online.py' + - label: "XPU GPQA Eval (GPT-OSS)" + depends_on: + - image-build-xpu + timeout_in_minutes: 60 + device: intel_gpu + agent_tags: + label: production + gpu: 1+ + mem: 24+ + no_plugin: true + env: + REGISTRY: "public.ecr.aws/q9t5s3a7" + REPO: "vllm-ci-test-repo" + VLLM_TEST_DEVICE: "xpu" + source_file_dependencies: + - vllm/ + - tests/evals/gpt_oss/ + - .buildkite/intel_jobs/test-intel.yaml + commands: + - >- + bash .buildkite/scripts/hardware_ci/run-intel-test.sh + 'pip install "gpt-oss[eval]==0.0.5" && + cd tests && + pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt' - label: "XPU compressed tensors FP8 test" depends_on: - image-build-xpu diff --git a/tests/evals/gpt_oss/configs/gpt-oss-20b-xpu-baseline.yaml b/tests/evals/gpt_oss/configs/gpt-oss-20b-xpu-baseline.yaml new file mode 100644 index 0000000000..78a583888b --- /dev/null +++ b/tests/evals/gpt_oss/configs/gpt-oss-20b-xpu-baseline.yaml @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +model_name: openai/gpt-oss-20b +metric_threshold: 0.568 +reasoning_effort: low diff --git a/tests/evals/gpt_oss/configs/gpt-oss-20b-xpu-triton-attn.yaml b/tests/evals/gpt_oss/configs/gpt-oss-20b-xpu-triton-attn.yaml new file mode 100644 index 0000000000..e711ffb331 --- /dev/null +++ b/tests/evals/gpt_oss/configs/gpt-oss-20b-xpu-triton-attn.yaml @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +model_name: openai/gpt-oss-20b +metric_threshold: 0.568 +reasoning_effort: low +server_args: "--attention-backend TRITON_ATTN" diff --git a/tests/evals/gpt_oss/configs/models-xpu.txt b/tests/evals/gpt_oss/configs/models-xpu.txt new file mode 100644 index 0000000000..a9de9266b1 --- /dev/null +++ b/tests/evals/gpt_oss/configs/models-xpu.txt @@ -0,0 +1,3 @@ +# Intel XPU model configurations for GPQA evaluation +gpt-oss-20b-xpu-baseline.yaml +gpt-oss-20b-xpu-triton-attn.yaml