From 73859fec5865700c5b2021b22890cb3b2005f66e Mon Sep 17 00:00:00 2001 From: Anton Peganov Date: Sun, 27 Sep 2026 22:43:56 +0200 Subject: [PATCH] [Frontend] Switch Python Harmony dependency to oss-harmony (#55128) Signed-off-by: Anton Peganov Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: OpenAI Codex --- .buildkite/intel_jobs/lm_eval_intel.yaml | 3 +- .buildkite/test-amd.yaml | 2 - .buildkite/test_areas/lm_eval.yaml | 4 -- docker/Dockerfile | 4 ++ pyproject.toml | 5 ++ requirements/common.txt | 2 +- requirements/test/cpu.txt | 41 ++++++++--- requirements/test/cuda.in | 2 +- requirements/test/cuda.txt | 42 ++++++++--- requirements/test/rocm.in | 2 +- requirements/test/rocm.txt | 38 +++++++--- requirements/test/xpu.in | 2 +- requirements/test/xpu.txt | 37 +++++++--- .../parser/test_harmony_render_parity.py | 18 +++++ tests/evals/gpt_oss/run_gpqa_eval.py | 34 +++++++++ tests/evals/gpt_oss/test_gpqa_correctness.py | 13 ++-- .../openai/parser/harmony_utils.py | 72 ++++++++++++++++++- 17 files changed, 265 insertions(+), 56 deletions(-) create mode 100644 tests/evals/gpt_oss/run_gpqa_eval.py diff --git a/.buildkite/intel_jobs/lm_eval_intel.yaml b/.buildkite/intel_jobs/lm_eval_intel.yaml index e1f649ad91..62d99cf769 100644 --- a/.buildkite/intel_jobs/lm_eval_intel.yaml +++ b/.buildkite/intel_jobs/lm_eval_intel.yaml @@ -23,8 +23,7 @@ steps: commands: - >- bash .buildkite/scripts/hardware_ci/run-intel-test.sh - 'uv pip install "gpt-oss[eval]==0.0.5" && - cd tests && + 'cd tests && pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt' - label: LM Eval Large Models(2 GPUs) key: lm-eval-large-models-2-gpus diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 1b2b854bc4..530fd0d27c 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -1287,7 +1287,6 @@ steps: - vllm/platforms/rocm.py - tests/evals/gpt_oss/ commands: - - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt - label: ":amd: (MI300) LM Eval Small Models Distributed" @@ -3584,7 +3583,6 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py commands: - - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt - label: ":amd: (MI355 DPX) LM Eval Watermarking" diff --git a/.buildkite/test_areas/lm_eval.yaml b/.buildkite/test_areas/lm_eval.yaml index c67cc08b94..9f99e1c53f 100644 --- a/.buildkite/test_areas/lm_eval.yaml +++ b/.buildkite/test_areas/lm_eval.yaml @@ -829,7 +829,6 @@ steps: - vllm/model_executor/layers/quantization - tests/evals/gpt_oss/ commands: - - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt - label: ":nvidia: (B200) GPQA Eval (GPT-OSS)" @@ -843,7 +842,6 @@ steps: - vllm/model_executor/layers/quantization - tests/evals/gpt_oss/ commands: - - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt mirror: amd: @@ -865,7 +863,6 @@ steps: - vllm/_aiter_ops.py - vllm/platforms/rocm.py commands: - - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt - label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval" @@ -940,7 +937,6 @@ steps: - vllm/model_executor/layers/quantization - tests/evals/gpt_oss/ commands: - - uv pip install --system 'gpt-oss[eval]==0.0.5' - pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt - label: ":nvidia: (H200 MIG 35GB) LM Eval KV-Offload" diff --git a/docker/Dockerfile b/docker/Dockerfile index 366a9a8752..6609ed7464 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -729,6 +729,8 @@ COPY requirements/lint.txt requirements/lint.txt COPY requirements/test/cuda.in requirements/test/cuda.in COPY requirements/test/cuda.txt requirements/test/cuda.txt COPY requirements/dev.txt requirements/dev.txt +# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below. +COPY pyproject.toml pyproject.toml COPY tools/use_existing_torch.py tools/use_existing_torch.py COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt RUN --mount=type=cache,target=/opt/uv/cache \ @@ -1143,6 +1145,8 @@ COPY requirements/lint.txt requirements/lint.txt COPY requirements/test/cuda.in requirements/test/cuda.in COPY requirements/test/cuda.txt requirements/test/cuda.txt COPY requirements/dev.txt requirements/dev.txt +# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below. +COPY pyproject.toml pyproject.toml COPY tools/use_existing_torch.py tools/use_existing_torch.py COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt RUN --mount=type=cache,target=/opt/uv/cache \ diff --git a/pyproject.toml b/pyproject.toml index 4a7c018929..2098e097bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -258,3 +258,8 @@ CANN = "CANN" [tool.uv] no-build-isolation-package = ["torch"] +# vLLM depends on oss-harmony, a fork that ships the same `openai_harmony` +# module without downloading tiktoken vocabularies at runtime. Installing +# openai-harmony alongside it silently overwrites those files, so keep it out +# of every resolution; gpt-oss pulls it in transitively. +exclude-dependencies = ["openai-harmony"] diff --git a/requirements/common.txt b/requirements/common.txt index b55f680b76..ab0060284c 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -49,7 +49,7 @@ pybase64 # fast base64 implementation cbor2 # Required for cross-language serialization of hashable objects ijson # Required for mistral streaming tool parser setproctitle # Used to set process names for better debugging and monitoring -openai-harmony >= 0.0.3 # Required for gpt-oss +oss-harmony >= 0.0.11 # Required for gpt-oss; includes embedded vocabularies anthropic >= 0.71.0 model-hosting-container-standards >= 0.1.14, < 1.0.0 mcp >= 2.0.0, < 3.0.0 diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index c1d4913953..ab3e42886c 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -76,7 +76,9 @@ black==24.10.0 blake3==1.0.9 # via -r requirements/test/../common.txt blobfile==3.0.0 - # via -r requirements/test/cuda.in + # via + # -r requirements/test/cuda.in + # gpt-oss bm25s==0.2.13 # via mteb boto3==1.35.57 @@ -111,7 +113,7 @@ chardet==5.2.0 # via mbstrdecoder charset-normalizer==3.4.0 # via requests -chz==0.3.0 +chz==0.4.0 # via gpt-oss click==8.4.2 # via @@ -190,7 +192,7 @@ distro==1.9.0 # openai dnspython==2.7.0 # via email-validator -docker==7.1.0 +docker==7.2.0 # via gpt-oss docopt==0.6.2 # via num2words @@ -291,7 +293,7 @@ googleapis-common-protos==1.70.0 # google-api-core # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -gpt-oss==0.0.8 +gpt-oss==0.0.9 # via -r requirements/test/cuda.in graphql-core==3.2.6 # via hypothesis-graphql @@ -402,6 +404,7 @@ jinja2==3.1.6 # datamodel-code-generator # fastapi # genai-perf + # gpt-oss # lm-eval # torch jiter==0.15.0 @@ -433,6 +436,10 @@ jsonschema-specifications==2024.10.1 # via jsonschema junit-xml==1.9 # via schemathesis +jupyter-client==8.10.0 + # via gpt-oss +jupyter-core==5.9.1 + # via jupyter-client kaldi-native-fbank==1.22.3 # via -r requirements/test/cuda.in kaleido==0.2.1 @@ -549,6 +556,7 @@ numpy==2.2.6 # evaluate # fastparquet # genai-perf + # gpt-oss # imagehash # imageio # librosa @@ -583,8 +591,6 @@ numpy==2.2.6 open-clip-torch==2.32.0 # via -r requirements/test/cuda.in openai==2.44.0 - # via -r requirements/test/../common.txt -openai-harmony==0.0.4 # via # -r requirements/test/../common.txt # gpt-oss @@ -641,6 +647,8 @@ optuna==3.6.1 # via genai-perf orjson==3.11.5 # via genai-perf +oss-harmony==0.0.11 + # via -r requirements/test/../common.txt outlines-core==0.2.14 # via -r requirements/test/../common.txt packaging==24.2 @@ -672,6 +680,7 @@ pandas==2.2.3 # evaluate # fastparquet # genai-perf + # gpt-oss # statsmodels partial-json-parser==0.2.1.1.post7 # via -r requirements/test/../common.txt @@ -702,6 +711,7 @@ pillow==10.4.0 platformdirs==4.3.6 # via # black + # jupyter-core # pooch # virtualenv plotly==5.24.1 @@ -794,7 +804,7 @@ pydantic==2.12.0 # model-hosting-container-standards # mteb # openai - # openai-harmony + # oss-harmony # pydantic-extra-types # pydantic-settings # ray @@ -855,6 +865,7 @@ pytest-timeout==2.3.1 python-dateutil==2.9.0.post0 # via # botocore + # jupyter-client # matplotlib # pandas # typepy @@ -898,7 +909,9 @@ pyyaml==6.0.2 # uvicorn # vocos pyzmq==27.1.0 - # via -r requirements/test/../common.txt + # via + # -r requirements/test/../common.txt + # jupyter-client rapidfuzz==3.12.1 # via jiwer ray==2.56.1 @@ -1077,7 +1090,7 @@ starlette-testclient==0.4.1 # via schemathesis statsmodels==0.14.4 # via genai-perf -structlog==25.4.0 +structlog==26.1.0 # via gpt-oss supervisor==4.3.0 # via model-hosting-container-standards @@ -1101,7 +1114,7 @@ tenacity==9.1.2 # schemathesis tensorizer==2.10.1 # via -r requirements/test/cuda.in -termcolor==3.1.0 +termcolor==3.3.0 # via gpt-oss threadpoolctl==3.5.0 # via scikit-learn @@ -1157,11 +1170,14 @@ torchvision==0.28.0+cpu # open-clip-torch # segmentation-models-pytorch # timm +tornado==6.5.10 + # via jupyter-client tqdm==4.67.3 # via # -r requirements/test/../common.txt # datasets # evaluate + # gpt-oss # huggingface-hub # lm-eval # mteb @@ -1174,6 +1190,10 @@ tqdm==4.67.3 # segmentation-models-pytorch # sentence-transformers # transformers +traitlets==5.16.1 + # via + # jupyter-client + # jupyter-core transformers==5.17.0 # via # -r requirements/test/../common.txt @@ -1226,6 +1246,7 @@ typing-extensions==4.15.0 # grpcio # httpx2 # huggingface-hub + # jupyter-client # librosa # lm-eval # mcp diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index 5e3faaebcc..271cdba155 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -66,7 +66,7 @@ decord==0.6.0; platform_machine == "x86_64" imagehash # Required for Prithvi tests segmentation-models-pytorch > 0.4.0 # Required for Prithvi tests -gpt-oss >= 0.0.7; python_version > '3.11' +gpt-oss[eval] >= 0.0.7; python_version > '3.11' perceptron # required for isaac test kaldi-native-fbank >= 1.18.7 # required for fireredasr2 test diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index 3207d395aa..ffbc639c4c 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -81,7 +81,9 @@ black==24.10.0 blake3==1.0.9 # via -r requirements/test/../common.txt blobfile==3.0.0 - # via -r requirements/test/cuda.in + # via + # -r requirements/test/cuda.in + # gpt-oss bm25s==0.2.13 # via mteb boto3==1.35.57 @@ -116,7 +118,7 @@ chardet==5.2.0 # via mbstrdecoder charset-normalizer==3.4.0 # via requests -chz==0.3.0 +chz==0.4.0 # via gpt-oss click==8.4.2 # via @@ -205,7 +207,7 @@ distro==1.9.0 # openai dnspython==2.7.0 # via email-validator -docker==7.1.0 +docker==7.2.0 # via gpt-oss docopt==0.6.2 # via num2words @@ -310,7 +312,7 @@ googleapis-common-protos==1.70.0 # google-api-core # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -gpt-oss==0.0.8 +gpt-oss==0.0.9 # via -r requirements/test/cuda.in graphql-core==3.2.6 # via hypothesis-graphql @@ -424,6 +426,7 @@ jinja2==3.1.6 # datamodel-code-generator # fastapi # genai-perf + # gpt-oss # lm-eval # torch jiter==0.15.0 @@ -456,6 +459,10 @@ jsonschema-specifications==2024.10.1 # via jsonschema junit-xml==1.9 # via schemathesis +jupyter-client==8.10.0 + # via gpt-oss +jupyter-core==5.9.1 + # via jupyter-client kaldi-native-fbank==1.22.3 # via -r requirements/test/cuda.in kaleido==0.2.1 @@ -584,6 +591,7 @@ numpy==2.2.6 # evaluate # fastparquet # genai-perf + # gpt-oss # imagehash # imageio # librosa @@ -662,10 +670,6 @@ openai==2.44.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt -openai-harmony==0.0.4 - # via - # -c requirements/common.txt - # -r requirements/test/../common.txt # gpt-oss opencensus==0.11.4 # via ray @@ -727,6 +731,10 @@ optuna==3.6.1 # via genai-perf orjson==3.11.5 # via genai-perf +oss-harmony==0.0.11 + # via + # -c requirements/common.txt + # -r requirements/test/../common.txt outlines-core==0.2.14 # via # -c requirements/common.txt @@ -760,6 +768,7 @@ pandas==2.2.3 # evaluate # fastparquet # genai-perf + # gpt-oss # statsmodels partial-json-parser==0.2.1.1.post7 # via -r requirements/test/../common.txt @@ -790,6 +799,7 @@ pillow==10.4.0 platformdirs==4.3.6 # via # black + # jupyter-core # pooch # virtualenv plotly==5.24.1 @@ -887,7 +897,7 @@ pydantic==2.12.0 # model-hosting-container-standards # mteb # openai - # openai-harmony + # oss-harmony # pydantic-extra-types # pydantic-settings # ray @@ -948,6 +958,7 @@ pytest-timeout==2.3.1 python-dateutil==2.9.0.post0 # via # botocore + # jupyter-client # matplotlib # pandas # typepy @@ -994,6 +1005,7 @@ pyzmq==27.1.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # jupyter-client rapidfuzz==3.12.1 # via jiwer ray==2.56.1 @@ -1177,7 +1189,7 @@ starlette-testclient==0.4.1 # via schemathesis statsmodels==0.14.4 # via genai-perf -structlog==25.4.0 +structlog==26.1.0 # via gpt-oss supervisor==4.3.0 # via model-hosting-container-standards @@ -1201,7 +1213,7 @@ tenacity==9.1.2 # schemathesis tensorizer==2.10.1 # via -r requirements/test/cuda.in -termcolor==3.1.0 +termcolor==3.3.0 # via gpt-oss threadpoolctl==3.5.0 # via scikit-learn @@ -1264,11 +1276,14 @@ torchvision==0.28.0+cu130 # open-clip-torch # segmentation-models-pytorch # timm +tornado==6.5.10 + # via jupyter-client tqdm==4.67.3 # via # -r requirements/test/../common.txt # datasets # evaluate + # gpt-oss # huggingface-hub # lm-eval # mteb @@ -1281,6 +1296,10 @@ tqdm==4.67.3 # segmentation-models-pytorch # sentence-transformers # transformers +traitlets==5.16.1 + # via + # jupyter-client + # jupyter-core transformers==5.17.0 # via # -c requirements/common.txt @@ -1337,6 +1356,7 @@ typing-extensions==4.15.0 # grpcio # httpx2 # huggingface-hub + # jupyter-client # librosa # lm-eval # mcp diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index c8c08fabe5..ffe72643aa 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -65,7 +65,7 @@ decord==0.6.0 imagehash # Required for Prithvi tests segmentation-models-pytorch>0.4.0 # Required for Prithvi tests -gpt-oss>=0.0.7; python_version > '3.11' +gpt-oss[eval]>=0.0.7; python_version > '3.11' perceptron # required for isaac test kaldi-native-fbank>=1.18.7 # required for fireredasr2 test diff --git a/requirements/test/rocm.txt b/requirements/test/rocm.txt index c961b6294a..36e1ba182d 100644 --- a/requirements/test/rocm.txt +++ b/requirements/test/rocm.txt @@ -80,7 +80,9 @@ black==26.3.1 blake3==1.0.8 # via -r requirements/test/../common.txt blobfile==3.0.0 - # via -r requirements/test/rocm.in + # via + # -r requirements/test/rocm.in + # gpt-oss bm25s==0.2.13 # via mteb boto3==1.42.74 @@ -198,7 +200,7 @@ distro==1.9.0 # openai dnspython==2.8.0 # via email-validator -docker==7.1.0 +docker==7.2.0 # via gpt-oss docopt==0.6.2 # via num2words @@ -300,7 +302,7 @@ googleapis-common-protos==1.73.0 # google-api-core # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -gpt-oss==0.0.8 +gpt-oss==0.0.9 # via -r requirements/test/rocm.in graphql-core==3.2.8 # via hypothesis-graphql @@ -415,6 +417,7 @@ jinja2==3.1.6 # datamodel-code-generator # fastapi # genai-perf + # gpt-oss # lm-eval # torch jiter==0.14.0 @@ -447,6 +450,10 @@ jsonschema-specifications==2025.9.1 # via jsonschema junit-xml==1.9 # via schemathesis +jupyter-client==8.10.0 + # via gpt-oss +jupyter-core==5.9.1 + # via jupyter-client kaldi-native-fbank==1.22.3 # via -r requirements/test/rocm.in kaleido==1.0.0 @@ -587,6 +594,7 @@ numpy==2.2.6 # evaluate # fastparquet # genai-perf + # gpt-oss # imagehash # imageio # librosa @@ -629,10 +637,6 @@ openai==2.31.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt -openai-harmony==0.0.8 - # via - # -c requirements/common.txt - # -r requirements/test/../common.txt # gpt-oss opencensus==0.11.4 # via ray @@ -699,6 +703,10 @@ orjson==3.11.7 # via # genai-perf # kaleido +oss-harmony==0.0.11 + # via + # -c requirements/common.txt + # -r requirements/test/../common.txt outlines-core==0.2.14 # via # -c requirements/common.txt @@ -734,6 +742,7 @@ pandas==3.0.1 # evaluate # fastparquet # genai-perf + # gpt-oss # statsmodels partial-json-parser==0.2.1.1.post7 # via -r requirements/test/../common.txt @@ -764,6 +773,7 @@ pillow==12.1.1 platformdirs==4.3.6 # via # black + # jupyter-core # pooch # python-discovery # virtualenv @@ -861,7 +871,7 @@ pydantic==2.12.5 # model-hosting-container-standards # mteb # openai - # openai-harmony + # oss-harmony # pydantic-extra-types # pydantic-settings # ray @@ -922,6 +932,7 @@ pytest-timeout==2.3.1 python-dateutil==2.9.0.post0 # via # botocore + # jupyter-client # matplotlib # pandas # typepy @@ -970,6 +981,7 @@ pyzmq==27.1.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # jupyter-client rapidfuzz==3.12.1 # via # -r requirements/test/rocm.in @@ -1159,7 +1171,7 @@ starlette-testclient==0.4.1 # via schemathesis statsmodels==0.14.6 # via genai-perf -structlog==25.5.0 +structlog==26.1.0 # via gpt-oss supervisor==4.3.0 # via model-hosting-container-standards @@ -1215,11 +1227,14 @@ tokenizers==0.23.1 # transformers torch-c-dlpack-ext==0.1.5 # via tilelang +tornado==6.5.10 + # via jupyter-client tqdm==4.67.3 # via # -r requirements/test/../common.txt # datasets # evaluate + # gpt-oss # huggingface-hub # lm-eval # mteb @@ -1233,6 +1248,10 @@ tqdm==4.67.3 # sentence-transformers # tilelang # transformers +traitlets==5.16.1 + # via + # jupyter-client + # jupyter-core transformers==5.17.0 # via # -c requirements/common.txt @@ -1287,6 +1306,7 @@ typing-extensions==4.15.0 # grpcio # httpx2 # huggingface-hub + # jupyter-client # librosa # lm-eval # mcp diff --git a/requirements/test/xpu.in b/requirements/test/xpu.in index d2ad5a0cfd..08a81cd20d 100644 --- a/requirements/test/xpu.in +++ b/requirements/test/xpu.in @@ -32,7 +32,7 @@ soundfile # --- Tool Parsing & Evaluation --- blobfile rapidfuzz -gpt-oss >= 0.0.7; python_version > '3.11' +gpt-oss[eval] >= 0.0.7; python_version > '3.11' schemathesis>=4.0.0 # Required for openai schema test. jiwer bm25s diff --git a/requirements/test/xpu.txt b/requirements/test/xpu.txt index ed8827f5db..e5f898adcc 100644 --- a/requirements/test/xpu.txt +++ b/requirements/test/xpu.txt @@ -67,7 +67,9 @@ azure-storage-blob==12.30.1 blake3==1.0.9 # via -r requirements/test/../common.txt blobfile==3.2.0 - # via -r requirements/test/xpu.in + # via + # -r requirements/test/xpu.in + # gpt-oss bm25s==0.3.10 # via # -r requirements/test/xpu.in @@ -257,7 +259,7 @@ googleapis-common-protos==1.75.1 # google-api-core # opentelemetry-exporter-otlp-proto-grpc # opentelemetry-exporter-otlp-proto-http -gpt-oss==0.0.8 +gpt-oss==0.0.9 # via -r requirements/test/xpu.in graphql-core==3.2.11 # via hypothesis-graphql @@ -377,6 +379,7 @@ jinja2==3.1.6 # via # -c requirements/xpu.txt # fastapi + # gpt-oss # lm-eval # torch jiter==0.16.0 @@ -406,6 +409,10 @@ jsonschema-rs==0.49.9 # via schemathesis jsonschema-specifications==2025.9.1 # via jsonschema +jupyter-client==8.10.0 + # via gpt-oss +jupyter-core==5.9.1 + # via jupyter-client kiwisolver==1.5.1 # via matplotlib lark==1.2.2 @@ -526,6 +533,7 @@ numpy==2.2.6 # datasets # decord # evaluate + # gpt-oss # librosa # lm-eval # matplotlib @@ -575,10 +583,6 @@ openai==3.2.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt -openai-harmony==0.0.8 - # via - # -c requirements/common.txt - # -r requirements/test/../common.txt # gpt-oss opencv-python-headless==5.0.0.93 # via @@ -632,6 +636,10 @@ opentelemetry-semantic-conventions-ai==0.5.1 # -c requirements/common.txt # -r requirements/test/../common.txt # -r requirements/test/xpu.in +oss-harmony==0.0.11 + # via + # -c requirements/common.txt + # -r requirements/test/../common.txt outlines-core==0.2.14 # via # -c requirements/common.txt @@ -656,6 +664,7 @@ pandas==3.0.5 # via # datasets # evaluate + # gpt-oss partial-json-parser==0.2.1.1.post7 # via -r requirements/test/../common.txt pathvalidate==3.3.1 @@ -667,7 +676,9 @@ pillow==12.3.0 # mistral-common # torchvision platformdirs==4.11.3 - # via pooch + # via + # jupyter-core + # pooch pluggy==1.6.0 # via # pytest @@ -747,7 +758,7 @@ pydantic==2.13.4 # model-hosting-container-standards # mteb # openai - # openai-harmony + # oss-harmony # pydantic-extra-types # pydantic-settings # xgrammar @@ -806,6 +817,7 @@ pytest-timeout==2.4.0 python-dateutil==2.9.0.post0 # via # botocore + # jupyter-client # matplotlib # pandas # typepy @@ -841,6 +853,7 @@ pyzmq==27.1.0 # via # -c requirements/common.txt # -r requirements/test/../common.txt + # jupyter-client rapidfuzz==3.14.5 # via # -r requirements/test/xpu.in @@ -1066,11 +1079,14 @@ torchvision==0.29.0+xpu # via # open-clip-torch # timm +tornado==6.5.10 + # via jupyter-client tqdm==4.70.0 # via # -r requirements/test/../common.txt # datasets # evaluate + # gpt-oss # huggingface-hub # lm-eval # modelscope @@ -1081,6 +1097,10 @@ tqdm==4.70.0 # pqdm # sentence-transformers # transformers +traitlets==5.16.1 + # via + # jupyter-client + # jupyter-core transformers==5.17.0 # via # -c requirements/common.txt @@ -1132,6 +1152,7 @@ typing-extensions==4.16.0 # grpcio # httpx2 # huggingface-hub + # jupyter-client # lm-eval # mcp # mcp-types diff --git a/tests/entrypoints/openai/parser/test_harmony_render_parity.py b/tests/entrypoints/openai/parser/test_harmony_render_parity.py index 5cb4461223..966ac5cf52 100644 --- a/tests/entrypoints/openai/parser/test_harmony_render_parity.py +++ b/tests/entrypoints/openai/parser/test_harmony_render_parity.py @@ -20,6 +20,7 @@ Each test: """ from openai.types.responses import ResponseFunctionToolCall +from openai_harmony import Message, Role from tests.entrypoints.openai.utils import verify_harmony_messages from vllm.entrypoints.openai.parser.harmony_utils import ( @@ -33,6 +34,23 @@ from vllm.entrypoints.openai.responses.harmony import ( response_previous_input_to_harmony, ) + +def test_render_uses_legacy_tool_call_header(): + tool_call = ( + Message.from_role_and_content(Role.ASSISTANT, "print('hello')") + .with_channel("commentary") + .with_recipient("python") + .with_content_type("code") + ) + + rendered = get_encoding().decode(render_for_completion([tool_call])) + + assert ( + "<|start|>assistant to=python<|channel|>commentary code<|message|>" in rendered + ) + assert "commentary to=python <|constrain|>code" not in rendered + + # Use a fixed date so the system message is deterministic across both paths. _DATE = "2025-01-01" diff --git a/tests/evals/gpt_oss/run_gpqa_eval.py b/tests/evals/gpt_oss/run_gpqa_eval.py new file mode 100644 index 0000000000..b9f94a59a4 --- /dev/null +++ b/tests/evals/gpt_oss/run_gpqa_eval.py @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Run the gpt-oss GPQA evaluator with an explicit output-token budget.""" + +import argparse + +from gpt_oss.evals.gpqa_eval import GPQAEval +from gpt_oss.evals.responses_sampler import ResponsesSampler + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--model", required=True) + parser.add_argument("--reasoning-effort", required=True) + parser.add_argument("--base-url", required=True) + parser.add_argument("--n-threads", required=True, type=int) + parser.add_argument("--max-output-tokens", required=True, type=int) + args = parser.parse_args() + + sampler = ResponsesSampler( + model=args.model, + reasoning_model=True, + reasoning_effort=args.reasoning_effort, + base_url=args.base_url, + max_tokens=args.max_output_tokens, + ) + result = GPQAEval(n_repeats=8, n_threads=args.n_threads)(sampler) + # gpt-oss returns a NumPy scalar. Convert it so the parent process gets a + # stable, dependency-independent representation that its regex can parse. + print({"metric": float(result.score)}) + + +if __name__ == "__main__": + main() diff --git a/tests/evals/gpt_oss/test_gpqa_correctness.py b/tests/evals/gpt_oss/test_gpqa_correctness.py index 99d799698a..a57f3a567a 100644 --- a/tests/evals/gpt_oss/test_gpqa_correctness.py +++ b/tests/evals/gpt_oss/test_gpqa_correctness.py @@ -21,6 +21,12 @@ from tests.utils import RemoteOpenAIServer TOL = 0.05 # Absolute tolerance for accuracy comparison +# gpt-oss >= 0.0.7 explicitly sends its 131072-token default as the output +# budget, which leaves no room for the prompt in GPT-OSS's 131072-token context. +# This test originally ran with a 32768-token model context, so retain that +# proven generation budget while using the current evaluator package. +MAX_OUTPUT_TOKENS = 32_768 + # Path to tiktoken encoding files TIKTOKEN_DATA_DIR = Path(__file__).parent / "data" @@ -50,10 +56,7 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa # Build the command to run the evaluation cmd = [ sys.executable, - "-m", - "gpt_oss.evals", - "--eval", - "gpqa", + str(Path(__file__).with_name("run_gpqa_eval.py")), "--model", model_name, "--reasoning-effort", @@ -62,6 +65,8 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa base_url, "--n-threads", "200", + "--max-output-tokens", + str(MAX_OUTPUT_TOKENS), ] try: diff --git a/vllm/entrypoints/openai/parser/harmony_utils.py b/vllm/entrypoints/openai/parser/harmony_utils.py index 873b9e92bd..dc3aef4a95 100644 --- a/vllm/entrypoints/openai/parser/harmony_utils.py +++ b/vllm/entrypoints/openai/parser/harmony_utils.py @@ -449,12 +449,80 @@ def parse_chat_input_to_harmony_message( def render_for_completion(messages: list[Message]) -> list[int]: messages = auto_drop_analysis_messages(messages) conversation = Conversation.from_messages(messages) - token_ids = get_encoding().render_conversation_for_completion( + encoding = get_encoding() + token_ids = encoding.render_conversation_for_completion( conversation, Role.ASSISTANT, config=RenderConversationConfig(auto_drop_analysis=False), ) - return token_ids + return _use_legacy_tool_call_headers(token_ids) + + +def _use_legacy_tool_call_headers(token_ids: list[int]) -> list[int]: + """Keep prior assistant tool calls in the format expected by GPT-OSS. + + oss-harmony 0.0.10 changed a rendered tool-call header from:: + + assistant to=python<|channel|>commentary code + + to:: + + assistant<|channel|>commentary to=python <|constrain|>code + + GPT-OSS can echo the latter as a malformed header with the recipient in + both positions. Rewrite only that unambiguous token-level header shape, + while retaining oss-harmony's embedded vocabulary and parser. + """ + encoding = get_encoding() + start = encoding.encode("<|start|>", allowed_special="all")[0] + channel_token = encoding.encode("<|channel|>", allowed_special="all")[0] + constrain = encoding.encode("<|constrain|>", allowed_special="all")[0] + message = encoding.encode("<|message|>", allowed_special="all")[0] + + result = token_ids.copy() + index = 0 + while index < len(result): + if result[index] != start: + index += 1 + continue + + try: + header_end = result.index(message, index + 1) + channel_index = result.index(channel_token, index + 1, header_end) + constrain_index = result.index(constrain, channel_index + 1, header_end) + except ValueError: + index += 1 + continue + + author = encoding.decode(result[index + 1 : channel_index]) + channel_and_recipient = encoding.decode( + result[channel_index + 1 : constrain_index] + ) + content_type = encoding.decode(result[constrain_index + 1 : header_end]) + if ( + author != "assistant" + or " to=" not in channel_and_recipient + or not channel_and_recipient.endswith(" ") + ): + index = header_end + 1 + continue + + channel, recipient = channel_and_recipient[:-1].split(" to=", 1) + if not channel or not recipient or not content_type: + index = header_end + 1 + continue + + legacy_header = ( + [start] + + encoding.encode(f"assistant to={recipient}") + + [channel_token] + + encoding.encode(f"{channel} {content_type}") + + [message] + ) + result[index : header_end + 1] = legacy_header + index += len(legacy_header) + + return result def get_streamable_parser_for_assistant() -> StreamableParser: