[Frontend] Switch Python Harmony dependency to oss-harmony (#55128)

Signed-off-by: Anton Peganov <apeganov@nvidia.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Anton Peganov
2026-09-28 00:43:56 +04:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> OpenAI Codex
parent 386ac2573e
commit 73859fec58
17 changed files with 265 additions and 56 deletions
+1 -2
View File
@@ -23,8 +23,7 @@ steps:
commands:
- >-
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
'uv pip install "gpt-oss[eval]==0.0.5" &&
cd tests &&
'cd tests &&
pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt'
- label: LM Eval Large Models(2 GPUs)
key: lm-eval-large-models-2-gpus
-2
View File
@@ -1287,7 +1287,6 @@ steps:
- vllm/platforms/rocm.py
- tests/evals/gpt_oss/
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt
- label: ":amd: (MI300) LM Eval Small Models Distributed"
@@ -3584,7 +3583,6 @@ steps:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
- label: ":amd: (MI355 DPX) LM Eval Watermarking"
-4
View File
@@ -829,7 +829,6 @@ steps:
- vllm/model_executor/layers/quantization
- tests/evals/gpt_oss/
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
- label: ":nvidia: (B200) GPQA Eval (GPT-OSS)"
@@ -843,7 +842,6 @@ steps:
- vllm/model_executor/layers/quantization
- tests/evals/gpt_oss/
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
mirror:
amd:
@@ -865,7 +863,6 @@ steps:
- vllm/_aiter_ops.py
- vllm/platforms/rocm.py
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
- label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
@@ -940,7 +937,6 @@ steps:
- vllm/model_executor/layers/quantization
- tests/evals/gpt_oss/
commands:
- uv pip install --system 'gpt-oss[eval]==0.0.5'
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt
- label: ":nvidia: (H200 MIG 35GB) LM Eval KV-Offload"
+4
View File
@@ -729,6 +729,8 @@ COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test/cuda.in requirements/test/cuda.in
COPY requirements/test/cuda.txt requirements/test/cuda.txt
COPY requirements/dev.txt requirements/dev.txt
# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below.
COPY pyproject.toml pyproject.toml
COPY tools/use_existing_torch.py tools/use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
RUN --mount=type=cache,target=/opt/uv/cache \
@@ -1143,6 +1145,8 @@ COPY requirements/lint.txt requirements/lint.txt
COPY requirements/test/cuda.in requirements/test/cuda.in
COPY requirements/test/cuda.txt requirements/test/cuda.txt
COPY requirements/dev.txt requirements/dev.txt
# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below.
COPY pyproject.toml pyproject.toml
COPY tools/use_existing_torch.py tools/use_existing_torch.py
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
RUN --mount=type=cache,target=/opt/uv/cache \
+5
View File
@@ -258,3 +258,8 @@ CANN = "CANN"
[tool.uv]
no-build-isolation-package = ["torch"]
# vLLM depends on oss-harmony, a fork that ships the same `openai_harmony`
# module without downloading tiktoken vocabularies at runtime. Installing
# openai-harmony alongside it silently overwrites those files, so keep it out
# of every resolution; gpt-oss pulls it in transitively.
exclude-dependencies = ["openai-harmony"]
+1 -1
View File
@@ -49,7 +49,7 @@ pybase64 # fast base64 implementation
cbor2 # Required for cross-language serialization of hashable objects
ijson # Required for mistral streaming tool parser
setproctitle # Used to set process names for better debugging and monitoring
openai-harmony >= 0.0.3 # Required for gpt-oss
oss-harmony >= 0.0.11 # Required for gpt-oss; includes embedded vocabularies
anthropic >= 0.71.0
model-hosting-container-standards >= 0.1.14, < 1.0.0
mcp >= 2.0.0, < 3.0.0
+31 -10
View File
@@ -76,7 +76,9 @@ black==24.10.0
blake3==1.0.9
# via -r requirements/test/../common.txt
blobfile==3.0.0
# via -r requirements/test/cuda.in
# via
# -r requirements/test/cuda.in
# gpt-oss
bm25s==0.2.13
# via mteb
boto3==1.35.57
@@ -111,7 +113,7 @@ chardet==5.2.0
# via mbstrdecoder
charset-normalizer==3.4.0
# via requests
chz==0.3.0
chz==0.4.0
# via gpt-oss
click==8.4.2
# via
@@ -190,7 +192,7 @@ distro==1.9.0
# openai
dnspython==2.7.0
# via email-validator
docker==7.1.0
docker==7.2.0
# via gpt-oss
docopt==0.6.2
# via num2words
@@ -291,7 +293,7 @@ googleapis-common-protos==1.70.0
# google-api-core
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
gpt-oss==0.0.8
gpt-oss==0.0.9
# via -r requirements/test/cuda.in
graphql-core==3.2.6
# via hypothesis-graphql
@@ -402,6 +404,7 @@ jinja2==3.1.6
# datamodel-code-generator
# fastapi
# genai-perf
# gpt-oss
# lm-eval
# torch
jiter==0.15.0
@@ -433,6 +436,10 @@ jsonschema-specifications==2024.10.1
# via jsonschema
junit-xml==1.9
# via schemathesis
jupyter-client==8.10.0
# via gpt-oss
jupyter-core==5.9.1
# via jupyter-client
kaldi-native-fbank==1.22.3
# via -r requirements/test/cuda.in
kaleido==0.2.1
@@ -549,6 +556,7 @@ numpy==2.2.6
# evaluate
# fastparquet
# genai-perf
# gpt-oss
# imagehash
# imageio
# librosa
@@ -583,8 +591,6 @@ numpy==2.2.6
open-clip-torch==2.32.0
# via -r requirements/test/cuda.in
openai==2.44.0
# via -r requirements/test/../common.txt
openai-harmony==0.0.4
# via
# -r requirements/test/../common.txt
# gpt-oss
@@ -641,6 +647,8 @@ optuna==3.6.1
# via genai-perf
orjson==3.11.5
# via genai-perf
oss-harmony==0.0.11
# via -r requirements/test/../common.txt
outlines-core==0.2.14
# via -r requirements/test/../common.txt
packaging==24.2
@@ -672,6 +680,7 @@ pandas==2.2.3
# evaluate
# fastparquet
# genai-perf
# gpt-oss
# statsmodels
partial-json-parser==0.2.1.1.post7
# via -r requirements/test/../common.txt
@@ -702,6 +711,7 @@ pillow==10.4.0
platformdirs==4.3.6
# via
# black
# jupyter-core
# pooch
# virtualenv
plotly==5.24.1
@@ -794,7 +804,7 @@ pydantic==2.12.0
# model-hosting-container-standards
# mteb
# openai
# openai-harmony
# oss-harmony
# pydantic-extra-types
# pydantic-settings
# ray
@@ -855,6 +865,7 @@ pytest-timeout==2.3.1
python-dateutil==2.9.0.post0
# via
# botocore
# jupyter-client
# matplotlib
# pandas
# typepy
@@ -898,7 +909,9 @@ pyyaml==6.0.2
# uvicorn
# vocos
pyzmq==27.1.0
# via -r requirements/test/../common.txt
# via
# -r requirements/test/../common.txt
# jupyter-client
rapidfuzz==3.12.1
# via jiwer
ray==2.56.1
@@ -1077,7 +1090,7 @@ starlette-testclient==0.4.1
# via schemathesis
statsmodels==0.14.4
# via genai-perf
structlog==25.4.0
structlog==26.1.0
# via gpt-oss
supervisor==4.3.0
# via model-hosting-container-standards
@@ -1101,7 +1114,7 @@ tenacity==9.1.2
# schemathesis
tensorizer==2.10.1
# via -r requirements/test/cuda.in
termcolor==3.1.0
termcolor==3.3.0
# via gpt-oss
threadpoolctl==3.5.0
# via scikit-learn
@@ -1157,11 +1170,14 @@ torchvision==0.28.0+cpu
# open-clip-torch
# segmentation-models-pytorch
# timm
tornado==6.5.10
# via jupyter-client
tqdm==4.67.3
# via
# -r requirements/test/../common.txt
# datasets
# evaluate
# gpt-oss
# huggingface-hub
# lm-eval
# mteb
@@ -1174,6 +1190,10 @@ tqdm==4.67.3
# segmentation-models-pytorch
# sentence-transformers
# transformers
traitlets==5.16.1
# via
# jupyter-client
# jupyter-core
transformers==5.17.0
# via
# -r requirements/test/../common.txt
@@ -1226,6 +1246,7 @@ typing-extensions==4.15.0
# grpcio
# httpx2
# huggingface-hub
# jupyter-client
# librosa
# lm-eval
# mcp
+1 -1
View File
@@ -66,7 +66,7 @@ decord==0.6.0; platform_machine == "x86_64"
imagehash # Required for Prithvi tests
segmentation-models-pytorch > 0.4.0 # Required for Prithvi tests
gpt-oss >= 0.0.7; python_version > '3.11'
gpt-oss[eval] >= 0.0.7; python_version > '3.11'
perceptron # required for isaac test
kaldi-native-fbank >= 1.18.7 # required for fireredasr2 test
+31 -11
View File
@@ -81,7 +81,9 @@ black==24.10.0
blake3==1.0.9
# via -r requirements/test/../common.txt
blobfile==3.0.0
# via -r requirements/test/cuda.in
# via
# -r requirements/test/cuda.in
# gpt-oss
bm25s==0.2.13
# via mteb
boto3==1.35.57
@@ -116,7 +118,7 @@ chardet==5.2.0
# via mbstrdecoder
charset-normalizer==3.4.0
# via requests
chz==0.3.0
chz==0.4.0
# via gpt-oss
click==8.4.2
# via
@@ -205,7 +207,7 @@ distro==1.9.0
# openai
dnspython==2.7.0
# via email-validator
docker==7.1.0
docker==7.2.0
# via gpt-oss
docopt==0.6.2
# via num2words
@@ -310,7 +312,7 @@ googleapis-common-protos==1.70.0
# google-api-core
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
gpt-oss==0.0.8
gpt-oss==0.0.9
# via -r requirements/test/cuda.in
graphql-core==3.2.6
# via hypothesis-graphql
@@ -424,6 +426,7 @@ jinja2==3.1.6
# datamodel-code-generator
# fastapi
# genai-perf
# gpt-oss
# lm-eval
# torch
jiter==0.15.0
@@ -456,6 +459,10 @@ jsonschema-specifications==2024.10.1
# via jsonschema
junit-xml==1.9
# via schemathesis
jupyter-client==8.10.0
# via gpt-oss
jupyter-core==5.9.1
# via jupyter-client
kaldi-native-fbank==1.22.3
# via -r requirements/test/cuda.in
kaleido==0.2.1
@@ -584,6 +591,7 @@ numpy==2.2.6
# evaluate
# fastparquet
# genai-perf
# gpt-oss
# imagehash
# imageio
# librosa
@@ -662,10 +670,6 @@ openai==2.44.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
openai-harmony==0.0.4
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
# gpt-oss
opencensus==0.11.4
# via ray
@@ -727,6 +731,10 @@ optuna==3.6.1
# via genai-perf
orjson==3.11.5
# via genai-perf
oss-harmony==0.0.11
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
outlines-core==0.2.14
# via
# -c requirements/common.txt
@@ -760,6 +768,7 @@ pandas==2.2.3
# evaluate
# fastparquet
# genai-perf
# gpt-oss
# statsmodels
partial-json-parser==0.2.1.1.post7
# via -r requirements/test/../common.txt
@@ -790,6 +799,7 @@ pillow==10.4.0
platformdirs==4.3.6
# via
# black
# jupyter-core
# pooch
# virtualenv
plotly==5.24.1
@@ -887,7 +897,7 @@ pydantic==2.12.0
# model-hosting-container-standards
# mteb
# openai
# openai-harmony
# oss-harmony
# pydantic-extra-types
# pydantic-settings
# ray
@@ -948,6 +958,7 @@ pytest-timeout==2.3.1
python-dateutil==2.9.0.post0
# via
# botocore
# jupyter-client
# matplotlib
# pandas
# typepy
@@ -994,6 +1005,7 @@ pyzmq==27.1.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
# jupyter-client
rapidfuzz==3.12.1
# via jiwer
ray==2.56.1
@@ -1177,7 +1189,7 @@ starlette-testclient==0.4.1
# via schemathesis
statsmodels==0.14.4
# via genai-perf
structlog==25.4.0
structlog==26.1.0
# via gpt-oss
supervisor==4.3.0
# via model-hosting-container-standards
@@ -1201,7 +1213,7 @@ tenacity==9.1.2
# schemathesis
tensorizer==2.10.1
# via -r requirements/test/cuda.in
termcolor==3.1.0
termcolor==3.3.0
# via gpt-oss
threadpoolctl==3.5.0
# via scikit-learn
@@ -1264,11 +1276,14 @@ torchvision==0.28.0+cu130
# open-clip-torch
# segmentation-models-pytorch
# timm
tornado==6.5.10
# via jupyter-client
tqdm==4.67.3
# via
# -r requirements/test/../common.txt
# datasets
# evaluate
# gpt-oss
# huggingface-hub
# lm-eval
# mteb
@@ -1281,6 +1296,10 @@ tqdm==4.67.3
# segmentation-models-pytorch
# sentence-transformers
# transformers
traitlets==5.16.1
# via
# jupyter-client
# jupyter-core
transformers==5.17.0
# via
# -c requirements/common.txt
@@ -1337,6 +1356,7 @@ typing-extensions==4.15.0
# grpcio
# httpx2
# huggingface-hub
# jupyter-client
# librosa
# lm-eval
# mcp
+1 -1
View File
@@ -65,7 +65,7 @@ decord==0.6.0
imagehash # Required for Prithvi tests
segmentation-models-pytorch>0.4.0 # Required for Prithvi tests
gpt-oss>=0.0.7; python_version > '3.11'
gpt-oss[eval]>=0.0.7; python_version > '3.11'
perceptron # required for isaac test
kaldi-native-fbank>=1.18.7 # required for fireredasr2 test
+29 -9
View File
@@ -80,7 +80,9 @@ black==26.3.1
blake3==1.0.8
# via -r requirements/test/../common.txt
blobfile==3.0.0
# via -r requirements/test/rocm.in
# via
# -r requirements/test/rocm.in
# gpt-oss
bm25s==0.2.13
# via mteb
boto3==1.42.74
@@ -198,7 +200,7 @@ distro==1.9.0
# openai
dnspython==2.8.0
# via email-validator
docker==7.1.0
docker==7.2.0
# via gpt-oss
docopt==0.6.2
# via num2words
@@ -300,7 +302,7 @@ googleapis-common-protos==1.73.0
# google-api-core
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
gpt-oss==0.0.8
gpt-oss==0.0.9
# via -r requirements/test/rocm.in
graphql-core==3.2.8
# via hypothesis-graphql
@@ -415,6 +417,7 @@ jinja2==3.1.6
# datamodel-code-generator
# fastapi
# genai-perf
# gpt-oss
# lm-eval
# torch
jiter==0.14.0
@@ -447,6 +450,10 @@ jsonschema-specifications==2025.9.1
# via jsonschema
junit-xml==1.9
# via schemathesis
jupyter-client==8.10.0
# via gpt-oss
jupyter-core==5.9.1
# via jupyter-client
kaldi-native-fbank==1.22.3
# via -r requirements/test/rocm.in
kaleido==1.0.0
@@ -587,6 +594,7 @@ numpy==2.2.6
# evaluate
# fastparquet
# genai-perf
# gpt-oss
# imagehash
# imageio
# librosa
@@ -629,10 +637,6 @@ openai==2.31.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
openai-harmony==0.0.8
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
# gpt-oss
opencensus==0.11.4
# via ray
@@ -699,6 +703,10 @@ orjson==3.11.7
# via
# genai-perf
# kaleido
oss-harmony==0.0.11
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
outlines-core==0.2.14
# via
# -c requirements/common.txt
@@ -734,6 +742,7 @@ pandas==3.0.1
# evaluate
# fastparquet
# genai-perf
# gpt-oss
# statsmodels
partial-json-parser==0.2.1.1.post7
# via -r requirements/test/../common.txt
@@ -764,6 +773,7 @@ pillow==12.1.1
platformdirs==4.3.6
# via
# black
# jupyter-core
# pooch
# python-discovery
# virtualenv
@@ -861,7 +871,7 @@ pydantic==2.12.5
# model-hosting-container-standards
# mteb
# openai
# openai-harmony
# oss-harmony
# pydantic-extra-types
# pydantic-settings
# ray
@@ -922,6 +932,7 @@ pytest-timeout==2.3.1
python-dateutil==2.9.0.post0
# via
# botocore
# jupyter-client
# matplotlib
# pandas
# typepy
@@ -970,6 +981,7 @@ pyzmq==27.1.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
# jupyter-client
rapidfuzz==3.12.1
# via
# -r requirements/test/rocm.in
@@ -1159,7 +1171,7 @@ starlette-testclient==0.4.1
# via schemathesis
statsmodels==0.14.6
# via genai-perf
structlog==25.5.0
structlog==26.1.0
# via gpt-oss
supervisor==4.3.0
# via model-hosting-container-standards
@@ -1215,11 +1227,14 @@ tokenizers==0.23.1
# transformers
torch-c-dlpack-ext==0.1.5
# via tilelang
tornado==6.5.10
# via jupyter-client
tqdm==4.67.3
# via
# -r requirements/test/../common.txt
# datasets
# evaluate
# gpt-oss
# huggingface-hub
# lm-eval
# mteb
@@ -1233,6 +1248,10 @@ tqdm==4.67.3
# sentence-transformers
# tilelang
# transformers
traitlets==5.16.1
# via
# jupyter-client
# jupyter-core
transformers==5.17.0
# via
# -c requirements/common.txt
@@ -1287,6 +1306,7 @@ typing-extensions==4.15.0
# grpcio
# httpx2
# huggingface-hub
# jupyter-client
# librosa
# lm-eval
# mcp
+1 -1
View File
@@ -32,7 +32,7 @@ soundfile
# --- Tool Parsing & Evaluation ---
blobfile
rapidfuzz
gpt-oss >= 0.0.7; python_version > '3.11'
gpt-oss[eval] >= 0.0.7; python_version > '3.11'
schemathesis>=4.0.0 # Required for openai schema test.
jiwer
bm25s
+29 -8
View File
@@ -67,7 +67,9 @@ azure-storage-blob==12.30.1
blake3==1.0.9
# via -r requirements/test/../common.txt
blobfile==3.2.0
# via -r requirements/test/xpu.in
# via
# -r requirements/test/xpu.in
# gpt-oss
bm25s==0.3.10
# via
# -r requirements/test/xpu.in
@@ -257,7 +259,7 @@ googleapis-common-protos==1.75.1
# google-api-core
# opentelemetry-exporter-otlp-proto-grpc
# opentelemetry-exporter-otlp-proto-http
gpt-oss==0.0.8
gpt-oss==0.0.9
# via -r requirements/test/xpu.in
graphql-core==3.2.11
# via hypothesis-graphql
@@ -377,6 +379,7 @@ jinja2==3.1.6
# via
# -c requirements/xpu.txt
# fastapi
# gpt-oss
# lm-eval
# torch
jiter==0.16.0
@@ -406,6 +409,10 @@ jsonschema-rs==0.49.9
# via schemathesis
jsonschema-specifications==2025.9.1
# via jsonschema
jupyter-client==8.10.0
# via gpt-oss
jupyter-core==5.9.1
# via jupyter-client
kiwisolver==1.5.1
# via matplotlib
lark==1.2.2
@@ -526,6 +533,7 @@ numpy==2.2.6
# datasets
# decord
# evaluate
# gpt-oss
# librosa
# lm-eval
# matplotlib
@@ -575,10 +583,6 @@ openai==3.2.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
openai-harmony==0.0.8
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
# gpt-oss
opencv-python-headless==5.0.0.93
# via
@@ -632,6 +636,10 @@ opentelemetry-semantic-conventions-ai==0.5.1
# -c requirements/common.txt
# -r requirements/test/../common.txt
# -r requirements/test/xpu.in
oss-harmony==0.0.11
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
outlines-core==0.2.14
# via
# -c requirements/common.txt
@@ -656,6 +664,7 @@ pandas==3.0.5
# via
# datasets
# evaluate
# gpt-oss
partial-json-parser==0.2.1.1.post7
# via -r requirements/test/../common.txt
pathvalidate==3.3.1
@@ -667,7 +676,9 @@ pillow==12.3.0
# mistral-common
# torchvision
platformdirs==4.11.3
# via pooch
# via
# jupyter-core
# pooch
pluggy==1.6.0
# via
# pytest
@@ -747,7 +758,7 @@ pydantic==2.13.4
# model-hosting-container-standards
# mteb
# openai
# openai-harmony
# oss-harmony
# pydantic-extra-types
# pydantic-settings
# xgrammar
@@ -806,6 +817,7 @@ pytest-timeout==2.4.0
python-dateutil==2.9.0.post0
# via
# botocore
# jupyter-client
# matplotlib
# pandas
# typepy
@@ -841,6 +853,7 @@ pyzmq==27.1.0
# via
# -c requirements/common.txt
# -r requirements/test/../common.txt
# jupyter-client
rapidfuzz==3.14.5
# via
# -r requirements/test/xpu.in
@@ -1066,11 +1079,14 @@ torchvision==0.29.0+xpu
# via
# open-clip-torch
# timm
tornado==6.5.10
# via jupyter-client
tqdm==4.70.0
# via
# -r requirements/test/../common.txt
# datasets
# evaluate
# gpt-oss
# huggingface-hub
# lm-eval
# modelscope
@@ -1081,6 +1097,10 @@ tqdm==4.70.0
# pqdm
# sentence-transformers
# transformers
traitlets==5.16.1
# via
# jupyter-client
# jupyter-core
transformers==5.17.0
# via
# -c requirements/common.txt
@@ -1132,6 +1152,7 @@ typing-extensions==4.16.0
# grpcio
# httpx2
# huggingface-hub
# jupyter-client
# lm-eval
# mcp
# mcp-types
@@ -20,6 +20,7 @@ Each test:
"""
from openai.types.responses import ResponseFunctionToolCall
from openai_harmony import Message, Role
from tests.entrypoints.openai.utils import verify_harmony_messages
from vllm.entrypoints.openai.parser.harmony_utils import (
@@ -33,6 +34,23 @@ from vllm.entrypoints.openai.responses.harmony import (
response_previous_input_to_harmony,
)
def test_render_uses_legacy_tool_call_header():
tool_call = (
Message.from_role_and_content(Role.ASSISTANT, "print('hello')")
.with_channel("commentary")
.with_recipient("python")
.with_content_type("code")
)
rendered = get_encoding().decode(render_for_completion([tool_call]))
assert (
"<|start|>assistant to=python<|channel|>commentary code<|message|>" in rendered
)
assert "commentary to=python <|constrain|>code" not in rendered
# Use a fixed date so the system message is deterministic across both paths.
_DATE = "2025-01-01"
+34
View File
@@ -0,0 +1,34 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Run the gpt-oss GPQA evaluator with an explicit output-token budget."""
import argparse
from gpt_oss.evals.gpqa_eval import GPQAEval
from gpt_oss.evals.responses_sampler import ResponsesSampler
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--model", required=True)
parser.add_argument("--reasoning-effort", required=True)
parser.add_argument("--base-url", required=True)
parser.add_argument("--n-threads", required=True, type=int)
parser.add_argument("--max-output-tokens", required=True, type=int)
args = parser.parse_args()
sampler = ResponsesSampler(
model=args.model,
reasoning_model=True,
reasoning_effort=args.reasoning_effort,
base_url=args.base_url,
max_tokens=args.max_output_tokens,
)
result = GPQAEval(n_repeats=8, n_threads=args.n_threads)(sampler)
# gpt-oss returns a NumPy scalar. Convert it so the parent process gets a
# stable, dependency-independent representation that its regex can parse.
print({"metric": float(result.score)})
if __name__ == "__main__":
main()
+9 -4
View File
@@ -21,6 +21,12 @@ from tests.utils import RemoteOpenAIServer
TOL = 0.05 # Absolute tolerance for accuracy comparison
# gpt-oss >= 0.0.7 explicitly sends its 131072-token default as the output
# budget, which leaves no room for the prompt in GPT-OSS's 131072-token context.
# This test originally ran with a 32768-token model context, so retain that
# proven generation budget while using the current evaluator package.
MAX_OUTPUT_TOKENS = 32_768
# Path to tiktoken encoding files
TIKTOKEN_DATA_DIR = Path(__file__).parent / "data"
@@ -50,10 +56,7 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa
# Build the command to run the evaluation
cmd = [
sys.executable,
"-m",
"gpt_oss.evals",
"--eval",
"gpqa",
str(Path(__file__).with_name("run_gpqa_eval.py")),
"--model",
model_name,
"--reasoning-effort",
@@ -62,6 +65,8 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa
base_url,
"--n-threads",
"200",
"--max-output-tokens",
str(MAX_OUTPUT_TOKENS),
]
try:
@@ -449,12 +449,80 @@ def parse_chat_input_to_harmony_message(
def render_for_completion(messages: list[Message]) -> list[int]:
messages = auto_drop_analysis_messages(messages)
conversation = Conversation.from_messages(messages)
token_ids = get_encoding().render_conversation_for_completion(
encoding = get_encoding()
token_ids = encoding.render_conversation_for_completion(
conversation,
Role.ASSISTANT,
config=RenderConversationConfig(auto_drop_analysis=False),
)
return token_ids
return _use_legacy_tool_call_headers(token_ids)
def _use_legacy_tool_call_headers(token_ids: list[int]) -> list[int]:
"""Keep prior assistant tool calls in the format expected by GPT-OSS.
oss-harmony 0.0.10 changed a rendered tool-call header from::
assistant to=python<|channel|>commentary code
to::
assistant<|channel|>commentary to=python <|constrain|>code
GPT-OSS can echo the latter as a malformed header with the recipient in
both positions. Rewrite only that unambiguous token-level header shape,
while retaining oss-harmony's embedded vocabulary and parser.
"""
encoding = get_encoding()
start = encoding.encode("<|start|>", allowed_special="all")[0]
channel_token = encoding.encode("<|channel|>", allowed_special="all")[0]
constrain = encoding.encode("<|constrain|>", allowed_special="all")[0]
message = encoding.encode("<|message|>", allowed_special="all")[0]
result = token_ids.copy()
index = 0
while index < len(result):
if result[index] != start:
index += 1
continue
try:
header_end = result.index(message, index + 1)
channel_index = result.index(channel_token, index + 1, header_end)
constrain_index = result.index(constrain, channel_index + 1, header_end)
except ValueError:
index += 1
continue
author = encoding.decode(result[index + 1 : channel_index])
channel_and_recipient = encoding.decode(
result[channel_index + 1 : constrain_index]
)
content_type = encoding.decode(result[constrain_index + 1 : header_end])
if (
author != "assistant"
or " to=" not in channel_and_recipient
or not channel_and_recipient.endswith(" ")
):
index = header_end + 1
continue
channel, recipient = channel_and_recipient[:-1].split(" to=", 1)
if not channel or not recipient or not content_type:
index = header_end + 1
continue
legacy_header = (
[start]
+ encoding.encode(f"assistant to={recipient}")
+ [channel_token]
+ encoding.encode(f"{channel} {content_type}")
+ [message]
)
result[index : header_end + 1] = legacy_header
index += len(legacy_header)
return result
def get_streamable_parser_for_assistant() -> StreamableParser: