[Frontend] Switch Python Harmony dependency to oss-harmony (#55128)
Signed-off-by: Anton Peganov <apeganov@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
OpenAI Codex
parent
386ac2573e
commit
73859fec58
@@ -23,8 +23,7 @@ steps:
|
||||
commands:
|
||||
- >-
|
||||
bash .buildkite/scripts/hardware_ci/run-intel-test.sh
|
||||
'uv pip install "gpt-oss[eval]==0.0.5" &&
|
||||
cd tests &&
|
||||
'cd tests &&
|
||||
pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-xpu.txt'
|
||||
- label: LM Eval Large Models(2 GPUs)
|
||||
key: lm-eval-large-models-2-gpus
|
||||
|
||||
@@ -1287,7 +1287,6 @@ steps:
|
||||
- vllm/platforms/rocm.py
|
||||
- tests/evals/gpt_oss/
|
||||
commands:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx942.txt
|
||||
|
||||
- label: ":amd: (MI300) LM Eval Small Models Distributed"
|
||||
@@ -3584,7 +3583,6 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
|
||||
|
||||
- label: ":amd: (MI355 DPX) LM Eval Watermarking"
|
||||
|
||||
@@ -829,7 +829,6 @@ steps:
|
||||
- vllm/model_executor/layers/quantization
|
||||
- tests/evals/gpt_oss/
|
||||
commands:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-h100.txt
|
||||
|
||||
- label: ":nvidia: (B200) GPQA Eval (GPT-OSS)"
|
||||
@@ -843,7 +842,6 @@ steps:
|
||||
- vllm/model_executor/layers/quantization
|
||||
- tests/evals/gpt_oss/
|
||||
commands:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-b200.txt
|
||||
mirror:
|
||||
amd:
|
||||
@@ -865,7 +863,6 @@ steps:
|
||||
- vllm/_aiter_ops.py
|
||||
- vllm/platforms/rocm.py
|
||||
commands:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-gfx950.txt
|
||||
|
||||
- label: ":nvidia: (B200) Qwen3.8-Flash-Next-FP8 Accuracy Eval"
|
||||
@@ -940,7 +937,6 @@ steps:
|
||||
- vllm/model_executor/layers/quantization
|
||||
- tests/evals/gpt_oss/
|
||||
commands:
|
||||
- uv pip install --system 'gpt-oss[eval]==0.0.5'
|
||||
- pytest -s -v evals/gpt_oss/test_gpqa_correctness.py --config-list-file=configs/models-spark.txt
|
||||
|
||||
- label: ":nvidia: (H200 MIG 35GB) LM Eval KV-Offload"
|
||||
|
||||
@@ -729,6 +729,8 @@ COPY requirements/lint.txt requirements/lint.txt
|
||||
COPY requirements/test/cuda.in requirements/test/cuda.in
|
||||
COPY requirements/test/cuda.txt requirements/test/cuda.txt
|
||||
COPY requirements/dev.txt requirements/dev.txt
|
||||
# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below.
|
||||
COPY pyproject.toml pyproject.toml
|
||||
COPY tools/use_existing_torch.py tools/use_existing_torch.py
|
||||
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
|
||||
RUN --mount=type=cache,target=/opt/uv/cache \
|
||||
@@ -1143,6 +1145,8 @@ COPY requirements/lint.txt requirements/lint.txt
|
||||
COPY requirements/test/cuda.in requirements/test/cuda.in
|
||||
COPY requirements/test/cuda.txt requirements/test/cuda.txt
|
||||
COPY requirements/dev.txt requirements/dev.txt
|
||||
# uv reads `[tool.uv] exclude-dependencies` from here when recompiling below.
|
||||
COPY pyproject.toml pyproject.toml
|
||||
COPY tools/use_existing_torch.py tools/use_existing_torch.py
|
||||
COPY --from=base /workspace/torch_lib_versions.txt torch_lib_versions.txt
|
||||
RUN --mount=type=cache,target=/opt/uv/cache \
|
||||
|
||||
@@ -258,3 +258,8 @@ CANN = "CANN"
|
||||
|
||||
[tool.uv]
|
||||
no-build-isolation-package = ["torch"]
|
||||
# vLLM depends on oss-harmony, a fork that ships the same `openai_harmony`
|
||||
# module without downloading tiktoken vocabularies at runtime. Installing
|
||||
# openai-harmony alongside it silently overwrites those files, so keep it out
|
||||
# of every resolution; gpt-oss pulls it in transitively.
|
||||
exclude-dependencies = ["openai-harmony"]
|
||||
|
||||
@@ -49,7 +49,7 @@ pybase64 # fast base64 implementation
|
||||
cbor2 # Required for cross-language serialization of hashable objects
|
||||
ijson # Required for mistral streaming tool parser
|
||||
setproctitle # Used to set process names for better debugging and monitoring
|
||||
openai-harmony >= 0.0.3 # Required for gpt-oss
|
||||
oss-harmony >= 0.0.11 # Required for gpt-oss; includes embedded vocabularies
|
||||
anthropic >= 0.71.0
|
||||
model-hosting-container-standards >= 0.1.14, < 1.0.0
|
||||
mcp >= 2.0.0, < 3.0.0
|
||||
|
||||
+31
-10
@@ -76,7 +76,9 @@ black==24.10.0
|
||||
blake3==1.0.9
|
||||
# via -r requirements/test/../common.txt
|
||||
blobfile==3.0.0
|
||||
# via -r requirements/test/cuda.in
|
||||
# via
|
||||
# -r requirements/test/cuda.in
|
||||
# gpt-oss
|
||||
bm25s==0.2.13
|
||||
# via mteb
|
||||
boto3==1.35.57
|
||||
@@ -111,7 +113,7 @@ chardet==5.2.0
|
||||
# via mbstrdecoder
|
||||
charset-normalizer==3.4.0
|
||||
# via requests
|
||||
chz==0.3.0
|
||||
chz==0.4.0
|
||||
# via gpt-oss
|
||||
click==8.4.2
|
||||
# via
|
||||
@@ -190,7 +192,7 @@ distro==1.9.0
|
||||
# openai
|
||||
dnspython==2.7.0
|
||||
# via email-validator
|
||||
docker==7.1.0
|
||||
docker==7.2.0
|
||||
# via gpt-oss
|
||||
docopt==0.6.2
|
||||
# via num2words
|
||||
@@ -291,7 +293,7 @@ googleapis-common-protos==1.70.0
|
||||
# google-api-core
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
gpt-oss==0.0.8
|
||||
gpt-oss==0.0.9
|
||||
# via -r requirements/test/cuda.in
|
||||
graphql-core==3.2.6
|
||||
# via hypothesis-graphql
|
||||
@@ -402,6 +404,7 @@ jinja2==3.1.6
|
||||
# datamodel-code-generator
|
||||
# fastapi
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# lm-eval
|
||||
# torch
|
||||
jiter==0.15.0
|
||||
@@ -433,6 +436,10 @@ jsonschema-specifications==2024.10.1
|
||||
# via jsonschema
|
||||
junit-xml==1.9
|
||||
# via schemathesis
|
||||
jupyter-client==8.10.0
|
||||
# via gpt-oss
|
||||
jupyter-core==5.9.1
|
||||
# via jupyter-client
|
||||
kaldi-native-fbank==1.22.3
|
||||
# via -r requirements/test/cuda.in
|
||||
kaleido==0.2.1
|
||||
@@ -549,6 +556,7 @@ numpy==2.2.6
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# imagehash
|
||||
# imageio
|
||||
# librosa
|
||||
@@ -583,8 +591,6 @@ numpy==2.2.6
|
||||
open-clip-torch==2.32.0
|
||||
# via -r requirements/test/cuda.in
|
||||
openai==2.44.0
|
||||
# via -r requirements/test/../common.txt
|
||||
openai-harmony==0.0.4
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# gpt-oss
|
||||
@@ -641,6 +647,8 @@ optuna==3.6.1
|
||||
# via genai-perf
|
||||
orjson==3.11.5
|
||||
# via genai-perf
|
||||
oss-harmony==0.0.11
|
||||
# via -r requirements/test/../common.txt
|
||||
outlines-core==0.2.14
|
||||
# via -r requirements/test/../common.txt
|
||||
packaging==24.2
|
||||
@@ -672,6 +680,7 @@ pandas==2.2.3
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# statsmodels
|
||||
partial-json-parser==0.2.1.1.post7
|
||||
# via -r requirements/test/../common.txt
|
||||
@@ -702,6 +711,7 @@ pillow==10.4.0
|
||||
platformdirs==4.3.6
|
||||
# via
|
||||
# black
|
||||
# jupyter-core
|
||||
# pooch
|
||||
# virtualenv
|
||||
plotly==5.24.1
|
||||
@@ -794,7 +804,7 @@ pydantic==2.12.0
|
||||
# model-hosting-container-standards
|
||||
# mteb
|
||||
# openai
|
||||
# openai-harmony
|
||||
# oss-harmony
|
||||
# pydantic-extra-types
|
||||
# pydantic-settings
|
||||
# ray
|
||||
@@ -855,6 +865,7 @@ pytest-timeout==2.3.1
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# botocore
|
||||
# jupyter-client
|
||||
# matplotlib
|
||||
# pandas
|
||||
# typepy
|
||||
@@ -898,7 +909,9 @@ pyyaml==6.0.2
|
||||
# uvicorn
|
||||
# vocos
|
||||
pyzmq==27.1.0
|
||||
# via -r requirements/test/../common.txt
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# jupyter-client
|
||||
rapidfuzz==3.12.1
|
||||
# via jiwer
|
||||
ray==2.56.1
|
||||
@@ -1077,7 +1090,7 @@ starlette-testclient==0.4.1
|
||||
# via schemathesis
|
||||
statsmodels==0.14.4
|
||||
# via genai-perf
|
||||
structlog==25.4.0
|
||||
structlog==26.1.0
|
||||
# via gpt-oss
|
||||
supervisor==4.3.0
|
||||
# via model-hosting-container-standards
|
||||
@@ -1101,7 +1114,7 @@ tenacity==9.1.2
|
||||
# schemathesis
|
||||
tensorizer==2.10.1
|
||||
# via -r requirements/test/cuda.in
|
||||
termcolor==3.1.0
|
||||
termcolor==3.3.0
|
||||
# via gpt-oss
|
||||
threadpoolctl==3.5.0
|
||||
# via scikit-learn
|
||||
@@ -1157,11 +1170,14 @@ torchvision==0.28.0+cpu
|
||||
# open-clip-torch
|
||||
# segmentation-models-pytorch
|
||||
# timm
|
||||
tornado==6.5.10
|
||||
# via jupyter-client
|
||||
tqdm==4.67.3
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# datasets
|
||||
# evaluate
|
||||
# gpt-oss
|
||||
# huggingface-hub
|
||||
# lm-eval
|
||||
# mteb
|
||||
@@ -1174,6 +1190,10 @@ tqdm==4.67.3
|
||||
# segmentation-models-pytorch
|
||||
# sentence-transformers
|
||||
# transformers
|
||||
traitlets==5.16.1
|
||||
# via
|
||||
# jupyter-client
|
||||
# jupyter-core
|
||||
transformers==5.17.0
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
@@ -1226,6 +1246,7 @@ typing-extensions==4.15.0
|
||||
# grpcio
|
||||
# httpx2
|
||||
# huggingface-hub
|
||||
# jupyter-client
|
||||
# librosa
|
||||
# lm-eval
|
||||
# mcp
|
||||
|
||||
@@ -66,7 +66,7 @@ decord==0.6.0; platform_machine == "x86_64"
|
||||
imagehash # Required for Prithvi tests
|
||||
segmentation-models-pytorch > 0.4.0 # Required for Prithvi tests
|
||||
|
||||
gpt-oss >= 0.0.7; python_version > '3.11'
|
||||
gpt-oss[eval] >= 0.0.7; python_version > '3.11'
|
||||
|
||||
perceptron # required for isaac test
|
||||
kaldi-native-fbank >= 1.18.7 # required for fireredasr2 test
|
||||
|
||||
+31
-11
@@ -81,7 +81,9 @@ black==24.10.0
|
||||
blake3==1.0.9
|
||||
# via -r requirements/test/../common.txt
|
||||
blobfile==3.0.0
|
||||
# via -r requirements/test/cuda.in
|
||||
# via
|
||||
# -r requirements/test/cuda.in
|
||||
# gpt-oss
|
||||
bm25s==0.2.13
|
||||
# via mteb
|
||||
boto3==1.35.57
|
||||
@@ -116,7 +118,7 @@ chardet==5.2.0
|
||||
# via mbstrdecoder
|
||||
charset-normalizer==3.4.0
|
||||
# via requests
|
||||
chz==0.3.0
|
||||
chz==0.4.0
|
||||
# via gpt-oss
|
||||
click==8.4.2
|
||||
# via
|
||||
@@ -205,7 +207,7 @@ distro==1.9.0
|
||||
# openai
|
||||
dnspython==2.7.0
|
||||
# via email-validator
|
||||
docker==7.1.0
|
||||
docker==7.2.0
|
||||
# via gpt-oss
|
||||
docopt==0.6.2
|
||||
# via num2words
|
||||
@@ -310,7 +312,7 @@ googleapis-common-protos==1.70.0
|
||||
# google-api-core
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
gpt-oss==0.0.8
|
||||
gpt-oss==0.0.9
|
||||
# via -r requirements/test/cuda.in
|
||||
graphql-core==3.2.6
|
||||
# via hypothesis-graphql
|
||||
@@ -424,6 +426,7 @@ jinja2==3.1.6
|
||||
# datamodel-code-generator
|
||||
# fastapi
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# lm-eval
|
||||
# torch
|
||||
jiter==0.15.0
|
||||
@@ -456,6 +459,10 @@ jsonschema-specifications==2024.10.1
|
||||
# via jsonschema
|
||||
junit-xml==1.9
|
||||
# via schemathesis
|
||||
jupyter-client==8.10.0
|
||||
# via gpt-oss
|
||||
jupyter-core==5.9.1
|
||||
# via jupyter-client
|
||||
kaldi-native-fbank==1.22.3
|
||||
# via -r requirements/test/cuda.in
|
||||
kaleido==0.2.1
|
||||
@@ -584,6 +591,7 @@ numpy==2.2.6
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# imagehash
|
||||
# imageio
|
||||
# librosa
|
||||
@@ -662,10 +670,6 @@ openai==2.44.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
openai-harmony==0.0.4
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# gpt-oss
|
||||
opencensus==0.11.4
|
||||
# via ray
|
||||
@@ -727,6 +731,10 @@ optuna==3.6.1
|
||||
# via genai-perf
|
||||
orjson==3.11.5
|
||||
# via genai-perf
|
||||
oss-harmony==0.0.11
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
outlines-core==0.2.14
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
@@ -760,6 +768,7 @@ pandas==2.2.3
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# statsmodels
|
||||
partial-json-parser==0.2.1.1.post7
|
||||
# via -r requirements/test/../common.txt
|
||||
@@ -790,6 +799,7 @@ pillow==10.4.0
|
||||
platformdirs==4.3.6
|
||||
# via
|
||||
# black
|
||||
# jupyter-core
|
||||
# pooch
|
||||
# virtualenv
|
||||
plotly==5.24.1
|
||||
@@ -887,7 +897,7 @@ pydantic==2.12.0
|
||||
# model-hosting-container-standards
|
||||
# mteb
|
||||
# openai
|
||||
# openai-harmony
|
||||
# oss-harmony
|
||||
# pydantic-extra-types
|
||||
# pydantic-settings
|
||||
# ray
|
||||
@@ -948,6 +958,7 @@ pytest-timeout==2.3.1
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# botocore
|
||||
# jupyter-client
|
||||
# matplotlib
|
||||
# pandas
|
||||
# typepy
|
||||
@@ -994,6 +1005,7 @@ pyzmq==27.1.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# jupyter-client
|
||||
rapidfuzz==3.12.1
|
||||
# via jiwer
|
||||
ray==2.56.1
|
||||
@@ -1177,7 +1189,7 @@ starlette-testclient==0.4.1
|
||||
# via schemathesis
|
||||
statsmodels==0.14.4
|
||||
# via genai-perf
|
||||
structlog==25.4.0
|
||||
structlog==26.1.0
|
||||
# via gpt-oss
|
||||
supervisor==4.3.0
|
||||
# via model-hosting-container-standards
|
||||
@@ -1201,7 +1213,7 @@ tenacity==9.1.2
|
||||
# schemathesis
|
||||
tensorizer==2.10.1
|
||||
# via -r requirements/test/cuda.in
|
||||
termcolor==3.1.0
|
||||
termcolor==3.3.0
|
||||
# via gpt-oss
|
||||
threadpoolctl==3.5.0
|
||||
# via scikit-learn
|
||||
@@ -1264,11 +1276,14 @@ torchvision==0.28.0+cu130
|
||||
# open-clip-torch
|
||||
# segmentation-models-pytorch
|
||||
# timm
|
||||
tornado==6.5.10
|
||||
# via jupyter-client
|
||||
tqdm==4.67.3
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# datasets
|
||||
# evaluate
|
||||
# gpt-oss
|
||||
# huggingface-hub
|
||||
# lm-eval
|
||||
# mteb
|
||||
@@ -1281,6 +1296,10 @@ tqdm==4.67.3
|
||||
# segmentation-models-pytorch
|
||||
# sentence-transformers
|
||||
# transformers
|
||||
traitlets==5.16.1
|
||||
# via
|
||||
# jupyter-client
|
||||
# jupyter-core
|
||||
transformers==5.17.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
@@ -1337,6 +1356,7 @@ typing-extensions==4.15.0
|
||||
# grpcio
|
||||
# httpx2
|
||||
# huggingface-hub
|
||||
# jupyter-client
|
||||
# librosa
|
||||
# lm-eval
|
||||
# mcp
|
||||
|
||||
@@ -65,7 +65,7 @@ decord==0.6.0
|
||||
imagehash # Required for Prithvi tests
|
||||
segmentation-models-pytorch>0.4.0 # Required for Prithvi tests
|
||||
|
||||
gpt-oss>=0.0.7; python_version > '3.11'
|
||||
gpt-oss[eval]>=0.0.7; python_version > '3.11'
|
||||
|
||||
perceptron # required for isaac test
|
||||
kaldi-native-fbank>=1.18.7 # required for fireredasr2 test
|
||||
|
||||
@@ -80,7 +80,9 @@ black==26.3.1
|
||||
blake3==1.0.8
|
||||
# via -r requirements/test/../common.txt
|
||||
blobfile==3.0.0
|
||||
# via -r requirements/test/rocm.in
|
||||
# via
|
||||
# -r requirements/test/rocm.in
|
||||
# gpt-oss
|
||||
bm25s==0.2.13
|
||||
# via mteb
|
||||
boto3==1.42.74
|
||||
@@ -198,7 +200,7 @@ distro==1.9.0
|
||||
# openai
|
||||
dnspython==2.8.0
|
||||
# via email-validator
|
||||
docker==7.1.0
|
||||
docker==7.2.0
|
||||
# via gpt-oss
|
||||
docopt==0.6.2
|
||||
# via num2words
|
||||
@@ -300,7 +302,7 @@ googleapis-common-protos==1.73.0
|
||||
# google-api-core
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
gpt-oss==0.0.8
|
||||
gpt-oss==0.0.9
|
||||
# via -r requirements/test/rocm.in
|
||||
graphql-core==3.2.8
|
||||
# via hypothesis-graphql
|
||||
@@ -415,6 +417,7 @@ jinja2==3.1.6
|
||||
# datamodel-code-generator
|
||||
# fastapi
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# lm-eval
|
||||
# torch
|
||||
jiter==0.14.0
|
||||
@@ -447,6 +450,10 @@ jsonschema-specifications==2025.9.1
|
||||
# via jsonschema
|
||||
junit-xml==1.9
|
||||
# via schemathesis
|
||||
jupyter-client==8.10.0
|
||||
# via gpt-oss
|
||||
jupyter-core==5.9.1
|
||||
# via jupyter-client
|
||||
kaldi-native-fbank==1.22.3
|
||||
# via -r requirements/test/rocm.in
|
||||
kaleido==1.0.0
|
||||
@@ -587,6 +594,7 @@ numpy==2.2.6
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# imagehash
|
||||
# imageio
|
||||
# librosa
|
||||
@@ -629,10 +637,6 @@ openai==2.31.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
openai-harmony==0.0.8
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# gpt-oss
|
||||
opencensus==0.11.4
|
||||
# via ray
|
||||
@@ -699,6 +703,10 @@ orjson==3.11.7
|
||||
# via
|
||||
# genai-perf
|
||||
# kaleido
|
||||
oss-harmony==0.0.11
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
outlines-core==0.2.14
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
@@ -734,6 +742,7 @@ pandas==3.0.1
|
||||
# evaluate
|
||||
# fastparquet
|
||||
# genai-perf
|
||||
# gpt-oss
|
||||
# statsmodels
|
||||
partial-json-parser==0.2.1.1.post7
|
||||
# via -r requirements/test/../common.txt
|
||||
@@ -764,6 +773,7 @@ pillow==12.1.1
|
||||
platformdirs==4.3.6
|
||||
# via
|
||||
# black
|
||||
# jupyter-core
|
||||
# pooch
|
||||
# python-discovery
|
||||
# virtualenv
|
||||
@@ -861,7 +871,7 @@ pydantic==2.12.5
|
||||
# model-hosting-container-standards
|
||||
# mteb
|
||||
# openai
|
||||
# openai-harmony
|
||||
# oss-harmony
|
||||
# pydantic-extra-types
|
||||
# pydantic-settings
|
||||
# ray
|
||||
@@ -922,6 +932,7 @@ pytest-timeout==2.3.1
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# botocore
|
||||
# jupyter-client
|
||||
# matplotlib
|
||||
# pandas
|
||||
# typepy
|
||||
@@ -970,6 +981,7 @@ pyzmq==27.1.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# jupyter-client
|
||||
rapidfuzz==3.12.1
|
||||
# via
|
||||
# -r requirements/test/rocm.in
|
||||
@@ -1159,7 +1171,7 @@ starlette-testclient==0.4.1
|
||||
# via schemathesis
|
||||
statsmodels==0.14.6
|
||||
# via genai-perf
|
||||
structlog==25.5.0
|
||||
structlog==26.1.0
|
||||
# via gpt-oss
|
||||
supervisor==4.3.0
|
||||
# via model-hosting-container-standards
|
||||
@@ -1215,11 +1227,14 @@ tokenizers==0.23.1
|
||||
# transformers
|
||||
torch-c-dlpack-ext==0.1.5
|
||||
# via tilelang
|
||||
tornado==6.5.10
|
||||
# via jupyter-client
|
||||
tqdm==4.67.3
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# datasets
|
||||
# evaluate
|
||||
# gpt-oss
|
||||
# huggingface-hub
|
||||
# lm-eval
|
||||
# mteb
|
||||
@@ -1233,6 +1248,10 @@ tqdm==4.67.3
|
||||
# sentence-transformers
|
||||
# tilelang
|
||||
# transformers
|
||||
traitlets==5.16.1
|
||||
# via
|
||||
# jupyter-client
|
||||
# jupyter-core
|
||||
transformers==5.17.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
@@ -1287,6 +1306,7 @@ typing-extensions==4.15.0
|
||||
# grpcio
|
||||
# httpx2
|
||||
# huggingface-hub
|
||||
# jupyter-client
|
||||
# librosa
|
||||
# lm-eval
|
||||
# mcp
|
||||
|
||||
@@ -32,7 +32,7 @@ soundfile
|
||||
# --- Tool Parsing & Evaluation ---
|
||||
blobfile
|
||||
rapidfuzz
|
||||
gpt-oss >= 0.0.7; python_version > '3.11'
|
||||
gpt-oss[eval] >= 0.0.7; python_version > '3.11'
|
||||
schemathesis>=4.0.0 # Required for openai schema test.
|
||||
jiwer
|
||||
bm25s
|
||||
|
||||
@@ -67,7 +67,9 @@ azure-storage-blob==12.30.1
|
||||
blake3==1.0.9
|
||||
# via -r requirements/test/../common.txt
|
||||
blobfile==3.2.0
|
||||
# via -r requirements/test/xpu.in
|
||||
# via
|
||||
# -r requirements/test/xpu.in
|
||||
# gpt-oss
|
||||
bm25s==0.3.10
|
||||
# via
|
||||
# -r requirements/test/xpu.in
|
||||
@@ -257,7 +259,7 @@ googleapis-common-protos==1.75.1
|
||||
# google-api-core
|
||||
# opentelemetry-exporter-otlp-proto-grpc
|
||||
# opentelemetry-exporter-otlp-proto-http
|
||||
gpt-oss==0.0.8
|
||||
gpt-oss==0.0.9
|
||||
# via -r requirements/test/xpu.in
|
||||
graphql-core==3.2.11
|
||||
# via hypothesis-graphql
|
||||
@@ -377,6 +379,7 @@ jinja2==3.1.6
|
||||
# via
|
||||
# -c requirements/xpu.txt
|
||||
# fastapi
|
||||
# gpt-oss
|
||||
# lm-eval
|
||||
# torch
|
||||
jiter==0.16.0
|
||||
@@ -406,6 +409,10 @@ jsonschema-rs==0.49.9
|
||||
# via schemathesis
|
||||
jsonschema-specifications==2025.9.1
|
||||
# via jsonschema
|
||||
jupyter-client==8.10.0
|
||||
# via gpt-oss
|
||||
jupyter-core==5.9.1
|
||||
# via jupyter-client
|
||||
kiwisolver==1.5.1
|
||||
# via matplotlib
|
||||
lark==1.2.2
|
||||
@@ -526,6 +533,7 @@ numpy==2.2.6
|
||||
# datasets
|
||||
# decord
|
||||
# evaluate
|
||||
# gpt-oss
|
||||
# librosa
|
||||
# lm-eval
|
||||
# matplotlib
|
||||
@@ -575,10 +583,6 @@ openai==3.2.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
openai-harmony==0.0.8
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# gpt-oss
|
||||
opencv-python-headless==5.0.0.93
|
||||
# via
|
||||
@@ -632,6 +636,10 @@ opentelemetry-semantic-conventions-ai==0.5.1
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# -r requirements/test/xpu.in
|
||||
oss-harmony==0.0.11
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
outlines-core==0.2.14
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
@@ -656,6 +664,7 @@ pandas==3.0.5
|
||||
# via
|
||||
# datasets
|
||||
# evaluate
|
||||
# gpt-oss
|
||||
partial-json-parser==0.2.1.1.post7
|
||||
# via -r requirements/test/../common.txt
|
||||
pathvalidate==3.3.1
|
||||
@@ -667,7 +676,9 @@ pillow==12.3.0
|
||||
# mistral-common
|
||||
# torchvision
|
||||
platformdirs==4.11.3
|
||||
# via pooch
|
||||
# via
|
||||
# jupyter-core
|
||||
# pooch
|
||||
pluggy==1.6.0
|
||||
# via
|
||||
# pytest
|
||||
@@ -747,7 +758,7 @@ pydantic==2.13.4
|
||||
# model-hosting-container-standards
|
||||
# mteb
|
||||
# openai
|
||||
# openai-harmony
|
||||
# oss-harmony
|
||||
# pydantic-extra-types
|
||||
# pydantic-settings
|
||||
# xgrammar
|
||||
@@ -806,6 +817,7 @@ pytest-timeout==2.4.0
|
||||
python-dateutil==2.9.0.post0
|
||||
# via
|
||||
# botocore
|
||||
# jupyter-client
|
||||
# matplotlib
|
||||
# pandas
|
||||
# typepy
|
||||
@@ -841,6 +853,7 @@ pyzmq==27.1.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
# -r requirements/test/../common.txt
|
||||
# jupyter-client
|
||||
rapidfuzz==3.14.5
|
||||
# via
|
||||
# -r requirements/test/xpu.in
|
||||
@@ -1066,11 +1079,14 @@ torchvision==0.29.0+xpu
|
||||
# via
|
||||
# open-clip-torch
|
||||
# timm
|
||||
tornado==6.5.10
|
||||
# via jupyter-client
|
||||
tqdm==4.70.0
|
||||
# via
|
||||
# -r requirements/test/../common.txt
|
||||
# datasets
|
||||
# evaluate
|
||||
# gpt-oss
|
||||
# huggingface-hub
|
||||
# lm-eval
|
||||
# modelscope
|
||||
@@ -1081,6 +1097,10 @@ tqdm==4.70.0
|
||||
# pqdm
|
||||
# sentence-transformers
|
||||
# transformers
|
||||
traitlets==5.16.1
|
||||
# via
|
||||
# jupyter-client
|
||||
# jupyter-core
|
||||
transformers==5.17.0
|
||||
# via
|
||||
# -c requirements/common.txt
|
||||
@@ -1132,6 +1152,7 @@ typing-extensions==4.16.0
|
||||
# grpcio
|
||||
# httpx2
|
||||
# huggingface-hub
|
||||
# jupyter-client
|
||||
# lm-eval
|
||||
# mcp
|
||||
# mcp-types
|
||||
|
||||
@@ -20,6 +20,7 @@ Each test:
|
||||
"""
|
||||
|
||||
from openai.types.responses import ResponseFunctionToolCall
|
||||
from openai_harmony import Message, Role
|
||||
|
||||
from tests.entrypoints.openai.utils import verify_harmony_messages
|
||||
from vllm.entrypoints.openai.parser.harmony_utils import (
|
||||
@@ -33,6 +34,23 @@ from vllm.entrypoints.openai.responses.harmony import (
|
||||
response_previous_input_to_harmony,
|
||||
)
|
||||
|
||||
|
||||
def test_render_uses_legacy_tool_call_header():
|
||||
tool_call = (
|
||||
Message.from_role_and_content(Role.ASSISTANT, "print('hello')")
|
||||
.with_channel("commentary")
|
||||
.with_recipient("python")
|
||||
.with_content_type("code")
|
||||
)
|
||||
|
||||
rendered = get_encoding().decode(render_for_completion([tool_call]))
|
||||
|
||||
assert (
|
||||
"<|start|>assistant to=python<|channel|>commentary code<|message|>" in rendered
|
||||
)
|
||||
assert "commentary to=python <|constrain|>code" not in rendered
|
||||
|
||||
|
||||
# Use a fixed date so the system message is deterministic across both paths.
|
||||
_DATE = "2025-01-01"
|
||||
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Run the gpt-oss GPQA evaluator with an explicit output-token budget."""
|
||||
|
||||
import argparse
|
||||
|
||||
from gpt_oss.evals.gpqa_eval import GPQAEval
|
||||
from gpt_oss.evals.responses_sampler import ResponsesSampler
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--model", required=True)
|
||||
parser.add_argument("--reasoning-effort", required=True)
|
||||
parser.add_argument("--base-url", required=True)
|
||||
parser.add_argument("--n-threads", required=True, type=int)
|
||||
parser.add_argument("--max-output-tokens", required=True, type=int)
|
||||
args = parser.parse_args()
|
||||
|
||||
sampler = ResponsesSampler(
|
||||
model=args.model,
|
||||
reasoning_model=True,
|
||||
reasoning_effort=args.reasoning_effort,
|
||||
base_url=args.base_url,
|
||||
max_tokens=args.max_output_tokens,
|
||||
)
|
||||
result = GPQAEval(n_repeats=8, n_threads=args.n_threads)(sampler)
|
||||
# gpt-oss returns a NumPy scalar. Convert it so the parent process gets a
|
||||
# stable, dependency-independent representation that its regex can parse.
|
||||
print({"metric": float(result.score)})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -21,6 +21,12 @@ from tests.utils import RemoteOpenAIServer
|
||||
|
||||
TOL = 0.05 # Absolute tolerance for accuracy comparison
|
||||
|
||||
# gpt-oss >= 0.0.7 explicitly sends its 131072-token default as the output
|
||||
# budget, which leaves no room for the prompt in GPT-OSS's 131072-token context.
|
||||
# This test originally ran with a 32768-token model context, so retain that
|
||||
# proven generation budget while using the current evaluator package.
|
||||
MAX_OUTPUT_TOKENS = 32_768
|
||||
|
||||
# Path to tiktoken encoding files
|
||||
TIKTOKEN_DATA_DIR = Path(__file__).parent / "data"
|
||||
|
||||
@@ -50,10 +56,7 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa
|
||||
# Build the command to run the evaluation
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"gpt_oss.evals",
|
||||
"--eval",
|
||||
"gpqa",
|
||||
str(Path(__file__).with_name("run_gpqa_eval.py")),
|
||||
"--model",
|
||||
model_name,
|
||||
"--reasoning-effort",
|
||||
@@ -62,6 +65,8 @@ def run_gpqa_eval(model_name: str, base_url: str, reasoning_effort: str) -> floa
|
||||
base_url,
|
||||
"--n-threads",
|
||||
"200",
|
||||
"--max-output-tokens",
|
||||
str(MAX_OUTPUT_TOKENS),
|
||||
]
|
||||
|
||||
try:
|
||||
|
||||
@@ -449,12 +449,80 @@ def parse_chat_input_to_harmony_message(
|
||||
def render_for_completion(messages: list[Message]) -> list[int]:
|
||||
messages = auto_drop_analysis_messages(messages)
|
||||
conversation = Conversation.from_messages(messages)
|
||||
token_ids = get_encoding().render_conversation_for_completion(
|
||||
encoding = get_encoding()
|
||||
token_ids = encoding.render_conversation_for_completion(
|
||||
conversation,
|
||||
Role.ASSISTANT,
|
||||
config=RenderConversationConfig(auto_drop_analysis=False),
|
||||
)
|
||||
return token_ids
|
||||
return _use_legacy_tool_call_headers(token_ids)
|
||||
|
||||
|
||||
def _use_legacy_tool_call_headers(token_ids: list[int]) -> list[int]:
|
||||
"""Keep prior assistant tool calls in the format expected by GPT-OSS.
|
||||
|
||||
oss-harmony 0.0.10 changed a rendered tool-call header from::
|
||||
|
||||
assistant to=python<|channel|>commentary code
|
||||
|
||||
to::
|
||||
|
||||
assistant<|channel|>commentary to=python <|constrain|>code
|
||||
|
||||
GPT-OSS can echo the latter as a malformed header with the recipient in
|
||||
both positions. Rewrite only that unambiguous token-level header shape,
|
||||
while retaining oss-harmony's embedded vocabulary and parser.
|
||||
"""
|
||||
encoding = get_encoding()
|
||||
start = encoding.encode("<|start|>", allowed_special="all")[0]
|
||||
channel_token = encoding.encode("<|channel|>", allowed_special="all")[0]
|
||||
constrain = encoding.encode("<|constrain|>", allowed_special="all")[0]
|
||||
message = encoding.encode("<|message|>", allowed_special="all")[0]
|
||||
|
||||
result = token_ids.copy()
|
||||
index = 0
|
||||
while index < len(result):
|
||||
if result[index] != start:
|
||||
index += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
header_end = result.index(message, index + 1)
|
||||
channel_index = result.index(channel_token, index + 1, header_end)
|
||||
constrain_index = result.index(constrain, channel_index + 1, header_end)
|
||||
except ValueError:
|
||||
index += 1
|
||||
continue
|
||||
|
||||
author = encoding.decode(result[index + 1 : channel_index])
|
||||
channel_and_recipient = encoding.decode(
|
||||
result[channel_index + 1 : constrain_index]
|
||||
)
|
||||
content_type = encoding.decode(result[constrain_index + 1 : header_end])
|
||||
if (
|
||||
author != "assistant"
|
||||
or " to=" not in channel_and_recipient
|
||||
or not channel_and_recipient.endswith(" ")
|
||||
):
|
||||
index = header_end + 1
|
||||
continue
|
||||
|
||||
channel, recipient = channel_and_recipient[:-1].split(" to=", 1)
|
||||
if not channel or not recipient or not content_type:
|
||||
index = header_end + 1
|
||||
continue
|
||||
|
||||
legacy_header = (
|
||||
[start]
|
||||
+ encoding.encode(f"assistant to={recipient}")
|
||||
+ [channel_token]
|
||||
+ encoding.encode(f"{channel} {content_type}")
|
||||
+ [message]
|
||||
)
|
||||
result[index : header_end + 1] = legacy_header
|
||||
index += len(legacy_header)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_streamable_parser_for_assistant() -> StreamableParser:
|
||||
|
||||
Reference in New Issue
Block a user