From 4a8abf37c75b4a2587bfdad48bc6b442dc71332a Mon Sep 17 00:00:00 2001 From: Ben Browning Date: Fri, 19 Jun 2026 14:05:18 -0400 Subject: [PATCH] [Test] Migrate test_openai_schema.py to schemathesis 4.x (#46173) Signed-off-by: Ben Browning --- requirements/test/cuda.in | 2 +- requirements/test/nightly-torch.txt | 2 +- requirements/test/rocm.in | 2 +- .../entrypoints/openai/test_openai_schema.py | 101 ++++++++++-------- 4 files changed, 60 insertions(+), 47 deletions(-) diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index 8d7ad7d0aa..a7fc65def8 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -40,7 +40,7 @@ lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test transformers==5.5.3 tokenizers==0.22.2 -schemathesis>=3.39.15 # Required for openai schema test. +schemathesis>=4.0.0 # Required for openai schema test. # quantization bitsandbytes==0.49.2 buildkite-test-collector==0.1.9 diff --git a/requirements/test/nightly-torch.txt b/requirements/test/nightly-torch.txt index 10eb7a6219..a58e0fa248 100644 --- a/requirements/test/nightly-torch.txt +++ b/requirements/test/nightly-torch.txt @@ -31,7 +31,7 @@ lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test transformers==5.5.3 tokenizers==0.22.2 -schemathesis>=3.39.15 # Required for openai schema test. +schemathesis>=4.0.0 # Required for openai schema test. # quantization bitsandbytes>=0.49.2 buildkite-test-collector==0.1.9 diff --git a/requirements/test/rocm.in b/requirements/test/rocm.in index ed10270f56..046ca09ff7 100644 --- a/requirements/test/rocm.in +++ b/requirements/test/rocm.in @@ -39,7 +39,7 @@ lm-eval[api]>=0.4.12 # required for model evaluation test mteb[bm25s]>=2, <3 # required for mteb test transformers==5.5.3 tokenizers==0.22.2 -schemathesis>=3.39.15 # Required for openai schema test +schemathesis>=4.0.0 # Required for openai schema test # quantization bitsandbytes==0.49.2 buildkite-test-collector==0.1.9 diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py index 56e4e9baf2..38ea2661c8 100644 --- a/tests/entrypoints/openai/test_openai_schema.py +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -6,15 +6,22 @@ from typing import Final import pytest import schemathesis from hypothesis import HealthCheck, settings -from schemathesis import GenerationConfig -from schemathesis.models import Case +from schemathesis import GenerationMode +from schemathesis.config import ( + ChecksConfig, + CoveragePhaseConfig, + GenerationConfig, + PhasesConfig, + PositiveDataAcceptanceConfig, + ProjectConfig, + ProjectsConfig, + SchemathesisConfig, +) from vllm.platforms import current_platform from ...utils import RemoteOpenAIServer -schemathesis.experimental.OPEN_API_3_1.enable() - MODEL_NAME = "HuggingFaceTB/SmolVLM-256M-Instruct" MAXIMUM_IMAGES = 2 _ROCM_TIMEOUT_MULTIPLIER = 3 if current_platform.is_rocm() else 1 @@ -44,21 +51,38 @@ def server(): @pytest.fixture(scope="module") def get_schema(server): # avoid generating null (\x00) bytes in strings during test case generation - return schemathesis.openapi.from_uri( + return schemathesis.openapi.from_url( f"{server.url_root}/openapi.json", - generation_config=GenerationConfig(allow_x00=False), + config=SchemathesisConfig( + projects=ProjectsConfig( + default=ProjectConfig( + generation=GenerationConfig( + allow_x00=False, + modes=[GenerationMode.POSITIVE], + ), + checks=ChecksConfig( + positive_data_acceptance=PositiveDataAcceptanceConfig( + enabled=False, + ), + ), + phases=PhasesConfig( + coverage=CoveragePhaseConfig(enabled=False), + ), + ), + ), + ), ) -schema = schemathesis.from_pytest_fixture("get_schema") +schema = schemathesis.pytest.from_fixture("get_schema") @schemathesis.hook -def before_generate_case(context: schemathesis.hooks.HookContext, strategy): +def before_generate_case(context: schemathesis.HookContext, strategy): op = context.operation assert op is not None - def no_invalid_types(case: schemathesis.models.Case): + def no_invalid_types(case: schemathesis.Case): """ Skips tool_calls with `"type": "custom"` which schemathesis incorrectly generates instead of the valid `"type": "function"`. @@ -68,39 +92,25 @@ def before_generate_case(context: schemathesis.hooks.HookContext, strategy): -d '{"messages": [{"role": "assistant", "tool_calls": [{"custom": {"input": "", "name": ""}, "id": "", "type": "custom"}]}]}' \ http://localhost:8000/v1/chat/completions """ # noqa: E501 - if hasattr(case, "body") and isinstance(case.body, dict): - if ( - "messages" in case.body - and isinstance(case.body["messages"], list) - and len(case.body["messages"]) > 0 - ): - for message in case.body["messages"]: - if not isinstance(message, dict): - continue + if ( + hasattr(case, "body") + and isinstance(case.body, dict) + and "messages" in case.body + and isinstance(case.body["messages"], list) + and len(case.body["messages"]) > 0 + ): + for message in case.body["messages"]: + if not isinstance(message, dict): + continue - tool_calls = message.get("tool_calls", []) - if isinstance(tool_calls, list): - for tool_call in tool_calls: - if isinstance(tool_call, dict): - if tool_call.get("type") != "function": - return False - if "custom" in tool_call: - return False - - # Sometimes structured_outputs.grammar is generated to be empty - # Causing a server error in EBNF grammar parsing - # https://github.com/vllm-project/vllm/pull/22587#issuecomment-3195253421 - structured_outputs = case.body.get("structured_outputs", {}) - grammar = ( - structured_outputs.get("grammar") - if isinstance(structured_outputs, dict) - else None - ) - - if grammar == "": - # Allow None (will be handled as no grammar) - # But skip empty strings - return False + tool_calls = message.get("tool_calls", []) + if isinstance(tool_calls, list): + for tool_call in tool_calls: + if isinstance(tool_call, dict): + if tool_call.get("type") != "function": + return False + if "custom" in tool_call: + return False return True @@ -108,7 +118,6 @@ def before_generate_case(context: schemathesis.hooks.HookContext, strategy): @schema.parametrize() -@schema.override(headers={"Content-Type": "application/json"}) @settings( deadline=LONG_TIMEOUT_SECONDS * 1000, max_examples=50, @@ -122,7 +131,7 @@ def before_generate_case(context: schemathesis.hooks.HookContext, strategy): # generating large-but-valid request bodies before vLLM is called. suppress_health_check=[HealthCheck.filter_too_much, HealthCheck.data_too_large], ) -def test_openapi_stateless(case: Case): +def test_openapi_stateless(case: schemathesis.Case): key = ( case.operation.method.upper(), case.operation.path, @@ -151,4 +160,8 @@ def test_openapi_stateless(case: Case): }.get(key, DEFAULT_TIMEOUT_SECONDS) # No need to verify SSL certificate for localhost - case.call_and_validate(verify=False, timeout=timeout) + case.call_and_validate( + verify=False, + timeout=timeout, + headers={"Content-Type": "application/json"}, + )