[Frontend] Expose /v1/systemone without an opt-in flag (#60677)

Signed-off-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Francisco Javier Arceo
2026-10-08 20:21:05 +00:00
committed by GitHub
co-authored by Codex Cyrus Leung
parent 72d59adc2c
commit ae9adec905
8 changed files with 56 additions and 33 deletions
@@ -3,10 +3,10 @@
The `/v1/systemone` endpoint answers a set of typed questions about a state and
returns a probability for every allowed answer.
The endpoint is off unless the server starts with
`--enable-structured-decisions`. With the flag, startup fails for a model that
does not support structured decisions, and with `--logprobs-mode raw_logits` or
`processed_logits`, since `label_mass` needs log probabilities.
The endpoint is available by default on generation models. Models without a
supported read strategy return 501. Supported architectures require
`--logprobs-mode raw_logprobs` or `processed_logprobs`, since `label_mass` needs
log probabilities.
## How it works
@@ -5,6 +5,7 @@ from typing import Any
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from vllm.entrypoints.generate.structured_decisions.api_router import (
register_structured_decisions_api_router,
@@ -16,6 +17,9 @@ from vllm.entrypoints.generate.structured_decisions.strategies import (
reply_label_ids,
select_read_strategy,
)
from vllm.entrypoints.serve.exception_handling.register import (
init_exception_handler,
)
def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
@@ -32,13 +36,30 @@ def test_strategy_selection():
select_read_strategy(model(qwen, "raw_logits"))
def test_route_needs_the_flag():
for enabled in (False, True):
app = FastAPI()
app.state.args = SimpleNamespace(enable_structured_decisions=enabled)
register_structured_decisions_api_router(app)
paths = {getattr(route, "path", None) for route in app.routes}
assert ("/v1/systemone" in paths) == enabled
def test_route_is_registered_by_default():
app = FastAPI()
register_structured_decisions_api_router(app)
paths = {getattr(route, "path", None) for route in app.routes}
assert "/v1/systemone" in paths
def test_unsupported_model_returns_501():
app = FastAPI()
app.state.args = SimpleNamespace(log_error_stack=False)
app.state.serving_structured_decisions = None
init_exception_handler(app)
register_structured_decisions_api_router(app)
with TestClient(app) as client:
response = client.post(
"/v1/systemone",
json={
"model": "unsupported",
"state": "x",
"questions": {"answer": {"type": "choice", "criteria": {"yes": None}}},
},
)
assert response.status_code == 501
def test_structured_state_keeps_unicode_in_prompt():
@@ -19,7 +19,6 @@ def server():
"--max-num-seqs",
"32",
"--enable-prefix-caching",
"--enable-structured-decisions",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
@@ -7,6 +7,7 @@ import pytest
import schemathesis
from hypothesis import HealthCheck, settings
from schemathesis import GenerationMode
from schemathesis.checks import not_a_server_error
from schemathesis.config import (
ChecksConfig,
CoveragePhaseConfig,
@@ -161,8 +162,13 @@ def test_openapi_stateless(case: schemathesis.Case):
}.get(key, DEFAULT_TIMEOUT_SECONDS)
# No need to verify SSL certificate for localhost
case.call_and_validate(
systemone = case.operation.path == "/v1/systemone"
response = case.call_and_validate(
verify=False,
timeout=timeout,
headers={"Content-Type": "application/json"},
excluded_checks=[not_a_server_error] if systemone else None,
)
if systemone:
# SmolVLM does not support structured decisions.
assert response.status_code < 500 or response.status_code == 501
+13 -12
View File
@@ -230,16 +230,11 @@ async def init_generate_state(
from .structured_decisions.serving import ServingStructuredDecisions
from .structured_decisions.strategies import ReadContext, select_read_strategy
strategy_cls = (
select_read_strategy(engine_client.model_config)
if "generate" in supported_tasks
and getattr(args, "enable_structured_decisions", False)
else None
)
state.serving_structured_decisions = (
ServingStructuredDecisions(
state.openai_serving_models,
strategy_cls(
strategy = None
if "generate" in supported_tasks:
try:
strategy_cls = select_read_strategy(engine_client.model_config)
strategy = strategy_cls(
ReadContext(
engine_client=engine_client,
online_renderer=state.online_renderer,
@@ -247,9 +242,15 @@ async def init_generate_state(
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=default_chat_template_kwargs,
)
),
)
except ValueError:
strategy = None
state.serving_structured_decisions = (
ServingStructuredDecisions(
state.openai_serving_models,
strategy,
request_logger=request_logger,
)
if strategy_cls is not None
if strategy is not None
else None
)
@@ -49,5 +49,4 @@ async def create_structured_decision(
def register_structured_decisions_api_router(app: FastAPI):
if getattr(app.state.args, "enable_structured_decisions", False):
app.include_router(router)
app.include_router(router)
@@ -219,17 +219,16 @@ LOGPROBS_MODES = frozenset({"raw_logprobs", "processed_logprobs"})
def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]:
"""Raises ValueError when the model cannot serve structured decisions. The
server opted in with --enable-structured-decisions, so startup fails."""
"""Raise ValueError when the model cannot serve structured decisions."""
if model_config.architecture not in NEXT_TOKEN_ARCHITECTURES:
raise ValueError(
"--enable-structured-decisions does not support "
"The structured decisions API does not support "
f"{model_config.architecture}. Supported architectures: "
f"{sorted(NEXT_TOKEN_ARCHITECTURES)}"
)
if model_config.logprobs_mode not in LOGPROBS_MODES:
raise ValueError(
"--enable-structured-decisions needs --logprobs-mode raw_logprobs or "
"Structured decisions need --logprobs-mode raw_logprobs or "
f"processed_logprobs, not {model_config.logprobs_mode}"
)
return NextTokenStrategy
-2
View File
@@ -163,8 +163,6 @@ class BaseFrontendArgs:
enable_tokenizer_info_endpoint: bool = False
"""Enable the `/tokenizer_info` endpoint. May expose chat
templates and other tokenizer configuration."""
enable_structured_decisions: bool = False
"""Enable the `/v1/systemone` structured decisions endpoint."""
enable_log_outputs: bool = False
"""If set to True, log model outputs (generations). Requires
`--enable-log-requests`. Output text and finish reasons are logged at INFO,