[Frontend] Expose /v1/systemone without an opt-in flag (#60677)
Signed-off-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Codex
Cyrus Leung
parent
72d59adc2c
commit
ae9adec905
@@ -3,10 +3,10 @@
|
||||
The `/v1/systemone` endpoint answers a set of typed questions about a state and
|
||||
returns a probability for every allowed answer.
|
||||
|
||||
The endpoint is off unless the server starts with
|
||||
`--enable-structured-decisions`. With the flag, startup fails for a model that
|
||||
does not support structured decisions, and with `--logprobs-mode raw_logits` or
|
||||
`processed_logits`, since `label_mass` needs log probabilities.
|
||||
The endpoint is available by default on generation models. Models without a
|
||||
supported read strategy return 501. Supported architectures require
|
||||
`--logprobs-mode raw_logprobs` or `processed_logprobs`, since `label_mass` needs
|
||||
log probabilities.
|
||||
|
||||
## How it works
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.api_router import (
|
||||
register_structured_decisions_api_router,
|
||||
@@ -16,6 +17,9 @@ from vllm.entrypoints.generate.structured_decisions.strategies import (
|
||||
reply_label_ids,
|
||||
select_read_strategy,
|
||||
)
|
||||
from vllm.entrypoints.serve.exception_handling.register import (
|
||||
init_exception_handler,
|
||||
)
|
||||
|
||||
|
||||
def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
|
||||
@@ -32,13 +36,30 @@ def test_strategy_selection():
|
||||
select_read_strategy(model(qwen, "raw_logits"))
|
||||
|
||||
|
||||
def test_route_needs_the_flag():
|
||||
for enabled in (False, True):
|
||||
app = FastAPI()
|
||||
app.state.args = SimpleNamespace(enable_structured_decisions=enabled)
|
||||
register_structured_decisions_api_router(app)
|
||||
paths = {getattr(route, "path", None) for route in app.routes}
|
||||
assert ("/v1/systemone" in paths) == enabled
|
||||
def test_route_is_registered_by_default():
|
||||
app = FastAPI()
|
||||
register_structured_decisions_api_router(app)
|
||||
paths = {getattr(route, "path", None) for route in app.routes}
|
||||
assert "/v1/systemone" in paths
|
||||
|
||||
|
||||
def test_unsupported_model_returns_501():
|
||||
app = FastAPI()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
app.state.serving_structured_decisions = None
|
||||
init_exception_handler(app)
|
||||
register_structured_decisions_api_router(app)
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.post(
|
||||
"/v1/systemone",
|
||||
json={
|
||||
"model": "unsupported",
|
||||
"state": "x",
|
||||
"questions": {"answer": {"type": "choice", "criteria": {"yes": None}}},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 501
|
||||
|
||||
|
||||
def test_structured_state_keeps_unicode_in_prompt():
|
||||
|
||||
@@ -19,7 +19,6 @@ def server():
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--enable-prefix-caching",
|
||||
"--enable-structured-decisions",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
@@ -7,6 +7,7 @@ import pytest
|
||||
import schemathesis
|
||||
from hypothesis import HealthCheck, settings
|
||||
from schemathesis import GenerationMode
|
||||
from schemathesis.checks import not_a_server_error
|
||||
from schemathesis.config import (
|
||||
ChecksConfig,
|
||||
CoveragePhaseConfig,
|
||||
@@ -161,8 +162,13 @@ def test_openapi_stateless(case: schemathesis.Case):
|
||||
}.get(key, DEFAULT_TIMEOUT_SECONDS)
|
||||
|
||||
# No need to verify SSL certificate for localhost
|
||||
case.call_and_validate(
|
||||
systemone = case.operation.path == "/v1/systemone"
|
||||
response = case.call_and_validate(
|
||||
verify=False,
|
||||
timeout=timeout,
|
||||
headers={"Content-Type": "application/json"},
|
||||
excluded_checks=[not_a_server_error] if systemone else None,
|
||||
)
|
||||
if systemone:
|
||||
# SmolVLM does not support structured decisions.
|
||||
assert response.status_code < 500 or response.status_code == 501
|
||||
|
||||
@@ -230,16 +230,11 @@ async def init_generate_state(
|
||||
from .structured_decisions.serving import ServingStructuredDecisions
|
||||
from .structured_decisions.strategies import ReadContext, select_read_strategy
|
||||
|
||||
strategy_cls = (
|
||||
select_read_strategy(engine_client.model_config)
|
||||
if "generate" in supported_tasks
|
||||
and getattr(args, "enable_structured_decisions", False)
|
||||
else None
|
||||
)
|
||||
state.serving_structured_decisions = (
|
||||
ServingStructuredDecisions(
|
||||
state.openai_serving_models,
|
||||
strategy_cls(
|
||||
strategy = None
|
||||
if "generate" in supported_tasks:
|
||||
try:
|
||||
strategy_cls = select_read_strategy(engine_client.model_config)
|
||||
strategy = strategy_cls(
|
||||
ReadContext(
|
||||
engine_client=engine_client,
|
||||
online_renderer=state.online_renderer,
|
||||
@@ -247,9 +242,15 @@ async def init_generate_state(
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
),
|
||||
)
|
||||
except ValueError:
|
||||
strategy = None
|
||||
state.serving_structured_decisions = (
|
||||
ServingStructuredDecisions(
|
||||
state.openai_serving_models,
|
||||
strategy,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
if strategy_cls is not None
|
||||
if strategy is not None
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -49,5 +49,4 @@ async def create_structured_decision(
|
||||
|
||||
|
||||
def register_structured_decisions_api_router(app: FastAPI):
|
||||
if getattr(app.state.args, "enable_structured_decisions", False):
|
||||
app.include_router(router)
|
||||
app.include_router(router)
|
||||
|
||||
@@ -219,17 +219,16 @@ LOGPROBS_MODES = frozenset({"raw_logprobs", "processed_logprobs"})
|
||||
|
||||
|
||||
def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]:
|
||||
"""Raises ValueError when the model cannot serve structured decisions. The
|
||||
server opted in with --enable-structured-decisions, so startup fails."""
|
||||
"""Raise ValueError when the model cannot serve structured decisions."""
|
||||
if model_config.architecture not in NEXT_TOKEN_ARCHITECTURES:
|
||||
raise ValueError(
|
||||
"--enable-structured-decisions does not support "
|
||||
"The structured decisions API does not support "
|
||||
f"{model_config.architecture}. Supported architectures: "
|
||||
f"{sorted(NEXT_TOKEN_ARCHITECTURES)}"
|
||||
)
|
||||
if model_config.logprobs_mode not in LOGPROBS_MODES:
|
||||
raise ValueError(
|
||||
"--enable-structured-decisions needs --logprobs-mode raw_logprobs or "
|
||||
"Structured decisions need --logprobs-mode raw_logprobs or "
|
||||
f"processed_logprobs, not {model_config.logprobs_mode}"
|
||||
)
|
||||
return NextTokenStrategy
|
||||
|
||||
@@ -163,8 +163,6 @@ class BaseFrontendArgs:
|
||||
enable_tokenizer_info_endpoint: bool = False
|
||||
"""Enable the `/tokenizer_info` endpoint. May expose chat
|
||||
templates and other tokenizer configuration."""
|
||||
enable_structured_decisions: bool = False
|
||||
"""Enable the `/v1/systemone` structured decisions endpoint."""
|
||||
enable_log_outputs: bool = False
|
||||
"""If set to True, log model outputs (generations). Requires
|
||||
`--enable-log-requests`. Output text and finish reasons are logged at INFO,
|
||||
|
||||
Reference in New Issue
Block a user