diff --git a/docs/serving/online_serving/structured_decisions.md b/docs/serving/online_serving/structured_decisions.md index dd26f21301..7de935a62f 100644 --- a/docs/serving/online_serving/structured_decisions.md +++ b/docs/serving/online_serving/structured_decisions.md @@ -3,10 +3,10 @@ The `/v1/systemone` endpoint answers a set of typed questions about a state and returns a probability for every allowed answer. -The endpoint is off unless the server starts with -`--enable-structured-decisions`. With the flag, startup fails for a model that -does not support structured decisions, and with `--logprobs-mode raw_logits` or -`processed_logits`, since `label_mass` needs log probabilities. +The endpoint is available by default on generation models. Models without a +supported read strategy return 501. Supported architectures require +`--logprobs-mode raw_logprobs` or `processed_logprobs`, since `label_mass` needs +log probabilities. ## How it works diff --git a/tests/entrypoints/generate/structured_decisions/test_strategies.py b/tests/entrypoints/generate/structured_decisions/test_strategies.py index 017788f7b1..51bff19c8c 100644 --- a/tests/entrypoints/generate/structured_decisions/test_strategies.py +++ b/tests/entrypoints/generate/structured_decisions/test_strategies.py @@ -5,6 +5,7 @@ from typing import Any import pytest from fastapi import FastAPI +from fastapi.testclient import TestClient from vllm.entrypoints.generate.structured_decisions.api_router import ( register_structured_decisions_api_router, @@ -16,6 +17,9 @@ from vllm.entrypoints.generate.structured_decisions.strategies import ( reply_label_ids, select_read_strategy, ) +from vllm.entrypoints.serve.exception_handling.register import ( + init_exception_handler, +) def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any: @@ -32,13 +36,30 @@ def test_strategy_selection(): select_read_strategy(model(qwen, "raw_logits")) -def test_route_needs_the_flag(): - for enabled in (False, True): - app = FastAPI() - app.state.args = SimpleNamespace(enable_structured_decisions=enabled) - register_structured_decisions_api_router(app) - paths = {getattr(route, "path", None) for route in app.routes} - assert ("/v1/systemone" in paths) == enabled +def test_route_is_registered_by_default(): + app = FastAPI() + register_structured_decisions_api_router(app) + paths = {getattr(route, "path", None) for route in app.routes} + assert "/v1/systemone" in paths + + +def test_unsupported_model_returns_501(): + app = FastAPI() + app.state.args = SimpleNamespace(log_error_stack=False) + app.state.serving_structured_decisions = None + init_exception_handler(app) + register_structured_decisions_api_router(app) + + with TestClient(app) as client: + response = client.post( + "/v1/systemone", + json={ + "model": "unsupported", + "state": "x", + "questions": {"answer": {"type": "choice", "criteria": {"yes": None}}}, + }, + ) + assert response.status_code == 501 def test_structured_state_keeps_unicode_in_prompt(): diff --git a/tests/entrypoints/generate/structured_decisions/test_structured_decisions_e2e.py b/tests/entrypoints/generate/structured_decisions/test_structured_decisions_e2e.py index e060b096ec..3e0f9bc0fb 100644 --- a/tests/entrypoints/generate/structured_decisions/test_structured_decisions_e2e.py +++ b/tests/entrypoints/generate/structured_decisions/test_structured_decisions_e2e.py @@ -19,7 +19,6 @@ def server(): "--max-num-seqs", "32", "--enable-prefix-caching", - "--enable-structured-decisions", ] with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: yield remote_server diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py index 18b01b85a8..a405550c4c 100644 --- a/tests/entrypoints/openai/test_openai_schema.py +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -7,6 +7,7 @@ import pytest import schemathesis from hypothesis import HealthCheck, settings from schemathesis import GenerationMode +from schemathesis.checks import not_a_server_error from schemathesis.config import ( ChecksConfig, CoveragePhaseConfig, @@ -161,8 +162,13 @@ def test_openapi_stateless(case: schemathesis.Case): }.get(key, DEFAULT_TIMEOUT_SECONDS) # No need to verify SSL certificate for localhost - case.call_and_validate( + systemone = case.operation.path == "/v1/systemone" + response = case.call_and_validate( verify=False, timeout=timeout, headers={"Content-Type": "application/json"}, + excluded_checks=[not_a_server_error] if systemone else None, ) + if systemone: + # SmolVLM does not support structured decisions. + assert response.status_code < 500 or response.status_code == 501 diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index 70869b61b5..67a2561924 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -230,16 +230,11 @@ async def init_generate_state( from .structured_decisions.serving import ServingStructuredDecisions from .structured_decisions.strategies import ReadContext, select_read_strategy - strategy_cls = ( - select_read_strategy(engine_client.model_config) - if "generate" in supported_tasks - and getattr(args, "enable_structured_decisions", False) - else None - ) - state.serving_structured_decisions = ( - ServingStructuredDecisions( - state.openai_serving_models, - strategy_cls( + strategy = None + if "generate" in supported_tasks: + try: + strategy_cls = select_read_strategy(engine_client.model_config) + strategy = strategy_cls( ReadContext( engine_client=engine_client, online_renderer=state.online_renderer, @@ -247,9 +242,15 @@ async def init_generate_state( chat_template_content_format=args.chat_template_content_format, default_chat_template_kwargs=default_chat_template_kwargs, ) - ), + ) + except ValueError: + strategy = None + state.serving_structured_decisions = ( + ServingStructuredDecisions( + state.openai_serving_models, + strategy, request_logger=request_logger, ) - if strategy_cls is not None + if strategy is not None else None ) diff --git a/vllm/entrypoints/generate/structured_decisions/api_router.py b/vllm/entrypoints/generate/structured_decisions/api_router.py index a0fbaa518e..5729b58b2e 100644 --- a/vllm/entrypoints/generate/structured_decisions/api_router.py +++ b/vllm/entrypoints/generate/structured_decisions/api_router.py @@ -49,5 +49,4 @@ async def create_structured_decision( def register_structured_decisions_api_router(app: FastAPI): - if getattr(app.state.args, "enable_structured_decisions", False): - app.include_router(router) + app.include_router(router) diff --git a/vllm/entrypoints/generate/structured_decisions/strategies.py b/vllm/entrypoints/generate/structured_decisions/strategies.py index 22233ff91b..864443cbb0 100644 --- a/vllm/entrypoints/generate/structured_decisions/strategies.py +++ b/vllm/entrypoints/generate/structured_decisions/strategies.py @@ -219,17 +219,16 @@ LOGPROBS_MODES = frozenset({"raw_logprobs", "processed_logprobs"}) def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]: - """Raises ValueError when the model cannot serve structured decisions. The - server opted in with --enable-structured-decisions, so startup fails.""" + """Raise ValueError when the model cannot serve structured decisions.""" if model_config.architecture not in NEXT_TOKEN_ARCHITECTURES: raise ValueError( - "--enable-structured-decisions does not support " + "The structured decisions API does not support " f"{model_config.architecture}. Supported architectures: " f"{sorted(NEXT_TOKEN_ARCHITECTURES)}" ) if model_config.logprobs_mode not in LOGPROBS_MODES: raise ValueError( - "--enable-structured-decisions needs --logprobs-mode raw_logprobs or " + "Structured decisions need --logprobs-mode raw_logprobs or " f"processed_logprobs, not {model_config.logprobs_mode}" ) return NextTokenStrategy diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index 18f8cb00b5..28fe79fd69 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -163,8 +163,6 @@ class BaseFrontendArgs: enable_tokenizer_info_endpoint: bool = False """Enable the `/tokenizer_info` endpoint. May expose chat templates and other tokenizer configuration.""" - enable_structured_decisions: bool = False - """Enable the `/v1/systemone` structured decisions endpoint.""" enable_log_outputs: bool = False """If set to True, log model outputs (generations). Requires `--enable-log-requests`. Output text and finish reasons are logged at INFO,