[Bugfix][Frontend] Default max_tokens server-side on /inference/v1/generate (#42329)

Signed-off-by: hallerite <git@hallerite.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
hallerite
2026-05-13 11:16:46 +02:00
committed by GitHub
co-authored by Claude Opus 4.7
parent f6e868fbdf
commit 97c4317bf5
5 changed files with 168 additions and 2 deletions
@@ -145,6 +145,7 @@ def test_openapi_stateless(case: Case):
("POST", "/v1/chat/completions/batch"): LONG_TIMEOUT_SECONDS,
("POST", "/v1/completions"): LONG_TIMEOUT_SECONDS,
("POST", "/v1/messages"): LONG_TIMEOUT_SECONDS,
("POST", "/inference/v1/generate"): LONG_TIMEOUT_SECONDS,
}.get(key, DEFAULT_TIMEOUT_SECONDS)
# No need to verify SSL certificate for localhost
@@ -0,0 +1,70 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the disagg request/response protocol.
These tests intentionally avoid spinning up a server — they exercise the
pydantic validators on ``GenerateRequest`` directly so they run fast and
fail loudly if the validator semantics ever drift.
"""
import json
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
from vllm.sampling_params import SamplingParams
def _base_payload() -> dict:
return {"token_ids": [1, 2, 3], "sampling_params": {}}
def test_omitted_max_tokens_is_not_provided():
"""Body without ``max_tokens`` must surface as 'not provided' so the
server can apply its own default instead of the dataclass 16."""
req = GenerateRequest.model_validate(_base_payload())
# SamplingParams' dataclass default leaks through the parsed instance —
# this is exactly the bug the server-side defaulting works around.
assert req.sampling_params.max_tokens == 16
assert not req.is_sampling_param_provided("max_tokens")
def test_explicit_max_tokens_is_provided():
"""Even when the client picks the same value as the dataclass default,
it must register as explicitly set so the server won't override it."""
payload = _base_payload()
payload["sampling_params"] = {"max_tokens": 16}
req = GenerateRequest.model_validate(payload)
assert req.sampling_params.max_tokens == 16
assert req.is_sampling_param_provided("max_tokens")
payload["sampling_params"] = {"max_tokens": 256}
req = GenerateRequest.model_validate(payload)
assert req.sampling_params.max_tokens == 256
assert req.is_sampling_param_provided("max_tokens")
def test_other_fields_tracked_independently():
payload = _base_payload()
payload["sampling_params"] = {"temperature": 0.0}
req = GenerateRequest.model_validate(payload)
assert not req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
def test_json_roundtrip_preserves_provided_keys():
payload = _base_payload()
payload["sampling_params"] = {"temperature": 0.5}
req = GenerateRequest.model_validate_json(json.dumps(payload))
assert not req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
def test_internal_instance_construction_treats_all_as_provided():
"""When internal callers build ``GenerateRequest`` from a pre-resolved
``SamplingParams`` instance, every field is considered explicitly set
so server-side defaulting can't clobber values resolved upstream."""
sp = SamplingParams(max_tokens=500, temperature=0.0)
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=sp)
assert req.is_sampling_param_provided("max_tokens")
assert req.is_sampling_param_provided("temperature")
# And keys we never touched should also count as provided in this path.
assert req.is_sampling_param_provided("top_p")
@@ -114,6 +114,36 @@ async def test_generate_endpoint(client):
assert "choices" in data
@pytest.mark.asyncio
async def test_generate_defaults_max_tokens_when_omitted(client):
"""Regression: omitting ``max_tokens`` must not silently cap at 16.
``SamplingParams.max_tokens`` defaults to 16. Before the server-side
defaulting was wired up, every request that didn't set ``max_tokens``
truncated mid-generation. The server should now fill it in from
``max_model_len - prompt_len`` (matching ``/v1/chat/completions``).
"""
payload = {
"model": MODEL_NAME,
"token_ids": [1, 2, 3],
"sampling_params": {
"temperature": 0.0,
"ignore_eos": True,
},
"stream": False,
}
resp = await client.post(GEN_ENDPOINT, json=payload)
resp.raise_for_status()
data = resp.json()
completion_tokens = len(data["choices"][0]["token_ids"])
# max_model_len=1024 in the test fixture, prompt is 3 tokens, so we
# should get ~1021 tokens of output (capped at max_model_len boundary).
assert completion_tokens > 16, (
f"expected server-side default to exceed the legacy 16-token cap, "
f"got {completion_tokens}"
)
@pytest.mark.asyncio
async def test_generate_stream(client):
payload = {
+40 -1
View File
@@ -2,7 +2,13 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
from pydantic import BaseModel, Field, field_validator
from pydantic import (
BaseModel,
Field,
PrivateAttr,
field_validator,
model_validator,
)
from vllm.config import ModelConfig
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs
@@ -108,6 +114,39 @@ class GenerateRequest(BaseModel):
description="KVTransfer parameters used for disaggregated serving.",
)
# Tracks which keys the caller explicitly set inside ``sampling_params``
# when the request was parsed from a JSON body. Lets the server tell
# "client said max_tokens=16" from "client said nothing → dataclass
# default 16" so it can apply server-side defaulting only in the latter
# case. ``None`` means the request was constructed with a pre-built
# ``SamplingParams`` instance (e.g. from internal callers that have
# already resolved values), in which case all fields are considered set.
_sampling_params_provided_keys: set[str] | None = PrivateAttr(default=None)
@model_validator(mode="wrap")
@classmethod
def _capture_sampling_params_provided_keys(cls, data: Any, handler):
provided: set[str] | None = None
if isinstance(data, dict):
sp = data.get("sampling_params")
if isinstance(sp, dict):
provided = set(sp.keys())
instance = handler(data)
instance._sampling_params_provided_keys = provided
return instance
def is_sampling_param_provided(self, name: str) -> bool:
"""Whether the caller explicitly set ``sampling_params.<name>``.
For requests parsed from a JSON body, this reflects the raw input
dict. For requests constructed with a pre-built ``SamplingParams``
instance, all fields are considered provided so server-side defaults
do not clobber values already resolved upstream.
"""
if self._sampling_params_provided_keys is None:
return True
return name in self._sampling_params_provided_keys
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
return TokenizeParams(
max_total_tokens=None,
+27 -1
View File
@@ -34,7 +34,7 @@ from vllm.entrypoints.serve.disagg.protocol import (
GenerateStreamResponse,
)
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
from vllm.entrypoints.utils import should_include_usage
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
from vllm.inputs import EngineInput, mm_input
from vllm.logger import init_logger
from vllm.logprobs import Logprob
@@ -81,6 +81,18 @@ class ServingTokens(OpenAIServing):
"step for incoming requests."
)
# Mirrors ``OpenAIServingChat`` so we can apply server-side
# ``max_tokens`` defaulting when the client omits it. Without this,
# ``SamplingParams.max_tokens`` falls back to its dataclass default
# of 16 and silently truncates every generation.
self.default_sampling_params = self.model_config.get_diff_sampling_param()
mc = self.model_config
self.override_max_tokens = (
self.default_sampling_params.get("max_tokens")
if mc.generation_config not in ("auto", "vllm")
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
)
async def serve_tokens(
self,
request: GenerateRequest,
@@ -150,6 +162,20 @@ class ServingTokens(OpenAIServing):
# Schedule the request and get the result generator.
result_generator: AsyncGenerator[RequestOutput, None] | None = None
sampling_params = request.sampling_params
# Apply server-side ``max_tokens`` defaulting when the client did
# not set it, matching the OpenAI-compat endpoints. ``SamplingParams``
# defaults ``max_tokens`` to 16, which would otherwise silently cap
# every generation that omits the field.
if not request.is_sampling_param_provided("max_tokens"):
sampling_params.max_tokens = get_max_tokens(
max_model_len=self.model_config.max_model_len,
max_tokens=None,
input_length=self._extract_prompt_len(engine_input),
default_sampling_params=self.default_sampling_params,
override_max_tokens=self.override_max_tokens,
)
if self.force_no_detokenize:
sampling_params.detokenize = False
if request.stream: