[Bugfix][Frontend] Default max_tokens server-side on /inference/v1/generate (#42329)
Signed-off-by: hallerite <git@hallerite.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
f6e868fbdf
commit
97c4317bf5
@@ -145,6 +145,7 @@ def test_openapi_stateless(case: Case):
|
||||
("POST", "/v1/chat/completions/batch"): LONG_TIMEOUT_SECONDS,
|
||||
("POST", "/v1/completions"): LONG_TIMEOUT_SECONDS,
|
||||
("POST", "/v1/messages"): LONG_TIMEOUT_SECONDS,
|
||||
("POST", "/inference/v1/generate"): LONG_TIMEOUT_SECONDS,
|
||||
}.get(key, DEFAULT_TIMEOUT_SECONDS)
|
||||
|
||||
# No need to verify SSL certificate for localhost
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the disagg request/response protocol.
|
||||
|
||||
These tests intentionally avoid spinning up a server — they exercise the
|
||||
pydantic validators on ``GenerateRequest`` directly so they run fast and
|
||||
fail loudly if the validator semantics ever drift.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from vllm.entrypoints.serve.disagg.protocol import GenerateRequest
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
|
||||
def _base_payload() -> dict:
|
||||
return {"token_ids": [1, 2, 3], "sampling_params": {}}
|
||||
|
||||
|
||||
def test_omitted_max_tokens_is_not_provided():
|
||||
"""Body without ``max_tokens`` must surface as 'not provided' so the
|
||||
server can apply its own default instead of the dataclass 16."""
|
||||
req = GenerateRequest.model_validate(_base_payload())
|
||||
# SamplingParams' dataclass default leaks through the parsed instance —
|
||||
# this is exactly the bug the server-side defaulting works around.
|
||||
assert req.sampling_params.max_tokens == 16
|
||||
assert not req.is_sampling_param_provided("max_tokens")
|
||||
|
||||
|
||||
def test_explicit_max_tokens_is_provided():
|
||||
"""Even when the client picks the same value as the dataclass default,
|
||||
it must register as explicitly set so the server won't override it."""
|
||||
payload = _base_payload()
|
||||
payload["sampling_params"] = {"max_tokens": 16}
|
||||
req = GenerateRequest.model_validate(payload)
|
||||
assert req.sampling_params.max_tokens == 16
|
||||
assert req.is_sampling_param_provided("max_tokens")
|
||||
|
||||
payload["sampling_params"] = {"max_tokens": 256}
|
||||
req = GenerateRequest.model_validate(payload)
|
||||
assert req.sampling_params.max_tokens == 256
|
||||
assert req.is_sampling_param_provided("max_tokens")
|
||||
|
||||
|
||||
def test_other_fields_tracked_independently():
|
||||
payload = _base_payload()
|
||||
payload["sampling_params"] = {"temperature": 0.0}
|
||||
req = GenerateRequest.model_validate(payload)
|
||||
assert not req.is_sampling_param_provided("max_tokens")
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
|
||||
|
||||
def test_json_roundtrip_preserves_provided_keys():
|
||||
payload = _base_payload()
|
||||
payload["sampling_params"] = {"temperature": 0.5}
|
||||
req = GenerateRequest.model_validate_json(json.dumps(payload))
|
||||
assert not req.is_sampling_param_provided("max_tokens")
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
|
||||
|
||||
def test_internal_instance_construction_treats_all_as_provided():
|
||||
"""When internal callers build ``GenerateRequest`` from a pre-resolved
|
||||
``SamplingParams`` instance, every field is considered explicitly set
|
||||
so server-side defaulting can't clobber values resolved upstream."""
|
||||
sp = SamplingParams(max_tokens=500, temperature=0.0)
|
||||
req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=sp)
|
||||
assert req.is_sampling_param_provided("max_tokens")
|
||||
assert req.is_sampling_param_provided("temperature")
|
||||
# And keys we never touched should also count as provided in this path.
|
||||
assert req.is_sampling_param_provided("top_p")
|
||||
@@ -114,6 +114,36 @@ async def test_generate_endpoint(client):
|
||||
assert "choices" in data
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_defaults_max_tokens_when_omitted(client):
|
||||
"""Regression: omitting ``max_tokens`` must not silently cap at 16.
|
||||
|
||||
``SamplingParams.max_tokens`` defaults to 16. Before the server-side
|
||||
defaulting was wired up, every request that didn't set ``max_tokens``
|
||||
truncated mid-generation. The server should now fill it in from
|
||||
``max_model_len - prompt_len`` (matching ``/v1/chat/completions``).
|
||||
"""
|
||||
payload = {
|
||||
"model": MODEL_NAME,
|
||||
"token_ids": [1, 2, 3],
|
||||
"sampling_params": {
|
||||
"temperature": 0.0,
|
||||
"ignore_eos": True,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
completion_tokens = len(data["choices"][0]["token_ids"])
|
||||
# max_model_len=1024 in the test fixture, prompt is 3 tokens, so we
|
||||
# should get ~1021 tokens of output (capped at max_model_len boundary).
|
||||
assert completion_tokens > 16, (
|
||||
f"expected server-side default to exceed the legacy 16-token cap, "
|
||||
f"got {completion_tokens}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_stream(client):
|
||||
payload = {
|
||||
|
||||
@@ -2,7 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
Field,
|
||||
PrivateAttr,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs
|
||||
@@ -108,6 +114,39 @@ class GenerateRequest(BaseModel):
|
||||
description="KVTransfer parameters used for disaggregated serving.",
|
||||
)
|
||||
|
||||
# Tracks which keys the caller explicitly set inside ``sampling_params``
|
||||
# when the request was parsed from a JSON body. Lets the server tell
|
||||
# "client said max_tokens=16" from "client said nothing → dataclass
|
||||
# default 16" so it can apply server-side defaulting only in the latter
|
||||
# case. ``None`` means the request was constructed with a pre-built
|
||||
# ``SamplingParams`` instance (e.g. from internal callers that have
|
||||
# already resolved values), in which case all fields are considered set.
|
||||
_sampling_params_provided_keys: set[str] | None = PrivateAttr(default=None)
|
||||
|
||||
@model_validator(mode="wrap")
|
||||
@classmethod
|
||||
def _capture_sampling_params_provided_keys(cls, data: Any, handler):
|
||||
provided: set[str] | None = None
|
||||
if isinstance(data, dict):
|
||||
sp = data.get("sampling_params")
|
||||
if isinstance(sp, dict):
|
||||
provided = set(sp.keys())
|
||||
instance = handler(data)
|
||||
instance._sampling_params_provided_keys = provided
|
||||
return instance
|
||||
|
||||
def is_sampling_param_provided(self, name: str) -> bool:
|
||||
"""Whether the caller explicitly set ``sampling_params.<name>``.
|
||||
|
||||
For requests parsed from a JSON body, this reflects the raw input
|
||||
dict. For requests constructed with a pre-built ``SamplingParams``
|
||||
instance, all fields are considered provided so server-side defaults
|
||||
do not clobber values already resolved upstream.
|
||||
"""
|
||||
if self._sampling_params_provided_keys is None:
|
||||
return True
|
||||
return name in self._sampling_params_provided_keys
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return TokenizeParams(
|
||||
max_total_tokens=None,
|
||||
|
||||
@@ -34,7 +34,7 @@ from vllm.entrypoints.serve.disagg.protocol import (
|
||||
GenerateStreamResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
from vllm.entrypoints.utils import should_include_usage
|
||||
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
||||
from vllm.inputs import EngineInput, mm_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.logprobs import Logprob
|
||||
@@ -81,6 +81,18 @@ class ServingTokens(OpenAIServing):
|
||||
"step for incoming requests."
|
||||
)
|
||||
|
||||
# Mirrors ``OpenAIServingChat`` so we can apply server-side
|
||||
# ``max_tokens`` defaulting when the client omits it. Without this,
|
||||
# ``SamplingParams.max_tokens`` falls back to its dataclass default
|
||||
# of 16 and silently truncates every generation.
|
||||
self.default_sampling_params = self.model_config.get_diff_sampling_param()
|
||||
mc = self.model_config
|
||||
self.override_max_tokens = (
|
||||
self.default_sampling_params.get("max_tokens")
|
||||
if mc.generation_config not in ("auto", "vllm")
|
||||
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
|
||||
)
|
||||
|
||||
async def serve_tokens(
|
||||
self,
|
||||
request: GenerateRequest,
|
||||
@@ -150,6 +162,20 @@ class ServingTokens(OpenAIServing):
|
||||
# Schedule the request and get the result generator.
|
||||
result_generator: AsyncGenerator[RequestOutput, None] | None = None
|
||||
sampling_params = request.sampling_params
|
||||
|
||||
# Apply server-side ``max_tokens`` defaulting when the client did
|
||||
# not set it, matching the OpenAI-compat endpoints. ``SamplingParams``
|
||||
# defaults ``max_tokens`` to 16, which would otherwise silently cap
|
||||
# every generation that omits the field.
|
||||
if not request.is_sampling_param_provided("max_tokens"):
|
||||
sampling_params.max_tokens = get_max_tokens(
|
||||
max_model_len=self.model_config.max_model_len,
|
||||
max_tokens=None,
|
||||
input_length=self._extract_prompt_len(engine_input),
|
||||
default_sampling_params=self.default_sampling_params,
|
||||
override_max_tokens=self.override_max_tokens,
|
||||
)
|
||||
|
||||
if self.force_no_detokenize:
|
||||
sampling_params.detokenize = False
|
||||
if request.stream:
|
||||
|
||||
Reference in New Issue
Block a user