diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py index 61ec8b5c50..d67c0ccd42 100644 --- a/tests/entrypoints/openai/test_openai_schema.py +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -145,6 +145,7 @@ def test_openapi_stateless(case: Case): ("POST", "/v1/chat/completions/batch"): LONG_TIMEOUT_SECONDS, ("POST", "/v1/completions"): LONG_TIMEOUT_SECONDS, ("POST", "/v1/messages"): LONG_TIMEOUT_SECONDS, + ("POST", "/inference/v1/generate"): LONG_TIMEOUT_SECONDS, }.get(key, DEFAULT_TIMEOUT_SECONDS) # No need to verify SSL certificate for localhost diff --git a/tests/entrypoints/serve/disagg/test_protocol.py b/tests/entrypoints/serve/disagg/test_protocol.py new file mode 100644 index 0000000000..414fc2a261 --- /dev/null +++ b/tests/entrypoints/serve/disagg/test_protocol.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the disagg request/response protocol. + +These tests intentionally avoid spinning up a server — they exercise the +pydantic validators on ``GenerateRequest`` directly so they run fast and +fail loudly if the validator semantics ever drift. +""" + +import json + +from vllm.entrypoints.serve.disagg.protocol import GenerateRequest +from vllm.sampling_params import SamplingParams + + +def _base_payload() -> dict: + return {"token_ids": [1, 2, 3], "sampling_params": {}} + + +def test_omitted_max_tokens_is_not_provided(): + """Body without ``max_tokens`` must surface as 'not provided' so the + server can apply its own default instead of the dataclass 16.""" + req = GenerateRequest.model_validate(_base_payload()) + # SamplingParams' dataclass default leaks through the parsed instance — + # this is exactly the bug the server-side defaulting works around. + assert req.sampling_params.max_tokens == 16 + assert not req.is_sampling_param_provided("max_tokens") + + +def test_explicit_max_tokens_is_provided(): + """Even when the client picks the same value as the dataclass default, + it must register as explicitly set so the server won't override it.""" + payload = _base_payload() + payload["sampling_params"] = {"max_tokens": 16} + req = GenerateRequest.model_validate(payload) + assert req.sampling_params.max_tokens == 16 + assert req.is_sampling_param_provided("max_tokens") + + payload["sampling_params"] = {"max_tokens": 256} + req = GenerateRequest.model_validate(payload) + assert req.sampling_params.max_tokens == 256 + assert req.is_sampling_param_provided("max_tokens") + + +def test_other_fields_tracked_independently(): + payload = _base_payload() + payload["sampling_params"] = {"temperature": 0.0} + req = GenerateRequest.model_validate(payload) + assert not req.is_sampling_param_provided("max_tokens") + assert req.is_sampling_param_provided("temperature") + + +def test_json_roundtrip_preserves_provided_keys(): + payload = _base_payload() + payload["sampling_params"] = {"temperature": 0.5} + req = GenerateRequest.model_validate_json(json.dumps(payload)) + assert not req.is_sampling_param_provided("max_tokens") + assert req.is_sampling_param_provided("temperature") + + +def test_internal_instance_construction_treats_all_as_provided(): + """When internal callers build ``GenerateRequest`` from a pre-resolved + ``SamplingParams`` instance, every field is considered explicitly set + so server-side defaulting can't clobber values resolved upstream.""" + sp = SamplingParams(max_tokens=500, temperature=0.0) + req = GenerateRequest(token_ids=[1, 2, 3], sampling_params=sp) + assert req.is_sampling_param_provided("max_tokens") + assert req.is_sampling_param_provided("temperature") + # And keys we never touched should also count as provided in this path. + assert req.is_sampling_param_provided("top_p") diff --git a/tests/entrypoints/serve/disagg/test_serving_tokens.py b/tests/entrypoints/serve/disagg/test_serving_tokens.py index 4ae7e0494a..3425bdef63 100644 --- a/tests/entrypoints/serve/disagg/test_serving_tokens.py +++ b/tests/entrypoints/serve/disagg/test_serving_tokens.py @@ -114,6 +114,36 @@ async def test_generate_endpoint(client): assert "choices" in data +@pytest.mark.asyncio +async def test_generate_defaults_max_tokens_when_omitted(client): + """Regression: omitting ``max_tokens`` must not silently cap at 16. + + ``SamplingParams.max_tokens`` defaults to 16. Before the server-side + defaulting was wired up, every request that didn't set ``max_tokens`` + truncated mid-generation. The server should now fill it in from + ``max_model_len - prompt_len`` (matching ``/v1/chat/completions``). + """ + payload = { + "model": MODEL_NAME, + "token_ids": [1, 2, 3], + "sampling_params": { + "temperature": 0.0, + "ignore_eos": True, + }, + "stream": False, + } + resp = await client.post(GEN_ENDPOINT, json=payload) + resp.raise_for_status() + data = resp.json() + completion_tokens = len(data["choices"][0]["token_ids"]) + # max_model_len=1024 in the test fixture, prompt is 3 tokens, so we + # should get ~1021 tokens of output (capped at max_model_len boundary). + assert completion_tokens > 16, ( + f"expected server-side default to exceed the legacy 16-token cap, " + f"got {completion_tokens}" + ) + + @pytest.mark.asyncio async def test_generate_stream(client): payload = { diff --git a/vllm/entrypoints/serve/disagg/protocol.py b/vllm/entrypoints/serve/disagg/protocol.py index 6336968246..13d0e0d21d 100644 --- a/vllm/entrypoints/serve/disagg/protocol.py +++ b/vllm/entrypoints/serve/disagg/protocol.py @@ -2,7 +2,13 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from typing import Any -from pydantic import BaseModel, Field, field_validator +from pydantic import ( + BaseModel, + Field, + PrivateAttr, + field_validator, + model_validator, +) from vllm.config import ModelConfig from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionLogProbs @@ -108,6 +114,39 @@ class GenerateRequest(BaseModel): description="KVTransfer parameters used for disaggregated serving.", ) + # Tracks which keys the caller explicitly set inside ``sampling_params`` + # when the request was parsed from a JSON body. Lets the server tell + # "client said max_tokens=16" from "client said nothing → dataclass + # default 16" so it can apply server-side defaulting only in the latter + # case. ``None`` means the request was constructed with a pre-built + # ``SamplingParams`` instance (e.g. from internal callers that have + # already resolved values), in which case all fields are considered set. + _sampling_params_provided_keys: set[str] | None = PrivateAttr(default=None) + + @model_validator(mode="wrap") + @classmethod + def _capture_sampling_params_provided_keys(cls, data: Any, handler): + provided: set[str] | None = None + if isinstance(data, dict): + sp = data.get("sampling_params") + if isinstance(sp, dict): + provided = set(sp.keys()) + instance = handler(data) + instance._sampling_params_provided_keys = provided + return instance + + def is_sampling_param_provided(self, name: str) -> bool: + """Whether the caller explicitly set ``sampling_params.``. + + For requests parsed from a JSON body, this reflects the raw input + dict. For requests constructed with a pre-built ``SamplingParams`` + instance, all fields are considered provided so server-side defaults + do not clobber values already resolved upstream. + """ + if self._sampling_params_provided_keys is None: + return True + return name in self._sampling_params_provided_keys + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: return TokenizeParams( max_total_tokens=None, diff --git a/vllm/entrypoints/serve/disagg/serving.py b/vllm/entrypoints/serve/disagg/serving.py index d510125fd0..9ef3198464 100644 --- a/vllm/entrypoints/serve/disagg/serving.py +++ b/vllm/entrypoints/serve/disagg/serving.py @@ -34,7 +34,7 @@ from vllm.entrypoints.serve.disagg.protocol import ( GenerateStreamResponse, ) from vllm.entrypoints.serve.render.serving import OpenAIServingRender -from vllm.entrypoints.utils import should_include_usage +from vllm.entrypoints.utils import get_max_tokens, should_include_usage from vllm.inputs import EngineInput, mm_input from vllm.logger import init_logger from vllm.logprobs import Logprob @@ -81,6 +81,18 @@ class ServingTokens(OpenAIServing): "step for incoming requests." ) + # Mirrors ``OpenAIServingChat`` so we can apply server-side + # ``max_tokens`` defaulting when the client omits it. Without this, + # ``SamplingParams.max_tokens`` falls back to its dataclass default + # of 16 and silently truncates every generation. + self.default_sampling_params = self.model_config.get_diff_sampling_param() + mc = self.model_config + self.override_max_tokens = ( + self.default_sampling_params.get("max_tokens") + if mc.generation_config not in ("auto", "vllm") + else getattr(mc, "override_generation_config", {}).get("max_new_tokens") + ) + async def serve_tokens( self, request: GenerateRequest, @@ -150,6 +162,20 @@ class ServingTokens(OpenAIServing): # Schedule the request and get the result generator. result_generator: AsyncGenerator[RequestOutput, None] | None = None sampling_params = request.sampling_params + + # Apply server-side ``max_tokens`` defaulting when the client did + # not set it, matching the OpenAI-compat endpoints. ``SamplingParams`` + # defaults ``max_tokens`` to 16, which would otherwise silently cap + # every generation that omits the field. + if not request.is_sampling_param_provided("max_tokens"): + sampling_params.max_tokens = get_max_tokens( + max_model_len=self.model_config.max_model_len, + max_tokens=None, + input_length=self._extract_prompt_len(engine_input), + default_sampling_params=self.default_sampling_params, + override_max_tokens=self.override_max_tokens, + ) + if self.force_no_detokenize: sampling_params.detokenize = False if request.stream: