From 92bdee05cb4ea5e94c4cce3eb0544f51d6eece8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Quentin=20Gallou=C3=A9dec?= <45557362+qgallouedec@users.noreply.github.com> Date: Wed, 19 Aug 2026 08:48:45 -0400 Subject: [PATCH] [Bugfix][Frontend] Return all choices from /inference/v1/generate when n > 1 (#52399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Quentin Gallouédec --- .../token_in_token_out/test_serving_tokens.py | 24 +++++++++++++++++++ .../scale_out/token_in_token_out/serving.py | 5 ++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py index a1491c1c3e..be33a182bc 100644 --- a/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py +++ b/tests/entrypoints/scale_out/token_in_token_out/test_serving_tokens.py @@ -183,6 +183,30 @@ async def test_generate_defaults_max_tokens_when_omitted(client): ) +@pytest.mark.asyncio +@pytest.mark.skipif( + envs.VLLM_USE_RUST_FRONTEND, + reason="parallel sampling (n > 1) is not supported by the Rust frontend", +) +async def test_generate_returns_all_choices_when_n_greater_than_one(client): + """Regression: ``n > 1`` must return ``n`` choices. + + Non-streaming requests kept ``SamplingParams``' default output kind, + ``CUMULATIVE``, so only the sequences updated during the last engine step + reached the response and the others were silently dropped. + """ + payload = { + "model": MODEL_NAME, + "token_ids": [1, 2, 3], + "sampling_params": {"max_tokens": 5, "temperature": 1.0, "n": 4}, + "stream": False, + } + resp = await client.post(GEN_ENDPOINT, json=payload) + resp.raise_for_status() + data = resp.json() + assert sorted(choice["index"] for choice in data["choices"]) == [0, 1, 2, 3] + + @pytest.mark.asyncio async def test_generate_stream(client): payload = { diff --git a/vllm/entrypoints/scale_out/token_in_token_out/serving.py b/vllm/entrypoints/scale_out/token_in_token_out/serving.py index 9e9ace877a..fbbb8052be 100644 --- a/vllm/entrypoints/scale_out/token_in_token_out/serving.py +++ b/vllm/entrypoints/scale_out/token_in_token_out/serving.py @@ -218,8 +218,9 @@ class ServingTokens(GenerateBaseServing): if self.force_no_detokenize: sampling_params.detokenize = False - if request.stream: - sampling_params.output_kind = RequestOutputKind.DELTA + sampling_params.output_kind = ( + RequestOutputKind.DELTA if request.stream else RequestOutputKind.FINAL_ONLY + ) self._log_inputs( request_id,