[Bugfix][Frontend] Return all choices from /inference/v1/generate when n > 1 (#52399)
Signed-off-by: Quentin Gallouédec <quentin.gallouedec@huggingface.co>
This commit is contained in:
@@ -183,6 +183,30 @@ async def test_generate_defaults_max_tokens_when_omitted(client):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
@pytest.mark.skipif(
|
||||||
|
envs.VLLM_USE_RUST_FRONTEND,
|
||||||
|
reason="parallel sampling (n > 1) is not supported by the Rust frontend",
|
||||||
|
)
|
||||||
|
async def test_generate_returns_all_choices_when_n_greater_than_one(client):
|
||||||
|
"""Regression: ``n > 1`` must return ``n`` choices.
|
||||||
|
|
||||||
|
Non-streaming requests kept ``SamplingParams``' default output kind,
|
||||||
|
``CUMULATIVE``, so only the sequences updated during the last engine step
|
||||||
|
reached the response and the others were silently dropped.
|
||||||
|
"""
|
||||||
|
payload = {
|
||||||
|
"model": MODEL_NAME,
|
||||||
|
"token_ids": [1, 2, 3],
|
||||||
|
"sampling_params": {"max_tokens": 5, "temperature": 1.0, "n": 4},
|
||||||
|
"stream": False,
|
||||||
|
}
|
||||||
|
resp = await client.post(GEN_ENDPOINT, json=payload)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
assert sorted(choice["index"] for choice in data["choices"]) == [0, 1, 2, 3]
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_generate_stream(client):
|
async def test_generate_stream(client):
|
||||||
payload = {
|
payload = {
|
||||||
|
|||||||
@@ -218,8 +218,9 @@ class ServingTokens(GenerateBaseServing):
|
|||||||
|
|
||||||
if self.force_no_detokenize:
|
if self.force_no_detokenize:
|
||||||
sampling_params.detokenize = False
|
sampling_params.detokenize = False
|
||||||
if request.stream:
|
sampling_params.output_kind = (
|
||||||
sampling_params.output_kind = RequestOutputKind.DELTA
|
RequestOutputKind.DELTA if request.stream else RequestOutputKind.FINAL_ONLY
|
||||||
|
)
|
||||||
|
|
||||||
self._log_inputs(
|
self._log_inputs(
|
||||||
request_id,
|
request_id,
|
||||||
|
|||||||
Reference in New Issue
Block a user