[Bugfix][Frontend] Avoid generation for empty streaming input (#59015)

Signed-off-by: Benedikt Falk <bfalk@nvidia.com>
This commit is contained in:
Benedikt Falk
2026-10-02 18:48:43 +00:00
committed by GitHub
parent 10f6ab01a9
commit c4973d94fe
3 changed files with 41 additions and 8 deletions
+2 -4
View File
@@ -491,7 +491,7 @@ async def test_streaming_input_per_chunk_sampling_params(engine: AsyncLLM):
async def test_streaming_input_empty_generator(engine: AsyncLLM):
"""Test behavior when the input generator yields nothing.
An empty generator should still produce a finished output.
An empty generator should finish without producing output.
"""
request_id = "test_empty_generator"
sampling_params = get_sampling_params(max_tokens=10)
@@ -505,9 +505,7 @@ async def test_streaming_input_empty_generator(engine: AsyncLLM):
async for output in engine.generate(empty_generator(), sampling_params, request_id):
outputs.append(output)
# Should still get a finished marker
assert len(outputs) >= 1, "Should receive at least one output"
assert outputs[-1].finished, "Should have a finished output"
assert outputs == []
@pytest.mark.asyncio(loop_scope="module")
@@ -8,7 +8,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
from vllm.engine.protocol import StreamingInput
from vllm.outputs import RequestOutput
from vllm.outputs import STREAM_FINISHED, RequestOutput
from vllm.sampling_params import RequestOutputKind, SamplingParams
from vllm.v1.engine.async_llm import AsyncLLM
from vllm.v1.engine.output_processor import RequestOutputCollector
@@ -170,3 +170,31 @@ async def test_generate_with_async_generator():
assert outputs[2].finished is True
# Both inputs were processed
assert inputs_received == ["Hello", " world"]
@pytest.mark.asyncio
async def test_empty_input_stream_finishes_without_engine_request():
llm = AsyncLLM.__new__(AsyncLLM)
llm._validate_streaming_input_sampling_params = MagicMock()
llm.get_supported_tasks = AsyncMock(return_value=("generate",))
llm._run_output_handler = MagicMock()
llm._add_request = AsyncMock()
final_req = MagicMock()
final_req.request_id = "request-int"
llm.input_processor = MagicMock()
llm.input_processor.process_inputs.return_value = final_req
async def empty_stream():
return
yield # pragma: no cover
queue = await llm._add_streaming_input_request(
"request", empty_stream(), SamplingParams(max_tokens=4)
)
task = queue._input_stream_task
assert task is not None
await task
assert queue.get_nowait() is STREAM_FINISHED
llm._add_request.assert_not_awaited()
+10 -3
View File
@@ -599,6 +599,8 @@ class AsyncLLM(EngineClient):
async def handle_inputs():
cancelled = False
errored = False
any_added = False
try:
async for input_chunk in input_stream:
sp = input_chunk.sampling_params
@@ -623,18 +625,23 @@ class AsyncLLM(EngineClient):
self.model_config, input_chunk.prompt
)
await self._add_request(req, prompt_text, None, 0, queue)
any_added = True
except (asyncio.CancelledError, GeneratorExit):
cancelled = True
except Exception as error:
# Wrap in InputStreamError so generate() can propagate it
# without wrapping in EngineGenerateError.
queue.put(InputStreamError(error))
errored = True
finally:
queue._input_stream_task = None
if not cancelled:
# Send empty final request to indicate that inputs have
# finished. Don't send if cancelled (session was aborted).
await self._add_request(final_req, None, None, 0, queue)
if any_added:
# Send empty final request to indicate that inputs have
# finished. Don't send if cancelled (session was aborted).
await self._add_request(final_req, None, None, 0, queue)
elif not errored:
queue.put(STREAM_FINISHED)
# Ensure output handler is running.
self._run_output_handler()