From c4973d94fed25a3bf32d601873755a9bcd951886 Mon Sep 17 00:00:00 2001 From: Benedikt Falk Date: Fri, 2 Oct 2026 20:48:43 +0200 Subject: [PATCH] [Bugfix][Frontend] Avoid generation for empty streaming input (#59015) Signed-off-by: Benedikt Falk --- tests/v1/e2e/general/test_streaming_input.py | 6 ++-- .../test_async_llm_streaming.py | 30 ++++++++++++++++++- vllm/v1/engine/async_llm.py | 13 ++++++-- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/tests/v1/e2e/general/test_streaming_input.py b/tests/v1/e2e/general/test_streaming_input.py index c71e7ad0a6..5a226e75b1 100644 --- a/tests/v1/e2e/general/test_streaming_input.py +++ b/tests/v1/e2e/general/test_streaming_input.py @@ -491,7 +491,7 @@ async def test_streaming_input_per_chunk_sampling_params(engine: AsyncLLM): async def test_streaming_input_empty_generator(engine: AsyncLLM): """Test behavior when the input generator yields nothing. - An empty generator should still produce a finished output. + An empty generator should finish without producing output. """ request_id = "test_empty_generator" sampling_params = get_sampling_params(max_tokens=10) @@ -505,9 +505,7 @@ async def test_streaming_input_empty_generator(engine: AsyncLLM): async for output in engine.generate(empty_generator(), sampling_params, request_id): outputs.append(output) - # Should still get a finished marker - assert len(outputs) >= 1, "Should receive at least one output" - assert outputs[-1].finished, "Should have a finished output" + assert outputs == [] @pytest.mark.asyncio(loop_scope="module") diff --git a/tests/v1/streaming_input/test_async_llm_streaming.py b/tests/v1/streaming_input/test_async_llm_streaming.py index b532eed15f..21a49024ce 100644 --- a/tests/v1/streaming_input/test_async_llm_streaming.py +++ b/tests/v1/streaming_input/test_async_llm_streaming.py @@ -8,7 +8,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest from vllm.engine.protocol import StreamingInput -from vllm.outputs import RequestOutput +from vllm.outputs import STREAM_FINISHED, RequestOutput from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.v1.engine.async_llm import AsyncLLM from vllm.v1.engine.output_processor import RequestOutputCollector @@ -170,3 +170,31 @@ async def test_generate_with_async_generator(): assert outputs[2].finished is True # Both inputs were processed assert inputs_received == ["Hello", " world"] + + +@pytest.mark.asyncio +async def test_empty_input_stream_finishes_without_engine_request(): + llm = AsyncLLM.__new__(AsyncLLM) + llm._validate_streaming_input_sampling_params = MagicMock() + llm.get_supported_tasks = AsyncMock(return_value=("generate",)) + llm._run_output_handler = MagicMock() + llm._add_request = AsyncMock() + + final_req = MagicMock() + final_req.request_id = "request-int" + llm.input_processor = MagicMock() + llm.input_processor.process_inputs.return_value = final_req + + async def empty_stream(): + return + yield # pragma: no cover + + queue = await llm._add_streaming_input_request( + "request", empty_stream(), SamplingParams(max_tokens=4) + ) + task = queue._input_stream_task + assert task is not None + await task + + assert queue.get_nowait() is STREAM_FINISHED + llm._add_request.assert_not_awaited() diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index b1cc2edbc4..4a7a91cb8e 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -599,6 +599,8 @@ class AsyncLLM(EngineClient): async def handle_inputs(): cancelled = False + errored = False + any_added = False try: async for input_chunk in input_stream: sp = input_chunk.sampling_params @@ -623,18 +625,23 @@ class AsyncLLM(EngineClient): self.model_config, input_chunk.prompt ) await self._add_request(req, prompt_text, None, 0, queue) + any_added = True except (asyncio.CancelledError, GeneratorExit): cancelled = True except Exception as error: # Wrap in InputStreamError so generate() can propagate it # without wrapping in EngineGenerateError. queue.put(InputStreamError(error)) + errored = True finally: queue._input_stream_task = None if not cancelled: - # Send empty final request to indicate that inputs have - # finished. Don't send if cancelled (session was aborted). - await self._add_request(final_req, None, None, 0, queue) + if any_added: + # Send empty final request to indicate that inputs have + # finished. Don't send if cancelled (session was aborted). + await self._add_request(final_req, None, None, 0, queue) + elif not errored: + queue.put(STREAM_FINISHED) # Ensure output handler is running. self._run_output_handler()