From f339e0e7571c0bd49d6d5f1acb9f21a35b65e4ed Mon Sep 17 00:00:00 2001 From: Denis Ermakov Date: Fri, 25 Sep 2026 18:33:19 +0200 Subject: [PATCH] [Bugfix][Frontend] Keep logprobs of parser-suppressed streaming chunks (#58583) Signed-off-by: errmakov Co-authored-by: Yanxiao Zhao <39199723+sdpkjc@users.noreply.github.com> Co-authored-by: Prakhar Agarwal <270064960+agarwalprakhar2511@users.noreply.github.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> --- .../chat_completion/test_serving_chat.py | 66 +++++++++++++++++++ .../openai/chat_completion/serving.py | 10 +-- 2 files changed, 72 insertions(+), 4 deletions(-) diff --git a/tests/entrypoints/openai/chat_completion/test_serving_chat.py b/tests/entrypoints/openai/chat_completion/test_serving_chat.py index 6f580a9c28..6c85b0876a 100644 --- a/tests/entrypoints/openai/chat_completion/test_serving_chat.py +++ b/tests/entrypoints/openai/chat_completion/test_serving_chat.py @@ -44,6 +44,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import get_encoding from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.exceptions import QueueOverflowError, VLLMValidationError from vllm.inputs import TokensPrompt +from vllm.logprobs import Logprob from vllm.multimodal.inputs import PlaceholderRange from vllm.outputs import CompletionOutput, RequestOutput from vllm.parser import HarmonyParser @@ -845,6 +846,71 @@ async def test_streaming_reasoning_usage_counts_across_deltas(): } +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("include_reasoning", "expected_tokens"), + [(True, ["", "answer"]), (False, [])], +) +async def test_streaming_logprobs_kept_when_parser_suppresses_delta( + include_reasoning: bool, expected_tokens: list[str] +): + """Logprobs of tokens the parser swallows (e.g. ) must still + be streamed, unless per-token metadata is hidden with the reasoning.""" + serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False) + serving.model_config = None + serving.return_tokens_as_token_ids = False + + parser = MagicMock() + parser.parse_delta.side_effect = [None, DeltaMessage(content="answer")] + parser.count_reasoning_tokens.return_value = 0 + serving.parser_cls = MagicMock(return_value=parser) + + def make_output(token_id: int, text: str, finish_reason: str | None): + request_output = _make_metrics_request_output( + metrics=None, token_ids=(token_id,) + ) + completion = request_output.outputs[0] + completion.text = text + completion.finish_reason = finish_reason + completion.logprobs = [ + {token_id: Logprob(logprob=-0.5, rank=1, decoded_token=text)} + ] + return request_output + + request = ChatCompletionRequest( + model="test-model", + messages=[{"role": "user", "content": "Test prompt"}], + max_tokens=10, + stream=True, + logprobs=True, + include_reasoning=include_reasoning, + ) + chunks: list[dict[str, Any]] = [] + async for line in serving.chat_completion_stream_generator( + request, + _stream_request_outputs( + make_output(10, "", None), make_output(11, "answer", "stop") + ), + "chatcmpl-test-id", + "test-model", + conversation=[{"role": "user", "content": "Test"}], + tokenizer=MagicMock(), + request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"), + ): + payload = line.removeprefix("data: ").strip() + if payload != "[DONE]": + chunks.append(json.loads(payload)) + + streamed_tokens = [ + entry["token"] + for chunk in chunks + for choice in chunk["choices"] + if choice.get("logprobs") + for entry in choice["logprobs"]["content"] + ] + assert streamed_tokens == expected_tokens + + @dataclass class MockEngine: model_config: MockModelConfig = field(default_factory=MockModelConfig) diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 45ea3203cd..61a7ac25ed 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -693,11 +693,13 @@ class OpenAIServingChat(GenerateBaseServing): logprobs = None if delta_message is None: - # NOTE: If return_token_ids is enabled, we still need to - # send a chunk with token_ids even if delta_message is None + # NOTE: If return_token_ids or logprobs are enabled, we + # still need to send a chunk even if delta_message is None # to ensure all tokens are included in the response - if output.finish_reason is None and ( - not request.return_token_ids or hide_stream_metadata + if ( + output.finish_reason is None + and logprobs is None + and (not request.return_token_ids or hide_stream_metadata) ): continue delta_message = DeltaMessage()