[Bugfix][Frontend] Keep logprobs of parser-suppressed streaming chunks (#58583)

Signed-off-by: errmakov <ide404@gmail.com>
Co-authored-by: Yanxiao Zhao <39199723+sdpkjc@users.noreply.github.com>
Co-authored-by: Prakhar Agarwal <270064960+agarwalprakhar2511@users.noreply.github.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Denis Ermakov
2026-09-25 16:33:19 +00:00
committed by GitHub
co-authored by Yanxiao Zhao Prakhar Agarwal Flora Feng
parent 6491f481a7
commit f339e0e757
2 changed files with 72 additions and 4 deletions
@@ -44,6 +44,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.exceptions import QueueOverflowError, VLLMValidationError
from vllm.inputs import TokensPrompt
from vllm.logprobs import Logprob
from vllm.multimodal.inputs import PlaceholderRange
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.parser import HarmonyParser
@@ -845,6 +846,71 @@ async def test_streaming_reasoning_usage_counts_across_deltas():
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
("include_reasoning", "expected_tokens"),
[(True, ["<tool_call>", "answer"]), (False, [])],
)
async def test_streaming_logprobs_kept_when_parser_suppresses_delta(
include_reasoning: bool, expected_tokens: list[str]
):
"""Logprobs of tokens the parser swallows (e.g. <tool_call>) must still
be streamed, unless per-token metadata is hidden with the reasoning."""
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
serving.model_config = None
serving.return_tokens_as_token_ids = False
parser = MagicMock()
parser.parse_delta.side_effect = [None, DeltaMessage(content="answer")]
parser.count_reasoning_tokens.return_value = 0
serving.parser_cls = MagicMock(return_value=parser)
def make_output(token_id: int, text: str, finish_reason: str | None):
request_output = _make_metrics_request_output(
metrics=None, token_ids=(token_id,)
)
completion = request_output.outputs[0]
completion.text = text
completion.finish_reason = finish_reason
completion.logprobs = [
{token_id: Logprob(logprob=-0.5, rank=1, decoded_token=text)}
]
return request_output
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Test prompt"}],
max_tokens=10,
stream=True,
logprobs=True,
include_reasoning=include_reasoning,
)
chunks: list[dict[str, Any]] = []
async for line in serving.chat_completion_stream_generator(
request,
_stream_request_outputs(
make_output(10, "<tool_call>", None), make_output(11, "answer", "stop")
),
"chatcmpl-test-id",
"test-model",
conversation=[{"role": "user", "content": "Test"}],
tokenizer=MagicMock(),
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
):
payload = line.removeprefix("data: ").strip()
if payload != "[DONE]":
chunks.append(json.loads(payload))
streamed_tokens = [
entry["token"]
for chunk in chunks
for choice in chunk["choices"]
if choice.get("logprobs")
for entry in choice["logprobs"]["content"]
]
assert streamed_tokens == expected_tokens
@dataclass
class MockEngine:
model_config: MockModelConfig = field(default_factory=MockModelConfig)
@@ -693,11 +693,13 @@ class OpenAIServingChat(GenerateBaseServing):
logprobs = None
if delta_message is None:
# NOTE: If return_token_ids is enabled, we still need to
# send a chunk with token_ids even if delta_message is None
# NOTE: If return_token_ids or logprobs are enabled, we
# still need to send a chunk even if delta_message is None
# to ensure all tokens are included in the response
if output.finish_reason is None and (
not request.return_token_ids or hide_stream_metadata
if (
output.finish_reason is None
and logprobs is None
and (not request.return_token_ids or hide_stream_metadata)
):
continue
delta_message = DeltaMessage()