[Bugfix][Frontend] Keep logprobs of parser-suppressed streaming chunks (#58583)
Signed-off-by: errmakov <ide404@gmail.com> Co-authored-by: Yanxiao Zhao <39199723+sdpkjc@users.noreply.github.com> Co-authored-by: Prakhar Agarwal <270064960+agarwalprakhar2511@users.noreply.github.com> Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
co-authored by
Yanxiao Zhao
Prakhar Agarwal
Flora Feng
parent
6491f481a7
commit
f339e0e757
@@ -44,6 +44,7 @@ from vllm.entrypoints.openai.parser.harmony_utils import get_encoding
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.exceptions import QueueOverflowError, VLLMValidationError
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.multimodal.inputs import PlaceholderRange
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.parser import HarmonyParser
|
||||
@@ -845,6 +846,71 @@ async def test_streaming_reasoning_usage_counts_across_deltas():
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("include_reasoning", "expected_tokens"),
|
||||
[(True, ["<tool_call>", "answer"]), (False, [])],
|
||||
)
|
||||
async def test_streaming_logprobs_kept_when_parser_suppresses_delta(
|
||||
include_reasoning: bool, expected_tokens: list[str]
|
||||
):
|
||||
"""Logprobs of tokens the parser swallows (e.g. <tool_call>) must still
|
||||
be streamed, unless per-token metadata is hidden with the reasoning."""
|
||||
serving = _build_minimal_metrics_serving_chat(enable_per_request_metrics=False)
|
||||
serving.model_config = None
|
||||
serving.return_tokens_as_token_ids = False
|
||||
|
||||
parser = MagicMock()
|
||||
parser.parse_delta.side_effect = [None, DeltaMessage(content="answer")]
|
||||
parser.count_reasoning_tokens.return_value = 0
|
||||
serving.parser_cls = MagicMock(return_value=parser)
|
||||
|
||||
def make_output(token_id: int, text: str, finish_reason: str | None):
|
||||
request_output = _make_metrics_request_output(
|
||||
metrics=None, token_ids=(token_id,)
|
||||
)
|
||||
completion = request_output.outputs[0]
|
||||
completion.text = text
|
||||
completion.finish_reason = finish_reason
|
||||
completion.logprobs = [
|
||||
{token_id: Logprob(logprob=-0.5, rank=1, decoded_token=text)}
|
||||
]
|
||||
return request_output
|
||||
|
||||
request = ChatCompletionRequest(
|
||||
model="test-model",
|
||||
messages=[{"role": "user", "content": "Test prompt"}],
|
||||
max_tokens=10,
|
||||
stream=True,
|
||||
logprobs=True,
|
||||
include_reasoning=include_reasoning,
|
||||
)
|
||||
chunks: list[dict[str, Any]] = []
|
||||
async for line in serving.chat_completion_stream_generator(
|
||||
request,
|
||||
_stream_request_outputs(
|
||||
make_output(10, "<tool_call>", None), make_output(11, "answer", "stop")
|
||||
),
|
||||
"chatcmpl-test-id",
|
||||
"test-model",
|
||||
conversation=[{"role": "user", "content": "Test"}],
|
||||
tokenizer=MagicMock(),
|
||||
request_metadata=RequestResponseMetadata(request_id="chatcmpl-test-id"),
|
||||
):
|
||||
payload = line.removeprefix("data: ").strip()
|
||||
if payload != "[DONE]":
|
||||
chunks.append(json.loads(payload))
|
||||
|
||||
streamed_tokens = [
|
||||
entry["token"]
|
||||
for chunk in chunks
|
||||
for choice in chunk["choices"]
|
||||
if choice.get("logprobs")
|
||||
for entry in choice["logprobs"]["content"]
|
||||
]
|
||||
assert streamed_tokens == expected_tokens
|
||||
|
||||
|
||||
@dataclass
|
||||
class MockEngine:
|
||||
model_config: MockModelConfig = field(default_factory=MockModelConfig)
|
||||
|
||||
@@ -693,11 +693,13 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
logprobs = None
|
||||
|
||||
if delta_message is None:
|
||||
# NOTE: If return_token_ids is enabled, we still need to
|
||||
# send a chunk with token_ids even if delta_message is None
|
||||
# NOTE: If return_token_ids or logprobs are enabled, we
|
||||
# still need to send a chunk even if delta_message is None
|
||||
# to ensure all tokens are included in the response
|
||||
if output.finish_reason is None and (
|
||||
not request.return_token_ids or hide_stream_metadata
|
||||
if (
|
||||
output.finish_reason is None
|
||||
and logprobs is None
|
||||
and (not request.return_token_ids or hide_stream_metadata)
|
||||
):
|
||||
continue
|
||||
delta_message = DeltaMessage()
|
||||
|
||||
Reference in New Issue
Block a user