[Frontend] Attach resolved logprobs to streaming derender chunks (#55029)
Signed-off-by: Shimi Bandiel <shimib@google.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
@@ -368,9 +369,30 @@ class OnlineDerenderer:
|
||||
tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special
|
||||
)
|
||||
|
||||
# NOTE: parser-configured servers dispatch to
|
||||
# _derender_chat_stream_parsed above and never reach this plain
|
||||
# path. That parsed path does not resolve logprobs yet; when it
|
||||
# does, mirror the generate chat streaming path, which suppresses
|
||||
# logprobs entirely when a parser is configured and reasoning is
|
||||
# hidden, because decoded logprob token text would leak hidden
|
||||
# reasoning.
|
||||
resolved_logprobs = None
|
||||
if choice.logprobs is not None:
|
||||
resolved_logprobs = _resolve_logprobs(
|
||||
choice.logprobs,
|
||||
tokenizer,
|
||||
initial_context_token_ids=state.logprob_context_token_ids,
|
||||
)
|
||||
|
||||
include_role = not updated_state.role_sent
|
||||
if include_role:
|
||||
updated_state = updated_state.model_copy(update={"role_sent": True})
|
||||
updated_state = updated_state.model_copy(
|
||||
update={
|
||||
"role_sent": True,
|
||||
"logprob_context_token_ids": _logprob_context_tail(
|
||||
state.logprob_context_token_ids, delta_tids
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
delta = DeltaMessage(
|
||||
role="assistant" if include_role else None,
|
||||
@@ -380,6 +402,7 @@ class OnlineDerenderer:
|
||||
ChatCompletionResponseStreamChoice(
|
||||
index=choice.index,
|
||||
delta=delta,
|
||||
logprobs=resolved_logprobs,
|
||||
finish_reason=choice.finish_reason,
|
||||
)
|
||||
)
|
||||
@@ -721,10 +744,32 @@ class OnlineDerenderer:
|
||||
new_text, updated_state = await self._detokenize_delta_async(
|
||||
tokenizer, delta_tids, updated_state, skip_special_tokens=skip_special
|
||||
)
|
||||
|
||||
completion_logprobs = None
|
||||
if choice.logprobs is not None:
|
||||
resolved = _resolve_logprobs(
|
||||
choice.logprobs,
|
||||
tokenizer,
|
||||
initial_context_token_ids=state.logprob_context_token_ids,
|
||||
)
|
||||
completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
|
||||
resolved, initial_text_offset=state.logprob_text_offset
|
||||
)
|
||||
|
||||
updated_state = updated_state.model_copy(
|
||||
update={
|
||||
"logprob_context_token_ids": _logprob_context_tail(
|
||||
state.logprob_context_token_ids, delta_tids
|
||||
),
|
||||
"logprob_text_offset": state.logprob_text_offset + len(new_text),
|
||||
}
|
||||
)
|
||||
|
||||
stream_choices.append(
|
||||
CompletionResponseStreamChoice(
|
||||
index=choice.index,
|
||||
text=new_text,
|
||||
logprobs=completion_logprobs,
|
||||
finish_reason=choice.finish_reason,
|
||||
)
|
||||
)
|
||||
@@ -750,6 +795,18 @@ class OnlineDerenderer:
|
||||
return chunk, updated_state
|
||||
|
||||
|
||||
# Number of preceding sampled token IDs `_correct_decoded_token` reads to
|
||||
# repair U+FFFD from byte-fallback tokenization.
|
||||
_LOGPROB_CONTEXT_WINDOW = 4
|
||||
|
||||
|
||||
def _logprob_context_tail(
|
||||
context_token_ids: list[int], delta_token_ids: list[int]
|
||||
) -> list[int]:
|
||||
"""Advance the carried logprob context by this chunk's sampled tokens."""
|
||||
return (list(context_token_ids) + list(delta_token_ids))[-_LOGPROB_CONTEXT_WINDOW:]
|
||||
|
||||
|
||||
def _parse_token_id_placeholder(token: str) -> int | None:
|
||||
"""Extract token ID from a 'token_id:N' placeholder string."""
|
||||
if not token.startswith("token_id:"):
|
||||
@@ -799,13 +856,20 @@ def _correct_decoded_token(
|
||||
|
||||
|
||||
def _resolve_logprobs(
|
||||
logprobs: ChatCompletionLogProbs, tokenizer: TokenizerLike
|
||||
logprobs: ChatCompletionLogProbs,
|
||||
tokenizer: TokenizerLike,
|
||||
initial_context_token_ids: Sequence[int] = (),
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Resolve token_id:N placeholders in a ChatCompletionLogProbs object."""
|
||||
"""Resolve token_id:N placeholders in a ChatCompletionLogProbs object.
|
||||
|
||||
``initial_context_token_ids`` seeds the byte-fallback correction context
|
||||
with sampled IDs from preceding chunks (streaming), so multi-byte
|
||||
characters split across chunk boundaries still resolve.
|
||||
"""
|
||||
if logprobs.content is None:
|
||||
return logprobs
|
||||
|
||||
context_token_ids: list[int] = []
|
||||
context_token_ids: list[int] = list(initial_context_token_ids)
|
||||
resolved_content = []
|
||||
|
||||
for entry in logprobs.content:
|
||||
@@ -845,9 +909,14 @@ def _resolve_logprobs(
|
||||
|
||||
def _convert_chat_logprobs_to_completion_logprobs(
|
||||
logprobs: ChatCompletionLogProbs,
|
||||
initial_text_offset: int = 0,
|
||||
) -> CompletionLogProbs:
|
||||
"""Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
|
||||
(parallel flat lists) as required by the /v1/completions response schema."""
|
||||
(parallel flat lists) as required by the /v1/completions response schema.
|
||||
|
||||
``initial_text_offset`` keeps ``text_offset`` absolute across streaming
|
||||
chunks, mirroring the generate streaming path.
|
||||
"""
|
||||
if logprobs.content is None:
|
||||
return CompletionLogProbs()
|
||||
|
||||
@@ -856,7 +925,7 @@ def _convert_chat_logprobs_to_completion_logprobs(
|
||||
top_logprobs_list: list[dict[str, float] | None] = []
|
||||
text_offset: list[int] = []
|
||||
|
||||
offset = 0
|
||||
offset = initial_text_offset
|
||||
for entry in logprobs.content:
|
||||
text_offset.append(offset)
|
||||
tokens.append(entry.token)
|
||||
|
||||
Reference in New Issue
Block a user