[Frontend] Apply each endpoint's top_logprobs cut in derender (#59518)
Signed-off-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Yin Li <kevinli.ai.work@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Yin Li
Cyrus Leung
parent
011fa68ca9
commit
3dcfaeb4e8
@@ -356,6 +356,32 @@ def decode_token_ids(
|
||||
return out
|
||||
|
||||
|
||||
def chat_top_logprobs_limit(
|
||||
top_logprobs: int | None, return_all: bool = False
|
||||
) -> int | None:
|
||||
"""How many of a position's logprob candidates `/v1/chat/completions`
|
||||
returns, in the engine's order (sampled token first): the first
|
||||
``top_logprobs``, none when it is unset, and all of them (``None``) for
|
||||
``top_logprobs=-1`` or ``logprob_token_ids`` (``return_all``). Shared with
|
||||
chat derender so the two cannot drift."""
|
||||
if return_all or top_logprobs == -1:
|
||||
return None
|
||||
return top_logprobs or 0
|
||||
|
||||
|
||||
def completion_top_logprobs_limit(
|
||||
logprobs: int | None, return_all: bool = False
|
||||
) -> int | None:
|
||||
"""How many of a position's logprob candidates `/v1/completions` returns:
|
||||
the first ``logprobs + 1`` (the OpenAI completions convention), and all of
|
||||
them (``None``) for ``logprobs=-1``, ``logprob_token_ids`` (``return_all``)
|
||||
or when ``logprobs`` is unset. Shared with completions derender so the two
|
||||
cannot drift."""
|
||||
if return_all or logprobs is None or logprobs == -1:
|
||||
return None
|
||||
return max(logprobs + 1, 0)
|
||||
|
||||
|
||||
def clamp_prompt_logprobs(
|
||||
prompt_logprobs: PromptLogprobs | None,
|
||||
) -> PromptLogprobs | None:
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
@@ -22,6 +23,7 @@ from vllm.entrypoints.generate.base.serving import (
|
||||
GenerateBaseServing,
|
||||
build_per_request_timing_metrics,
|
||||
build_spec_decoding_metrics,
|
||||
chat_top_logprobs_limit,
|
||||
clamp_prompt_logprobs,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
@@ -1231,10 +1233,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
logprob=max(p[1].logprob, -9999.0),
|
||||
bytes=list(token.encode("utf-8", errors="replace")),
|
||||
)
|
||||
for i, p in enumerate(logprobs.items())
|
||||
if return_all
|
||||
or top_logprobs == -1
|
||||
or (top_logprobs is not None and i < top_logprobs)
|
||||
for p in itertools.islice(
|
||||
logprobs.items(), chat_top_logprobs_limit(top_logprobs, return_all)
|
||||
)
|
||||
]
|
||||
|
||||
def _create_chat_logprobs(
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, AsyncIterator
|
||||
from collections.abc import Sequence as GenericSequence
|
||||
@@ -19,6 +20,7 @@ from vllm.entrypoints.generate.base.serving import (
|
||||
build_per_request_timing_metrics,
|
||||
build_spec_decoding_metrics,
|
||||
clamp_prompt_logprobs,
|
||||
completion_top_logprobs_limit,
|
||||
format_token_id_placeholder,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import (
|
||||
@@ -721,10 +723,12 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
tokenizer,
|
||||
return_as_token_id=should_return_as_token_id,
|
||||
): max(top_lp[1].logprob, -9999.0)
|
||||
for i, top_lp in enumerate(step_top_logprobs.items())
|
||||
if logprob_token_ids
|
||||
or num_output_top_logprobs == -1
|
||||
or num_output_top_logprobs >= i
|
||||
for top_lp in itertools.islice(
|
||||
step_top_logprobs.items(),
|
||||
completion_top_logprobs_limit(
|
||||
num_output_top_logprobs, bool(logprob_token_ids)
|
||||
),
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -424,14 +424,12 @@ class ServingTokens(GenerateBaseServing):
|
||||
logprobs = self._create_text_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
else:
|
||||
logprobs = self._create_tokens_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
@@ -599,14 +597,12 @@ class ServingTokens(GenerateBaseServing):
|
||||
logprobs = self._create_text_logprobs(
|
||||
token_ids=delta_token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
tokenizer=tokenizer,
|
||||
)
|
||||
else:
|
||||
logprobs = self._create_tokens_logprobs(
|
||||
token_ids=delta_token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=sampling_params.logprobs,
|
||||
)
|
||||
else:
|
||||
logprobs = None
|
||||
@@ -728,14 +724,15 @@ class ServingTokens(GenerateBaseServing):
|
||||
self,
|
||||
token_ids: GenericSequence[int],
|
||||
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
||||
num_output_top_logprobs: int | None = None,
|
||||
tokenizer: TokenizerLike | None = None,
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Create OpenAI-style logprobs for ``output_mode="text"``.
|
||||
|
||||
With a ``tokenizer`` the tokens are decoded strings and carry ``bytes``;
|
||||
without one (``--return-tokens-as-token-ids``) they are ``token_id:N``
|
||||
placeholders.
|
||||
placeholders. Like tokens mode, ``top_logprobs`` carries every candidate
|
||||
the engine returned (the sampled token first, then the top k), not cut
|
||||
to the requested count.
|
||||
"""
|
||||
logprobs_content: list[ChatCompletionLogProbsContent] = []
|
||||
|
||||
@@ -757,14 +754,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
bytes=token_bytes,
|
||||
top_logprobs=[
|
||||
_top_logprob(top_id, logprob, tokenizer)
|
||||
for i, (top_id, logprob) in enumerate(
|
||||
step_top_logprobs.items()
|
||||
)
|
||||
if num_output_top_logprobs is not None
|
||||
and (
|
||||
num_output_top_logprobs == -1
|
||||
or i < max(num_output_top_logprobs, 1)
|
||||
)
|
||||
for top_id, logprob in step_top_logprobs.items()
|
||||
],
|
||||
)
|
||||
)
|
||||
@@ -775,10 +765,16 @@ class ServingTokens(GenerateBaseServing):
|
||||
self,
|
||||
token_ids: GenericSequence[int],
|
||||
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
||||
num_output_top_logprobs: int | None = None,
|
||||
) -> GenerateLogProbs:
|
||||
"""Create generate-shaped logprobs (integer token ids, no tokenizer).
|
||||
|
||||
``top_logprobs`` carries every candidate the engine returned for the
|
||||
position: the sampled token first, then the top k, so k entries when the
|
||||
sampled token is in the top k and k + 1 when it is not. It is not cut to
|
||||
the requested count: `/v1/chat/completions` and
|
||||
`/v1/completions` cut differently, and derender applies each endpoint's
|
||||
cut the way the coupled server does.
|
||||
|
||||
The engine reports rank 0 for a sampled token whose logprob is NaN; that
|
||||
is not a rank, so it is sent as ``None`` (the logprob is clamped).
|
||||
"""
|
||||
@@ -806,14 +802,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
logprob=_clamp_logprob(top_logprob.logprob),
|
||||
rank=top_logprob.rank or None,
|
||||
)
|
||||
for rank_index, (top_token_id, top_logprob) in enumerate(
|
||||
step_top_logprobs.items()
|
||||
)
|
||||
if num_output_top_logprobs is not None
|
||||
and (
|
||||
num_output_top_logprobs == -1
|
||||
or rank_index < max(num_output_top_logprobs, 1)
|
||||
)
|
||||
for top_token_id, top_logprob in step_top_logprobs.items()
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
@@ -9,7 +9,11 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
DeltaMessage,
|
||||
ToolCall,
|
||||
)
|
||||
from vllm.entrypoints.generate.base.serving import decode_token_ids
|
||||
from vllm.entrypoints.generate.base.serving import (
|
||||
chat_top_logprobs_limit,
|
||||
completion_top_logprobs_limit,
|
||||
decode_token_ids,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionLogProb,
|
||||
ChatCompletionLogProbs,
|
||||
@@ -204,12 +208,17 @@ class OnlineDerenderer:
|
||||
and chat_request.return_token_ids
|
||||
and (not has_parser or chat_request.include_reasoning)
|
||||
)
|
||||
top_limit = _chat_top_logprobs_limit(chat_request)
|
||||
for choice in generate_response.choices:
|
||||
if not choice.token_ids:
|
||||
raise ValueError(f"choice {choice.index} has empty or null token_ids")
|
||||
|
||||
resolved_logprobs = (
|
||||
_resolve_logprobs(choice.logprobs, tokenizer)
|
||||
_resolve_logprobs(
|
||||
choice.logprobs,
|
||||
tokenizer,
|
||||
top_limit=top_limit,
|
||||
)
|
||||
if choice.logprobs is not None
|
||||
else None
|
||||
)
|
||||
@@ -486,6 +495,7 @@ class OnlineDerenderer:
|
||||
updated_state = state
|
||||
|
||||
return_token_ids = chat_request is not None and chat_request.return_token_ids
|
||||
top_limit = _chat_top_logprobs_limit(chat_request)
|
||||
for choice in generate_chunk.choices:
|
||||
delta_tids = choice.token_ids or []
|
||||
new_text, updated_state = await self._detokenize_delta_async(
|
||||
@@ -511,6 +521,7 @@ class OnlineDerenderer:
|
||||
choice.logprobs,
|
||||
tokenizer,
|
||||
initial_context_token_ids=state.logprob_context_token_ids,
|
||||
top_limit=top_limit,
|
||||
)
|
||||
|
||||
include_role = not updated_state.role_sent
|
||||
@@ -794,6 +805,7 @@ class OnlineDerenderer:
|
||||
total_prompt_tokens = 0
|
||||
total_completion_tokens = 0
|
||||
index = 0
|
||||
top_limit = _completion_top_logprobs_limit(completion_request)
|
||||
|
||||
for gen, pt, seed_ids_override in zip(
|
||||
generate_responses, prompt_tokens_list, prompt_token_ids_list
|
||||
@@ -822,9 +834,13 @@ class OnlineDerenderer:
|
||||
)
|
||||
completion_logprobs = None
|
||||
if choice.logprobs is not None:
|
||||
resolved = _resolve_logprobs(choice.logprobs, tokenizer)
|
||||
resolved = _resolve_logprobs(
|
||||
choice.logprobs,
|
||||
tokenizer,
|
||||
top_limit=top_limit,
|
||||
)
|
||||
completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
|
||||
resolved
|
||||
resolved, choice.logprobs
|
||||
)
|
||||
choices.append(
|
||||
CompletionResponseChoice(
|
||||
@@ -897,6 +913,7 @@ class OnlineDerenderer:
|
||||
stream_choices: list[CompletionResponseStreamChoice] = []
|
||||
updated_state = state
|
||||
|
||||
top_limit = _completion_top_logprobs_limit(completion_request)
|
||||
for choice in generate_chunk.choices:
|
||||
delta_tids = choice.token_ids or []
|
||||
new_text, updated_state = await self._detokenize_delta_async(
|
||||
@@ -913,9 +930,12 @@ class OnlineDerenderer:
|
||||
choice.logprobs,
|
||||
tokenizer,
|
||||
initial_context_token_ids=state.logprob_context_token_ids,
|
||||
top_limit=top_limit,
|
||||
)
|
||||
completion_logprobs = _convert_chat_logprobs_to_completion_logprobs(
|
||||
resolved, initial_text_offset=state.logprob_text_offset
|
||||
resolved,
|
||||
choice.logprobs,
|
||||
initial_text_offset=state.logprob_text_offset,
|
||||
)
|
||||
|
||||
updated_state = updated_state.model_copy(
|
||||
@@ -1067,10 +1087,38 @@ def _correct_decoded_token(
|
||||
return ""
|
||||
|
||||
|
||||
def _chat_top_logprobs_limit(chat_request: ChatCompletionRequest | None) -> int | None:
|
||||
"""The cut `/v1/chat/completions` applies (`chat_top_logprobs_limit`, the
|
||||
same helper its `_get_top_logprobs` uses). None keeps every candidate, which
|
||||
is also what happens without a `chat_request` or with one that does not ask
|
||||
for logprobs: it then says nothing about the cut, so the candidates
|
||||
generate returned are passed through."""
|
||||
if chat_request is None or not chat_request.logprobs:
|
||||
return None
|
||||
return chat_top_logprobs_limit(
|
||||
chat_request.top_logprobs, bool(chat_request.logprob_token_ids)
|
||||
)
|
||||
|
||||
|
||||
def _completion_top_logprobs_limit(
|
||||
completion_request: CompletionRequest | None,
|
||||
) -> int | None:
|
||||
"""The cut `/v1/completions` applies (`completion_top_logprobs_limit`, the
|
||||
same helper `_create_completion_logprobs` uses). None keeps every candidate,
|
||||
which is also what happens without a `completion_request` or with one that
|
||||
does not ask for logprobs (`logprobs` unset)."""
|
||||
if completion_request is None:
|
||||
return None
|
||||
return completion_top_logprobs_limit(
|
||||
completion_request.logprobs, bool(completion_request.logprob_token_ids)
|
||||
)
|
||||
|
||||
|
||||
def _resolve_logprobs(
|
||||
logprobs: GenerateLogProbs,
|
||||
tokenizer: TokenizerLike,
|
||||
initial_context_token_ids: Sequence[int] = (),
|
||||
top_limit: int | None = None,
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Convert generate's integer-id logprobs to the OpenAI chat shape.
|
||||
|
||||
@@ -1080,6 +1128,11 @@ def _resolve_logprobs(
|
||||
ids as context). ``initial_context_token_ids`` seeds that context with
|
||||
sampled IDs from preceding chunks (streaming), so multi-byte characters
|
||||
split across chunk boundaries still resolve.
|
||||
|
||||
Generate returns every candidate the engine produced for a position (the
|
||||
sampled token first, then the top k). ``top_limit`` applies the target
|
||||
endpoint's cut before decoding, so each derender endpoint returns what its
|
||||
coupled counterpart would; None keeps all of them.
|
||||
"""
|
||||
if logprobs.content is None:
|
||||
return ChatCompletionLogProbs()
|
||||
@@ -1088,9 +1141,12 @@ def _resolve_logprobs(
|
||||
resolved_content = []
|
||||
|
||||
for entry in logprobs.content:
|
||||
# One batch per position: the sampled id and its top-k ids.
|
||||
top_entries = (
|
||||
entry.top_logprobs if top_limit is None else entry.top_logprobs[:top_limit]
|
||||
)
|
||||
# One batch per position: the sampled id and its kept top-k ids.
|
||||
(token_str, token_bytes), *top_decoded = decode_token_ids(
|
||||
[entry.token_id, *(top.token_id for top in entry.top_logprobs)],
|
||||
[entry.token_id, *(top.token_id for top in top_entries)],
|
||||
tokenizer,
|
||||
)
|
||||
|
||||
@@ -1101,7 +1157,7 @@ def _resolve_logprobs(
|
||||
token_bytes = list(token_str.encode("utf-8"))
|
||||
|
||||
resolved_top = []
|
||||
for top, (top_str, top_bytes) in zip(entry.top_logprobs, top_decoded):
|
||||
for top, (top_str, top_bytes) in zip(top_entries, top_decoded):
|
||||
if top_str.endswith("\ufffd"):
|
||||
top_str = _correct_decoded_token(
|
||||
top.token_id, context_token_ids, tokenizer
|
||||
@@ -1131,11 +1187,17 @@ def _resolve_logprobs(
|
||||
|
||||
def _convert_chat_logprobs_to_completion_logprobs(
|
||||
logprobs: ChatCompletionLogProbs,
|
||||
generate_logprobs: GenerateLogProbs,
|
||||
initial_text_offset: int = 0,
|
||||
) -> CompletionLogProbs:
|
||||
"""Convert ChatCompletionLogProbs (per-token objects) to CompletionLogProbs
|
||||
(parallel flat lists) as required by the /v1/completions response schema.
|
||||
|
||||
``generate_logprobs`` is what ``logprobs`` was resolved from. A position
|
||||
with no candidates there had no logprobs in the engine output, which
|
||||
/v1/completions reports as ``None``; every other position is a dict of
|
||||
the candidates the cut kept.
|
||||
|
||||
``initial_text_offset`` keeps ``text_offset`` absolute across streaming
|
||||
chunks, mirroring the generate streaming path.
|
||||
"""
|
||||
@@ -1148,13 +1210,13 @@ def _convert_chat_logprobs_to_completion_logprobs(
|
||||
text_offset: list[int] = []
|
||||
|
||||
offset = initial_text_offset
|
||||
for entry in logprobs.content:
|
||||
for entry, source in zip(logprobs.content, generate_logprobs.content or ()):
|
||||
text_offset.append(offset)
|
||||
tokens.append(entry.token)
|
||||
token_logprobs.append(entry.logprob)
|
||||
top_logprobs_list.append(
|
||||
{t.token: t.logprob for t in entry.top_logprobs}
|
||||
if entry.top_logprobs
|
||||
if source.top_logprobs
|
||||
else None
|
||||
)
|
||||
offset += len(entry.token)
|
||||
|
||||
Reference in New Issue
Block a user