[Frontend] Expose logprob_token_ids on Python OpenAI endpoints (#43463)
Signed-off-by: Lang Zhao <lang.zhao@galileo.ai> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude
parent
7738ef35b8
commit
8b8af2caf7
@@ -251,12 +251,15 @@ class OpenAIServingChatBatch(OpenAIServingChat):
|
||||
for output in final_res.outputs:
|
||||
self._raise_if_error(output.finish_reason, request_id)
|
||||
|
||||
if request.logprobs and request.top_logprobs is not None:
|
||||
if request.logprobs and (
|
||||
request.top_logprobs is not None or request.logprob_token_ids
|
||||
):
|
||||
assert output.logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_chat_logprobs(
|
||||
token_ids=output.token_ids,
|
||||
top_logprobs=output.logprobs,
|
||||
num_output_top_logprobs=request.top_logprobs,
|
||||
logprob_token_ids=request.logprob_token_ids,
|
||||
tokenizer=tokenizer,
|
||||
return_as_token_id=request.return_tokens_as_token_ids,
|
||||
)
|
||||
|
||||
@@ -270,6 +270,18 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
),
|
||||
)
|
||||
prompt_logprobs: int | None = None
|
||||
logprob_token_ids: list[int] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Specific vocab token IDs to return logprobs for at each generated "
|
||||
"position, in addition to the sampled token. More efficient than "
|
||||
"`top_logprobs=-1` when only a small fixed label set is needed "
|
||||
"(e.g. multilabel scoring "
|
||||
"where each label corresponds to a known vocab id). When set, "
|
||||
"this explicit token selection takes precedence over the natural "
|
||||
"top-k selected by `top_logprobs`. Requires `logprobs=True`."
|
||||
),
|
||||
)
|
||||
allowed_token_ids: list[int] | None = None
|
||||
bad_words: list[str] = Field(default_factory=list)
|
||||
# --8<-- [end:chat-completion-sampling-params]
|
||||
@@ -690,8 +702,13 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
seed=self.seed,
|
||||
stop=self.stop,
|
||||
stop_token_ids=stop_token_ids,
|
||||
logprobs=self.top_logprobs if self.logprobs else None,
|
||||
logprobs=(
|
||||
self.top_logprobs
|
||||
if self.logprobs and not self.logprob_token_ids
|
||||
else None
|
||||
),
|
||||
prompt_logprobs=prompt_logprobs,
|
||||
logprob_token_ids=self.logprob_token_ids or None,
|
||||
ignore_eos=self.ignore_eos,
|
||||
max_tokens=max_tokens,
|
||||
min_tokens=self.min_tokens,
|
||||
@@ -756,6 +773,18 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_logprobs(cls, data):
|
||||
if data.get("logprob_token_ids") and data.get("use_beam_search"):
|
||||
raise VLLMValidationError(
|
||||
"`logprob_token_ids` is not supported with beam search.",
|
||||
parameter="logprob_token_ids",
|
||||
)
|
||||
|
||||
if data.get("logprob_token_ids") and not data.get("logprobs"):
|
||||
raise VLLMValidationError(
|
||||
"when using `logprob_token_ids`, `logprobs` must be set to true.",
|
||||
parameter="logprob_token_ids",
|
||||
)
|
||||
|
||||
if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
|
||||
if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
|
||||
raise VLLMValidationError(
|
||||
@@ -1009,6 +1038,14 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
|
||||
logit_bias: dict[str, float] | None = None
|
||||
logprobs: bool | None = False
|
||||
top_logprobs: int | None = 0
|
||||
logprob_token_ids: list[int] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Specific vocab token IDs to return logprobs for at each generated "
|
||||
"position, in addition to the sampled token. Requires "
|
||||
"`logprobs=True`."
|
||||
),
|
||||
)
|
||||
max_tokens: int | None = None
|
||||
max_completion_tokens: int | None = None
|
||||
n: int | None = 1
|
||||
@@ -1058,6 +1095,10 @@ class BatchChatCompletionRequest(OpenAIBaseModel):
|
||||
"Batch chat completions do not support beam search. "
|
||||
"Please set `use_beam_search` to False."
|
||||
)
|
||||
if data.get("logprob_token_ids") and not data.get("logprobs"):
|
||||
raise ValueError(
|
||||
"when using `logprob_token_ids`, `logprobs` must be set to true."
|
||||
)
|
||||
response_format = data.get("response_format")
|
||||
rf_type = (
|
||||
response_format.get("type")
|
||||
|
||||
@@ -568,13 +568,16 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
if finish_reason_sent[i]:
|
||||
continue
|
||||
|
||||
if request.logprobs and request.top_logprobs is not None:
|
||||
if request.logprobs and (
|
||||
request.top_logprobs is not None or request.logprob_token_ids
|
||||
):
|
||||
assert output.logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_chat_logprobs(
|
||||
token_ids=output.token_ids,
|
||||
top_logprobs=output.logprobs,
|
||||
tokenizer=tokenizer,
|
||||
num_output_top_logprobs=request.top_logprobs,
|
||||
logprob_token_ids=request.logprob_token_ids,
|
||||
return_as_token_id=request.return_tokens_as_token_ids,
|
||||
)
|
||||
else:
|
||||
@@ -862,12 +865,15 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
token_ids = output.token_ids
|
||||
out_logprobs = output.logprobs
|
||||
|
||||
if request.logprobs and request.top_logprobs is not None:
|
||||
if request.logprobs and (
|
||||
request.top_logprobs is not None or request.logprob_token_ids
|
||||
):
|
||||
assert out_logprobs is not None, "Did not output logprobs"
|
||||
logprobs = self._create_chat_logprobs(
|
||||
token_ids=token_ids,
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=request.top_logprobs,
|
||||
logprob_token_ids=request.logprob_token_ids,
|
||||
tokenizer=tokenizer,
|
||||
return_as_token_id=request.return_tokens_as_token_ids,
|
||||
)
|
||||
@@ -1101,6 +1107,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
top_logprobs: int | None,
|
||||
tokenizer: TokenizerLike | None,
|
||||
should_return_as_token_id: bool,
|
||||
return_all: bool = False,
|
||||
) -> list[ChatCompletionLogProb]:
|
||||
return [
|
||||
ChatCompletionLogProb(
|
||||
@@ -1116,7 +1123,9 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
bytes=list(token.encode("utf-8", errors="replace")),
|
||||
)
|
||||
for i, p in enumerate(logprobs.items())
|
||||
if (top_logprobs and i < top_logprobs or top_logprobs == -1)
|
||||
if return_all
|
||||
or top_logprobs == -1
|
||||
or (top_logprobs is not None and i < top_logprobs)
|
||||
]
|
||||
|
||||
def _create_chat_logprobs(
|
||||
@@ -1125,6 +1134,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
||||
tokenizer: TokenizerLike | None,
|
||||
num_output_top_logprobs: int | None = None,
|
||||
logprob_token_ids: list[int] | None = None,
|
||||
return_as_token_id: bool | None = None,
|
||||
) -> ChatCompletionLogProbs:
|
||||
"""Create OpenAI-style logprobs."""
|
||||
@@ -1177,6 +1187,7 @@ class OpenAIServingChat(GenerateBaseServing):
|
||||
num_output_top_logprobs,
|
||||
tokenizer,
|
||||
should_return_as_token_id,
|
||||
return_all=bool(logprob_token_ids),
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -95,6 +95,19 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
)
|
||||
allowed_token_ids: list[int] | None = None
|
||||
prompt_logprobs: int | None = None
|
||||
logprob_token_ids: list[int] | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Specific vocab token IDs to return logprobs for at each generated "
|
||||
"position, in addition to the sampled token. More efficient than "
|
||||
"requesting the full vocab when only a small fixed label set is "
|
||||
"needed (e.g. multilabel "
|
||||
"scoring where each label corresponds to a known vocab id). When "
|
||||
"set, this explicit token selection takes precedence over the "
|
||||
"natural top-k selected by `logprobs`. Requires `logprobs` to be "
|
||||
"set."
|
||||
),
|
||||
)
|
||||
bad_words: list[str] = Field(default_factory=list)
|
||||
# --8<-- [end:completion-sampling-params]
|
||||
|
||||
@@ -368,11 +381,12 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
seed=self.seed,
|
||||
stop=self.stop,
|
||||
stop_token_ids=stop_token_ids,
|
||||
logprobs=self.logprobs,
|
||||
logprobs=None if self.logprob_token_ids else self.logprobs,
|
||||
ignore_eos=self.ignore_eos,
|
||||
max_tokens=max_tokens if not echo_without_generation else 1,
|
||||
min_tokens=self.min_tokens,
|
||||
prompt_logprobs=prompt_logprobs,
|
||||
logprob_token_ids=self.logprob_token_ids or None,
|
||||
skip_special_tokens=self.skip_special_tokens,
|
||||
spaces_between_special_tokens=self.spaces_between_special_tokens,
|
||||
include_stop_str_in_output=self.include_stop_str_in_output,
|
||||
@@ -459,6 +473,29 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_logprobs(cls, data):
|
||||
if data.get("logprob_token_ids") and data.get("use_beam_search"):
|
||||
raise VLLMValidationError(
|
||||
"`logprob_token_ids` is not supported with beam search.",
|
||||
parameter="logprob_token_ids",
|
||||
)
|
||||
|
||||
if (
|
||||
data.get("logprob_token_ids")
|
||||
and data.get("echo")
|
||||
and data.get("max_tokens") == 0
|
||||
):
|
||||
raise VLLMValidationError(
|
||||
"`logprob_token_ids` is not supported when `echo=True` and "
|
||||
"`max_tokens=0` because no output tokens are generated.",
|
||||
parameter="logprob_token_ids",
|
||||
)
|
||||
|
||||
if data.get("logprob_token_ids") and data.get("logprobs") is None:
|
||||
raise VLLMValidationError(
|
||||
"when using `logprob_token_ids`, `logprobs` must be set.",
|
||||
parameter="logprob_token_ids",
|
||||
)
|
||||
|
||||
if (prompt_logprobs := data.get("prompt_logprobs")) is not None:
|
||||
if data.get("stream") and (prompt_logprobs > 0 or prompt_logprobs == -1):
|
||||
raise VLLMValidationError(
|
||||
|
||||
@@ -386,6 +386,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
top_logprobs=out_logprobs,
|
||||
num_output_top_logprobs=request.logprobs,
|
||||
tokenizer=tokenizer,
|
||||
logprob_token_ids=request.logprob_token_ids,
|
||||
initial_text_offset=previous_text_lens[i],
|
||||
return_as_token_id=request.return_tokens_as_token_ids,
|
||||
)
|
||||
@@ -560,6 +561,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
top_logprobs=out_logprobs,
|
||||
tokenizer=tokenizer,
|
||||
num_output_top_logprobs=request.logprobs,
|
||||
logprob_token_ids=request.logprob_token_ids,
|
||||
return_as_token_id=request.return_tokens_as_token_ids,
|
||||
)
|
||||
else:
|
||||
@@ -653,6 +655,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
||||
num_output_top_logprobs: int,
|
||||
tokenizer: TokenizerLike | None,
|
||||
logprob_token_ids: list[int] | None = None,
|
||||
initial_text_offset: int = 0,
|
||||
return_as_token_id: bool | None = None,
|
||||
) -> CompletionLogProbs:
|
||||
@@ -717,7 +720,7 @@ class OpenAIServingCompletion(GenerateBaseServing):
|
||||
return_as_token_id=should_return_as_token_id,
|
||||
): max(top_lp[1].logprob, -9999.0)
|
||||
for i, top_lp in enumerate(step_top_logprobs.items())
|
||||
if num_output_top_logprobs >= i
|
||||
if logprob_token_ids or num_output_top_logprobs >= i
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user