[API] Add token offsets to render endpoints (/v1/.../render) (#44226)
Signed-off-by: HyunKyun Moon <mhg5303@gmail.com>
This commit is contained in:
@@ -382,6 +382,21 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"need to map generated text back to input tokens."
|
||||
),
|
||||
)
|
||||
return_token_offsets: bool | None = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, return char-level (start, end) offsets for each "
|
||||
"token relative to the tokenized source string in the "
|
||||
"`token_offsets` field of the rendered response. Only "
|
||||
"supported on the `/v1/completions/render` and "
|
||||
"`/v1/chat/completions/render` endpoints; ignored on regular "
|
||||
"generation endpoints. Honored only for Fast (Rust-backed) "
|
||||
"tokenizers; otherwise `token_offsets` is null. For chat "
|
||||
"requests, offsets are relative to the templated prompt "
|
||||
"string (after applying the chat template). Multimodal "
|
||||
"inputs and pre-tokenized inputs always yield null."
|
||||
),
|
||||
)
|
||||
return_prompt_text: bool | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
@@ -524,6 +539,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
needs_detokenization=bool(self.echo and not self.return_token_ids),
|
||||
max_total_tokens_param="max_model_len",
|
||||
max_output_tokens_param=max_output_tokens_param,
|
||||
return_token_offsets=bool(self.return_token_offsets),
|
||||
)
|
||||
|
||||
# Default sampling parameters for chat completion requests
|
||||
|
||||
Reference in New Issue
Block a user