[API] Add token offsets to render endpoints (/v1/.../render) (#44226)

Signed-off-by: HyunKyun Moon <mhg5303@gmail.com>
This commit is contained in:
Hyunkyun Moon
2026-06-26 05:02:52 -07:00
committed by GitHub
parent d980a3cc6e
commit 950ee4c2e4
13 changed files with 518 additions and 22 deletions
@@ -152,6 +152,21 @@ class CompletionRequest(OpenAIBaseModel):
"need to map generated text back to input tokens."
),
)
return_token_offsets: bool | None = Field(
default=False,
description=(
"If true, return char-level (start, end) offsets for each "
"token relative to the tokenized source string in the "
"`token_offsets` field of the rendered response. Only "
"supported on the `/v1/completions/render` and "
"`/v1/chat/completions/render` endpoints; ignored on regular "
"generation endpoints. Honored only for Fast (Rust-backed) "
"tokenizers; otherwise `token_offsets` is null. For chat "
"requests, offsets are relative to the templated prompt "
"string (after applying the chat template). Multimodal "
"inputs and pre-tokenized inputs always yield null."
),
)
cache_salt: str | None = Field(
default=None,
@@ -209,6 +224,7 @@ class CompletionRequest(OpenAIBaseModel):
needs_detokenization=bool(self.echo and not self.return_token_ids),
max_total_tokens_param="max_model_len",
max_output_tokens_param="max_tokens",
return_token_offsets=bool(self.return_token_offsets),
)
# Default sampling parameters for completion requests