[API] Add token offsets to render endpoints (/v1/.../render) (#44226)

Signed-off-by: HyunKyun Moon <mhg5303@gmail.com>
This commit is contained in:
Hyunkyun Moon
2026-06-26 05:02:52 -07:00
committed by GitHub
parent d980a3cc6e
commit 950ee4c2e4
13 changed files with 518 additions and 22 deletions
@@ -382,6 +382,21 @@ class ChatCompletionRequest(OpenAIBaseModel):
"need to map generated text back to input tokens."
),
)
return_token_offsets: bool | None = Field(
default=False,
description=(
"If true, return char-level (start, end) offsets for each "
"token relative to the tokenized source string in the "
"`token_offsets` field of the rendered response. Only "
"supported on the `/v1/completions/render` and "
"`/v1/chat/completions/render` endpoints; ignored on regular "
"generation endpoints. Honored only for Fast (Rust-backed) "
"tokenizers; otherwise `token_offsets` is null. For chat "
"requests, offsets are relative to the templated prompt "
"string (after applying the chat template). Multimodal "
"inputs and pre-tokenized inputs always yield null."
),
)
return_prompt_text: bool | None = Field(
default=None,
description=(
@@ -524,6 +539,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
needs_detokenization=bool(self.echo and not self.return_token_ids),
max_total_tokens_param="max_model_len",
max_output_tokens_param=max_output_tokens_param,
return_token_offsets=bool(self.return_token_offsets),
)
# Default sampling parameters for chat completion requests
@@ -152,6 +152,21 @@ class CompletionRequest(OpenAIBaseModel):
"need to map generated text back to input tokens."
),
)
return_token_offsets: bool | None = Field(
default=False,
description=(
"If true, return char-level (start, end) offsets for each "
"token relative to the tokenized source string in the "
"`token_offsets` field of the rendered response. Only "
"supported on the `/v1/completions/render` and "
"`/v1/chat/completions/render` endpoints; ignored on regular "
"generation endpoints. Honored only for Fast (Rust-backed) "
"tokenizers; otherwise `token_offsets` is null. For chat "
"requests, offsets are relative to the templated prompt "
"string (after applying the chat template). Multimodal "
"inputs and pre-tokenized inputs always yield null."
),
)
cache_salt: str | None = Field(
default=None,
@@ -209,6 +224,7 @@ class CompletionRequest(OpenAIBaseModel):
needs_detokenization=bool(self.echo and not self.return_token_ids),
max_total_tokens_param="max_model_len",
max_output_tokens_param="max_tokens",
return_token_offsets=bool(self.return_token_offsets),
)
# Default sampling parameters for completion requests
@@ -82,6 +82,13 @@ class GenerateRequest(BaseModel):
raise ValueError("token_ids must not contain negative values")
return v
token_offsets: list[tuple[int, int]] | None = None
"""Char-level (start, end) offsets per token, relative to the
tokenized source string. Present only when the request set
`return_token_offsets=True` and the renderer was able to compute
them (Fast tokenizer, text input, no multimodal data). List length
equals `token_ids` length when present. None otherwise."""
features: MultiModalFeatures | None = None
"""Multimodal hashes and placeholder positions (populated for MM inputs)."""
+2
View File
@@ -139,6 +139,7 @@ class ServingRender(BaseServing):
stream_options=(request.stream_options if request.stream else None),
cache_salt=request.cache_salt,
priority=request.priority,
token_offsets=engine_input.get("prompt_token_offsets"),
)
async def render_completion_request(
@@ -194,6 +195,7 @@ class ServingRender(BaseServing):
stream_options=(request.stream_options if request.stream else None),
cache_salt=request.cache_salt,
priority=request.priority,
token_offsets=engine_input.get("prompt_token_offsets"),
)
)