[API] Add token offsets to render endpoints (/v1/.../render) (#44226)
Signed-off-by: HyunKyun Moon <mhg5303@gmail.com>
This commit is contained in:
@@ -382,6 +382,21 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"need to map generated text back to input tokens."
|
||||
),
|
||||
)
|
||||
return_token_offsets: bool | None = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, return char-level (start, end) offsets for each "
|
||||
"token relative to the tokenized source string in the "
|
||||
"`token_offsets` field of the rendered response. Only "
|
||||
"supported on the `/v1/completions/render` and "
|
||||
"`/v1/chat/completions/render` endpoints; ignored on regular "
|
||||
"generation endpoints. Honored only for Fast (Rust-backed) "
|
||||
"tokenizers; otherwise `token_offsets` is null. For chat "
|
||||
"requests, offsets are relative to the templated prompt "
|
||||
"string (after applying the chat template). Multimodal "
|
||||
"inputs and pre-tokenized inputs always yield null."
|
||||
),
|
||||
)
|
||||
return_prompt_text: bool | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
@@ -524,6 +539,7 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
needs_detokenization=bool(self.echo and not self.return_token_ids),
|
||||
max_total_tokens_param="max_model_len",
|
||||
max_output_tokens_param=max_output_tokens_param,
|
||||
return_token_offsets=bool(self.return_token_offsets),
|
||||
)
|
||||
|
||||
# Default sampling parameters for chat completion requests
|
||||
|
||||
@@ -152,6 +152,21 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
"need to map generated text back to input tokens."
|
||||
),
|
||||
)
|
||||
return_token_offsets: bool | None = Field(
|
||||
default=False,
|
||||
description=(
|
||||
"If true, return char-level (start, end) offsets for each "
|
||||
"token relative to the tokenized source string in the "
|
||||
"`token_offsets` field of the rendered response. Only "
|
||||
"supported on the `/v1/completions/render` and "
|
||||
"`/v1/chat/completions/render` endpoints; ignored on regular "
|
||||
"generation endpoints. Honored only for Fast (Rust-backed) "
|
||||
"tokenizers; otherwise `token_offsets` is null. For chat "
|
||||
"requests, offsets are relative to the templated prompt "
|
||||
"string (after applying the chat template). Multimodal "
|
||||
"inputs and pre-tokenized inputs always yield null."
|
||||
),
|
||||
)
|
||||
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
@@ -209,6 +224,7 @@ class CompletionRequest(OpenAIBaseModel):
|
||||
needs_detokenization=bool(self.echo and not self.return_token_ids),
|
||||
max_total_tokens_param="max_model_len",
|
||||
max_output_tokens_param="max_tokens",
|
||||
return_token_offsets=bool(self.return_token_offsets),
|
||||
)
|
||||
|
||||
# Default sampling parameters for completion requests
|
||||
|
||||
@@ -82,6 +82,13 @@ class GenerateRequest(BaseModel):
|
||||
raise ValueError("token_ids must not contain negative values")
|
||||
return v
|
||||
|
||||
token_offsets: list[tuple[int, int]] | None = None
|
||||
"""Char-level (start, end) offsets per token, relative to the
|
||||
tokenized source string. Present only when the request set
|
||||
`return_token_offsets=True` and the renderer was able to compute
|
||||
them (Fast tokenizer, text input, no multimodal data). List length
|
||||
equals `token_ids` length when present. None otherwise."""
|
||||
|
||||
features: MultiModalFeatures | None = None
|
||||
"""Multimodal hashes and placeholder positions (populated for MM inputs)."""
|
||||
|
||||
|
||||
@@ -139,6 +139,7 @@ class ServingRender(BaseServing):
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
|
||||
async def render_completion_request(
|
||||
@@ -194,6 +195,7 @@ class ServingRender(BaseServing):
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user