[API] Add token offsets to render endpoints (/v1/.../render) (#44226)
Signed-off-by: HyunKyun Moon <mhg5303@gmail.com>
This commit is contained in:
@@ -82,6 +82,13 @@ class GenerateRequest(BaseModel):
|
||||
raise ValueError("token_ids must not contain negative values")
|
||||
return v
|
||||
|
||||
token_offsets: list[tuple[int, int]] | None = None
|
||||
"""Char-level (start, end) offsets per token, relative to the
|
||||
tokenized source string. Present only when the request set
|
||||
`return_token_offsets=True` and the renderer was able to compute
|
||||
them (Fast tokenizer, text input, no multimodal data). List length
|
||||
equals `token_ids` length when present. None otherwise."""
|
||||
|
||||
features: MultiModalFeatures | None = None
|
||||
"""Multimodal hashes and placeholder positions (populated for MM inputs)."""
|
||||
|
||||
|
||||
@@ -139,6 +139,7 @@ class ServingRender(BaseServing):
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
|
||||
async def render_completion_request(
|
||||
@@ -194,6 +195,7 @@ class ServingRender(BaseServing):
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user