[API] Add token offsets to render endpoints (/v1/.../render) (#44226)
Signed-off-by: HyunKyun Moon <mhg5303@gmail.com>
This commit is contained in:
@@ -139,6 +139,7 @@ class ServingRender(BaseServing):
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
|
||||
async def render_completion_request(
|
||||
@@ -194,6 +195,7 @@ class ServingRender(BaseServing):
|
||||
stream_options=(request.stream_options if request.stream else None),
|
||||
cache_salt=request.cache_salt,
|
||||
priority=request.priority,
|
||||
token_offsets=engine_input.get("prompt_token_offsets"),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user