[Frontend] Return rendered prompt text in chat completion response (#42052)
Signed-off-by: Wang, Zhipeng | RASIA <zhipeng.wang@rakuten.com> Co-authored-by: Wang, Zhipeng | RASIA <zhipeng.wang@rakuten.com> Co-authored-by: Cursor <cursor@cursor.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
Wang, Zhipeng | RASIA <zhipeng.wang@rakuten.com>
Cursor
Chauncey
parent
5536fc0c01
commit
581b5e9afc
@@ -111,6 +111,9 @@ class ChatCompletionResponse(OpenAIBaseModel):
|
||||
# vLLM-specific fields that are not in OpenAI spec
|
||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||
prompt_token_ids: list[int] | None = None
|
||||
# Rendered prompt text from chat templating (only set when
|
||||
# ``return_prompt_text=True`` on the request).
|
||||
prompt_text: str | None = None
|
||||
kv_transfer_params: dict[str, Any] | None = Field(
|
||||
default=None, description="KVTransfer parameters."
|
||||
)
|
||||
@@ -138,6 +141,9 @@ class ChatCompletionStreamResponse(OpenAIBaseModel):
|
||||
system_fingerprint: str | None = None
|
||||
# not part of the OpenAI spec but for tracing the tokens
|
||||
prompt_token_ids: list[int] | None = None
|
||||
# Rendered prompt text from chat templating (only set when
|
||||
# ``return_prompt_text=True`` on the request); only sent on the first chunk.
|
||||
prompt_text: str | None = None
|
||||
|
||||
|
||||
class ChatCompletionToolsParam(OpenAIBaseModel):
|
||||
@@ -352,6 +358,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
|
||||
"need to map generated text back to input tokens."
|
||||
),
|
||||
)
|
||||
return_prompt_text: bool | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"If true, the response will include ``prompt_text`` containing the "
|
||||
"prompt string produced by chat templating. In streaming mode it "
|
||||
"is sent only on the first chunk. This is useful for inspecting "
|
||||
"exactly what was fed into the model."
|
||||
),
|
||||
)
|
||||
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
|
||||
@@ -508,6 +508,9 @@ class OpenAIServingChat(OpenAIServing):
|
||||
# the role
|
||||
role = self.get_chat_request_role(request)
|
||||
|
||||
# ``res.prompt`` is the rendered chat-templated prompt
|
||||
prompt_text = res.prompt if request.return_prompt_text else None
|
||||
|
||||
# NOTE num_choices defaults to 1 so this usually executes
|
||||
# once per request
|
||||
for i in range(num_choices):
|
||||
@@ -533,6 +536,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
if request.return_token_ids
|
||||
else None
|
||||
),
|
||||
prompt_text=prompt_text,
|
||||
)
|
||||
|
||||
# if continuous usage stats are requested, add it
|
||||
@@ -1371,6 +1375,9 @@ class OpenAIServingChat(OpenAIServing):
|
||||
if final_res.prompt_routed_experts is not None:
|
||||
prompt_routed_experts = final_res.prompt_routed_experts.tolist()
|
||||
|
||||
# ``final_res.prompt`` is the rendered chat-templated prompt text
|
||||
prompt_text = final_res.prompt if request.return_prompt_text else None
|
||||
|
||||
response = ChatCompletionResponse(
|
||||
id=request_id,
|
||||
created=created_time,
|
||||
@@ -1382,6 +1389,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
prompt_token_ids=(
|
||||
final_res.prompt_token_ids if request.return_token_ids else None
|
||||
),
|
||||
prompt_text=prompt_text,
|
||||
kv_transfer_params=final_res.kv_transfer_params,
|
||||
prompt_routed_experts=prompt_routed_experts,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user