[Misc] Include matched stop string/token in responses (#2976)
Co-authored-by: Sahil Suneja <sahilsuneja@gmail.com>
This commit is contained in:
co-authored by
Sahil Suneja
parent
3a243095e5
commit
dfeb2ecc3a
@@ -338,6 +338,13 @@ class CompletionResponseChoice(BaseModel):
|
||||
text: str
|
||||
logprobs: Optional[LogProbs] = None
|
||||
finish_reason: Optional[Literal["stop", "length"]] = None
|
||||
stop_reason: Union[None, int, str] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"The stop string or token id that caused the completion "
|
||||
"to stop, None if the completion finished for some other reason "
|
||||
"including encountering the EOS token"),
|
||||
)
|
||||
|
||||
|
||||
class CompletionResponse(BaseModel):
|
||||
@@ -354,6 +361,13 @@ class CompletionResponseStreamChoice(BaseModel):
|
||||
text: str
|
||||
logprobs: Optional[LogProbs] = None
|
||||
finish_reason: Optional[Literal["stop", "length"]] = None
|
||||
stop_reason: Union[None, int, str] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"The stop string or token id that caused the completion "
|
||||
"to stop, None if the completion finished for some other reason "
|
||||
"including encountering the EOS token"),
|
||||
)
|
||||
|
||||
|
||||
class CompletionStreamResponse(BaseModel):
|
||||
@@ -375,6 +389,7 @@ class ChatCompletionResponseChoice(BaseModel):
|
||||
message: ChatMessage
|
||||
logprobs: Optional[LogProbs] = None
|
||||
finish_reason: Optional[Literal["stop", "length"]] = None
|
||||
stop_reason: Union[None, int, str] = None
|
||||
|
||||
|
||||
class ChatCompletionResponse(BaseModel):
|
||||
@@ -396,6 +411,7 @@ class ChatCompletionResponseStreamChoice(BaseModel):
|
||||
delta: DeltaMessage
|
||||
logprobs: Optional[LogProbs] = None
|
||||
finish_reason: Optional[Literal["stop", "length"]] = None
|
||||
stop_reason: Union[None, int, str] = None
|
||||
|
||||
|
||||
class ChatCompletionStreamResponse(BaseModel):
|
||||
|
||||
@@ -220,7 +220,8 @@ class OpenAIServingChat(OpenAIServing):
|
||||
index=i,
|
||||
delta=DeltaMessage(content=delta_text),
|
||||
logprobs=logprobs,
|
||||
finish_reason=output.finish_reason)
|
||||
finish_reason=output.finish_reason,
|
||||
stop_reason=output.stop_reason)
|
||||
chunk = ChatCompletionStreamResponse(
|
||||
id=request_id,
|
||||
object=chunk_object_type,
|
||||
@@ -278,6 +279,7 @@ class OpenAIServingChat(OpenAIServing):
|
||||
message=ChatMessage(role=role, content=output.text),
|
||||
logprobs=logprobs,
|
||||
finish_reason=output.finish_reason,
|
||||
stop_reason=output.stop_reason,
|
||||
)
|
||||
choices.append(choice_data)
|
||||
|
||||
|
||||
@@ -266,6 +266,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
previous_texts[i] = output.text
|
||||
previous_num_tokens[i] = len(output.token_ids)
|
||||
finish_reason = output.finish_reason
|
||||
stop_reason = output.stop_reason
|
||||
if output.finish_reason is not None: # return final usage
|
||||
prompt_tokens = len(res.prompt_token_ids)
|
||||
completion_tokens = len(output.token_ids)
|
||||
@@ -286,6 +287,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
text=delta_text,
|
||||
logprobs=logprobs,
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=stop_reason,
|
||||
)
|
||||
],
|
||||
usage=final_usage,
|
||||
@@ -342,6 +344,7 @@ class OpenAIServingCompletion(OpenAIServing):
|
||||
text=output_text,
|
||||
logprobs=logprobs,
|
||||
finish_reason=output.finish_reason,
|
||||
stop_reason=output.stop_reason,
|
||||
)
|
||||
choices.append(choice_data)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user