[Frontend] Report stop_reason on /inference/v1/generate and pass it through derender (#60251)
Signed-off-by: Hyeonki Hong <hhk7734@gmail.com> Signed-off-by: Shimi Bandiel <shimib@google.com> Signed-off-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Hyeonki Hong <hhk7734@gmail.com> Co-authored-by: Mimir <mimir@nousresearch.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Bugen Zhao <i@bugenzhao.com>
This commit is contained in:
co-authored by
Hyeonki Hong
Mimir
Claude
Bugen Zhao
parent
32b05addc1
commit
8d7b93fec2
@@ -400,6 +400,9 @@ class GenerateChoiceBase(BaseModel):
|
||||
index: int
|
||||
# per OpenAI spec this is the default
|
||||
finish_reason: str | None = "stop"
|
||||
# Stop string or token id that ended generation (repetition detection
|
||||
# sets it too); None on EOS, length and abort.
|
||||
stop_reason: int | str | None = None
|
||||
token_ids: list[int] | None = None
|
||||
# Per-token expert routing decisions, base64-encoded `.npy` bytes
|
||||
# (numpy serialization). Shape after decode:
|
||||
@@ -443,6 +446,9 @@ class GenerateStreamChoiceBase(BaseModel):
|
||||
|
||||
index: int
|
||||
finish_reason: str | None = None
|
||||
# Stop string or token id that ended generation (repetition detection
|
||||
# sets it too); None on EOS, length and abort.
|
||||
stop_reason: int | str | None = None
|
||||
token_ids: list[int] | None = None
|
||||
routed_experts: str | None = None
|
||||
sampling_mask: list[list[int]] | None = None
|
||||
|
||||
@@ -450,6 +450,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
index=output.index,
|
||||
logprobs=logprobs,
|
||||
finish_reason=output.finish_reason if output.finish_reason else "stop",
|
||||
stop_reason=output.stop_reason,
|
||||
token_ids=as_list(output.token_ids),
|
||||
routed_experts=routed_experts_b64,
|
||||
sampling_mask=sampling_mask,
|
||||
@@ -623,6 +624,7 @@ class ServingTokens(GenerateBaseServing):
|
||||
index=i,
|
||||
logprobs=logprobs,
|
||||
finish_reason=finish_reason,
|
||||
stop_reason=output.stop_reason,
|
||||
token_ids=as_list(delta_token_ids),
|
||||
routed_experts=routed_experts_b64,
|
||||
sampling_mask=sampling_mask,
|
||||
|
||||
Reference in New Issue
Block a user