[Bugfix] Expose usage field in GenerateResponse for disaggregated serving (#42748)
Signed-off-by: AIvashov <ivashov.aleksey@proton.me> Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Co-authored-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
co-authored by
NickLucche
parent
e7d0fcbc09
commit
024b06b0dc
@@ -221,8 +221,10 @@ class GenerateResponse(BaseModel):
|
|||||||
"through out the inference process and return in response."
|
"through out the inference process and return in response."
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
model: str | None = None
|
||||||
|
created: int | None = None
|
||||||
choices: list[GenerateResponseChoice]
|
choices: list[GenerateResponseChoice]
|
||||||
|
usage: UsageInfo | None = Field(default=None)
|
||||||
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
prompt_logprobs: list[dict[int, Logprob] | None] | None = None
|
||||||
|
|
||||||
kv_transfer_params: dict[str, Any] | None = Field(
|
kv_transfer_params: dict[str, Any] | None = Field(
|
||||||
|
|||||||
@@ -320,7 +320,7 @@ class ServingTokens(OpenAIServing):
|
|||||||
request_metadata.final_usage_info = usage
|
request_metadata.final_usage_info = usage
|
||||||
|
|
||||||
response = GenerateResponse(
|
response = GenerateResponse(
|
||||||
id=request_id,
|
request_id=request_id,
|
||||||
created=created_time,
|
created=created_time,
|
||||||
model=model_name,
|
model=model_name,
|
||||||
choices=choices,
|
choices=choices,
|
||||||
|
|||||||
Reference in New Issue
Block a user