[Bugfix][Frontend] Return logprobs in Cohere Chat v2 responses (#59072)
Signed-off-by: Mohammad Hijjawi <mohammad.hijjawi1997@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
2342343746
commit
ef014bdf06
@@ -58,6 +58,7 @@ from vllm.entrypoints.cohere.protocol import (
|
||||
CohereChatV2Request,
|
||||
CohereChatV2Response,
|
||||
CohereFinishReason,
|
||||
CohereLogprobItem,
|
||||
CohereUsage,
|
||||
CohereUsageBilledUnits,
|
||||
CohereUsageTokens,
|
||||
@@ -79,6 +80,7 @@ from vllm.entrypoints.generate.base.protocol import (
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ChatCompletionResponse,
|
||||
ChatCompletionResponseChoice,
|
||||
ChatCompletionStreamResponse,
|
||||
ChatCompletionToolsParam,
|
||||
ChatMessage,
|
||||
@@ -562,6 +564,8 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
frequency_penalty=request.frequency_penalty,
|
||||
presence_penalty=request.presence_penalty,
|
||||
logprobs=request.logprobs,
|
||||
# Cohere logprob items carry token ids alongside the logprobs.
|
||||
return_token_ids=request.logprobs,
|
||||
priority=request.priority or 0,
|
||||
kv_transfer_params=request.kv_transfer_params,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
@@ -1091,9 +1095,29 @@ class CohereServingChatV2(OpenAIServingChat):
|
||||
),
|
||||
message=assistant_msg,
|
||||
usage=usage,
|
||||
logprobs=self._build_logprobs(choice),
|
||||
kv_transfer_params=response.kv_transfer_params,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _build_logprobs(
|
||||
choice: ChatCompletionResponseChoice,
|
||||
) -> list[CohereLogprobItem] | None:
|
||||
if (
|
||||
choice.logprobs is None
|
||||
or not choice.logprobs.content
|
||||
or choice.token_ids is None
|
||||
):
|
||||
return None
|
||||
return [
|
||||
CohereLogprobItem(
|
||||
text=entry.token, token_ids=[token_id], logprobs=[entry.logprob]
|
||||
)
|
||||
for entry, token_id in zip(
|
||||
choice.logprobs.content, choice.token_ids, strict=True
|
||||
)
|
||||
]
|
||||
|
||||
def _create_chat_message(self, *args: Any, **kwargs: Any) -> ChatMessage:
|
||||
"""Route response construction through the citation-carrying subclass.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user