[Bugfix][Frontend] Return logprobs in Cohere Chat v2 responses (#59072)

Signed-off-by: Mohammad Hijjawi <mohammad.hijjawi1997@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Mohammad Hijjawi
2026-10-07 04:13:52 +00:00
committed by GitHub
co-authored by Cyrus Leung
parent 2342343746
commit ef014bdf06
2 changed files with 49 additions and 0 deletions
+24
View File
@@ -58,6 +58,7 @@ from vllm.entrypoints.cohere.protocol import (
CohereChatV2Request,
CohereChatV2Response,
CohereFinishReason,
CohereLogprobItem,
CohereUsage,
CohereUsageBilledUnits,
CohereUsageTokens,
@@ -79,6 +80,7 @@ from vllm.entrypoints.generate.base.protocol import (
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionResponseChoice,
ChatCompletionStreamResponse,
ChatCompletionToolsParam,
ChatMessage,
@@ -562,6 +564,8 @@ class CohereServingChatV2(OpenAIServingChat):
frequency_penalty=request.frequency_penalty,
presence_penalty=request.presence_penalty,
logprobs=request.logprobs,
# Cohere logprob items carry token ids alongside the logprobs.
return_token_ids=request.logprobs,
priority=request.priority or 0,
kv_transfer_params=request.kv_transfer_params,
chat_template_kwargs=request.chat_template_kwargs,
@@ -1091,9 +1095,29 @@ class CohereServingChatV2(OpenAIServingChat):
),
message=assistant_msg,
usage=usage,
logprobs=self._build_logprobs(choice),
kv_transfer_params=response.kv_transfer_params,
)
@staticmethod
def _build_logprobs(
choice: ChatCompletionResponseChoice,
) -> list[CohereLogprobItem] | None:
if (
choice.logprobs is None
or not choice.logprobs.content
or choice.token_ids is None
):
return None
return [
CohereLogprobItem(
text=entry.token, token_ids=[token_id], logprobs=[entry.logprob]
)
for entry, token_id in zip(
choice.logprobs.content, choice.token_ids, strict=True
)
]
def _create_chat_message(self, *args: Any, **kwargs: Any) -> ChatMessage:
"""Route response construction through the citation-carrying subclass.