Aaron HaoandIsotr0py
2d6b3489b9
Signed-off-by: ahao-anyscale <ahao@anyscale.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
1599 lines
69 KiB
Python
1599 lines
69 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
import asyncio
|
|
import io
|
|
import json
|
|
import time
|
|
from collections.abc import AsyncGenerator, AsyncIterator
|
|
from collections.abc import Sequence as GenericSequence
|
|
from http import HTTPStatus
|
|
from typing import TYPE_CHECKING, Any, Final
|
|
|
|
import numpy as np
|
|
import pybase64 as base64
|
|
from fastapi import Request
|
|
|
|
from vllm.engine.protocol import EngineClient
|
|
from vllm.entrypoints.chat_utils import (
|
|
ChatTemplateContentFormatOption,
|
|
ConversationMessage,
|
|
get_history_tool_calls_cnt,
|
|
get_tool_call_id_type,
|
|
make_tool_call_id,
|
|
)
|
|
from vllm.entrypoints.logger import RequestLogger
|
|
from vllm.entrypoints.openai.chat_completion.protocol import (
|
|
ChatCompletionLogProb,
|
|
ChatCompletionLogProbs,
|
|
ChatCompletionLogProbsContent,
|
|
ChatCompletionNamedToolChoiceParam,
|
|
ChatCompletionRequest,
|
|
ChatCompletionResponse,
|
|
ChatCompletionResponseChoice,
|
|
ChatCompletionResponseStreamChoice,
|
|
ChatCompletionStreamResponse,
|
|
ChatMessage,
|
|
)
|
|
from vllm.entrypoints.openai.chat_completion.stream_harmony import (
|
|
TokenState,
|
|
extract_harmony_streaming_delta,
|
|
)
|
|
from vllm.entrypoints.openai.engine.protocol import (
|
|
DeltaFunctionCall,
|
|
DeltaMessage,
|
|
DeltaToolCall,
|
|
ErrorResponse,
|
|
FunctionCall,
|
|
PromptTokenUsageInfo,
|
|
RequestResponseMetadata,
|
|
ToolCall,
|
|
UsageInfo,
|
|
)
|
|
from vllm.entrypoints.openai.engine.serving import (
|
|
GenerationError,
|
|
OpenAIServing,
|
|
clamp_prompt_logprobs,
|
|
)
|
|
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
|
from vllm.entrypoints.openai.parser.harmony_utils import (
|
|
get_stop_tokens_for_assistant_actions,
|
|
get_streamable_parser_for_assistant,
|
|
parse_chat_output,
|
|
)
|
|
from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls
|
|
from vllm.entrypoints.utils import get_max_tokens, should_include_usage
|
|
from vllm.inputs import EngineInput
|
|
from vllm.logger import init_logger
|
|
from vllm.logprobs import Logprob
|
|
from vllm.outputs import CompletionOutput, RequestOutput
|
|
from vllm.parser import ParserManager
|
|
from vllm.parser.abstract_parser import Parser
|
|
from vllm.reasoning import ReasoningParser
|
|
from vllm.renderers import ChatParams
|
|
from vllm.sampling_params import BeamSearchParams, SamplingParams
|
|
from vllm.tokenizers import TokenizerLike
|
|
from vllm.utils.collection_utils import as_list
|
|
from vllm.utils.mistral import is_mistral_tokenizer, is_mistral_tool_parser
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class OpenAIServingChat(OpenAIServing):
|
|
def __init__(
|
|
self,
|
|
engine_client: EngineClient,
|
|
models: OpenAIServingModels,
|
|
response_role: str,
|
|
*,
|
|
openai_serving_render: "OpenAIServingRender",
|
|
request_logger: RequestLogger | None,
|
|
chat_template: str | None,
|
|
chat_template_content_format: ChatTemplateContentFormatOption,
|
|
trust_request_chat_template: bool = False,
|
|
return_tokens_as_token_ids: bool = False,
|
|
reasoning_parser: str = "",
|
|
enable_auto_tools: bool = False,
|
|
exclude_tools_when_tool_choice_none: bool = False,
|
|
tool_parser: str | None = None,
|
|
enable_prompt_tokens_details: bool = False,
|
|
enable_force_include_usage: bool = False,
|
|
enable_log_outputs: bool = False,
|
|
enable_log_deltas: bool = True,
|
|
default_chat_template_kwargs: dict[str, Any] | None = None,
|
|
) -> None:
|
|
super().__init__(
|
|
engine_client=engine_client,
|
|
models=models,
|
|
request_logger=request_logger,
|
|
return_tokens_as_token_ids=return_tokens_as_token_ids,
|
|
)
|
|
|
|
self.openai_serving_render = openai_serving_render
|
|
self.response_role = response_role
|
|
self.chat_template = chat_template
|
|
self.chat_template_content_format: Final = chat_template_content_format
|
|
self.trust_request_chat_template = trust_request_chat_template
|
|
self.default_chat_template_kwargs = default_chat_template_kwargs or {}
|
|
self.enable_log_outputs = enable_log_outputs
|
|
self.enable_log_deltas = enable_log_deltas
|
|
|
|
# set up reasoning parser
|
|
self.reasoning_parser_cls = ParserManager.get_reasoning_parser(
|
|
reasoning_parser_name=reasoning_parser
|
|
)
|
|
# set up tool use
|
|
self.enable_auto_tools: bool = enable_auto_tools
|
|
self.tool_parser = ParserManager.get_tool_parser(
|
|
tool_parser_name=tool_parser,
|
|
enable_auto_tools=enable_auto_tools,
|
|
model_name=self.model_config.model,
|
|
)
|
|
self.parser_cls = ParserManager.get_parser(
|
|
tool_parser_name=tool_parser,
|
|
reasoning_parser_name=reasoning_parser,
|
|
enable_auto_tools=enable_auto_tools,
|
|
model_name=self.model_config.model,
|
|
)
|
|
if (
|
|
is_mistral_tool_parser(self.tool_parser)
|
|
and self.reasoning_parser_cls is not None
|
|
):
|
|
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
|
|
|
|
MistralToolParser.model_can_reason = True
|
|
|
|
self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none
|
|
|
|
self.enable_prompt_tokens_details = enable_prompt_tokens_details
|
|
self.enable_force_include_usage = enable_force_include_usage
|
|
self.default_sampling_params = self.model_config.get_diff_sampling_param()
|
|
mc = self.model_config
|
|
self.override_max_tokens = (
|
|
self.default_sampling_params.get("max_tokens")
|
|
if mc.generation_config not in ("auto", "vllm")
|
|
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
|
|
)
|
|
self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
|
|
if self.use_harmony:
|
|
if "stop_token_ids" not in self.default_sampling_params:
|
|
self.default_sampling_params["stop_token_ids"] = []
|
|
self.default_sampling_params["stop_token_ids"].extend(
|
|
get_stop_tokens_for_assistant_actions()
|
|
)
|
|
|
|
self.tool_call_id_type = get_tool_call_id_type(self.model_config)
|
|
|
|
# NOTE(woosuk): While OpenAI's chat completion API supports browsing
|
|
# for some models, currently vLLM doesn't support it. Please use the
|
|
# Responses API instead.
|
|
self.supports_browsing = False
|
|
self.browser_tool = None
|
|
# NOTE(woosuk): Chat completion API does not support code interpreter.
|
|
# Please use the Responses API instead.
|
|
self.supports_code_interpreter = False
|
|
self.python_tool = None
|
|
|
|
def warmup(self) -> None:
|
|
self.renderer.warmup(
|
|
ChatParams(
|
|
chat_template=self.chat_template,
|
|
chat_template_content_format=self.chat_template_content_format,
|
|
chat_template_kwargs=self.default_chat_template_kwargs,
|
|
)
|
|
)
|
|
|
|
def _effective_chat_template_kwargs(
|
|
self, request: ChatCompletionRequest
|
|
) -> dict[str, Any]:
|
|
return (
|
|
request.build_chat_params(
|
|
self.chat_template,
|
|
self.chat_template_content_format,
|
|
)
|
|
.with_defaults(self.default_chat_template_kwargs)
|
|
.chat_template_kwargs
|
|
)
|
|
|
|
async def render_chat_request(
|
|
self,
|
|
request: ChatCompletionRequest,
|
|
) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse:
|
|
"""
|
|
Validate the model and preprocess a chat completion request.
|
|
|
|
Delegates preprocessing logic to OpenAIServingRender, adding the
|
|
engine-aware checks (LoRA model validation, engine health).
|
|
|
|
Returns:
|
|
A tuple of (conversation, engine_inputs) on success,
|
|
or an ErrorResponse on failure.
|
|
"""
|
|
error_check_ret = await self._check_model(request)
|
|
if error_check_ret is not None:
|
|
logger.error("Error with model %s", error_check_ret)
|
|
return error_check_ret
|
|
|
|
# If the engine is dead, raise the engine's DEAD_ERROR.
|
|
# This is required for the streaming case, where we return a
|
|
# success status before we actually start generating text :).
|
|
if self.engine_client.errored:
|
|
raise self.engine_client.dead_error
|
|
|
|
return await self.openai_serving_render.render_chat(request)
|
|
|
|
async def create_chat_completion(
|
|
self,
|
|
request: ChatCompletionRequest,
|
|
raw_request: Request | None = None,
|
|
) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
|
|
"""
|
|
Chat Completion API similar to OpenAI's API.
|
|
|
|
See https://platform.openai.com/docs/api-reference/chat/create
|
|
for the API specification. This API mimics the OpenAI
|
|
Chat Completion API.
|
|
"""
|
|
return await self._with_kv_transfer_rejection_cleanup(
|
|
self._create_chat_completion(request, raw_request), request, raw_request
|
|
)
|
|
|
|
async def _create_chat_completion(
|
|
self,
|
|
request: ChatCompletionRequest,
|
|
raw_request: Request | None = None,
|
|
) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse:
|
|
# Streaming response
|
|
tokenizer = self.renderer.tokenizer
|
|
assert tokenizer is not None
|
|
chat_template_kwargs = self._effective_chat_template_kwargs(request)
|
|
reasoning_parser: ReasoningParser | None = None
|
|
if self.reasoning_parser_cls:
|
|
reasoning_parser = self.reasoning_parser_cls(
|
|
tokenizer,
|
|
chat_template_kwargs=chat_template_kwargs, # type: ignore[call-arg]
|
|
)
|
|
result = await self.render_chat_request(request)
|
|
if isinstance(result, ErrorResponse):
|
|
return result
|
|
|
|
conversation, engine_inputs = result
|
|
|
|
request_id = (
|
|
f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
|
|
)
|
|
|
|
request_metadata = RequestResponseMetadata(request_id=request_id)
|
|
if raw_request:
|
|
raw_request.state.request_metadata = request_metadata
|
|
|
|
lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
|
|
|
|
model_name = self.models.model_name(lora_request)
|
|
|
|
# Extract data_parallel_rank from header (router can inject it)
|
|
data_parallel_rank = self._get_data_parallel_rank(raw_request)
|
|
|
|
# Schedule the request and get the result generator.
|
|
max_model_len = self.model_config.max_model_len
|
|
generators: list[AsyncGenerator[RequestOutput, None]] = []
|
|
for i, engine_input in enumerate(engine_inputs):
|
|
prompt_token_ids = self._extract_prompt_components(engine_input).token_ids
|
|
|
|
# If we are creating sub requests for multiple prompts, ensure that they
|
|
# have unique request ids.
|
|
sub_request_id = (
|
|
request_id if len(engine_inputs) == 1 else f"{request_id}_{i}"
|
|
)
|
|
|
|
max_tokens = get_max_tokens(
|
|
max_model_len,
|
|
request.max_completion_tokens
|
|
if request.max_completion_tokens is not None
|
|
else request.max_tokens,
|
|
self._extract_prompt_len(engine_input),
|
|
self.default_sampling_params,
|
|
self.override_max_tokens,
|
|
truncate_prompt_tokens=request.truncate_prompt_tokens,
|
|
)
|
|
|
|
sampling_params: SamplingParams | BeamSearchParams
|
|
if request.use_beam_search:
|
|
sampling_params = request.to_beam_search_params(
|
|
max_tokens, self.default_sampling_params
|
|
)
|
|
else:
|
|
sampling_params = request.to_sampling_params(
|
|
max_tokens,
|
|
self.default_sampling_params,
|
|
)
|
|
|
|
self._log_inputs(
|
|
sub_request_id,
|
|
engine_input,
|
|
params=sampling_params,
|
|
lora_request=lora_request,
|
|
)
|
|
|
|
trace_headers = (
|
|
None
|
|
if raw_request is None
|
|
else await self._get_trace_headers(raw_request.headers)
|
|
)
|
|
|
|
if isinstance(sampling_params, BeamSearchParams):
|
|
generator = self.beam_search(
|
|
prompt=engine_input,
|
|
request_id=sub_request_id,
|
|
params=sampling_params,
|
|
lora_request=lora_request,
|
|
trace_headers=trace_headers,
|
|
)
|
|
else:
|
|
if not request.include_reasoning:
|
|
reasoning_ended = True
|
|
elif request._grammar_from_tool_parser:
|
|
# The Mistral grammar already includes an optional
|
|
# `think?` rule that handles both reasoning and
|
|
# non-reasoning outputs.
|
|
reasoning_ended = True
|
|
elif reasoning_parser:
|
|
reasoning_ended = reasoning_parser.is_reasoning_end(
|
|
prompt_token_ids or []
|
|
)
|
|
else:
|
|
reasoning_ended = None
|
|
|
|
generator = self.engine_client.generate(
|
|
engine_input,
|
|
sampling_params,
|
|
sub_request_id,
|
|
lora_request=lora_request,
|
|
trace_headers=trace_headers,
|
|
priority=request.priority,
|
|
data_parallel_rank=data_parallel_rank,
|
|
reasoning_ended=reasoning_ended,
|
|
reasoning_parser_kwargs={
|
|
"chat_template_kwargs": chat_template_kwargs,
|
|
}
|
|
if reasoning_parser
|
|
else None,
|
|
)
|
|
|
|
generators.append(generator)
|
|
|
|
assert len(generators) == 1
|
|
(result_generator,) = generators
|
|
|
|
if request.stream:
|
|
return self.chat_completion_stream_generator(
|
|
request,
|
|
result_generator,
|
|
request_id,
|
|
model_name,
|
|
conversation,
|
|
tokenizer,
|
|
request_metadata,
|
|
reasoning_parser,
|
|
chat_template_kwargs=chat_template_kwargs,
|
|
)
|
|
|
|
return await self.chat_completion_full_generator(
|
|
request,
|
|
result_generator,
|
|
request_id,
|
|
model_name,
|
|
conversation,
|
|
tokenizer,
|
|
request_metadata,
|
|
reasoning_parser,
|
|
)
|
|
|
|
def get_chat_request_role(self, request: ChatCompletionRequest) -> str:
|
|
if request.add_generation_prompt:
|
|
return self.response_role
|
|
return request.messages[-1]["role"]
|
|
|
|
async def chat_completion_stream_generator(
|
|
self,
|
|
request: ChatCompletionRequest,
|
|
result_generator: AsyncIterator[RequestOutput],
|
|
request_id: str,
|
|
model_name: str,
|
|
conversation: list[ConversationMessage],
|
|
tokenizer: TokenizerLike,
|
|
request_metadata: RequestResponseMetadata,
|
|
reasoning_parser: ReasoningParser | None = None,
|
|
chat_template_kwargs: dict[str, Any] | None = None,
|
|
) -> AsyncGenerator[str, None]:
|
|
created_time = int(time.time())
|
|
chunk_object_type: Final = "chat.completion.chunk"
|
|
first_iteration = True
|
|
|
|
# Send response for each token for each request.n (index)
|
|
num_choices = 1 if request.n is None else request.n
|
|
previous_num_tokens = [0] * num_choices
|
|
finish_reason_sent = [False] * num_choices
|
|
num_prompt_tokens = 0
|
|
num_cached_tokens = None
|
|
if self.use_harmony:
|
|
harmony_parsers = [
|
|
get_streamable_parser_for_assistant() for _ in range(num_choices)
|
|
]
|
|
harmony_tools_streamed = [False] * num_choices
|
|
tools_streamed = [False] * num_choices
|
|
|
|
is_mistral_grammar_path = request._grammar_from_tool_parser
|
|
|
|
if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam):
|
|
tool_choice_function_name = request.tool_choice.function.name
|
|
else:
|
|
tool_choice_function_name = None
|
|
|
|
# Determine whether tools are in use with "auto" tool choice
|
|
tool_choice_auto = (
|
|
not tool_choice_function_name
|
|
and self._should_stream_with_auto_tool_parsing(request)
|
|
)
|
|
|
|
all_previous_token_ids: list[list[int]] | None
|
|
if self.tool_call_id_type == "kimi_k2":
|
|
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
|
|
else:
|
|
history_tool_call_cnt = 0
|
|
|
|
# Always track previous_texts for comprehensive output logging
|
|
previous_texts = [""] * num_choices
|
|
|
|
# Only one of these will be used, thus previous_texts and
|
|
# all_previous_token_ids will not be used twice in the same iteration.
|
|
if (
|
|
is_mistral_grammar_path
|
|
or tool_choice_auto
|
|
or tool_choice_function_name
|
|
or request.tool_choice == "required"
|
|
or reasoning_parser
|
|
):
|
|
all_previous_token_ids = [[] for _ in range(num_choices)]
|
|
reasoning_end_arr = [False] * num_choices
|
|
prompt_is_reasoning_end_arr: list[bool | None] = [None] * num_choices
|
|
else:
|
|
all_previous_token_ids = None
|
|
|
|
try:
|
|
if self.parser_cls is not None:
|
|
if tokenizer is None:
|
|
raise ValueError(
|
|
"Tokenizer not available when `skip_tokenizer_init=True`"
|
|
)
|
|
parsers: list[Parser | None] = [
|
|
self.parser_cls(
|
|
tokenizer,
|
|
request.tools,
|
|
chat_template_kwargs=chat_template_kwargs,
|
|
)
|
|
for _ in range(num_choices)
|
|
]
|
|
for p in parsers:
|
|
if p is not None:
|
|
p._stream_state.tool_call_id_type = self.tool_call_id_type
|
|
p._stream_state.history_tool_call_cnt = history_tool_call_cnt
|
|
else:
|
|
parsers = [None] * num_choices
|
|
except Exception as e:
|
|
logger.exception("Error in parser creation.")
|
|
data = self.create_streaming_error_response(e)
|
|
yield f"data: {data}\n\n"
|
|
yield "data: [DONE]\n\n"
|
|
return
|
|
|
|
stream_options = request.stream_options
|
|
include_usage, include_continuous_usage = should_include_usage(
|
|
stream_options, self.enable_force_include_usage
|
|
)
|
|
|
|
try:
|
|
async for res in result_generator:
|
|
if res.prompt_token_ids is not None:
|
|
num_prompt_tokens = len(res.prompt_token_ids)
|
|
if res.encoder_prompt_token_ids is not None:
|
|
num_prompt_tokens += len(res.encoder_prompt_token_ids)
|
|
|
|
# We need to do it here, because if there are exceptions in
|
|
# the result_generator, it needs to be sent as the FIRST
|
|
# response (by the try...catch).
|
|
if first_iteration:
|
|
num_cached_tokens = res.num_cached_tokens
|
|
# Send first response for each request.n (index) with
|
|
# the role
|
|
role = self.get_chat_request_role(request)
|
|
|
|
# ``res.prompt`` is the rendered chat-templated prompt
|
|
prompt_text = res.prompt if request.return_prompt_text else None
|
|
|
|
# NOTE num_choices defaults to 1 so this usually executes
|
|
# once per request
|
|
for i in range(num_choices):
|
|
choice_data = ChatCompletionResponseStreamChoice(
|
|
index=i,
|
|
delta=DeltaMessage(
|
|
role=role,
|
|
content="",
|
|
),
|
|
logprobs=None,
|
|
finish_reason=None,
|
|
)
|
|
|
|
# return prompt_token_ids at the first chunk ever
|
|
chunk = ChatCompletionStreamResponse(
|
|
id=request_id,
|
|
object=chunk_object_type,
|
|
created=created_time,
|
|
choices=[choice_data],
|
|
model=model_name,
|
|
prompt_token_ids=(
|
|
res.prompt_token_ids
|
|
if request.return_token_ids
|
|
else None
|
|
),
|
|
prompt_text=prompt_text,
|
|
)
|
|
|
|
# if continuous usage stats are requested, add it
|
|
if include_continuous_usage:
|
|
chunk.usage = UsageInfo(
|
|
prompt_tokens=num_prompt_tokens,
|
|
completion_tokens=0,
|
|
total_tokens=num_prompt_tokens,
|
|
)
|
|
|
|
data = chunk.model_dump_json(exclude_unset=True)
|
|
yield f"data: {data}\n\n"
|
|
|
|
# Send response to echo the input portion of the
|
|
# last message
|
|
if request.echo:
|
|
last_msg_content: str | list[dict[str, str]] = ""
|
|
if (
|
|
conversation
|
|
and "content" in conversation[-1]
|
|
and conversation[-1].get("role") == role
|
|
):
|
|
last_msg_content = conversation[-1]["content"] or ""
|
|
|
|
if last_msg_content:
|
|
for i in range(num_choices):
|
|
choice_data = ChatCompletionResponseStreamChoice(
|
|
index=i,
|
|
delta=DeltaMessage(content=last_msg_content),
|
|
logprobs=None,
|
|
finish_reason=None,
|
|
)
|
|
chunk = ChatCompletionStreamResponse(
|
|
id=request_id,
|
|
object=chunk_object_type,
|
|
created=created_time,
|
|
choices=[choice_data],
|
|
model=model_name,
|
|
)
|
|
if include_continuous_usage:
|
|
chunk.usage = UsageInfo(
|
|
prompt_tokens=num_prompt_tokens,
|
|
completion_tokens=0,
|
|
total_tokens=num_prompt_tokens,
|
|
)
|
|
|
|
data = chunk.model_dump_json(exclude_unset=True)
|
|
yield f"data: {data}\n\n"
|
|
first_iteration = False
|
|
|
|
for output in res.outputs:
|
|
i = output.index
|
|
parser = parsers[i]
|
|
tool_parser = parser.tool_parser if parser is not None else None
|
|
|
|
if (
|
|
reasoning_parser
|
|
and res.prompt_token_ids
|
|
and prompt_is_reasoning_end_arr[i] is None
|
|
):
|
|
# only check once per choice, because prompt_token_ids
|
|
# are the same for all deltas in that choice
|
|
prompt_is_reasoning_end_arr[i] = (
|
|
reasoning_parser.is_reasoning_end(res.prompt_token_ids)
|
|
)
|
|
if finish_reason_sent[i]:
|
|
continue
|
|
|
|
if request.logprobs and request.top_logprobs is not None:
|
|
assert output.logprobs is not None, "Did not output logprobs"
|
|
logprobs = self._create_chat_logprobs(
|
|
token_ids=output.token_ids,
|
|
top_logprobs=output.logprobs,
|
|
tokenizer=tokenizer,
|
|
num_output_top_logprobs=request.top_logprobs,
|
|
return_as_token_id=request.return_tokens_as_token_ids,
|
|
)
|
|
else:
|
|
logprobs = None
|
|
|
|
if self.use_harmony:
|
|
harmony_parser = harmony_parsers[i]
|
|
prev_recipient = harmony_parser.current_recipient
|
|
|
|
# Track accumulated content per token with their state
|
|
token_states: list[TokenState] = []
|
|
for token_id in output.token_ids:
|
|
harmony_parser.process(token_id)
|
|
token_delta = harmony_parser.last_content_delta or ""
|
|
token_states.append(
|
|
TokenState(
|
|
harmony_parser.current_channel,
|
|
harmony_parser.current_recipient,
|
|
token_delta,
|
|
)
|
|
)
|
|
delta_text = "".join(delta for _, _, delta in token_states)
|
|
cur_channel = harmony_parser.current_channel
|
|
|
|
# handle the case where several tokens where generated at once
|
|
# including the final token, leading to a delta in the text
|
|
# but the current channel to be empty (start state)
|
|
if not cur_channel and delta_text:
|
|
cur_channel = "final"
|
|
else:
|
|
delta_text = output.text
|
|
|
|
if (
|
|
not delta_text
|
|
and not output.token_ids
|
|
and not previous_num_tokens[i]
|
|
):
|
|
# Chunked prefill case, don't return empty chunks
|
|
continue
|
|
|
|
delta_message: DeltaMessage | None
|
|
|
|
# just update previous_texts and previous_token_ids
|
|
if (
|
|
is_mistral_grammar_path
|
|
or tool_choice_auto
|
|
or tool_choice_function_name
|
|
or request.tool_choice == "required"
|
|
or reasoning_parser
|
|
):
|
|
assert previous_texts is not None
|
|
assert all_previous_token_ids is not None
|
|
previous_text = previous_texts[i]
|
|
previous_token_ids = all_previous_token_ids[i]
|
|
current_text = previous_text + delta_text
|
|
# avoid the None + list error.
|
|
if previous_token_ids:
|
|
current_token_ids = previous_token_ids + as_list(
|
|
output.token_ids
|
|
)
|
|
else:
|
|
current_token_ids = as_list(output.token_ids)
|
|
|
|
if self.use_harmony:
|
|
delta_message, tools_streamed_flag = (
|
|
extract_harmony_streaming_delta(
|
|
harmony_parser=harmony_parser,
|
|
token_states=token_states,
|
|
prev_recipient=prev_recipient,
|
|
include_reasoning=request.include_reasoning,
|
|
)
|
|
)
|
|
harmony_tools_streamed[i] |= tools_streamed_flag
|
|
# Mistral grammar path: combined reasoning + tool streaming
|
|
elif is_mistral_grammar_path:
|
|
from vllm.tool_parsers.mistral_tool_parser import (
|
|
MistralToolParser,
|
|
)
|
|
|
|
assert tool_parser is not None
|
|
assert isinstance(tool_parser, MistralToolParser)
|
|
assert reasoning_end_arr is not None
|
|
output_token_ids = as_list(output.token_ids)
|
|
result = tool_parser.extract_maybe_reasoning_and_tool_streaming(
|
|
reasoning_parser=reasoning_parser,
|
|
previous_text=previous_text,
|
|
current_text=current_text,
|
|
delta_text=delta_text,
|
|
previous_token_ids=previous_token_ids,
|
|
current_token_ids=current_token_ids,
|
|
output_token_ids=output_token_ids,
|
|
reasoning_ended=reasoning_end_arr[i],
|
|
prompt_is_reasoning_end=(prompt_is_reasoning_end_arr[i]),
|
|
request=request,
|
|
)
|
|
delta_message = result.delta_message
|
|
reasoning_end_arr[i] = result.reasoning_ended
|
|
current_text = result.current_text
|
|
current_token_ids = result.current_token_ids
|
|
if result.tools_called:
|
|
tools_streamed[i] = True
|
|
|
|
elif parser is not None:
|
|
delta_message = parser.parse_delta(
|
|
delta_text=delta_text,
|
|
delta_token_ids=as_list(output.token_ids),
|
|
request=request,
|
|
prompt_token_ids=res.prompt_token_ids,
|
|
)
|
|
if delta_message and delta_message.tool_calls:
|
|
tools_streamed[i] = True
|
|
# handle streaming just a content delta (no parsers)
|
|
else:
|
|
delta_message = DeltaMessage(content=delta_text)
|
|
|
|
# update the previous values for the next iteration
|
|
if (
|
|
is_mistral_grammar_path
|
|
or tool_choice_auto
|
|
or tool_choice_function_name
|
|
or request.tool_choice == "required"
|
|
or reasoning_parser
|
|
) and not self.use_harmony:
|
|
assert previous_texts is not None
|
|
assert all_previous_token_ids is not None
|
|
previous_texts[i] = current_text
|
|
all_previous_token_ids[i] = current_token_ids
|
|
else:
|
|
# Update for comprehensive logging even in simple case
|
|
assert previous_texts is not None
|
|
previous_texts[i] += delta_text
|
|
|
|
# set the previous values for the next iteration
|
|
previous_num_tokens[i] += len(output.token_ids)
|
|
|
|
# if the message delta is None (e.g. because it was a
|
|
# "control token" for tool calls or the parser otherwise
|
|
# wasn't ready to send a token, then
|
|
# get the next token without streaming a chunk
|
|
if delta_message is None:
|
|
# NOTE: If return_token_ids is enabled, we still need to
|
|
# send a chunk with token_ids even if delta_message is None
|
|
# to ensure all tokens are included in the response
|
|
if (
|
|
output.finish_reason is None
|
|
and not request.return_token_ids
|
|
):
|
|
continue
|
|
delta_message = DeltaMessage()
|
|
|
|
# Log streaming delta if output logging is enabled
|
|
if self.enable_log_outputs and self.request_logger:
|
|
delta_content_parts = []
|
|
if delta_message.content:
|
|
delta_content_parts.append(delta_message.content)
|
|
if delta_message.reasoning:
|
|
reasoning = delta_message.reasoning
|
|
delta_content_parts.append(f"[reasoning: {reasoning}]")
|
|
if delta_message.tool_calls:
|
|
tool_args = "".join(
|
|
tc.function.arguments
|
|
for tc in delta_message.tool_calls
|
|
if tc.function and tc.function.arguments
|
|
)
|
|
if tool_args:
|
|
delta_content_parts.append(f"[tool_calls: {tool_args}]")
|
|
|
|
if delta_content_parts and self.enable_log_deltas:
|
|
delta_content = " ".join(delta_content_parts)
|
|
self.request_logger.log_outputs(
|
|
request_id=request_id,
|
|
outputs=delta_content,
|
|
output_token_ids=as_list(output.token_ids),
|
|
finish_reason=output.finish_reason,
|
|
is_streaming=True,
|
|
delta=True,
|
|
)
|
|
|
|
if output.finish_reason is None:
|
|
# Send token-by-token response for each request.n
|
|
choice_data = ChatCompletionResponseStreamChoice(
|
|
index=i,
|
|
delta=delta_message,
|
|
logprobs=logprobs,
|
|
finish_reason=None,
|
|
token_ids=(
|
|
as_list(output.token_ids)
|
|
if request.return_token_ids
|
|
else None
|
|
),
|
|
)
|
|
|
|
# if the model is finished generating
|
|
else:
|
|
# check for error finish reason and abort streaming
|
|
# finish_reason='error' indicates a retryable error
|
|
self._raise_if_error(output.finish_reason, request_id)
|
|
|
|
# check to make sure we haven't "forgotten" to stream
|
|
# any tokens that were generated but previously
|
|
# matched by partial json parsing
|
|
# only happens if we are NOT using structured outputs
|
|
index = 0
|
|
auto_tools_called = False
|
|
if tool_parser:
|
|
auto_tools_called = len(tool_parser.prev_tool_call_arr) > 0
|
|
index = (
|
|
len(tool_parser.prev_tool_call_arr) - 1
|
|
if auto_tools_called
|
|
else 0
|
|
)
|
|
should_check = (
|
|
self._should_check_for_unstreamed_tool_arg_tokens(
|
|
delta_message, output
|
|
)
|
|
)
|
|
# only check if there are any tool calls
|
|
# detected by partial parsing
|
|
if should_check and tool_parser and auto_tools_called:
|
|
latest_delta_len = 0
|
|
if (
|
|
isinstance(
|
|
delta_message.tool_calls[0].function,
|
|
DeltaFunctionCall,
|
|
)
|
|
) and isinstance(
|
|
delta_message.tool_calls[0].function.arguments, str
|
|
):
|
|
latest_delta_len = len(
|
|
delta_message.tool_calls[0].function.arguments
|
|
)
|
|
|
|
# get the expected call based on partial JSON
|
|
# parsing which "autocompletes" the JSON.
|
|
# Tool parsers (e.g. Qwen3Coder) store
|
|
# arguments as a JSON string in
|
|
# prev_tool_call_arr. Calling json.dumps()
|
|
# on an already-serialized string would
|
|
# double-serialize it (e.g. '{"k":1}' becomes
|
|
# '"{\\"k\\":1}"'), which then causes the
|
|
# replace() below to fail and append the
|
|
# entire double-serialized string as a
|
|
# spurious final delta.
|
|
args = tool_parser.prev_tool_call_arr[index].get(
|
|
"arguments", {}
|
|
)
|
|
if isinstance(args, str):
|
|
expected_call = args
|
|
else:
|
|
expected_call = json.dumps(args, ensure_ascii=False)
|
|
|
|
# get what we've streamed so far for arguments
|
|
# for the current tool
|
|
actual_call = tool_parser.streamed_args_for_tool[index]
|
|
if latest_delta_len > 0:
|
|
actual_call = actual_call[:-latest_delta_len]
|
|
|
|
# check to see if there's anything left to stream
|
|
remaining_call = expected_call.replace(actual_call, "", 1)
|
|
# set that as a delta message
|
|
delta_message = self._create_remaining_args_delta(
|
|
delta_message, remaining_call, index
|
|
)
|
|
|
|
# Send the finish response for each request.n only once
|
|
# In OpenAI's API, when a tool is called, the
|
|
# finish_reason is:
|
|
# "tool_calls" for "auto" or "required" tool calls,
|
|
# and "stop" for named tool calls.
|
|
if (
|
|
auto_tools_called
|
|
or (tools_streamed[i] and not tool_choice_function_name)
|
|
or (self.use_harmony and harmony_tools_streamed[i])
|
|
):
|
|
finish_reason_ = "tool_calls"
|
|
else:
|
|
finish_reason_ = (
|
|
output.finish_reason if output.finish_reason else "stop"
|
|
)
|
|
choice_data = ChatCompletionResponseStreamChoice(
|
|
index=i,
|
|
delta=delta_message,
|
|
logprobs=logprobs,
|
|
finish_reason=finish_reason_,
|
|
stop_reason=output.stop_reason,
|
|
token_ids=(
|
|
as_list(output.token_ids)
|
|
if request.return_token_ids
|
|
else None
|
|
),
|
|
)
|
|
|
|
finish_reason_sent[i] = True
|
|
|
|
choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
|
|
chunk = ChatCompletionStreamResponse(
|
|
id=request_id,
|
|
object=chunk_object_type,
|
|
created=created_time,
|
|
choices=[choice_data],
|
|
model=model_name,
|
|
)
|
|
# Stamp the fingerprint on terminal chunks only (those with
|
|
# finish_reason set). When ``include_usage`` is on, the
|
|
# trailing usage chunk below overrides this as the true
|
|
# final message.
|
|
if (
|
|
not include_usage
|
|
and self.system_fingerprint is not None
|
|
and choice_data.finish_reason is not None
|
|
):
|
|
chunk.system_fingerprint = self.system_fingerprint
|
|
|
|
# handle usage stats if requested & if continuous
|
|
if include_continuous_usage:
|
|
completion_tokens = previous_num_tokens[i]
|
|
chunk.usage = UsageInfo(
|
|
prompt_tokens=num_prompt_tokens,
|
|
completion_tokens=completion_tokens,
|
|
total_tokens=num_prompt_tokens + completion_tokens,
|
|
)
|
|
|
|
data = chunk.model_dump_json(exclude_unset=True)
|
|
yield f"data: {data}\n\n"
|
|
|
|
# once the final token is handled, if stream_options.include_usage
|
|
# is sent, send the usage
|
|
if include_usage:
|
|
completion_tokens = sum(previous_num_tokens)
|
|
final_usage = UsageInfo(
|
|
prompt_tokens=num_prompt_tokens,
|
|
completion_tokens=completion_tokens,
|
|
total_tokens=num_prompt_tokens + completion_tokens,
|
|
)
|
|
if self.enable_prompt_tokens_details and num_cached_tokens:
|
|
final_usage.prompt_tokens_details = PromptTokenUsageInfo(
|
|
cached_tokens=num_cached_tokens
|
|
)
|
|
|
|
final_usage_chunk = ChatCompletionStreamResponse(
|
|
id=request_id,
|
|
object=chunk_object_type,
|
|
created=created_time,
|
|
choices=[],
|
|
model=model_name,
|
|
usage=final_usage,
|
|
system_fingerprint=self.system_fingerprint,
|
|
)
|
|
final_usage_data = final_usage_chunk.model_dump_json(
|
|
exclude_unset=True, exclude_none=True
|
|
)
|
|
yield f"data: {final_usage_data}\n\n"
|
|
|
|
# report to FastAPI middleware aggregate usage across all choices
|
|
num_completion_tokens = sum(previous_num_tokens)
|
|
request_metadata.final_usage_info = UsageInfo(
|
|
prompt_tokens=num_prompt_tokens,
|
|
completion_tokens=num_completion_tokens,
|
|
total_tokens=num_prompt_tokens + num_completion_tokens,
|
|
)
|
|
|
|
# Log complete streaming response if output logging is enabled
|
|
if self.enable_log_outputs and self.request_logger:
|
|
# Log the complete response for each choice
|
|
for i in range(num_choices):
|
|
full_text = (
|
|
previous_texts[i]
|
|
if previous_texts and i < len(previous_texts)
|
|
else f"<streaming_complete: {previous_num_tokens[i]} tokens>"
|
|
)
|
|
self.request_logger.log_outputs(
|
|
request_id=request_id,
|
|
outputs=full_text,
|
|
output_token_ids=None, # Consider also logging all token IDs
|
|
finish_reason="streaming_complete",
|
|
is_streaming=True,
|
|
delta=False,
|
|
)
|
|
|
|
except GenerationError as e:
|
|
yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n"
|
|
except Exception as e:
|
|
logger.exception("Error in chat completion stream generator.")
|
|
data = self.create_streaming_error_response(e)
|
|
yield f"data: {data}\n\n"
|
|
# Send the final done message after all response.n are finished
|
|
yield "data: [DONE]\n\n"
|
|
|
|
async def chat_completion_full_generator(
|
|
self,
|
|
request: ChatCompletionRequest,
|
|
result_generator: AsyncIterator[RequestOutput],
|
|
request_id: str,
|
|
model_name: str,
|
|
conversation: list[ConversationMessage],
|
|
tokenizer: TokenizerLike,
|
|
request_metadata: RequestResponseMetadata,
|
|
reasoning_parser: ReasoningParser | None = None,
|
|
) -> ErrorResponse | ChatCompletionResponse:
|
|
created_time = int(time.time())
|
|
final_res: RequestOutput | None = None
|
|
|
|
try:
|
|
async for res in result_generator:
|
|
final_res = res
|
|
except asyncio.CancelledError:
|
|
return self.create_error_response("Client disconnected")
|
|
|
|
if final_res is None:
|
|
return self.create_error_response(
|
|
"No output received from the engine.",
|
|
err_type="InternalServerError",
|
|
status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
|
|
)
|
|
|
|
choices: list[ChatCompletionResponseChoice] = []
|
|
if self.tool_call_id_type == "kimi_k2":
|
|
history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
|
|
else:
|
|
history_tool_call_cnt = 0
|
|
|
|
role = self.get_chat_request_role(request)
|
|
for output in final_res.outputs:
|
|
# check for error finish reason and raise GenerationError
|
|
# finish_reason='error' indicates a retryable request-level internal error
|
|
self._raise_if_error(output.finish_reason, request_id)
|
|
token_ids = output.token_ids
|
|
out_logprobs = output.logprobs
|
|
tool_call_info = None
|
|
|
|
if request.logprobs and request.top_logprobs is not None:
|
|
assert out_logprobs is not None, "Did not output logprobs"
|
|
logprobs = self._create_chat_logprobs(
|
|
token_ids=token_ids,
|
|
top_logprobs=out_logprobs,
|
|
num_output_top_logprobs=request.top_logprobs,
|
|
tokenizer=tokenizer,
|
|
return_as_token_id=request.return_tokens_as_token_ids,
|
|
)
|
|
else:
|
|
logprobs = None
|
|
|
|
if self.use_harmony:
|
|
reasoning, content, _ = parse_chat_output(token_ids)
|
|
if not request.include_reasoning:
|
|
reasoning = None
|
|
|
|
if self.tool_parser is not None:
|
|
if tokenizer is None:
|
|
raise ValueError(
|
|
"Tokenizer not available when `skip_tokenizer_init=True`"
|
|
)
|
|
|
|
tool_parser = self.tool_parser(tokenizer, request.tools)
|
|
# NOTE: We use token_ids for openai tool parser
|
|
tool_call_info = tool_parser.extract_tool_calls(
|
|
"",
|
|
request=request,
|
|
token_ids=token_ids, # type: ignore
|
|
)
|
|
content = tool_call_info.content
|
|
message = ChatMessage(
|
|
role=role,
|
|
reasoning=reasoning,
|
|
content=content,
|
|
tool_calls=tool_call_info.tool_calls,
|
|
)
|
|
else:
|
|
message = ChatMessage(
|
|
role=role,
|
|
reasoning=reasoning,
|
|
content=content,
|
|
)
|
|
|
|
# Encode routed_experts for transport. JSON can't carry raw
|
|
# bytes, so we write the ndarray as a ``.npy`` byte stream
|
|
# and base64-encode it. ``pybase64`` is ~3x faster than the
|
|
# stdlib ``base64`` on large payloads thanks to SIMD.
|
|
routed_experts_b64 = None
|
|
if output.routed_experts is not None:
|
|
buf = io.BytesIO()
|
|
np.save(buf, output.routed_experts)
|
|
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode(
|
|
"ascii"
|
|
)
|
|
|
|
choice_data = ChatCompletionResponseChoice(
|
|
index=output.index,
|
|
message=message,
|
|
logprobs=logprobs,
|
|
finish_reason=(
|
|
"tool_calls"
|
|
if (tool_call_info is not None and tool_call_info.tools_called)
|
|
else output.finish_reason
|
|
if output.finish_reason
|
|
else "stop"
|
|
),
|
|
stop_reason=output.stop_reason,
|
|
token_ids=(
|
|
as_list(output.token_ids) if request.return_token_ids else None
|
|
),
|
|
routed_experts=routed_experts_b64,
|
|
)
|
|
choices.append(choice_data)
|
|
continue
|
|
|
|
if reasoning_parser:
|
|
# If the reasoning parser is enabled,
|
|
# tool calls are extracted exclusively from the content.
|
|
reasoning, content = reasoning_parser.extract_reasoning(
|
|
output.text, request=request
|
|
)
|
|
if not request.include_reasoning:
|
|
reasoning = None
|
|
else:
|
|
reasoning = None
|
|
content = output.text
|
|
|
|
auto_tools_called = False
|
|
# if auto tools are not enabled, and a named tool choice using
|
|
# outlines is not being used
|
|
tool_calls, content = self._parse_tool_calls_from_content(
|
|
request=request,
|
|
tokenizer=tokenizer,
|
|
content=content,
|
|
enable_auto_tools=self.enable_auto_tools,
|
|
tool_parser_cls=self.tool_parser,
|
|
)
|
|
if is_mistral_tokenizer(tokenizer):
|
|
from vllm.tool_parsers.mistral_tool_parser import MistralToolCall
|
|
|
|
tool_call_class: type[ToolCall] = MistralToolCall
|
|
else:
|
|
tool_call_class = ToolCall
|
|
|
|
use_mistral_tool_parser = request._grammar_from_tool_parser
|
|
if use_mistral_tool_parser:
|
|
from vllm.tool_parsers.mistral_tool_parser import MistralToolParser
|
|
|
|
tool_call_items = MistralToolParser.build_non_streaming_tool_calls(
|
|
tool_calls
|
|
)
|
|
if tool_call_items:
|
|
auto_tools_called = (
|
|
request.tool_choice is None or request.tool_choice == "auto"
|
|
)
|
|
message = ChatMessage(
|
|
role=role,
|
|
reasoning=reasoning,
|
|
content=content,
|
|
tool_calls=tool_call_items,
|
|
)
|
|
|
|
elif (not self.enable_auto_tools or not self.tool_parser) and (
|
|
not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
|
|
and request.tool_choice != "required"
|
|
):
|
|
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
|
|
|
elif (
|
|
request.tool_choice
|
|
and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam
|
|
):
|
|
tool_call_class_items = []
|
|
tool_calls = tool_calls or []
|
|
for idx, tc in enumerate(tool_calls):
|
|
# Use native ID if available (e.g., Kimi K2),
|
|
# otherwise generate ID with correct id_type
|
|
if tc.id:
|
|
tool_call_class_items.append(
|
|
tool_call_class(id=tc.id, function=tc)
|
|
)
|
|
else:
|
|
# Generate ID using the correct format (kimi_k2 or random),
|
|
# but leave it to the class if it's Mistral to preserve
|
|
# 9-char IDs
|
|
if is_mistral_tokenizer(tokenizer):
|
|
tool_call_class_items.append(tool_call_class(function=tc))
|
|
else:
|
|
generated_id = make_tool_call_id(
|
|
id_type=self.tool_call_id_type,
|
|
func_name=tc.name,
|
|
idx=history_tool_call_cnt,
|
|
)
|
|
tool_call_class_items.append(
|
|
tool_call_class(id=generated_id, function=tc)
|
|
)
|
|
history_tool_call_cnt += 1
|
|
message = ChatMessage(
|
|
role=role,
|
|
reasoning=reasoning,
|
|
content="",
|
|
tool_calls=tool_call_class_items,
|
|
)
|
|
|
|
elif request.tool_choice and request.tool_choice == "required":
|
|
tool_call_class_items = []
|
|
tool_calls = tool_calls or []
|
|
for idx, tool_call in enumerate(tool_calls):
|
|
# Use native ID if available,
|
|
# otherwise generate ID with correct id_type
|
|
if tool_call.id:
|
|
tool_call_class_items.append(
|
|
tool_call_class(id=tool_call.id, function=tool_call)
|
|
)
|
|
else:
|
|
# Generate ID using the correct format (kimi_k2 or random),
|
|
# but leave it to the class if it's Mistral to preserve
|
|
# 9-char IDs
|
|
if is_mistral_tokenizer(tokenizer):
|
|
tool_call_class_items.append(
|
|
tool_call_class(function=tool_call)
|
|
)
|
|
else:
|
|
generated_id = make_tool_call_id(
|
|
id_type=self.tool_call_id_type,
|
|
func_name=tool_call.name,
|
|
idx=history_tool_call_cnt,
|
|
)
|
|
tool_call_class_items.append(
|
|
tool_call_class(id=generated_id, function=tool_call)
|
|
)
|
|
history_tool_call_cnt += 1
|
|
message = ChatMessage(
|
|
role=role,
|
|
content="",
|
|
tool_calls=tool_call_class_items,
|
|
reasoning=reasoning,
|
|
)
|
|
|
|
# if the request doesn't use tool choice
|
|
# OR specifies to not use a tool
|
|
elif not request.tool_choice or request.tool_choice == "none":
|
|
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
|
|
|
# handle when there are tools and tool choice is auto
|
|
elif (
|
|
request.tools
|
|
and (request.tool_choice == "auto" or request.tool_choice is None)
|
|
and self.enable_auto_tools
|
|
and self.tool_parser
|
|
):
|
|
# In the OpenAI API the finish_reason is "tools_called"
|
|
# if the tool choice is auto and the model produced a tool
|
|
# call. The same is not true for named function calls
|
|
auto_tools_called = tool_calls is not None and len(tool_calls) > 0
|
|
if tool_calls:
|
|
tool_call_items = []
|
|
for idx, tc in enumerate(tool_calls):
|
|
# Use native ID if available (e.g., Kimi K2),
|
|
# otherwise generate ID with correct id_type
|
|
if tc.id:
|
|
tool_call_items.append(
|
|
tool_call_class(id=tc.id, function=tc)
|
|
)
|
|
else:
|
|
# Generate ID using the correct format (kimi_k2 or random),
|
|
# but leave it to the class if it's Mistral to preserve
|
|
# 9-char IDs
|
|
if is_mistral_tokenizer(tokenizer):
|
|
tool_call_items.append(tool_call_class(function=tc))
|
|
else:
|
|
generated_id = make_tool_call_id(
|
|
id_type=self.tool_call_id_type,
|
|
func_name=tc.name,
|
|
idx=history_tool_call_cnt,
|
|
)
|
|
tool_call_items.append(
|
|
tool_call_class(id=generated_id, function=tc)
|
|
)
|
|
history_tool_call_cnt += 1
|
|
message = ChatMessage(
|
|
role=role,
|
|
reasoning=reasoning,
|
|
content=content,
|
|
tool_calls=tool_call_items,
|
|
)
|
|
|
|
else:
|
|
# FOR NOW make it a chat message; we will have to detect
|
|
# the type to make it later.
|
|
ret_content = content
|
|
|
|
# try to use content return from tool parser first,
|
|
# tool parser may do some modify for the content.
|
|
if content and len(content) > 0:
|
|
ret_content = content
|
|
message = ChatMessage(
|
|
role=role,
|
|
reasoning=reasoning,
|
|
content=ret_content,
|
|
)
|
|
|
|
# undetermined case that is still important to handle
|
|
else:
|
|
logger.error(
|
|
"Error in chat_completion_full_generator - cannot determine"
|
|
" if tools should be extracted. Returning a standard chat "
|
|
"completion."
|
|
)
|
|
message = ChatMessage(role=role, reasoning=reasoning, content=content)
|
|
# In OpenAI's API, when a tool is called, the finish_reason is:
|
|
# "tool_calls" for "auto" or "required" tool calls,
|
|
# and "stop" for named tool calls.
|
|
is_finish_reason_tool_calls = auto_tools_called or (
|
|
request.tool_choice
|
|
and request.tool_choice == "required"
|
|
and output.finish_reason == "stop"
|
|
)
|
|
|
|
# Encode routed_experts for transport. JSON can't carry raw
|
|
# bytes, so we write the ndarray as a ``.npy`` byte stream
|
|
# and base64-encode it. ``pybase64`` is ~3x faster than the
|
|
# stdlib ``base64`` on large payloads thanks to SIMD.
|
|
routed_experts_b64 = None
|
|
if output.routed_experts is not None:
|
|
buf = io.BytesIO()
|
|
np.save(buf, output.routed_experts)
|
|
routed_experts_b64 = base64.b64encode(buf.getvalue()).decode("ascii")
|
|
|
|
choice_data = ChatCompletionResponseChoice(
|
|
index=output.index,
|
|
message=message,
|
|
logprobs=logprobs,
|
|
finish_reason="tool_calls"
|
|
if is_finish_reason_tool_calls
|
|
else output.finish_reason
|
|
if output.finish_reason
|
|
else "stop",
|
|
stop_reason=output.stop_reason,
|
|
token_ids=(
|
|
as_list(output.token_ids) if request.return_token_ids else None
|
|
),
|
|
routed_experts=routed_experts_b64,
|
|
)
|
|
choice_data = maybe_filter_parallel_tool_calls(choice_data, request)
|
|
|
|
choices.append(choice_data)
|
|
|
|
if request.echo:
|
|
last_msg_content: str | list[dict[str, str]] = ""
|
|
if (
|
|
conversation
|
|
and "content" in conversation[-1]
|
|
and conversation[-1].get("role") == role
|
|
):
|
|
last_msg_content = conversation[-1]["content"] or ""
|
|
if isinstance(last_msg_content, list):
|
|
last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
|
|
|
|
for choice in choices:
|
|
full_message = last_msg_content + (choice.message.content or "")
|
|
choice.message.content = full_message
|
|
|
|
assert final_res.prompt_token_ids is not None
|
|
num_prompt_tokens = len(final_res.prompt_token_ids)
|
|
if final_res.encoder_prompt_token_ids is not None:
|
|
num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
|
|
num_generated_tokens = sum(
|
|
len(output.token_ids) for output in final_res.outputs
|
|
)
|
|
usage = UsageInfo(
|
|
prompt_tokens=num_prompt_tokens,
|
|
completion_tokens=num_generated_tokens,
|
|
total_tokens=num_prompt_tokens + num_generated_tokens,
|
|
)
|
|
if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
|
|
usage.prompt_tokens_details = PromptTokenUsageInfo(
|
|
cached_tokens=final_res.num_cached_tokens
|
|
)
|
|
|
|
request_metadata.final_usage_info = usage
|
|
|
|
# ``final_res.prompt`` is the rendered chat-templated prompt text
|
|
prompt_text = final_res.prompt if request.return_prompt_text else None
|
|
|
|
response = ChatCompletionResponse(
|
|
id=request_id,
|
|
created=created_time,
|
|
model=model_name,
|
|
choices=choices,
|
|
usage=usage,
|
|
system_fingerprint=self.system_fingerprint,
|
|
prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs),
|
|
prompt_token_ids=(
|
|
final_res.prompt_token_ids if request.return_token_ids else None
|
|
),
|
|
prompt_text=prompt_text,
|
|
kv_transfer_params=final_res.kv_transfer_params,
|
|
)
|
|
|
|
# Log complete response if output logging is enabled
|
|
if self.enable_log_outputs and self.request_logger:
|
|
for choice in choices:
|
|
output_text = ""
|
|
if choice.message.content:
|
|
output_text = choice.message.content
|
|
elif choice.message.tool_calls:
|
|
# For tool calls, log the function name and arguments
|
|
tool_call_descriptions = []
|
|
for tc in choice.message.tool_calls: # type: ignore
|
|
function_call: FunctionCall = tc.function # type: ignore
|
|
tool_call_descriptions.append(
|
|
f"{function_call.name}({function_call.arguments})"
|
|
)
|
|
tool_calls_str = ", ".join(tool_call_descriptions)
|
|
output_text = f"[tool_calls: {tool_calls_str}]"
|
|
|
|
if output_text:
|
|
# Get the corresponding output token IDs
|
|
output_token_ids = None
|
|
if choice.index < len(final_res.outputs):
|
|
output_token_ids = final_res.outputs[choice.index].token_ids
|
|
|
|
self.request_logger.log_outputs(
|
|
request_id=request_id,
|
|
outputs=output_text,
|
|
output_token_ids=output_token_ids,
|
|
finish_reason=choice.finish_reason,
|
|
is_streaming=False,
|
|
delta=False,
|
|
)
|
|
|
|
return response
|
|
|
|
def _get_top_logprobs(
|
|
self,
|
|
logprobs: dict[int, Logprob],
|
|
top_logprobs: int | None,
|
|
tokenizer: TokenizerLike | None,
|
|
should_return_as_token_id: bool,
|
|
) -> list[ChatCompletionLogProb]:
|
|
return [
|
|
ChatCompletionLogProb(
|
|
token=(
|
|
token := self._get_decoded_token(
|
|
p[1],
|
|
p[0],
|
|
tokenizer,
|
|
return_as_token_id=should_return_as_token_id,
|
|
)
|
|
),
|
|
logprob=max(p[1].logprob, -9999.0),
|
|
bytes=list(token.encode("utf-8", errors="replace")),
|
|
)
|
|
for i, p in enumerate(logprobs.items())
|
|
if (top_logprobs and i < top_logprobs or top_logprobs == -1)
|
|
]
|
|
|
|
def _create_chat_logprobs(
|
|
self,
|
|
token_ids: GenericSequence[int],
|
|
top_logprobs: GenericSequence[dict[int, Logprob] | None],
|
|
tokenizer: TokenizerLike | None,
|
|
num_output_top_logprobs: int | None = None,
|
|
return_as_token_id: bool | None = None,
|
|
) -> ChatCompletionLogProbs:
|
|
"""Create OpenAI-style logprobs."""
|
|
logprobs_content: list[ChatCompletionLogProbsContent] = []
|
|
|
|
should_return_as_token_id = (
|
|
return_as_token_id
|
|
if return_as_token_id is not None
|
|
else self.return_tokens_as_token_ids
|
|
)
|
|
for i, token_id in enumerate(token_ids):
|
|
step_top_logprobs = top_logprobs[i]
|
|
if step_top_logprobs is None or step_top_logprobs.get(token_id) is None:
|
|
if should_return_as_token_id:
|
|
token = f"token_id:{token_id}"
|
|
else:
|
|
if tokenizer is None:
|
|
raise ValueError(
|
|
"Unable to get tokenizer because `skip_tokenizer_init=True`"
|
|
)
|
|
|
|
token = tokenizer.decode(token_id)
|
|
|
|
logprobs_content.append(
|
|
ChatCompletionLogProbsContent(
|
|
token=token,
|
|
bytes=list(token.encode("utf-8", errors="replace")),
|
|
)
|
|
)
|
|
else:
|
|
step_token = step_top_logprobs[token_id]
|
|
step_decoded = step_token.decoded_token
|
|
|
|
logprobs_content.append(
|
|
ChatCompletionLogProbsContent(
|
|
token=self._get_decoded_token(
|
|
step_token,
|
|
token_id,
|
|
tokenizer,
|
|
should_return_as_token_id,
|
|
),
|
|
logprob=max(step_token.logprob, -9999.0),
|
|
bytes=(
|
|
None
|
|
if step_decoded is None
|
|
else list(step_decoded.encode("utf-8", errors="replace"))
|
|
),
|
|
top_logprobs=self._get_top_logprobs(
|
|
step_top_logprobs,
|
|
num_output_top_logprobs,
|
|
tokenizer,
|
|
should_return_as_token_id,
|
|
),
|
|
)
|
|
)
|
|
|
|
return ChatCompletionLogProbs(content=logprobs_content)
|
|
|
|
def _should_stream_with_auto_tool_parsing(self, request: ChatCompletionRequest):
|
|
"""
|
|
Utility function to check if streamed tokens should go through the tool
|
|
call parser that was configured.
|
|
|
|
We only want to do this IF user-provided tools are set, a tool parser
|
|
is configured, "auto" tool choice is enabled, and the request's tool
|
|
choice field indicates that "auto" tool choice should be used.
|
|
"""
|
|
return (
|
|
request.tools
|
|
and self.tool_parser
|
|
and self.enable_auto_tools
|
|
and request.tool_choice in ["auto", None]
|
|
)
|
|
|
|
def _should_check_for_unstreamed_tool_arg_tokens(
|
|
self,
|
|
delta_message: DeltaMessage | None,
|
|
output: CompletionOutput,
|
|
) -> bool:
|
|
"""
|
|
Check to see if we should check for unstreamed tool arguments tokens.
|
|
This is only applicable when auto tool parsing is enabled, the delta
|
|
is a tool call with arguments.
|
|
"""
|
|
|
|
return bool(
|
|
# if there is a delta message that includes tool calls which
|
|
# include a function that has arguments
|
|
output.finish_reason is not None
|
|
and self.enable_auto_tools
|
|
and self.tool_parser
|
|
and delta_message
|
|
and delta_message.tool_calls
|
|
and delta_message.tool_calls[0]
|
|
and delta_message.tool_calls[0].function
|
|
and delta_message.tool_calls[0].function.arguments is not None
|
|
)
|
|
|
|
@staticmethod
|
|
def _create_remaining_args_delta(
|
|
delta_message: DeltaMessage,
|
|
remaining_call: str,
|
|
index: int,
|
|
) -> DeltaMessage:
|
|
"""
|
|
Create a delta message for remaining tool arguments, preserving
|
|
id/type/name from the original delta.
|
|
"""
|
|
original_tc = next(
|
|
(tc for tc in delta_message.tool_calls if tc.index == index),
|
|
None,
|
|
)
|
|
original_fn = original_tc.function if original_tc else None
|
|
return DeltaMessage(
|
|
tool_calls=[
|
|
DeltaToolCall(
|
|
index=index,
|
|
id=original_tc.id if original_tc else None,
|
|
type=original_tc.type if original_tc else None,
|
|
function=DeltaFunctionCall(
|
|
name=original_fn.name if original_fn else None,
|
|
arguments=remaining_call,
|
|
),
|
|
)
|
|
]
|
|
)
|