# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio import time from collections.abc import AsyncGenerator, AsyncIterator from collections.abc import Sequence as GenericSequence from http import HTTPStatus from typing import Any, Final, cast from fastapi import Request from vllm.engine.protocol import EngineClient from vllm.entrypoints.chat_utils import ( ChatTemplateContentFormatOption, ConversationMessage, make_tool_call_id, ) from vllm.entrypoints.generate.base.protocol import ( DeltaMessage, FunctionCall, PerRequestMetrics, RequestResponseMetadata, ToolCall, ) from vllm.entrypoints.generate.base.serving import ( GenerateBaseServing, build_per_request_timing_metrics, build_spec_decoding_metrics, clamp_prompt_logprobs, format_token_id_placeholder, ) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionLogProb, ChatCompletionLogProbs, ChatCompletionLogProbsContent, ChatCompletionNamedToolChoiceParam, ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse, ChatMessage, ) from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.engine.protocol import ( CompletionTokenUsageInfo, ErrorResponse, PromptTokenUsageInfo, UsageInfo, ) from vllm.entrypoints.serve.utils.api_utils import get_max_tokens, should_include_usage from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.entrypoints.serve.utils.tool_calls_utils import ( maybe_filter_parallel_tool_calls, ) from vllm.exceptions import GenerationError from vllm.inputs import EngineInput, MultiModalPlaceholders from vllm.logger import init_logger from vllm.logprobs import Logprob from vllm.outputs import RequestOutput from vllm.parser import ParserManager from vllm.parser.abstract_parser import Parser from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import BeamSearchParams, SamplingParams from vllm.tokenizers import TokenizerLike from vllm.utils.collection_utils import as_list from vllm.utils.serial_utils import numpy2base64 logger = init_logger(__name__) def _get_mm_token_counts(engine_input: EngineInput) -> dict[str, int]: """Sum per-modality placeholder tokens from ``mm_placeholders``. Keyed by modality name; ``PlaceholderRange.length`` is the placeholder's prompt token span, so each sum matches the placeholder tokens already counted in ``usage.prompt_tokens``. """ mm_placeholders = cast( MultiModalPlaceholders | None, engine_input.get("mm_placeholders") ) return { modality: sum(p.length for p in ranges) for modality, ranges in (mm_placeholders or {}).items() if ranges } def _make_prompt_tokens_details( enable_prompt_tokens_details: bool, num_cached_tokens: int | None, num_cache_creation_tokens: int | None, mm_token_counts: dict[str, int] | None, ) -> PromptTokenUsageInfo | None: """Build ``prompt_tokens_details`` from cached + multimodal token counts.""" if not enable_prompt_tokens_details: return None if ( num_cached_tokens is None and num_cache_creation_tokens is None and not mm_token_counts ): return None return PromptTokenUsageInfo( cached_tokens=num_cached_tokens, created_cache_tokens=num_cache_creation_tokens, multimodal_tokens=mm_token_counts or None, ) def _make_completion_tokens_details( reasoning_tokens: int, ) -> CompletionTokenUsageInfo: return CompletionTokenUsageInfo(reasoning_tokens=reasoning_tokens) class OpenAIServingChat(GenerateBaseServing): def __init__( self, engine_client: EngineClient, models: OpenAIServingModels, response_role: str, *, online_renderer: "OnlineRenderer", request_logger: RequestLogger | None, chat_template: str | None, chat_template_content_format: ChatTemplateContentFormatOption, trust_request_chat_template: bool = False, return_tokens_as_token_ids: bool = False, reasoning_parser: str = "", enable_auto_tools: bool = False, exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, enable_log_outputs: bool = False, enable_log_deltas: bool = True, default_chat_template_kwargs: dict[str, Any] | None = None, enable_per_request_metrics: bool = False, ) -> None: super().__init__( engine_client=engine_client, models=models, request_logger=request_logger, return_tokens_as_token_ids=return_tokens_as_token_ids, ) self.online_renderer = online_renderer self.response_role = response_role self.chat_template = chat_template self.chat_template_content_format: Final = chat_template_content_format self.trust_request_chat_template = trust_request_chat_template self.default_chat_template_kwargs = default_chat_template_kwargs or {} self.enable_log_outputs = enable_log_outputs self.enable_log_deltas = enable_log_deltas self.enable_auto_tools: bool = enable_auto_tools self._include_reasoning_tokens_details = bool(reasoning_parser) self.parser_cls = ParserManager.get_parser( tool_parser_name=tool_parser, reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", ) self.exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none self.enable_prompt_tokens_details = enable_prompt_tokens_details self.enable_force_include_usage = enable_force_include_usage self.enable_per_request_metrics = enable_per_request_metrics self.default_sampling_params = self.model_config.get_diff_sampling_param() mc = self.model_config self.override_max_tokens = ( self.default_sampling_params.get("max_tokens") if mc.generation_config not in ("auto", "vllm") else getattr(mc, "override_generation_config", {}).get("max_new_tokens") ) # NOTE(woosuk): While OpenAI's chat completion API supports browsing # for some models, currently vLLM doesn't support it. Please use the # Responses API instead. self.supports_browsing = False self.browser_tool = None # NOTE(woosuk): Chat completion API does not support code interpreter. # Please use the Responses API instead. self.supports_code_interpreter = False self.python_tool = None def _effective_chat_template_kwargs( self, request: ChatCompletionRequest ) -> dict[str, Any]: return ( request.build_chat_params( self.chat_template, self.chat_template_content_format, ) .with_defaults(self.default_chat_template_kwargs) .chat_template_kwargs ) def _engine_chat_template_kwargs( self, chat_template_kwargs: dict[str, Any] ) -> dict[str, Any]: """Subclass hook to narrow ``chat_template_kwargs`` for the engine. The same dict is used twice: to build the API-server-side parser instance, and to populate ``EngineCoreRequest.reasoning_parser_kwargs`` for the engine-core side. The latter crosses ZMQ as msgpack, so a handler that stashes request-scoped state only the API-server-side parser needs (values msgpack can't encode, or payloads not worth shipping) can drop those entries here without affecting the in-process parser. Must not mutate the argument -- the caller still needs the full dict. The default forwards it unchanged. """ return chat_template_kwargs async def render_chat_request( self, request: ChatCompletionRequest, ) -> tuple[list[ConversationMessage], list[EngineInput]] | ErrorResponse: """ Validate the model and preprocess a chat completion request. Delegates preprocessing logic to OnlineRenderer, adding the engine-aware checks (LoRA model validation, engine health). Returns: A tuple of (conversation, engine_inputs) on success, or an ErrorResponse on failure. """ error_check_ret = await self._check_model(request) if error_check_ret is not None: logger.error("Error with model %s", error_check_ret) return error_check_ret self._preflight(request.n or 1) return await self.online_renderer.render_chat(request) async def create_chat_completion( self, request: ChatCompletionRequest, raw_request: Request | None = None, ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse: """ Chat Completion API similar to OpenAI's API. See https://platform.openai.com/docs/api-reference/chat/create for the API specification. This API mimics the OpenAI Chat Completion API. """ return await self._with_kv_transfer_rejection_cleanup( self._create_chat_completion(request, raw_request), request, raw_request ) async def _create_chat_completion( self, request: ChatCompletionRequest, raw_request: Request | None = None, ) -> AsyncGenerator[str, None] | ChatCompletionResponse | ErrorResponse: # Streaming response tokenizer = self.renderer.tokenizer assert tokenizer is not None chat_template_kwargs = self._effective_chat_template_kwargs(request) parser: Parser | None = None if self.parser_cls is not None: parser = self.parser_cls( tokenizer, request.tools, chat_template_kwargs=chat_template_kwargs, model_config=self.model_config, ) result = await self.render_chat_request(request) if isinstance(result, ErrorResponse): return result conversation, engine_inputs = result request_id = ( f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}" ) request_metadata = RequestResponseMetadata(request_id=request_id) if raw_request: raw_request.state.request_metadata = request_metadata lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True) model_name = self.models.model_name(lora_request) # Extract data_parallel_rank from header (router can inject it) data_parallel_rank = self._get_data_parallel_rank(raw_request) # Schedule the request and get the result generator. max_model_len = self.model_config.max_model_len generators: list[AsyncGenerator[RequestOutput, None]] = [] mm_token_counts: dict[str, int] | None = None for i, engine_input in enumerate(engine_inputs): prompt_token_ids = self._extract_prompt_components(engine_input).token_ids mm_token_counts = _get_mm_token_counts(engine_input) # If we are creating sub requests for multiple prompts, ensure that they # have unique request ids. sub_request_id = ( request_id if len(engine_inputs) == 1 else f"{request_id}_{i}" ) max_tokens = get_max_tokens( max_model_len, request.max_completion_tokens if request.max_completion_tokens is not None else request.max_tokens, self._extract_prompt_len(engine_input), self.default_sampling_params, self.override_max_tokens, truncate_prompt_tokens=request.truncate_prompt_tokens, ) sampling_params: SamplingParams | BeamSearchParams if request.use_beam_search: sampling_params = request.to_beam_search_params( max_tokens, self.default_sampling_params ) else: sampling_params = request.to_sampling_params( max_tokens, self.default_sampling_params, ) self._log_inputs( sub_request_id, engine_input, params=sampling_params, lora_request=lora_request, ) trace_headers = ( None if raw_request is None else await self._get_trace_headers(raw_request.headers) ) session_id = self._get_session_id(request, raw_request) if isinstance(sampling_params, BeamSearchParams): generator = self.beam_search( prompt=engine_input, request_id=sub_request_id, params=sampling_params, lora_request=lora_request, trace_headers=trace_headers, session_id=session_id, ) else: if not request.include_reasoning: reasoning_ended = True elif request._grammar_from_parser: # The Mistral grammar already includes an optional # `think?` rule that handles both reasoning and # non-reasoning outputs. reasoning_ended = True elif parser is not None and parser.reasoning_parser is not None: reasoning_ended = parser.is_reasoning_end(prompt_token_ids or []) else: reasoning_ended = None generator = self.engine_client.generate( engine_input, sampling_params, sub_request_id, lora_request=lora_request, trace_headers=trace_headers, priority=self._get_priority(request, raw_request), data_parallel_rank=data_parallel_rank, session_id=session_id, reasoning_ended=reasoning_ended, reasoning_parser_kwargs={ "chat_template_kwargs": self._engine_chat_template_kwargs( chat_template_kwargs ), } if parser is not None and parser.reasoning_parser is not None else None, ) generators.append(generator) assert len(generators) == 1 (result_generator,) = generators if request.stream: return self.chat_completion_stream_generator( request, result_generator, request_id, model_name, conversation, tokenizer, request_metadata, chat_template_kwargs=chat_template_kwargs, mm_token_counts=mm_token_counts, ) return await self.chat_completion_full_generator( request, result_generator, request_id, model_name, conversation, tokenizer, request_metadata, parser=parser, mm_token_counts=mm_token_counts, ) def get_chat_request_role(self, request: ChatCompletionRequest) -> str: if request.add_generation_prompt: return self.response_role return request.messages[-1]["role"] def _create_chat_message(self, *args: Any, **kwargs: Any) -> ChatMessage: """Construct the response :class:`ChatMessage` for the non-streaming path. The full-generator calls this at every construction site so subclasses can swap in a specialized :class:`ChatMessage` subclass (e.g. :class:`CohereServingChatV2` returning :class:`CohereChatMessage`) without duplicating the branchy tool-choice / auto-tools logic that decides which fields are populated. The default returns a plain :class:`ChatMessage`. """ return ChatMessage(*args, **kwargs) def _finalize_response_message( self, message: ChatMessage, *, parser: Parser | None, ) -> ChatMessage: """Subclass hook to enrich a fully-constructed :class:`ChatMessage`. Default is a no-op. Subclasses that need to surface parser-side extras (e.g. :class:`CohereServingChatV2` reading grounding citations off the reasoning parser and populating :class:`CohereChatMessage.citations`) override this to inspect ``parser`` and mutate/replace ``message``. """ return message async def chat_completion_stream_generator( self, request: ChatCompletionRequest, result_generator: AsyncIterator[RequestOutput], request_id: str, model_name: str, conversation: list[ConversationMessage], tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, chat_template_kwargs: dict[str, Any] | None = None, mm_token_counts: dict[str, int] | None = None, ) -> AsyncGenerator[str, None]: created_time = int(time.time()) chunk_object_type: Final = "chat.completion.chunk" first_iteration = True # Send response for each token for each request.n (index) num_choices = 1 if request.n is None else request.n previous_num_tokens = [0] * num_choices # TODO: Remove once all reasoning parsers use the Parser Engine. generated_token_ids: list[list[int]] = [[] for _ in range(num_choices)] previous_reasoning_tokens = [0] * num_choices finish_reason_sent = [False] * num_choices num_prompt_tokens = 0 num_cached_tokens = None num_cache_creation_tokens = None tools_streamed = [False] * num_choices if isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam): tool_choice_function_name = request.tool_choice.function.name else: tool_choice_function_name = None previous_texts = [""] * num_choices try: if self.parser_cls is not None: if tokenizer is None: raise ValueError( "Tokenizer not available when `skip_tokenizer_init=True`" ) parsers: list[Parser | None] = [ self.parser_cls( tokenizer, request.tools, chat_template_kwargs=chat_template_kwargs, model_config=self.model_config, ) for _ in range(num_choices) ] else: parsers = [None] * num_choices except Exception as e: logger.exception("Error in parser creation.") data = self.create_streaming_error_response(e) yield f"data: {data}\n\n" yield "data: [DONE]\n\n" return stream_options = request.stream_options include_usage, include_continuous_usage = should_include_usage( stream_options, self.enable_force_include_usage ) last_res: RequestOutput | None = None try: async for res in result_generator: last_res = res if res.prompt_token_ids is not None: num_prompt_tokens = len(res.prompt_token_ids) if res.encoder_prompt_token_ids is not None: num_prompt_tokens += len(res.encoder_prompt_token_ids) # We need to do it here, because if there are exceptions in # the result_generator, it needs to be sent as the FIRST # response (by the try...catch). if first_iteration: num_cached_tokens = res.num_cached_tokens num_cache_creation_tokens = res.num_cache_creation_tokens # Send first response for each request.n (index) with # the role role = self.get_chat_request_role(request) # ``res.prompt`` is the rendered chat-templated prompt prompt_text = res.prompt if request.return_prompt_text else None # NOTE num_choices defaults to 1 so this usually executes # once per request for i in range(num_choices): choice_data = ChatCompletionResponseStreamChoice( index=i, delta=DeltaMessage( role=role, content="", ), logprobs=None, finish_reason=None, ) # return prompt_token_ids at the first chunk ever chunk = ChatCompletionStreamResponse( id=request_id, object=chunk_object_type, created=created_time, choices=[choice_data], model=model_name, prompt_token_ids=( res.prompt_token_ids if request.return_token_ids else None ), prompt_text=prompt_text, ) # if continuous usage stats are requested, add it if include_continuous_usage: chunk.usage = UsageInfo( prompt_tokens=num_prompt_tokens, completion_tokens=0, total_tokens=num_prompt_tokens, completion_tokens_details=( _make_completion_tokens_details(0) if self._include_reasoning_tokens_details else None ), ) data = chunk.model_dump_json(exclude_unset=True) yield f"data: {data}\n\n" # Send response to echo the input portion of the # last message if request.echo: last_msg_content: str | list[dict[str, str]] = "" if ( conversation and "content" in conversation[-1] and conversation[-1].get("role") == role ): last_msg_content = conversation[-1]["content"] or "" if last_msg_content: for i in range(num_choices): choice_data = ChatCompletionResponseStreamChoice( index=i, delta=DeltaMessage(content=last_msg_content), logprobs=None, finish_reason=None, ) chunk = ChatCompletionStreamResponse( id=request_id, object=chunk_object_type, created=created_time, choices=[choice_data], model=model_name, ) if include_continuous_usage: chunk.usage = UsageInfo( prompt_tokens=num_prompt_tokens, completion_tokens=0, total_tokens=num_prompt_tokens, completion_tokens_details=( _make_completion_tokens_details(0) if self._include_reasoning_tokens_details else None ), ) data = chunk.model_dump_json(exclude_unset=True) yield f"data: {data}\n\n" first_iteration = False for output in res.outputs: i = output.index parser = parsers[i] if finish_reason_sent[i]: continue if request.logprobs and ( request.top_logprobs is not None or request.logprob_token_ids ): assert output.logprobs is not None, "Did not output logprobs" logprobs = self._create_chat_logprobs( token_ids=output.token_ids, top_logprobs=output.logprobs, tokenizer=tokenizer, num_output_top_logprobs=request.top_logprobs, logprob_token_ids=request.logprob_token_ids, return_as_token_id=request.return_tokens_as_token_ids, ) else: logprobs = None delta_text = output.text if ( not delta_text and not output.token_ids and not previous_num_tokens[i] ): # Chunked prefill case, don't return empty chunks continue delta_message: DeltaMessage | None if parser is not None: delta_message = parser.parse_delta( delta_text=delta_text, delta_token_ids=as_list(output.token_ids), request=request, prompt_token_ids=res.prompt_token_ids, finished=output.finish_reason is not None, ) if delta_message is not None and delta_message.tool_calls: tools_streamed[i] = True # handle streaming just a content delta (no parsers) else: delta_message = DeltaMessage(content=delta_text) previous_texts[i] += delta_text # set the previous values for the next iteration previous_num_tokens[i] += len(output.token_ids) if parser is not None: generated_token_ids[i].extend(output.token_ids) previous_reasoning_tokens[i] = parser.count_reasoning_tokens( tuple(generated_token_ids[i]) ) # if the message delta is None (e.g. because it was a # "control token" for tool calls or the parser otherwise # wasn't ready to send a token, then # get the next token without streaming a chunk # When reasoning is hidden, suppress per-token # metadata (logprobs, token_ids) on every chunk to # prevent leaking reasoning tokens through decoded # token text in logprob entries or raw token IDs. hide_stream_metadata = ( not request.include_reasoning and parser is not None ) if hide_stream_metadata: logprobs = None if delta_message is None: # NOTE: If return_token_ids is enabled, we still need to # send a chunk with token_ids even if delta_message is None # to ensure all tokens are included in the response if output.finish_reason is None and ( not request.return_token_ids or hide_stream_metadata ): continue delta_message = DeltaMessage() # Log streaming delta if output logging is enabled if self.enable_log_outputs and self.request_logger: delta_content_parts = [] if delta_message.content: delta_content_parts.append(delta_message.content) if delta_message.reasoning: reasoning = delta_message.reasoning delta_content_parts.append(f"[reasoning: {reasoning}]") if delta_message.tool_calls: tool_args = "".join( tc.function.arguments for tc in delta_message.tool_calls if tc.function and tc.function.arguments ) if tool_args: delta_content_parts.append(f"[tool_calls: {tool_args}]") if delta_content_parts and self.enable_log_deltas: delta_content = " ".join(delta_content_parts) self.request_logger.log_outputs( request_id=request_id, outputs=delta_content, output_token_ids=as_list(output.token_ids), finish_reason=output.finish_reason, is_streaming=True, delta=True, ) include_token_ids = ( request.return_token_ids and not hide_stream_metadata ) if output.finish_reason is None: # Send token-by-token response for each request.n choice_data = ChatCompletionResponseStreamChoice( index=i, delta=delta_message, logprobs=logprobs, finish_reason=None, token_ids=( as_list(output.token_ids) if include_token_ids else None ), ) # if the model is finished generating else: # check for error finish reason and abort streaming # finish_reason='error' indicates a retryable error self._raise_if_error(output.finish_reason, request_id) # Send the finish response for each request.n only once # In OpenAI's API, when a tool is called, the # finish_reason is: # "tool_calls" for "auto" or "required" tool calls, # and "stop" for named tool calls. if tools_streamed[i] and not tool_choice_function_name: finish_reason_ = "tool_calls" else: finish_reason_ = ( output.finish_reason if output.finish_reason else "stop" ) choice_data = ChatCompletionResponseStreamChoice( index=i, delta=delta_message, logprobs=logprobs, finish_reason=finish_reason_, stop_reason=output.stop_reason, token_ids=( as_list(output.token_ids) if include_token_ids else None ), ) finish_reason_sent[i] = True choice_data = maybe_filter_parallel_tool_calls(choice_data, request) chunk = ChatCompletionStreamResponse( id=request_id, object=chunk_object_type, created=created_time, choices=[choice_data], model=model_name, ) # Stamp the fingerprint on terminal chunks only (those with # finish_reason set). When ``include_usage`` is on, the # trailing usage chunk below overrides this as the true # final message. if ( not include_usage and self.system_fingerprint is not None and choice_data.finish_reason is not None ): chunk.system_fingerprint = self.system_fingerprint # handle usage stats if requested & if continuous if include_continuous_usage: completion_tokens = previous_num_tokens[i] chunk.usage = UsageInfo( prompt_tokens=num_prompt_tokens, completion_tokens=completion_tokens, total_tokens=num_prompt_tokens + completion_tokens, completion_tokens_details=( _make_completion_tokens_details( previous_reasoning_tokens[i] ) if self._include_reasoning_tokens_details else None ), ) data = chunk.model_dump_json(exclude_unset=True) yield f"data: {data}\n\n" # once the final token is handled, if stream_options.include_usage # is sent, send the usage if include_usage: completion_tokens = sum(previous_num_tokens) final_usage = UsageInfo( prompt_tokens=num_prompt_tokens, completion_tokens=completion_tokens, total_tokens=num_prompt_tokens + completion_tokens, completion_tokens_details=_make_completion_tokens_details( sum(previous_reasoning_tokens) ) if self._include_reasoning_tokens_details else None, ) final_usage.prompt_tokens_details = _make_prompt_tokens_details( self.enable_prompt_tokens_details, num_cached_tokens, num_cache_creation_tokens, mm_token_counts, ) # In streaming, metrics ride on this final usage chunk, which is # only emitted when usage reporting is enabled (i.e. # ``stream_options.include_usage=true`` or # ``--enable-force-include-usage``). stream_per_request_metrics: PerRequestMetrics | None = None # See note in chat_completion_full_generator: suppress for n>1. if (request.n or 1) == 1: if self.enable_per_request_metrics: last_metrics = ( last_res.metrics if last_res is not None else None ) stream_per_request_metrics = build_per_request_timing_metrics( last_metrics, completion_tokens ) spec_stats = build_spec_decoding_metrics(last_res) if spec_stats is not None: if stream_per_request_metrics is None: stream_per_request_metrics = PerRequestMetrics() stream_per_request_metrics.speculative_decoding = spec_stats final_usage_chunk = ChatCompletionStreamResponse( id=request_id, object=chunk_object_type, created=created_time, choices=[], model=model_name, usage=final_usage, system_fingerprint=self.system_fingerprint, metrics=stream_per_request_metrics, ) final_usage_data = final_usage_chunk.model_dump_json( exclude_unset=True, exclude_none=True ) yield f"data: {final_usage_data}\n\n" # report to FastAPI middleware aggregate usage across all choices num_completion_tokens = sum(previous_num_tokens) request_metadata.final_usage_info = UsageInfo( prompt_tokens=num_prompt_tokens, completion_tokens=num_completion_tokens, total_tokens=num_prompt_tokens + num_completion_tokens, completion_tokens_details=_make_completion_tokens_details( sum(previous_reasoning_tokens) ) if self._include_reasoning_tokens_details else None, ) # Log complete streaming response if output logging is enabled if self.enable_log_outputs and self.request_logger: # Log the complete response for each choice for i in range(num_choices): full_text = ( previous_texts[i] if previous_texts and i < len(previous_texts) else f"" ) self.request_logger.log_outputs( request_id=request_id, outputs=full_text, output_token_ids=None, # Consider also logging all token IDs finish_reason="streaming_complete", is_streaming=True, delta=False, ) except GenerationError as e: yield f"data: {self._convert_generation_error_to_streaming_response(e)}\n\n" except Exception as e: logger.exception("Error in chat completion stream generator.") data = self.create_streaming_error_response(e) yield f"data: {data}\n\n" # Send the final done message after all response.n are finished yield "data: [DONE]\n\n" async def chat_completion_full_generator( self, request: ChatCompletionRequest, result_generator: AsyncIterator[RequestOutput], request_id: str, model_name: str, conversation: list[ConversationMessage], tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, parser: Parser | None = None, mm_token_counts: dict[str, int] | None = None, ) -> ErrorResponse | ChatCompletionResponse: created_time = int(time.time()) final_res: RequestOutput | None = None try: async for res in result_generator: final_res = res except asyncio.CancelledError: return self.create_error_response("Client disconnected") if final_res is None: return self.create_error_response( "No output received from the engine.", err_type="InternalServerError", status_code=HTTPStatus.INTERNAL_SERVER_ERROR, ) choices: list[ChatCompletionResponseChoice] = [] total_reasoning_tokens = 0 role = self.get_chat_request_role(request) tool_parser_cls = ( self.parser_cls.tool_parser_cls if self.parser_cls is not None else None ) for output in final_res.outputs: # check for error finish reason and raise GenerationError # finish_reason='error' indicates a retryable request-level internal error self._raise_if_error(output.finish_reason, request_id) token_ids = output.token_ids out_logprobs = output.logprobs if request.logprobs and ( request.top_logprobs is not None or request.logprob_token_ids ): assert out_logprobs is not None, "Did not output logprobs" logprobs = self._create_chat_logprobs( token_ids=token_ids, top_logprobs=out_logprobs, num_output_top_logprobs=request.top_logprobs, logprob_token_ids=request.logprob_token_ids, tokenizer=tokenizer, return_as_token_id=request.return_tokens_as_token_ids, ) else: logprobs = None if parser is not None: reasoning, content, tool_calls = parser.parse( output.text, request, enable_auto_tools=self.enable_auto_tools, model_output_token_ids=token_ids, ) suppress_metadata = not request.include_reasoning and parser is not None if not request.include_reasoning: reasoning = None total_reasoning_tokens += parser.count_reasoning_tokens(token_ids) if suppress_metadata: logprobs = None else: reasoning = None content = output.text tool_calls = [] suppress_metadata = False auto_tools_called = False is_named_tool_choice = ( request.tool_choice is not None and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam ) is_required_tool_choice = request.tool_choice == "required" # All six construction sites route through ``self._create_chat_message`` # so subclasses can swap in a specialized :class:`ChatMessage` # (e.g. the Cohere v2 handler's ``CohereChatMessage``) without # having to duplicate this branch logic. if (not self.enable_auto_tools or not tool_parser_cls) and ( not is_named_tool_choice and not is_required_tool_choice ): message = self._create_chat_message( role=role, reasoning=reasoning, content=content ) elif is_named_tool_choice or is_required_tool_choice: message = self._create_chat_message( role=role, reasoning=reasoning, content=content or "", tool_calls=[ ToolCall(id=tc.id or make_tool_call_id(), function=tc) for tc in (tool_calls or []) ], ) # if the request doesn't use tool choice # OR specifies to not use a tool elif not request.tool_choice or request.tool_choice == "none": message = self._create_chat_message( role=role, reasoning=reasoning, content=content ) # handle when there are tools and tool choice is auto elif ( request.tools and (request.tool_choice == "auto" or request.tool_choice is None) and self.enable_auto_tools and tool_parser_cls ): auto_tools_called = tool_calls is not None and len(tool_calls) > 0 if tool_calls: message = self._create_chat_message( role=role, reasoning=reasoning, content=content, tool_calls=[ ToolCall(id=tc.id or make_tool_call_id(), function=tc) for tc in tool_calls ], ) else: message = self._create_chat_message( role=role, reasoning=reasoning, content=content, ) # undetermined case that is still important to handle else: logger.error( "Error in chat_completion_full_generator - cannot determine" " if tools should be extracted. Returning a standard chat " "completion." ) message = self._create_chat_message( role=role, reasoning=reasoning, content=content ) # Subclass hook: enrich the constructed message with any # parser-side extras that don't fit through the plain # ``(reasoning, content, tool_calls)`` tuple. Base is a no-op; # citation-aware handlers use this to surface grounding # metadata cached on the reasoning parser. message = self._finalize_response_message(message, parser=parser) # In OpenAI's API, when a tool is called, the finish_reason is: # "tool_calls" for "auto" or "required" tool calls, # and "stop" for named tool calls. is_finish_reason_tool_calls = auto_tools_called or ( request.tool_choice and request.tool_choice == "required" and output.finish_reason == "stop" ) routed_experts_b64 = ( numpy2base64(output.routed_experts) if output.routed_experts is not None else None ) choice_data = ChatCompletionResponseChoice( index=output.index, message=message, logprobs=logprobs, finish_reason="tool_calls" if is_finish_reason_tool_calls else output.finish_reason if output.finish_reason else "stop", stop_reason=output.stop_reason, token_ids=( as_list(output.token_ids) if request.return_token_ids and not suppress_metadata else None ), routed_experts=routed_experts_b64, ) choice_data = maybe_filter_parallel_tool_calls(choice_data, request) choices.append(choice_data) if request.echo: last_msg_content: str | list[dict[str, str]] = "" if ( conversation and "content" in conversation[-1] and conversation[-1].get("role") == role ): last_msg_content = conversation[-1]["content"] or "" if isinstance(last_msg_content, list): last_msg_content = "\n".join(msg["text"] for msg in last_msg_content) for choice in choices: full_message = last_msg_content + (choice.message.content or "") choice.message.content = full_message assert final_res.prompt_token_ids is not None num_prompt_tokens = len(final_res.prompt_token_ids) if final_res.encoder_prompt_token_ids is not None: num_prompt_tokens += len(final_res.encoder_prompt_token_ids) num_generated_tokens = sum( len(output.token_ids) for output in final_res.outputs ) usage = UsageInfo( prompt_tokens=num_prompt_tokens, completion_tokens=num_generated_tokens, total_tokens=num_prompt_tokens + num_generated_tokens, completion_tokens_details=_make_completion_tokens_details( total_reasoning_tokens ) if self._include_reasoning_tokens_details else None, ) usage.prompt_tokens_details = _make_prompt_tokens_details( self.enable_prompt_tokens_details, final_res.num_cached_tokens, final_res.num_cache_creation_tokens, mm_token_counts, ) request_metadata.final_usage_info = usage per_request_metrics: PerRequestMetrics | None = None # Per-request metrics (timing + spec-decode acceptance) describe a single # generation stream. For n>1 the stats belong to only one of the n # sequences, so they cannot be attributed to the request; suppress. if (request.n or 1) == 1: if self.enable_per_request_metrics: per_request_metrics = build_per_request_timing_metrics( final_res.metrics, num_generated_tokens ) spec_stats = build_spec_decoding_metrics(final_res) if spec_stats is not None: if per_request_metrics is None: per_request_metrics = PerRequestMetrics() per_request_metrics.speculative_decoding = spec_stats # ``final_res.prompt`` is the rendered chat-templated prompt text prompt_text = final_res.prompt if request.return_prompt_text else None response = ChatCompletionResponse( id=request_id, created=created_time, model=model_name, choices=choices, usage=usage, system_fingerprint=self.system_fingerprint, prompt_logprobs=clamp_prompt_logprobs(final_res.prompt_logprobs), prompt_token_ids=( final_res.prompt_token_ids if request.return_token_ids else None ), prompt_text=prompt_text, kv_transfer_params=final_res.kv_transfer_params, ec_transfer_params=final_res.ec_transfer_params, metrics=per_request_metrics, ) # Log complete response if output logging is enabled if self.enable_log_outputs and self.request_logger: for choice in choices: output_text = "" if choice.message.content: output_text = choice.message.content elif choice.message.tool_calls: # For tool calls, log the function name and arguments tool_call_descriptions = [] for tc in choice.message.tool_calls: # type: ignore function_call: FunctionCall = tc.function # type: ignore tool_call_descriptions.append( f"{function_call.name}({function_call.arguments})" ) tool_calls_str = ", ".join(tool_call_descriptions) output_text = f"[tool_calls: {tool_calls_str}]" if output_text: # Get the corresponding output token IDs output_token_ids = None if choice.index < len(final_res.outputs): output_token_ids = final_res.outputs[choice.index].token_ids self.request_logger.log_outputs( request_id=request_id, outputs=output_text, output_token_ids=output_token_ids, finish_reason=choice.finish_reason, is_streaming=False, delta=False, ) return response def _get_top_logprobs( self, logprobs: dict[int, Logprob], top_logprobs: int | None, tokenizer: TokenizerLike | None, should_return_as_token_id: bool, return_all: bool = False, ) -> list[ChatCompletionLogProb]: return [ ChatCompletionLogProb( token=( token := self._get_decoded_token( p[1], p[0], tokenizer, return_as_token_id=should_return_as_token_id, ) ), logprob=max(p[1].logprob, -9999.0), bytes=list(token.encode("utf-8", errors="replace")), ) for i, p in enumerate(logprobs.items()) if return_all or top_logprobs == -1 or (top_logprobs is not None and i < top_logprobs) ] def _create_chat_logprobs( self, token_ids: GenericSequence[int], top_logprobs: GenericSequence[dict[int, Logprob] | None], tokenizer: TokenizerLike | None, num_output_top_logprobs: int | None = None, logprob_token_ids: list[int] | None = None, return_as_token_id: bool | None = None, ) -> ChatCompletionLogProbs: """Create OpenAI-style logprobs.""" logprobs_content: list[ChatCompletionLogProbsContent] = [] should_return_as_token_id = ( return_as_token_id if return_as_token_id is not None else self.return_tokens_as_token_ids ) for i, token_id in enumerate(token_ids): step_top_logprobs = top_logprobs[i] if step_top_logprobs is None or step_top_logprobs.get(token_id) is None: if should_return_as_token_id: token = format_token_id_placeholder(token_id) else: if tokenizer is None: raise ValueError( "Unable to get tokenizer because `skip_tokenizer_init=True`" ) token = tokenizer.decode(token_id) logprobs_content.append( ChatCompletionLogProbsContent( token=token, bytes=list(token.encode("utf-8", errors="replace")), ) ) else: step_token = step_top_logprobs[token_id] step_decoded = step_token.decoded_token logprobs_content.append( ChatCompletionLogProbsContent( token=self._get_decoded_token( step_token, token_id, tokenizer, should_return_as_token_id, ), logprob=max(step_token.logprob, -9999.0), bytes=( None if step_decoded is None else list(step_decoded.encode("utf-8", errors="replace")) ), top_logprobs=self._get_top_logprobs( step_top_logprobs, num_output_top_logprobs, tokenizer, should_return_as_token_id, return_all=bool(logprob_token_ids), ), ) ) return ChatCompletionLogProbs(content=logprobs_content)