# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Adapted from # https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/openai/chat_completion/serving.py """Anthropic Messages API serving handler.""" import json import logging import time import uuid from collections.abc import AsyncGenerator from typing import Any, Literal, get_args import jinja2 from fastapi import Request from vllm.engine.protocol import EngineClient from vllm.entrypoints.anthropic.protocol import ( AnthropicContentBlock, AnthropicContextManagement, AnthropicCountTokensRequest, AnthropicCountTokensResponse, AnthropicDelta, AnthropicDisabledThinkingEffort, AnthropicDisabledThinkingEffortOption, AnthropicEffort, AnthropicError, AnthropicMessagesRequest, AnthropicMessagesResponse, AnthropicOutputConfig, AnthropicStreamEvent, AnthropicThinkingConfig, AnthropicTool, AnthropicToolChangeDefinition, AnthropicUsage, ) from vllm.entrypoints.generate.base.protocol import ( JsonSchemaResponseFormat, ResponseFormat, StreamOptions, ) from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedToolChoiceParam, ChatCompletionRequest, ChatCompletionResponse, ChatCompletionStreamResponse, ChatCompletionToolsParam, ) from vllm.entrypoints.openai.chat_completion.serving import OpenAIServingChat from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo from vllm.entrypoints.serve.exception_handling.utils import sanitize_message from vllm.entrypoints.serve.utils.request_logger import RequestLogger from vllm.logger import init_logger from vllm.renderers.chat_utils import ChatTemplateContentFormatOption from vllm.renderers.hf import HfRenderer, resolve_chat_template from vllm.renderers.online_renderer import OnlineRenderer logger = init_logger(__name__) def _build_anthropic_usage( usage: UsageInfo | None, ) -> AnthropicUsage: """Build an AnthropicUsage from UsageInfo. Anthropic defines ``total_input == input_tokens + cache_read + cache_creation``. vLLM's ``prompt_tokens`` is the total, so ``input_tokens = prompt_tokens - cache_read - cache_creation``. Cache fields are taken from ``UsageInfo.prompt_tokens_details``. When cache info is absent (e.g. ``--enable-prompt-tokens-details`` off, or a streaming chunk that hasn't carried it yet), cache fields are left **unset** so ``exclude_unset=True`` serialization omits them entirely. ``completion_tokens`` follows ``UsageInfo`` and may be ``None`` on intermediate stream chunks; we coerce to ``0`` for the wire format. """ kwargs = {} if usage is None: kwargs["input_tokens"] = 0 kwargs["output_tokens"] = 0 else: kwargs["output_tokens"] = usage.completion_tokens or 0 input_tokens = usage.prompt_tokens if (details := usage.prompt_tokens_details) is not None: if (cache_read := details.cached_tokens) is not None: input_tokens -= cache_read kwargs["cache_read_input_tokens"] = cache_read if (cache_creation := details.created_cache_tokens) is not None: input_tokens -= cache_creation kwargs["cache_creation_input_tokens"] = cache_creation kwargs["input_tokens"] = max(0, input_tokens) return AnthropicUsage(**kwargs) def _to_anthropic_stop_reason( finish_reason: str | None, stop_reason: int | str | None, tool_used: bool, tool_complete: bool, ) -> tuple[ Literal["end_turn", "max_tokens", "stop_sequence", "tool_use"] | None, str | None, ]: """Map an OpenAI finish reason to Anthropic ``(stop_reason, stop_sequence)``. Anthropic reports ``tool_use`` whenever the model invoked a tool. The chat layer reports ``finish_reason="stop"`` for a named (forced) tool call, so ``tool_used`` records whether a ``tool_use`` block was produced. ``max_tokens`` always wins. A matched stop string wins unless the tool calls are complete (``tool_complete``): a call it cut short must not run. """ if finish_reason == "length": return "max_tokens", None if finish_reason not in ("stop", "tool_calls"): return None, None if tool_complete: return "tool_use", None # vLLM reports the matched stop string in stop_reason (a str); # an int stop-token-id or None (natural EOS) is not a stop sequence. if isinstance(stop_reason, str): return "stop_sequence", stop_reason if finish_reason == "tool_calls" or tool_used: return "tool_use", None return "end_turn", None def _is_json(text: str) -> bool: try: json.loads(text) except json.JSONDecodeError: return False return True def wrap_data_with_event(data: str, event: str): return f"event: {event}\ndata: {data}\n\n" class AnthropicServingMessages(OpenAIServingChat): """Handler for Anthropic Messages API requests.""" def __init__( self, engine_client: EngineClient, models: OpenAIServingModels, response_role: str, *, online_renderer: "OnlineRenderer", request_logger: RequestLogger | None, chat_template: str | None, chat_template_content_format: ChatTemplateContentFormatOption, return_tokens_as_token_ids: bool = False, reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, tool_strict_level: str | None = None, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, enable_log_outputs: bool = False, enable_log_deltas: bool = True, default_chat_template_kwargs: dict[str, Any] | None = None, disabled_thinking_effort: AnthropicDisabledThinkingEffortOption = "auto", ): super().__init__( engine_client=engine_client, models=models, response_role=response_role, online_renderer=online_renderer, request_logger=request_logger, chat_template=chat_template, chat_template_content_format=chat_template_content_format, return_tokens_as_token_ids=return_tokens_as_token_ids, reasoning_parser=reasoning_parser, enable_auto_tools=enable_auto_tools, tool_strict_level=tool_strict_level, tool_parser=tool_parser, enable_prompt_tokens_details=enable_prompt_tokens_details, enable_force_include_usage=enable_force_include_usage, enable_log_outputs=enable_log_outputs, enable_log_deltas=enable_log_deltas, default_chat_template_kwargs=default_chat_template_kwargs, ) self._merge_inline_system = self._should_merge_inline_system(online_renderer) # Resolved lazily from the renderer when "auto". self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = ( None if disabled_thinking_effort == "auto" else disabled_thinking_effort ) async def _get_disabled_thinking_effort(self) -> AnthropicDisabledThinkingEffort: if self._disabled_thinking_effort is None: self._disabled_thinking_effort = ( await self._probe_disabled_thinking_effort() ) logger.info( "Anthropic thinking.type=disabled maps to reasoning_effort=%r", self._disabled_thinking_effort, ) return self._disabled_thinking_effort async def _probe_disabled_thinking_effort(self) -> AnthropicDisabledThinkingEffort: """Use ``low`` if the renderer rejects or ignores ``none``. ``none`` is ignored when it renders the same prompt as a thinking effort, e.g. GLM-5.3 treats unknown efforts as ``max``. """ none_prompt = await self._render_probe_prompt("none") if none_prompt is None: return "low" for effort in get_args(AnthropicEffort): if await self._render_probe_prompt(effort) == none_prompt: return "low" return "none" async def _render_probe_prompt( self, effort: AnthropicDisabledThinkingEffort ) -> tuple[list[int] | None, str | None] | None: request = ChatCompletionRequest( messages=[{"role": "user", "content": "Hi"}], reasoning_effort=effort, ) try: result = await self.online_renderer.render_chat(request) except Exception: logger.debug("Rendering reasoning_effort=%r failed", effort, exc_info=True) return None if isinstance(result, ErrorResponse): return None _, (engine_input,) = result components = self._extract_prompt_components(engine_input) return components.token_ids, components.text @classmethod def _should_merge_inline_system(cls, online_renderer: OnlineRenderer) -> bool: """Probe the chat templates the renderer applies, not the CLI override. Both the tool and non-tool templates are checked since they may differ. """ renderer = online_renderer.renderer tool_variants: tuple[list[dict[str, Any]] | None, ...] = (None, []) if not isinstance(renderer, HfRenderer) or renderer.tokenizer is None: merge = cls._detect_merge_inline_system(online_renderer.chat_template) else: merge = any( cls._detect_merge_inline_system( resolve_chat_template( renderer.tokenizer, online_renderer.chat_template, tools, model_config=online_renderer.model_config, ) ) for tools in tool_variants ) if merge: logger.warning_once( "The chat template requires system-first ordering, so inline " "system messages in /v1/messages requests (e.g. Claude Code's " "per-turn reminders) are merged into the leading system prompt. " "Each new one changes the prompt prefix, so the rest of the " "conversation misses the prefix cache. Pass a --chat-template " "that accepts non-leading system messages to avoid this." ) return merge @staticmethod def _detect_merge_inline_system(chat_template: str | None) -> bool: """Auto-detect whether the chat template requires system-first ordering. Renders a [system, user, system, user] conversation against the template; if it raises (e.g. Qwen's ``loop.first`` guard), the model needs inline system messages merged into the leading block. """ if not chat_template: return True try: env = jinja2.sandbox.ImmutableSandboxedEnvironment( trim_blocks=True, lstrip_blocks=True, extensions=[jinja2.ext.loopcontrols], ) env.from_string(chat_template).render( messages=[ {"role": "system", "content": "t"}, {"role": "user", "content": "t"}, {"role": "system", "content": "t"}, {"role": "user", "content": "t"}, ], add_generation_prompt=False, ) return False except jinja2.TemplateError: return True @staticmethod def _convert_image_source_to_url(source: dict[str, Any]) -> str: """Convert an Anthropic image source to an OpenAI-compatible URL. Anthropic supports two image source types: - base64: {"type": "base64", "media_type": "image/jpeg", "data": "..."} - url: {"type": "url", "url": "https://..."} For base64 sources, this constructs a proper data URI that downstream processors (e.g. vLLM's media connector) can handle. """ source_type = source.get("type") if source_type == "url": return source.get("url", "") # Default to base64 processing if type is "base64" # or missing, ensuring a proper data URI is always # constructed for non-URL sources. media_type = source.get("media_type", "image/jpeg") data = source.get("data", "") return f"data:{media_type};base64,{data}" @classmethod def to_chat_completion_request( cls, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, *, merge_inline_system: bool = False, disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none", ) -> ChatCompletionRequest: """Convert Anthropic message format to OpenAI format.""" openai_messages: list[dict[str, Any]] = [] cls._convert_system_message( anthropic_request, openai_messages, merge_inline_system=merge_inline_system, ) cls._convert_messages( anthropic_request.messages, openai_messages, merge_inline_system=merge_inline_system, ) req = cls._build_base_request(anthropic_request, openai_messages) cls._handle_streaming_options(req, anthropic_request) cls._handle_output_config(req, anthropic_request) cls._handle_thinking(req, anthropic_request, disabled_thinking_effort) cls._convert_tool_choice(anthropic_request, req) cls._convert_tools(anthropic_request, req) return req @classmethod def _convert_system_message( cls, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, openai_messages: list[dict[str, Any]], *, merge_inline_system: bool = False, ) -> None: """Convert Anthropic system message to OpenAI format.""" system_parts: list[str] = [] # Top-level system field if anthropic_request.system: if isinstance(anthropic_request.system, str): system_parts.append(anthropic_request.system) else: for block in anthropic_request.system: if block.type == "text" and block.text: # Strip Claude Code's attribution header which contains # a per-request hash that defeats prefix caching. if block.text.startswith("x-anthropic-billing-header"): continue system_parts.append(block.text) # When the template requires system-first ordering, extract inline # system messages from the messages array and merge them into the # top-level block so the template doesn't reject them. if merge_inline_system: for msg in anthropic_request.messages: if msg.role != "system": continue text = cls._extract_system_text(msg) if text: system_parts.append(text) if system_parts: openai_messages.append({"role": "system", "content": "".join(system_parts)}) @classmethod def _extract_system_text(cls, msg) -> str | None: """Extract text from a system message, stripping billing headers.""" if isinstance(msg.content, str): text = msg.content if text.startswith("x-anthropic-billing-header"): return None return text parts: list[str] = [] for block in msg.content: if block.type == "text" and block.text: if block.text.startswith("x-anthropic-billing-header"): continue parts.append(block.text) return "".join(parts) if parts else None @classmethod def _convert_messages( cls, messages: list, openai_messages: list[dict[str, Any]], *, merge_inline_system: bool = False, ) -> None: """Convert Anthropic messages to OpenAI format.""" for msg in messages: # Handle system messages in-place: extract text, strip billing # headers, and only emit if there is real content. This avoids # going through _convert_block / _convert_message_content which # doesn't strip billing headers and may produce messages with # no "content" key. if msg.role == "system": if merge_inline_system: continue # already merged into top-level by _convert_system_message text = cls._extract_system_text(msg) if text: openai_messages.append({"role": "system", "content": text}) continue openai_msg: dict[str, Any] = {"role": msg.role} # type: ignore if isinstance(msg.content, str): openai_msg["content"] = msg.content else: cls._convert_message_content(msg, openai_msg, openai_messages) if not (msg.role == "user" and "content" not in openai_msg): openai_messages.append(openai_msg) @classmethod def _convert_message_content( cls, msg, openai_msg: dict[str, Any], openai_messages: list[dict[str, Any]], ) -> None: """Convert complex message content blocks.""" content_parts: list[dict[str, Any]] = [] tool_calls: list[dict[str, Any]] = [] reasoning_parts: list[str] = [] for block in msg.content: cls._convert_block( block, msg.role, content_parts, tool_calls, reasoning_parts, openai_messages, ) if reasoning_parts: openai_msg["reasoning"] = "".join(reasoning_parts) if tool_calls: openai_msg["tool_calls"] = tool_calls # type: ignore if content_parts: if len(content_parts) == 1 and content_parts[0]["type"] == "text": openai_msg["content"] = content_parts[0]["text"] else: openai_msg["content"] = content_parts # type: ignore elif not tool_calls and not reasoning_parts: return @classmethod def _convert_block( cls, block, role: str, content_parts: list[dict[str, Any]], tool_calls: list[dict[str, Any]], reasoning_parts: list[str], openai_messages: list[dict[str, Any]], ) -> None: """Convert individual content block.""" if block.type == "text" and block.text: content_parts.append({"type": "text", "text": block.text}) elif block.type == "image" and block.source: image_url = cls._convert_image_source_to_url(block.source) content_parts.append({"type": "image_url", "image_url": {"url": image_url}}) elif block.type == "thinking" and block.thinking is not None: reasoning_parts.append(block.thinking) elif block.type == "redacted_thinking": # Redacted thinking blocks contain safety-filtered reasoning. # We skip them as the content is opaque (base64 'data' field), # but accepting the block prevents a validation error when the # client echoes back the full assistant message. pass elif block.type == "tool_use": cls._convert_tool_use_block(block, tool_calls) elif block.type == "tool_result": cls._convert_tool_result_block(block, role, openai_messages, content_parts) elif block.type == "tool_reference": # Tool references are expanded during tool_result processing # when they appear inside tool_result content. pass @classmethod def _convert_tool_use_block(cls, block, tool_calls: list[dict[str, Any]]) -> None: """Convert tool_use block to OpenAI function call format.""" tool_call = { "id": block.id or f"call_{int(time.time())}", "type": "function", "function": { "name": block.name or "", "arguments": json.dumps(block.input or {}), }, } tool_calls.append(tool_call) @classmethod def _convert_tool_result_block( cls, block, role: str, openai_messages: list[dict[str, Any]], content_parts: list[dict[str, Any]], ) -> None: """Convert tool_result block to OpenAI format.""" if role == "user": cls._convert_user_tool_result(block, openai_messages) else: tool_result_text = str(block.content) if block.content else "" content_parts.append( {"type": "text", "text": f"Tool result: {tool_result_text}"} ) @classmethod def _convert_user_tool_result( cls, block, openai_messages: list[dict[str, Any]] ) -> None: """Convert user tool_result with text and image support.""" tool_text = "" tool_image_urls: list[str] = [] tool_reference: list[dict[str, Any]] = [] if isinstance(block.content, str): tool_text = block.content elif isinstance(block.content, list): text_parts: list[str] = [] for item in block.content: if not isinstance(item, dict): continue item_type = item.get("type") if item_type == "text": text_parts.append(item.get("text", "")) elif item_type == "image": source = item.get("source", {}) url = cls._convert_image_source_to_url(source) if url: tool_image_urls.append(url) elif item_type == "tool_reference": ref_name = item.get("tool_name") or item.get("name") if ref_name: tool_reference.append( {"type": "tool_reference", "name": ref_name} ) tool_text = "\n".join(text_parts) openai_messages.append( { "role": "tool", "tool_call_id": block.tool_use_id or "", "content": tool_text or "", } ) if tool_image_urls: openai_messages.append( { "role": "user", "content": [ # type: ignore[dict-item] {"type": "image_url", "image_url": {"url": img}} for img in tool_image_urls ], } ) if tool_reference: openai_messages.append( { "role": "tool", "tool_call_id": block.tool_use_id or "", "content": tool_reference, # type: ignore[dict-item] } ) @classmethod def _build_base_request( cls, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, openai_messages: list[dict[str, Any]], ) -> ChatCompletionRequest: """Build base ChatCompletionRequest.""" if isinstance(anthropic_request, AnthropicCountTokensRequest): return ChatCompletionRequest( model=anthropic_request.model, messages=openai_messages, chat_template_kwargs=anthropic_request.chat_template_kwargs, ) return ChatCompletionRequest( model=anthropic_request.model, messages=openai_messages, max_tokens=anthropic_request.max_tokens, max_completion_tokens=anthropic_request.max_tokens, stop=anthropic_request.stop_sequences, temperature=anthropic_request.temperature, top_p=anthropic_request.top_p, top_k=anthropic_request.top_k, watermarking=anthropic_request.watermarking, cache_salt=anthropic_request.cache_salt, kv_transfer_params=anthropic_request.kv_transfer_params, ec_transfer_params=anthropic_request.ec_transfer_params, vllm_xargs=anthropic_request.vllm_xargs, chat_template_kwargs=anthropic_request.chat_template_kwargs, ) @classmethod def _handle_thinking( cls, req: ChatCompletionRequest, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none", ) -> None: """Handle extended-thinking configuration. ``display`` is intentionally ignored: suppressing reasoning would mark it ended for structured outputs and drop it from multi-turn history. """ thinking: AnthropicThinkingConfig | None = anthropic_request.thinking if thinking is None: return if thinking.type == "disabled": # "none" clears enable_thinking for templates that honor it; models # that cannot disable thinking are configured with a low effort. req.reasoning_effort = disabled_thinking_effort elif thinking.type == "enabled": req.thinking_token_budget = thinking.budget_tokens # "adaptive" pins nothing: the model chooses depth beneath the ceiling # already set from output_config.effort. @classmethod def _handle_output_config( cls, req: ChatCompletionRequest, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, ) -> None: """Handle output configuration such as output format and effort.""" output_config: AnthropicOutputConfig | None = anthropic_request.output_config if output_config and output_config.format and output_config.format.json_schema: req.response_format = ResponseFormat( type=output_config.format.type, json_schema=JsonSchemaResponseFormat( schema=output_config.format.json_schema, name=output_config.format.type, ), ) if output_config and output_config.effort is not None: req.reasoning_effort = output_config.effort @classmethod def _handle_streaming_options( cls, req: ChatCompletionRequest, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, ) -> None: """Handle streaming configuration.""" if isinstance(anthropic_request, AnthropicCountTokensRequest): return if anthropic_request.stream: req.stream = anthropic_request.stream req.stream_options = StreamOptions.model_validate( {"include_usage": True, "continuous_usage_stats": True} ) @classmethod def _convert_tool_choice( cls, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, req: ChatCompletionRequest, ) -> None: """Convert Anthropic tool_choice to OpenAI format.""" if anthropic_request.tool_choice is None: req.tool_choice = None return req.parallel_tool_calls = ( not anthropic_request.tool_choice.disable_parallel_tool_use ) tool_choice_type = anthropic_request.tool_choice.type if tool_choice_type == "auto": req.tool_choice = "auto" elif tool_choice_type == "any": req.tool_choice = "required" elif tool_choice_type == "none": req.tool_choice = "none" elif tool_choice_type == "tool": req.tool_choice = ChatCompletionNamedToolChoiceParam.model_validate( { "type": "function", "function": {"name": anthropic_request.tool_choice.name}, } ) @classmethod def _resolve_tools( cls, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, ) -> list[AnthropicTool] | None: """Apply tool_addition/tool_removal blocks in ``messages`` to ``tools``. Chat templates see a single tool list, so this resolves the tools in effect at the end of the conversation: a tool added by reference is no longer deferred, a tool added by value is declared, and a removed tool is dropped. """ changes = [ block for msg in anthropic_request.messages if not isinstance(msg.content, str) for block in msg.content if block.type in ("tool_addition", "tool_removal") ] if not changes: return anthropic_request.tools tools = {tool.name: tool for tool in anthropic_request.tools or []} removed: set[str] = set() for block in changes: change = block.tool assert change is not None if isinstance(change, AnthropicToolChangeDefinition): name = change.definition.name tools[name] = change.definition else: name = change.name if name not in tools: raise ValueError( f"{block.type} references tool {name!r}, " "which is not declared in tools" ) if block.type == "tool_addition": tools[name] = tools[name].model_copy(update={"defer_loading": None}) if block.type == "tool_removal": removed.add(name) else: removed.discard(name) return [tool for name, tool in tools.items() if name not in removed] @classmethod def _convert_tools( cls, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, req: ChatCompletionRequest, ) -> None: """Convert Anthropic tools to OpenAI format.""" anthropic_tools = cls._resolve_tools(anthropic_request) if anthropic_tools is None: return tools = [] for tool in anthropic_tools: tools.append( ChatCompletionToolsParam.model_validate( { "type": "function", "function": { "name": tool.name, "description": tool.description, "parameters": tool.input_schema, "strict": tool.strict, "defer_loading": tool.defer_loading, }, } ) ) if req.tool_choice is None: req.tool_choice = "auto" req.tools = tools async def create_messages( self, request: AnthropicMessagesRequest, raw_request: Request | None = None, ) -> AsyncGenerator[str, None] | AnthropicMessagesResponse | ErrorResponse: """Messages API similar to Anthropic's API. See https://docs.anthropic.com/en/api/messages for the API specification. This API mimics the Anthropic messages API. """ if logger.isEnabledFor(logging.DEBUG): logger.debug("Received messages request %s", request.model_dump_json()) disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none" if request.thinking is not None and request.thinking.type == "disabled": disabled_thinking_effort = await self._get_disabled_thinking_effort() chat_req = self.to_chat_completion_request( request, merge_inline_system=self._merge_inline_system, disabled_thinking_effort=disabled_thinking_effort, ) if logger.isEnabledFor(logging.DEBUG): logger.debug("Convert to OpenAI request %s", chat_req.model_dump_json()) generator = await self.create_chat_completion(chat_req, raw_request) if isinstance(generator, ErrorResponse): return generator elif isinstance(generator, ChatCompletionResponse): return self.messages_full_converter(generator) return generator def chat_completion_stream_generator( self, *args: Any, **kwargs: Any ) -> AsyncGenerator[str, None]: return self.message_stream_converter( self.chat_completion_stream_generator_raw(*args, **kwargs) ) def messages_full_converter( self, generator: ChatCompletionResponse, ) -> AnthropicMessagesResponse: result = AnthropicMessagesResponse( id=generator.id, content=[], model=generator.model, usage=_build_anthropic_usage(generator.usage), kv_transfer_params=generator.kv_transfer_params, ec_transfer_params=generator.ec_transfer_params, ) choice = generator.choices[0] result.stop_reason, result.stop_sequence = _to_anthropic_stop_reason( choice.finish_reason, choice.stop_reason, tool_used=bool(choice.message.tool_calls), # Their arguments are json-decoded below, so parsed calls are complete. tool_complete=bool(choice.message.tool_calls), ) content: list[AnthropicContentBlock] = [] if choice.message.reasoning: content.append( AnthropicContentBlock( type="thinking", thinking=choice.message.reasoning, signature=uuid.uuid4().hex, ) ) if choice.message.content: content.append( AnthropicContentBlock( type="text", text=choice.message.content, ) ) for tool_call in choice.message.tool_calls: anthropic_tool_call = AnthropicContentBlock( type="tool_use", id=tool_call.id, name=tool_call.function.name, input=json.loads(tool_call.function.arguments), ) content += [anthropic_tool_call] # Anthropic's canonical shape for an empty completion is a single # empty text block, not []. Some strict clients assume content[0] # exists, so emit one here. if not content: content.append(AnthropicContentBlock(type="text", text="")) result.content = content return result async def message_stream_converter( self, generator: AsyncGenerator[ChatCompletionStreamResponse | ErrorResponse, None], ) -> AsyncGenerator[str, None]: try: class _ActiveBlockState: def __init__(self) -> None: self.content_block_index = 0 self.block_type: str | None = None self.block_index: int | None = None self.block_signature: str | None = None self.signature_emitted: bool = False self.tool_use_id: str | None = None self.pending_content: list[str] = [] def reset(self) -> None: self.block_type = None self.block_index = None self.block_signature = None self.signature_emitted = False self.tool_use_id = None self.pending_content.clear() def start(self, block: AnthropicContentBlock) -> None: self.block_type = block.type self.block_index = self.content_block_index if block.type == "thinking": self.block_signature = uuid.uuid4().hex self.signature_emitted = False self.tool_use_id = None elif block.type == "tool_use": self.block_signature = None self.signature_emitted = True self.tool_use_id = block.id else: self.block_signature = None self.signature_emitted = True self.tool_use_id = None first_item = True finish_reason = None # Matched stop string, when generation stopped on one (a str); # int stop-token-id / None are not stop sequences. stop_sequence: int | str | None = None tool_used = False # Arguments of the latest tool_use block; only the last call can be # cut short by a stop string. last_tool_args: list[str] = [] state = _ActiveBlockState() # Map from tool call index to tool_use_id tool_index_to_id: dict[int, str] = {} def stop_active_block(): events: list[str] = [] if state.block_type is None: return events if ( state.block_type == "thinking" and state.block_signature is not None and not state.signature_emitted ): chunk = AnthropicStreamEvent( index=state.block_index, type="content_block_delta", delta=AnthropicDelta( type="signature_delta", signature=state.block_signature, ), ) data = chunk.model_dump_json(exclude_unset=True) events.append(wrap_data_with_event(data, "content_block_delta")) state.signature_emitted = True stop_chunk = AnthropicStreamEvent( index=state.block_index, type="content_block_stop", ) data = stop_chunk.model_dump_json(exclude_unset=True) events.append(wrap_data_with_event(data, "content_block_stop")) state.reset() state.content_block_index += 1 return events def start_block(block: AnthropicContentBlock): chunk = AnthropicStreamEvent( index=state.content_block_index, type="content_block_start", content_block=block, ) data = chunk.model_dump_json(exclude_unset=True) event = wrap_data_with_event(data, "content_block_start") state.start(block) return event def stop_and_flush() -> list[str]: buffered = list(state.pending_content) state.pending_content.clear() events = stop_active_block() if not buffered: return events text = "".join(buffered) events.append(start_block(AnthropicContentBlock(type="text", text=""))) pc_chunk = AnthropicStreamEvent( index=state.block_index, type="content_block_delta", delta=AnthropicDelta(type="text_delta", text=text), ) pc_data = pc_chunk.model_dump_json(exclude_unset=True) events.append(wrap_data_with_event(pc_data, "content_block_delta")) events.extend(stop_active_block()) return events async for origin_chunk in generator: if isinstance(origin_chunk, ErrorResponse): raise RuntimeError(origin_chunk.error.message) if first_item: chunk = AnthropicStreamEvent( type="message_start", message=AnthropicMessagesResponse( id=origin_chunk.id, # Set explicitly: this event is serialized # with exclude_unset=True, which drops # default-valued fields, while strict # Anthropic SDK clients require # message.type/role (issue #45367). type="message", role="assistant", content=[], model=origin_chunk.model, stop_reason=None, stop_sequence=None, usage=_build_anthropic_usage(origin_chunk.usage), ), ) first_item = False data = chunk.model_dump_json(exclude_unset=True) yield wrap_data_with_event(data, "message_start") continue # last chunk including usage info if len(origin_chunk.choices) == 0: for event in stop_and_flush(): yield event anthropic_stop_reason, anthropic_stop_sequence = ( _to_anthropic_stop_reason( finish_reason or "stop", stop_sequence, tool_used=tool_used, tool_complete=( tool_used and _is_json("".join(last_tool_args)) ), ) ) stop_delta = AnthropicDelta( stop_reason=anthropic_stop_reason, # Set explicitly so exclude_unset=True keeps it. stop_sequence=anthropic_stop_sequence, ) chunk = AnthropicStreamEvent( type="message_delta", delta=stop_delta, usage=_build_anthropic_usage(origin_chunk.usage), ) data = chunk.model_dump_json(exclude_unset=True) yield wrap_data_with_event(data, "message_delta") continue if origin_chunk.choices[0].finish_reason is not None: finish_reason = origin_chunk.choices[0].finish_reason stop_sequence = origin_chunk.choices[0].stop_reason # continue # thinking / text content reasoning_delta = origin_chunk.choices[0].delta.reasoning if reasoning_delta is not None: if reasoning_delta == "": pass else: if state.block_type != "thinking": for event in stop_and_flush(): yield event start_event = start_block( AnthropicContentBlock(type="thinking", thinking="") ) yield start_event chunk = AnthropicStreamEvent( index=( state.block_index if state.block_index is not None else state.content_block_index ), type="content_block_delta", delta=AnthropicDelta( type="thinking_delta", thinking=reasoning_delta, ), ) data = chunk.model_dump_json(exclude_unset=True) yield wrap_data_with_event(data, "content_block_delta") if origin_chunk.choices[0].delta.content is not None: if origin_chunk.choices[0].delta.content == "": pass elif state.block_type == "tool_use": state.pending_content.append( origin_chunk.choices[0].delta.content ) else: if state.block_type != "text": for event in stop_and_flush(): yield event start_event = start_block( AnthropicContentBlock(type="text", text="") ) yield start_event chunk = AnthropicStreamEvent( index=( state.block_index if state.block_index is not None else state.content_block_index ), type="content_block_delta", delta=AnthropicDelta( type="text_delta", text=origin_chunk.choices[0].delta.content, ), ) data = chunk.model_dump_json(exclude_unset=True) yield wrap_data_with_event(data, "content_block_delta") # tool calls - process all tool calls in the delta if len(origin_chunk.choices[0].delta.tool_calls) > 0: for tool_call in origin_chunk.choices[0].delta.tool_calls: if tool_call.id is not None: # Update mapping for incremental updates tool_index_to_id[tool_call.index] = tool_call.id # Only create new block if different tool call # AND has a name tool_name = ( tool_call.function.name if tool_call.function else None ) if ( state.tool_use_id != tool_call.id and tool_name is not None ): for event in stop_and_flush(): yield event tool_used = True last_tool_args = [] start_event = start_block( AnthropicContentBlock( type="tool_use", id=tool_call.id, name=tool_name, input={}, ) ) yield start_event # Handle initial arguments if present if ( tool_call.function and tool_call.function.arguments and state.tool_use_id == tool_call.id ): last_tool_args.append(tool_call.function.arguments) chunk = AnthropicStreamEvent( index=( state.block_index if state.block_index is not None else state.content_block_index ), type="content_block_delta", delta=AnthropicDelta( type="input_json_delta", partial_json=tool_call.function.arguments, ), ) data = chunk.model_dump_json(exclude_unset=True) yield wrap_data_with_event(data, "content_block_delta") else: # Incremental update - use index to find tool_use_id tool_use_id = tool_index_to_id.get(tool_call.index) if ( tool_use_id is not None and tool_call.function and tool_call.function.arguments and state.tool_use_id == tool_use_id ): last_tool_args.append(tool_call.function.arguments) chunk = AnthropicStreamEvent( index=( state.block_index if state.block_index is not None else state.content_block_index ), type="content_block_delta", delta=AnthropicDelta( type="input_json_delta", partial_json=tool_call.function.arguments, ), ) data = chunk.model_dump_json(exclude_unset=True) yield wrap_data_with_event(data, "content_block_delta") continue for event in stop_and_flush(): yield event stop_message = AnthropicStreamEvent(type="message_stop") data = stop_message.model_dump_json(exclude_unset=True, exclude_none=True) yield wrap_data_with_event(data, "message_stop") except Exception as e: logger.exception("Error in message stream converter.") for event in stop_active_block(): yield event error_response = AnthropicStreamEvent( type="error", error=AnthropicError( type="internal_error", message=sanitize_message(str(e)) ), ) data = error_response.model_dump_json(exclude_unset=True) yield wrap_data_with_event(data, "error") async def count_tokens( self, request: AnthropicCountTokensRequest, raw_request: Request | None = None, ) -> AnthropicCountTokensResponse | ErrorResponse: """Implements Anthropic's messages.count_tokens endpoint.""" disabled_thinking_effort: AnthropicDisabledThinkingEffort = "none" if request.thinking is not None and request.thinking.type == "disabled": disabled_thinking_effort = await self._get_disabled_thinking_effort() chat_req = self.to_chat_completion_request( request, merge_inline_system=self._merge_inline_system, disabled_thinking_effort=disabled_thinking_effort, ) result = await self.render_chat_request(chat_req) if isinstance(result, ErrorResponse): return result _, engine_inputs = result input_tokens = sum( # type: ignore len(engine_input["prompt_token_ids"]) # type: ignore[typeddict-item, misc] for engine_input in engine_inputs if "prompt_token_ids" in engine_input ) response = AnthropicCountTokensResponse( input_tokens=input_tokens, context_management=AnthropicContextManagement( original_input_tokens=input_tokens ), ) return response