From 4ce2d0145312809ef6122ccb7be8ae7cafa462a9 Mon Sep 17 00:00:00 2001 From: MrFan <642664360@qq.com> Date: Fri, 19 Jun 2026 04:19:11 +0800 Subject: [PATCH] fix(anthropic): auto-detect template support for mid-conversation system messages (#46025) Signed-off-by: felix0080 Signed-off-by: Ben Browning Co-authored-by: felix0080 Co-authored-by: Ben Browning --- .../test_anthropic_messages_conversion.py | 47 +++++++++++ vllm/entrypoints/anthropic/serving.py | 79 +++++++++++++++++-- 2 files changed, 120 insertions(+), 6 deletions(-) diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index 2fb0f21c87..4663a6565d 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -1096,3 +1096,50 @@ class TestMessageStartIncludesTypeAndRole: message = events[0][1]["message"] assert message["type"] == "message" assert message["role"] == "assistant" + + +# ====================================================================== +# Auto-detection of system-first template requirement +# ====================================================================== + + +Q35_TEMPLATE = ( + "{%- for message in messages %}" + "{%- if message.role == 'system' %}" + "{%- if not loop.first %}" + "{{- raise_exception('System message must be at the beginning.') }}" + "{%- endif %}" + "{%- endif %}" + "{%- endfor %}" +) + + +class TestDetectMergeInlineSystem: + """Verify _detect_merge_inline_system auto-detection. + + Tests three scenarios: + 1. Template with system-first guard (e.g. Qwen) → merge needed + 2. Template without restrictions → no merge, cache-friendly + 3. No template provided → safe default: merge + """ + + def test_qwen_template_requires_merge(self): + """Template with loop.first guard rejects mid-conversation system.""" + assert ( + AnthropicServingMessages._detect_merge_inline_system(Q35_TEMPLATE) is True + ) + + def test_no_restriction_no_merge(self): + """Template without restriction accepts mid-conversation system.""" + assert ( + AnthropicServingMessages._detect_merge_inline_system( + "{%- for message in messages %}" + "{{- message.role }}: {{ message.content }}\n" + "{%- endfor %}" + ) + is False + ) + + def test_no_template_defaults_merge(self): + """No chat_template → conservative default: merge.""" + assert AnthropicServingMessages._detect_merge_inline_system(None) is True diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 5a7e8ae95e..9d5852428d 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -12,6 +12,7 @@ import uuid from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any +import jinja2 from fastapi import Request from vllm.engine.protocol import EngineClient @@ -99,6 +100,36 @@ class AnthropicServingMessages(OpenAIServingChat): "length": "max_tokens", "tool_calls": "tool_use", } + self._merge_inline_system = self._detect_merge_inline_system(chat_template) + + @staticmethod + def _detect_merge_inline_system(chat_template: str | None) -> bool: + """Auto-detect whether the chat template requires system-first ordering. + + Renders a [system, user, system, user] conversation against the + template; if it raises (e.g. Qwen's ``loop.first`` guard), the + model needs inline system messages merged into the leading block. + """ + if not chat_template: + return True + try: + env = jinja2.sandbox.ImmutableSandboxedEnvironment( + trim_blocks=True, + lstrip_blocks=True, + extensions=[jinja2.ext.loopcontrols], + ) + env.from_string(chat_template).render( + messages=[ + {"role": "system", "content": "t"}, + {"role": "user", "content": "t"}, + {"role": "system", "content": "t"}, + {"role": "user", "content": "t"}, + ], + add_generation_prompt=False, + ) + return False + except jinja2.TemplateError: + return True @staticmethod def _convert_image_source_to_url(source: dict[str, Any]) -> str: @@ -123,13 +154,24 @@ class AnthropicServingMessages(OpenAIServingChat): @classmethod def _convert_anthropic_to_openai_request( - cls, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest + cls, + anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, + *, + merge_inline_system: bool = False, ) -> ChatCompletionRequest: """Convert Anthropic message format to OpenAI format""" openai_messages: list[dict[str, Any]] = [] - cls._convert_system_message(anthropic_request, openai_messages) - cls._convert_messages(anthropic_request.messages, openai_messages) + cls._convert_system_message( + anthropic_request, + openai_messages, + merge_inline_system=merge_inline_system, + ) + cls._convert_messages( + anthropic_request.messages, + openai_messages, + merge_inline_system=merge_inline_system, + ) req = cls._build_base_request(anthropic_request, openai_messages) cls._handle_streaming_options(req, anthropic_request) cls._handle_output_config(req, anthropic_request) @@ -142,6 +184,8 @@ class AnthropicServingMessages(OpenAIServingChat): cls, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest, openai_messages: list[dict[str, Any]], + *, + merge_inline_system: bool = False, ) -> None: """Convert Anthropic system message to OpenAI format""" system_parts: list[str] = [] @@ -159,6 +203,17 @@ class AnthropicServingMessages(OpenAIServingChat): continue system_parts.append(block.text) + # When the template requires system-first ordering, extract inline + # system messages from the messages array and merge them into the + # top-level block so the template doesn't reject them. + if merge_inline_system: + for msg in anthropic_request.messages: + if msg.role != "system": + continue + text = cls._extract_system_text(msg) + if text: + system_parts.append(text) + if system_parts: openai_messages.append({"role": "system", "content": "".join(system_parts)}) @@ -180,7 +235,11 @@ class AnthropicServingMessages(OpenAIServingChat): @classmethod def _convert_messages( - cls, messages: list, openai_messages: list[dict[str, Any]] + cls, + messages: list, + openai_messages: list[dict[str, Any]], + *, + merge_inline_system: bool = False, ) -> None: """Convert Anthropic messages to OpenAI format""" for msg in messages: @@ -190,6 +249,8 @@ class AnthropicServingMessages(OpenAIServingChat): # doesn't strip billing headers and may produce messages with # no "content" key. if msg.role == "system": + if merge_inline_system: + continue # already merged into top-level by _convert_system_message text = cls._extract_system_text(msg) if text: openai_messages.append({"role": "system", "content": text}) @@ -497,7 +558,10 @@ class AnthropicServingMessages(OpenAIServingChat): """ if logger.isEnabledFor(logging.DEBUG): logger.debug("Received messages request %s", request.model_dump_json()) - chat_req = self._convert_anthropic_to_openai_request(request) + chat_req = self._convert_anthropic_to_openai_request( + request, + merge_inline_system=self._merge_inline_system, + ) if logger.isEnabledFor(logging.DEBUG): logger.debug("Convert to OpenAI request %s", chat_req.model_dump_json()) generator = await self.create_chat_completion(chat_req, raw_request) @@ -905,7 +969,10 @@ class AnthropicServingMessages(OpenAIServingChat): raw_request: Request | None = None, ) -> AnthropicCountTokensResponse | ErrorResponse: """Implements Anthropic's messages.count_tokens endpoint.""" - chat_req = self._convert_anthropic_to_openai_request(request) + chat_req = self._convert_anthropic_to_openai_request( + request, + merge_inline_system=self._merge_inline_system, + ) result = await self.render_chat_request(chat_req) if isinstance(result, ErrorResponse): return result