fix(anthropic): auto-detect template support for mid-conversation system messages (#46025)

Signed-off-by: felix0080 <felix0080@users.noreply.github.com>
Signed-off-by: Ben Browning <bbrownin@redhat.com>
Co-authored-by: felix0080 <felix0080@users.noreply.github.com>
Co-authored-by: Ben Browning <bbrownin@redhat.com>

tests/entrypoints/anthropic/test_anthropic_messages_conversion.py

@@ -1096,3 +1096,50 @@ class TestMessageStartIncludesTypeAndRole:
         message = events[0][1]["message"]
         assert message["type"] == "message"
         assert message["role"] == "assistant"
+
+
+# ======================================================================
+# Auto-detection of system-first template requirement
+# ======================================================================
+
+
+Q35_TEMPLATE = (
+    "{%- for message in messages %}"
+    "{%- if message.role == 'system' %}"
+    "{%- if not loop.first %}"
+    "{{- raise_exception('System message must be at the beginning.') }}"
+    "{%- endif %}"
+    "{%- endif %}"
+    "{%- endfor %}"
+)
+
+
+class TestDetectMergeInlineSystem:
+    """Verify _detect_merge_inline_system auto-detection.
+
+    Tests three scenarios:
+    1. Template with system-first guard (e.g. Qwen) → merge needed
+    2. Template without restrictions → no merge, cache-friendly
+    3. No template provided → safe default: merge
+    """
+
+    def test_qwen_template_requires_merge(self):
+        """Template with loop.first guard rejects mid-conversation system."""
+        assert (
+            AnthropicServingMessages._detect_merge_inline_system(Q35_TEMPLATE) is True
+        )
+
+    def test_no_restriction_no_merge(self):
+        """Template without restriction accepts mid-conversation system."""
+        assert (
+            AnthropicServingMessages._detect_merge_inline_system(
+                "{%- for message in messages %}"
+                "{{- message.role }}: {{ message.content }}\n"
+                "{%- endfor %}"
+            )
+            is False
+        )
+
+    def test_no_template_defaults_merge(self):
+        """No chat_template → conservative default: merge."""
+        assert AnthropicServingMessages._detect_merge_inline_system(None) is True

vllm/entrypoints/anthropic/serving.py

@@ -12,6 +12,7 @@ import uuid
 from collections.abc import AsyncGenerator
 from typing import TYPE_CHECKING, Any
 
+import jinja2
 from fastapi import Request
 
 from vllm.engine.protocol import EngineClient
@@ -99,6 +100,36 @@ class AnthropicServingMessages(OpenAIServingChat):
             "length": "max_tokens",
             "tool_calls": "tool_use",
         }
+        self._merge_inline_system = self._detect_merge_inline_system(chat_template)
+
+    @staticmethod
+    def _detect_merge_inline_system(chat_template: str | None) -> bool:
+        """Auto-detect whether the chat template requires system-first ordering.
+
+        Renders a [system, user, system, user] conversation against the
+        template; if it raises (e.g. Qwen's ``loop.first`` guard), the
+        model needs inline system messages merged into the leading block.
+        """
+        if not chat_template:
+            return True
+        try:
+            env = jinja2.sandbox.ImmutableSandboxedEnvironment(
+                trim_blocks=True,
+                lstrip_blocks=True,
+                extensions=[jinja2.ext.loopcontrols],
+            )
+            env.from_string(chat_template).render(
+                messages=[
+                    {"role": "system", "content": "t"},
+                    {"role": "user", "content": "t"},
+                    {"role": "system", "content": "t"},
+                    {"role": "user", "content": "t"},
+                ],
+                add_generation_prompt=False,
+            )
+            return False
+        except jinja2.TemplateError:
+            return True
 
     @staticmethod
     def _convert_image_source_to_url(source: dict[str, Any]) -> str:
@@ -123,13 +154,24 @@ class AnthropicServingMessages(OpenAIServingChat):
 
     @classmethod
     def _convert_anthropic_to_openai_request(
-        cls, anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest
+        cls,
+        anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
+        *,
+        merge_inline_system: bool = False,
     ) -> ChatCompletionRequest:
         """Convert Anthropic message format to OpenAI format"""
         openai_messages: list[dict[str, Any]] = []
 
-        cls._convert_system_message(anthropic_request, openai_messages)
-        cls._convert_messages(anthropic_request.messages, openai_messages)
+        cls._convert_system_message(
+            anthropic_request,
+            openai_messages,
+            merge_inline_system=merge_inline_system,
+        )
+        cls._convert_messages(
+            anthropic_request.messages,
+            openai_messages,
+            merge_inline_system=merge_inline_system,
+        )
         req = cls._build_base_request(anthropic_request, openai_messages)
         cls._handle_streaming_options(req, anthropic_request)
         cls._handle_output_config(req, anthropic_request)
@@ -142,6 +184,8 @@ class AnthropicServingMessages(OpenAIServingChat):
         cls,
         anthropic_request: AnthropicMessagesRequest | AnthropicCountTokensRequest,
         openai_messages: list[dict[str, Any]],
+        *,
+        merge_inline_system: bool = False,
     ) -> None:
         """Convert Anthropic system message to OpenAI format"""
         system_parts: list[str] = []
@@ -159,6 +203,17 @@ class AnthropicServingMessages(OpenAIServingChat):
                             continue
                         system_parts.append(block.text)
 
+        # When the template requires system-first ordering, extract inline
+        # system messages from the messages array and merge them into the
+        # top-level block so the template doesn't reject them.
+        if merge_inline_system:
+            for msg in anthropic_request.messages:
+                if msg.role != "system":
+                    continue
+                text = cls._extract_system_text(msg)
+                if text:
+                    system_parts.append(text)
+
         if system_parts:
             openai_messages.append({"role": "system", "content": "".join(system_parts)})
 
@@ -180,7 +235,11 @@ class AnthropicServingMessages(OpenAIServingChat):
 
     @classmethod
     def _convert_messages(
-        cls, messages: list, openai_messages: list[dict[str, Any]]
+        cls,
+        messages: list,
+        openai_messages: list[dict[str, Any]],
+        *,
+        merge_inline_system: bool = False,
     ) -> None:
         """Convert Anthropic messages to OpenAI format"""
         for msg in messages:
@@ -190,6 +249,8 @@ class AnthropicServingMessages(OpenAIServingChat):
             # doesn't strip billing headers and may produce messages with
             # no "content" key.
             if msg.role == "system":
+                if merge_inline_system:
+                    continue  # already merged into top-level by _convert_system_message
                 text = cls._extract_system_text(msg)
                 if text:
                     openai_messages.append({"role": "system", "content": text})
@@ -497,7 +558,10 @@ class AnthropicServingMessages(OpenAIServingChat):
         """
         if logger.isEnabledFor(logging.DEBUG):
             logger.debug("Received messages request %s", request.model_dump_json())
-        chat_req = self._convert_anthropic_to_openai_request(request)
+        chat_req = self._convert_anthropic_to_openai_request(
+            request,
+            merge_inline_system=self._merge_inline_system,
+        )
         if logger.isEnabledFor(logging.DEBUG):
             logger.debug("Convert to OpenAI request %s", chat_req.model_dump_json())
         generator = await self.create_chat_completion(chat_req, raw_request)
@@ -905,7 +969,10 @@ class AnthropicServingMessages(OpenAIServingChat):
         raw_request: Request | None = None,
     ) -> AnthropicCountTokensResponse | ErrorResponse:
         """Implements Anthropic's messages.count_tokens endpoint."""
-        chat_req = self._convert_anthropic_to_openai_request(request)
+        chat_req = self._convert_anthropic_to_openai_request(
+            request,
+            merge_inline_system=self._merge_inline_system,
+        )
         result = await self.render_chat_request(chat_req)
         if isinstance(result, ErrorResponse):
             return result