[Bugfix][Frontend] Detect Anthropic inline-system merge against the resolved chat template (#58754)

Signed-off-by: Robert Shaw <robshaw@redhat.com>
Co-authored-by: Robert Shaw <robshaw@redhat.com>

tests/entrypoints/anthropic/test_anthropic_messages_conversion.py

@@ -1383,6 +1383,11 @@ Q35_TEMPLATE = (
     "{%- endfor %}"
 )
 
+PERMISSIVE_TEMPLATE = (
+    "{%- for message in messages %}{{- message.role }}: {{ message.content }}\n"
+    "{%- endfor %}"
+)
+
 
 class TestDetectMergeInlineSystem:
     """Verify _detect_merge_inline_system auto-detection.
@@ -1402,11 +1407,7 @@ class TestDetectMergeInlineSystem:
     def test_no_restriction_no_merge(self):
         """Template without restriction accepts mid-conversation system."""
         assert (
-            AnthropicServingMessages._detect_merge_inline_system(
-                "{%- for message in messages %}"
-                "{{- message.role }}: {{ message.content }}\n"
-                "{%- endfor %}"
-            )
+            AnthropicServingMessages._detect_merge_inline_system(PERMISSIVE_TEMPLATE)
             is False
         )
 
@@ -1414,6 +1415,38 @@ class TestDetectMergeInlineSystem:
         """No chat_template → conservative default: merge."""
         assert AnthropicServingMessages._detect_merge_inline_system(None) is True
 
+    @pytest.mark.parametrize(
+        ("resolved", "expected"),
+        [(PERMISSIVE_TEMPLATE, False), (Q35_TEMPLATE, True)],
+    )
+    def test_probes_resolved_template_without_cli_override(
+        self, monkeypatch, resolved, expected
+    ):
+        """Without --chat-template, probe the tokenizer's template (#58727).
+
+        Merging defeats prefix caching, so it is warned about at startup.
+        """
+        import vllm.entrypoints.anthropic.serving as serving_mod
+        from vllm.renderers.hf import HfRenderer
+
+        monkeypatch.setattr(
+            serving_mod, "resolve_chat_template", lambda *a, **kw: resolved
+        )
+        mock_logger = MagicMock()
+        monkeypatch.setattr(serving_mod, "logger", mock_logger)
+        renderer = MagicMock(spec=HfRenderer)
+        renderer.tokenizer = MagicMock()
+        online_renderer = SimpleNamespace(
+            renderer=renderer,
+            chat_template=None,
+            model_config=None,
+        )
+        assert (
+            AnthropicServingMessages._should_merge_inline_system(online_renderer)
+            is expected
+        )
+        assert mock_logger.warning.called is expected
+
 
 # ======================================================================
 # Full (non-streaming) response conversion: messages_full_converter

tests/entrypoints/openai/chat_completion/test_chat_error.py

@@ -45,6 +45,8 @@ class MockModelConfig:
     trust_remote_code = False
     tokenizer_mode = "auto"
     max_model_len = 100
+    revision = None
+    code_revision = None
     tokenizer_revision = None
     multimodal_config = MultiModalConfig()
     hf_config = MockHFConfig()

tests/entrypoints/openai/completion/test_completion_error.py

@@ -56,6 +56,8 @@ class MockModelConfig:
     trust_remote_code = False
     tokenizer_mode = "auto"
     max_model_len = 100
+    revision = None
+    code_revision = None
     tokenizer_revision = None
     multimodal_config = MultiModalConfig()
     hf_config = MockHFConfig()

tests/entrypoints/scale_out/render/test_render.py

@@ -651,9 +651,9 @@ async def test_messages_render_system_and_multi_turn(client):
 async def test_messages_render_merges_inline_system(client):
     """Inline system messages merge into the leading system block.
 
-    Without a --chat-template arg the /v1/messages server path detects
-    merge_inline_system=True, so render must produce the same tokens as
-    the manually pre-merged request.
+    The model's chat template rejects non-leading system messages, so the
+    /v1/messages server path detects merge_inline_system=True and render must
+    produce the same tokens as the manually pre-merged request.
     """
     inline = await client.post(
         "/v1/messages/render",

vllm/entrypoints/anthropic/serving.py

@@ -51,9 +51,11 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
 from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
 from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
 from vllm.entrypoints.serve.utils.request_logger import RequestLogger
+from vllm.logger import init_logger
+from vllm.renderers.hf import HfRenderer, resolve_chat_template
 from vllm.renderers.online_renderer import OnlineRenderer
 
-logger = logging.getLogger(__name__)
+logger = init_logger(__name__)
 
 
 def _build_anthropic_usage(
@@ -144,7 +146,7 @@ class AnthropicServingMessages(OpenAIServingChat):
             "length": "max_tokens",
             "tool_calls": "tool_use",
         }
-        self._merge_inline_system = self._detect_merge_inline_system(chat_template)
+        self._merge_inline_system = self._should_merge_inline_system(online_renderer)
         # Resolved lazily from the renderer when "auto".
         self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = (
             None if disabled_thinking_effort == "auto" else disabled_thinking_effort
@@ -193,6 +195,39 @@ class AnthropicServingMessages(OpenAIServingChat):
         components = self._extract_prompt_components(engine_input)
         return components.token_ids, components.text
 
+    @classmethod
+    def _should_merge_inline_system(cls, online_renderer: OnlineRenderer) -> bool:
+        """Probe the chat templates the renderer applies, not the CLI override.
+
+        Both the tool and non-tool templates are checked since they may differ.
+        """
+        renderer = online_renderer.renderer
+        tool_variants: tuple[list[dict[str, Any]] | None, ...] = (None, [])
+        if not isinstance(renderer, HfRenderer) or renderer.tokenizer is None:
+            merge = cls._detect_merge_inline_system(online_renderer.chat_template)
+        else:
+            merge = any(
+                cls._detect_merge_inline_system(
+                    resolve_chat_template(
+                        renderer.tokenizer,
+                        online_renderer.chat_template,
+                        tools,
+                        model_config=online_renderer.model_config,
+                    )
+                )
+                for tools in tool_variants
+            )
+        if merge:
+            logger.warning(
+                "The chat template requires system-first ordering, so inline "
+                "system messages in /v1/messages requests (e.g. Claude Code's "
+                "per-turn reminders) are merged into the leading system prompt. "
+                "Each new one changes the prompt prefix, so the rest of the "
+                "conversation misses the prefix cache. Pass a --chat-template "
+                "that accepts non-leading system messages to avoid this."
+            )
+        return merge
+
     @staticmethod
     def _detect_merge_inline_system(chat_template: str | None) -> bool:
         """Auto-detect whether the chat template requires system-first ordering.

vllm/entrypoints/scale_out/render/serving.py

@@ -57,9 +57,7 @@ class ServingRender(BaseServing):
         self.tool_server = tool_server
 
         self._merge_inline_system = (
-            AnthropicServingMessages._detect_merge_inline_system(
-                online_renderer.chat_template
-            )
+            AnthropicServingMessages._should_merge_inline_system(online_renderer)
         )
 
         self._placeholder_metadata_parser: MultiModalDataParser | None = None