diff --git a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py index a6cfb15d60..aa68b6bff0 100644 --- a/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py +++ b/tests/entrypoints/anthropic/test_anthropic_messages_conversion.py @@ -1383,6 +1383,11 @@ Q35_TEMPLATE = ( "{%- endfor %}" ) +PERMISSIVE_TEMPLATE = ( + "{%- for message in messages %}{{- message.role }}: {{ message.content }}\n" + "{%- endfor %}" +) + class TestDetectMergeInlineSystem: """Verify _detect_merge_inline_system auto-detection. @@ -1402,11 +1407,7 @@ class TestDetectMergeInlineSystem: def test_no_restriction_no_merge(self): """Template without restriction accepts mid-conversation system.""" assert ( - AnthropicServingMessages._detect_merge_inline_system( - "{%- for message in messages %}" - "{{- message.role }}: {{ message.content }}\n" - "{%- endfor %}" - ) + AnthropicServingMessages._detect_merge_inline_system(PERMISSIVE_TEMPLATE) is False ) @@ -1414,6 +1415,38 @@ class TestDetectMergeInlineSystem: """No chat_template → conservative default: merge.""" assert AnthropicServingMessages._detect_merge_inline_system(None) is True + @pytest.mark.parametrize( + ("resolved", "expected"), + [(PERMISSIVE_TEMPLATE, False), (Q35_TEMPLATE, True)], + ) + def test_probes_resolved_template_without_cli_override( + self, monkeypatch, resolved, expected + ): + """Without --chat-template, probe the tokenizer's template (#58727). + + Merging defeats prefix caching, so it is warned about at startup. + """ + import vllm.entrypoints.anthropic.serving as serving_mod + from vllm.renderers.hf import HfRenderer + + monkeypatch.setattr( + serving_mod, "resolve_chat_template", lambda *a, **kw: resolved + ) + mock_logger = MagicMock() + monkeypatch.setattr(serving_mod, "logger", mock_logger) + renderer = MagicMock(spec=HfRenderer) + renderer.tokenizer = MagicMock() + online_renderer = SimpleNamespace( + renderer=renderer, + chat_template=None, + model_config=None, + ) + assert ( + AnthropicServingMessages._should_merge_inline_system(online_renderer) + is expected + ) + assert mock_logger.warning.called is expected + # ====================================================================== # Full (non-streaming) response conversion: messages_full_converter diff --git a/tests/entrypoints/openai/chat_completion/test_chat_error.py b/tests/entrypoints/openai/chat_completion/test_chat_error.py index eafcdffaa0..a4126ba5f7 100644 --- a/tests/entrypoints/openai/chat_completion/test_chat_error.py +++ b/tests/entrypoints/openai/chat_completion/test_chat_error.py @@ -45,6 +45,8 @@ class MockModelConfig: trust_remote_code = False tokenizer_mode = "auto" max_model_len = 100 + revision = None + code_revision = None tokenizer_revision = None multimodal_config = MultiModalConfig() hf_config = MockHFConfig() diff --git a/tests/entrypoints/openai/completion/test_completion_error.py b/tests/entrypoints/openai/completion/test_completion_error.py index 060ae01a1d..05811555a9 100644 --- a/tests/entrypoints/openai/completion/test_completion_error.py +++ b/tests/entrypoints/openai/completion/test_completion_error.py @@ -56,6 +56,8 @@ class MockModelConfig: trust_remote_code = False tokenizer_mode = "auto" max_model_len = 100 + revision = None + code_revision = None tokenizer_revision = None multimodal_config = MultiModalConfig() hf_config = MockHFConfig() diff --git a/tests/entrypoints/scale_out/render/test_render.py b/tests/entrypoints/scale_out/render/test_render.py index e65a26a7d1..0daf4d405e 100644 --- a/tests/entrypoints/scale_out/render/test_render.py +++ b/tests/entrypoints/scale_out/render/test_render.py @@ -651,9 +651,9 @@ async def test_messages_render_system_and_multi_turn(client): async def test_messages_render_merges_inline_system(client): """Inline system messages merge into the leading system block. - Without a --chat-template arg the /v1/messages server path detects - merge_inline_system=True, so render must produce the same tokens as - the manually pre-merged request. + The model's chat template rejects non-leading system messages, so the + /v1/messages server path detects merge_inline_system=True and render must + produce the same tokens as the manually pre-merged request. """ inline = await client.post( "/v1/messages/render", diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index 08f208f824..22484c744d 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -51,9 +51,11 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo from vllm.entrypoints.serve.exception_handling.utils import sanitize_message from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.logger import init_logger +from vllm.renderers.hf import HfRenderer, resolve_chat_template from vllm.renderers.online_renderer import OnlineRenderer -logger = logging.getLogger(__name__) +logger = init_logger(__name__) def _build_anthropic_usage( @@ -144,7 +146,7 @@ class AnthropicServingMessages(OpenAIServingChat): "length": "max_tokens", "tool_calls": "tool_use", } - self._merge_inline_system = self._detect_merge_inline_system(chat_template) + self._merge_inline_system = self._should_merge_inline_system(online_renderer) # Resolved lazily from the renderer when "auto". self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = ( None if disabled_thinking_effort == "auto" else disabled_thinking_effort @@ -193,6 +195,39 @@ class AnthropicServingMessages(OpenAIServingChat): components = self._extract_prompt_components(engine_input) return components.token_ids, components.text + @classmethod + def _should_merge_inline_system(cls, online_renderer: OnlineRenderer) -> bool: + """Probe the chat templates the renderer applies, not the CLI override. + + Both the tool and non-tool templates are checked since they may differ. + """ + renderer = online_renderer.renderer + tool_variants: tuple[list[dict[str, Any]] | None, ...] = (None, []) + if not isinstance(renderer, HfRenderer) or renderer.tokenizer is None: + merge = cls._detect_merge_inline_system(online_renderer.chat_template) + else: + merge = any( + cls._detect_merge_inline_system( + resolve_chat_template( + renderer.tokenizer, + online_renderer.chat_template, + tools, + model_config=online_renderer.model_config, + ) + ) + for tools in tool_variants + ) + if merge: + logger.warning( + "The chat template requires system-first ordering, so inline " + "system messages in /v1/messages requests (e.g. Claude Code's " + "per-turn reminders) are merged into the leading system prompt. " + "Each new one changes the prompt prefix, so the rest of the " + "conversation misses the prefix cache. Pass a --chat-template " + "that accepts non-leading system messages to avoid this." + ) + return merge + @staticmethod def _detect_merge_inline_system(chat_template: str | None) -> bool: """Auto-detect whether the chat template requires system-first ordering. diff --git a/vllm/entrypoints/scale_out/render/serving.py b/vllm/entrypoints/scale_out/render/serving.py index 7973ac3c3d..117ede0f9b 100644 --- a/vllm/entrypoints/scale_out/render/serving.py +++ b/vllm/entrypoints/scale_out/render/serving.py @@ -57,9 +57,7 @@ class ServingRender(BaseServing): self.tool_server = tool_server self._merge_inline_system = ( - AnthropicServingMessages._detect_merge_inline_system( - online_renderer.chat_template - ) + AnthropicServingMessages._should_merge_inline_system(online_renderer) ) self._placeholder_metadata_parser: MultiModalDataParser | None = None