[Bugfix][Frontend] Detect Anthropic inline-system merge against the resolved chat template (#58754)

Signed-off-by: Robert Shaw <robshaw@redhat.com>
Co-authored-by: Robert Shaw <robshaw@redhat.com>
This commit is contained in:
Robert Shaw
2026-09-26 14:00:56 +00:00
committed by GitHub
co-authored by Robert Shaw
parent d64f277c54
commit dfab504333
6 changed files with 83 additions and 13 deletions
+37 -2
View File
@@ -51,9 +51,11 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger
from vllm.renderers.hf import HfRenderer, resolve_chat_template
from vllm.renderers.online_renderer import OnlineRenderer
logger = logging.getLogger(__name__)
logger = init_logger(__name__)
def _build_anthropic_usage(
@@ -144,7 +146,7 @@ class AnthropicServingMessages(OpenAIServingChat):
"length": "max_tokens",
"tool_calls": "tool_use",
}
self._merge_inline_system = self._detect_merge_inline_system(chat_template)
self._merge_inline_system = self._should_merge_inline_system(online_renderer)
# Resolved lazily from the renderer when "auto".
self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = (
None if disabled_thinking_effort == "auto" else disabled_thinking_effort
@@ -193,6 +195,39 @@ class AnthropicServingMessages(OpenAIServingChat):
components = self._extract_prompt_components(engine_input)
return components.token_ids, components.text
@classmethod
def _should_merge_inline_system(cls, online_renderer: OnlineRenderer) -> bool:
"""Probe the chat templates the renderer applies, not the CLI override.
Both the tool and non-tool templates are checked since they may differ.
"""
renderer = online_renderer.renderer
tool_variants: tuple[list[dict[str, Any]] | None, ...] = (None, [])
if not isinstance(renderer, HfRenderer) or renderer.tokenizer is None:
merge = cls._detect_merge_inline_system(online_renderer.chat_template)
else:
merge = any(
cls._detect_merge_inline_system(
resolve_chat_template(
renderer.tokenizer,
online_renderer.chat_template,
tools,
model_config=online_renderer.model_config,
)
)
for tools in tool_variants
)
if merge:
logger.warning(
"The chat template requires system-first ordering, so inline "
"system messages in /v1/messages requests (e.g. Claude Code's "
"per-turn reminders) are merged into the leading system prompt. "
"Each new one changes the prompt prefix, so the rest of the "
"conversation misses the prefix cache. Pass a --chat-template "
"that accepts non-leading system messages to avoid this."
)
return merge
@staticmethod
def _detect_merge_inline_system(chat_template: str | None) -> bool:
"""Auto-detect whether the chat template requires system-first ordering.
+1 -3
View File
@@ -57,9 +57,7 @@ class ServingRender(BaseServing):
self.tool_server = tool_server
self._merge_inline_system = (
AnthropicServingMessages._detect_merge_inline_system(
online_renderer.chat_template
)
AnthropicServingMessages._should_merge_inline_system(online_renderer)
)
self._placeholder_metadata_parser: MultiModalDataParser | None = None