[Bugfix][Frontend] Detect Anthropic inline-system merge against the resolved chat template (#58754)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
d64f277c54
commit
dfab504333
@@ -51,9 +51,11 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers.hf import HfRenderer, resolve_chat_template
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _build_anthropic_usage(
|
||||
@@ -144,7 +146,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
"length": "max_tokens",
|
||||
"tool_calls": "tool_use",
|
||||
}
|
||||
self._merge_inline_system = self._detect_merge_inline_system(chat_template)
|
||||
self._merge_inline_system = self._should_merge_inline_system(online_renderer)
|
||||
# Resolved lazily from the renderer when "auto".
|
||||
self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = (
|
||||
None if disabled_thinking_effort == "auto" else disabled_thinking_effort
|
||||
@@ -193,6 +195,39 @@ class AnthropicServingMessages(OpenAIServingChat):
|
||||
components = self._extract_prompt_components(engine_input)
|
||||
return components.token_ids, components.text
|
||||
|
||||
@classmethod
|
||||
def _should_merge_inline_system(cls, online_renderer: OnlineRenderer) -> bool:
|
||||
"""Probe the chat templates the renderer applies, not the CLI override.
|
||||
|
||||
Both the tool and non-tool templates are checked since they may differ.
|
||||
"""
|
||||
renderer = online_renderer.renderer
|
||||
tool_variants: tuple[list[dict[str, Any]] | None, ...] = (None, [])
|
||||
if not isinstance(renderer, HfRenderer) or renderer.tokenizer is None:
|
||||
merge = cls._detect_merge_inline_system(online_renderer.chat_template)
|
||||
else:
|
||||
merge = any(
|
||||
cls._detect_merge_inline_system(
|
||||
resolve_chat_template(
|
||||
renderer.tokenizer,
|
||||
online_renderer.chat_template,
|
||||
tools,
|
||||
model_config=online_renderer.model_config,
|
||||
)
|
||||
)
|
||||
for tools in tool_variants
|
||||
)
|
||||
if merge:
|
||||
logger.warning(
|
||||
"The chat template requires system-first ordering, so inline "
|
||||
"system messages in /v1/messages requests (e.g. Claude Code's "
|
||||
"per-turn reminders) are merged into the leading system prompt. "
|
||||
"Each new one changes the prompt prefix, so the rest of the "
|
||||
"conversation misses the prefix cache. Pass a --chat-template "
|
||||
"that accepts non-leading system messages to avoid this."
|
||||
)
|
||||
return merge
|
||||
|
||||
@staticmethod
|
||||
def _detect_merge_inline_system(chat_template: str | None) -> bool:
|
||||
"""Auto-detect whether the chat template requires system-first ordering.
|
||||
|
||||
@@ -57,9 +57,7 @@ class ServingRender(BaseServing):
|
||||
self.tool_server = tool_server
|
||||
|
||||
self._merge_inline_system = (
|
||||
AnthropicServingMessages._detect_merge_inline_system(
|
||||
online_renderer.chat_template
|
||||
)
|
||||
AnthropicServingMessages._should_merge_inline_system(online_renderer)
|
||||
)
|
||||
|
||||
self._placeholder_metadata_parser: MultiModalDataParser | None = None
|
||||
|
||||
Reference in New Issue
Block a user