[Bugfix][Frontend] Detect Anthropic inline-system merge against the resolved chat template (#58754)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
d64f277c54
commit
dfab504333
@@ -1383,6 +1383,11 @@ Q35_TEMPLATE = (
|
|||||||
"{%- endfor %}"
|
"{%- endfor %}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
PERMISSIVE_TEMPLATE = (
|
||||||
|
"{%- for message in messages %}{{- message.role }}: {{ message.content }}\n"
|
||||||
|
"{%- endfor %}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestDetectMergeInlineSystem:
|
class TestDetectMergeInlineSystem:
|
||||||
"""Verify _detect_merge_inline_system auto-detection.
|
"""Verify _detect_merge_inline_system auto-detection.
|
||||||
@@ -1402,11 +1407,7 @@ class TestDetectMergeInlineSystem:
|
|||||||
def test_no_restriction_no_merge(self):
|
def test_no_restriction_no_merge(self):
|
||||||
"""Template without restriction accepts mid-conversation system."""
|
"""Template without restriction accepts mid-conversation system."""
|
||||||
assert (
|
assert (
|
||||||
AnthropicServingMessages._detect_merge_inline_system(
|
AnthropicServingMessages._detect_merge_inline_system(PERMISSIVE_TEMPLATE)
|
||||||
"{%- for message in messages %}"
|
|
||||||
"{{- message.role }}: {{ message.content }}\n"
|
|
||||||
"{%- endfor %}"
|
|
||||||
)
|
|
||||||
is False
|
is False
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -1414,6 +1415,38 @@ class TestDetectMergeInlineSystem:
|
|||||||
"""No chat_template → conservative default: merge."""
|
"""No chat_template → conservative default: merge."""
|
||||||
assert AnthropicServingMessages._detect_merge_inline_system(None) is True
|
assert AnthropicServingMessages._detect_merge_inline_system(None) is True
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("resolved", "expected"),
|
||||||
|
[(PERMISSIVE_TEMPLATE, False), (Q35_TEMPLATE, True)],
|
||||||
|
)
|
||||||
|
def test_probes_resolved_template_without_cli_override(
|
||||||
|
self, monkeypatch, resolved, expected
|
||||||
|
):
|
||||||
|
"""Without --chat-template, probe the tokenizer's template (#58727).
|
||||||
|
|
||||||
|
Merging defeats prefix caching, so it is warned about at startup.
|
||||||
|
"""
|
||||||
|
import vllm.entrypoints.anthropic.serving as serving_mod
|
||||||
|
from vllm.renderers.hf import HfRenderer
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
serving_mod, "resolve_chat_template", lambda *a, **kw: resolved
|
||||||
|
)
|
||||||
|
mock_logger = MagicMock()
|
||||||
|
monkeypatch.setattr(serving_mod, "logger", mock_logger)
|
||||||
|
renderer = MagicMock(spec=HfRenderer)
|
||||||
|
renderer.tokenizer = MagicMock()
|
||||||
|
online_renderer = SimpleNamespace(
|
||||||
|
renderer=renderer,
|
||||||
|
chat_template=None,
|
||||||
|
model_config=None,
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
AnthropicServingMessages._should_merge_inline_system(online_renderer)
|
||||||
|
is expected
|
||||||
|
)
|
||||||
|
assert mock_logger.warning.called is expected
|
||||||
|
|
||||||
|
|
||||||
# ======================================================================
|
# ======================================================================
|
||||||
# Full (non-streaming) response conversion: messages_full_converter
|
# Full (non-streaming) response conversion: messages_full_converter
|
||||||
|
|||||||
@@ -45,6 +45,8 @@ class MockModelConfig:
|
|||||||
trust_remote_code = False
|
trust_remote_code = False
|
||||||
tokenizer_mode = "auto"
|
tokenizer_mode = "auto"
|
||||||
max_model_len = 100
|
max_model_len = 100
|
||||||
|
revision = None
|
||||||
|
code_revision = None
|
||||||
tokenizer_revision = None
|
tokenizer_revision = None
|
||||||
multimodal_config = MultiModalConfig()
|
multimodal_config = MultiModalConfig()
|
||||||
hf_config = MockHFConfig()
|
hf_config = MockHFConfig()
|
||||||
|
|||||||
@@ -56,6 +56,8 @@ class MockModelConfig:
|
|||||||
trust_remote_code = False
|
trust_remote_code = False
|
||||||
tokenizer_mode = "auto"
|
tokenizer_mode = "auto"
|
||||||
max_model_len = 100
|
max_model_len = 100
|
||||||
|
revision = None
|
||||||
|
code_revision = None
|
||||||
tokenizer_revision = None
|
tokenizer_revision = None
|
||||||
multimodal_config = MultiModalConfig()
|
multimodal_config = MultiModalConfig()
|
||||||
hf_config = MockHFConfig()
|
hf_config = MockHFConfig()
|
||||||
|
|||||||
@@ -651,9 +651,9 @@ async def test_messages_render_system_and_multi_turn(client):
|
|||||||
async def test_messages_render_merges_inline_system(client):
|
async def test_messages_render_merges_inline_system(client):
|
||||||
"""Inline system messages merge into the leading system block.
|
"""Inline system messages merge into the leading system block.
|
||||||
|
|
||||||
Without a --chat-template arg the /v1/messages server path detects
|
The model's chat template rejects non-leading system messages, so the
|
||||||
merge_inline_system=True, so render must produce the same tokens as
|
/v1/messages server path detects merge_inline_system=True and render must
|
||||||
the manually pre-merged request.
|
produce the same tokens as the manually pre-merged request.
|
||||||
"""
|
"""
|
||||||
inline = await client.post(
|
inline = await client.post(
|
||||||
"/v1/messages/render",
|
"/v1/messages/render",
|
||||||
|
|||||||
@@ -51,9 +51,11 @@ from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
|||||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse, UsageInfo
|
||||||
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
from vllm.entrypoints.serve.exception_handling.utils import sanitize_message
|
||||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||||
|
from vllm.logger import init_logger
|
||||||
|
from vllm.renderers.hf import HfRenderer, resolve_chat_template
|
||||||
from vllm.renderers.online_renderer import OnlineRenderer
|
from vllm.renderers.online_renderer import OnlineRenderer
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
def _build_anthropic_usage(
|
def _build_anthropic_usage(
|
||||||
@@ -144,7 +146,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
|||||||
"length": "max_tokens",
|
"length": "max_tokens",
|
||||||
"tool_calls": "tool_use",
|
"tool_calls": "tool_use",
|
||||||
}
|
}
|
||||||
self._merge_inline_system = self._detect_merge_inline_system(chat_template)
|
self._merge_inline_system = self._should_merge_inline_system(online_renderer)
|
||||||
# Resolved lazily from the renderer when "auto".
|
# Resolved lazily from the renderer when "auto".
|
||||||
self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = (
|
self._disabled_thinking_effort: AnthropicDisabledThinkingEffort | None = (
|
||||||
None if disabled_thinking_effort == "auto" else disabled_thinking_effort
|
None if disabled_thinking_effort == "auto" else disabled_thinking_effort
|
||||||
@@ -193,6 +195,39 @@ class AnthropicServingMessages(OpenAIServingChat):
|
|||||||
components = self._extract_prompt_components(engine_input)
|
components = self._extract_prompt_components(engine_input)
|
||||||
return components.token_ids, components.text
|
return components.token_ids, components.text
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _should_merge_inline_system(cls, online_renderer: OnlineRenderer) -> bool:
|
||||||
|
"""Probe the chat templates the renderer applies, not the CLI override.
|
||||||
|
|
||||||
|
Both the tool and non-tool templates are checked since they may differ.
|
||||||
|
"""
|
||||||
|
renderer = online_renderer.renderer
|
||||||
|
tool_variants: tuple[list[dict[str, Any]] | None, ...] = (None, [])
|
||||||
|
if not isinstance(renderer, HfRenderer) or renderer.tokenizer is None:
|
||||||
|
merge = cls._detect_merge_inline_system(online_renderer.chat_template)
|
||||||
|
else:
|
||||||
|
merge = any(
|
||||||
|
cls._detect_merge_inline_system(
|
||||||
|
resolve_chat_template(
|
||||||
|
renderer.tokenizer,
|
||||||
|
online_renderer.chat_template,
|
||||||
|
tools,
|
||||||
|
model_config=online_renderer.model_config,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
for tools in tool_variants
|
||||||
|
)
|
||||||
|
if merge:
|
||||||
|
logger.warning(
|
||||||
|
"The chat template requires system-first ordering, so inline "
|
||||||
|
"system messages in /v1/messages requests (e.g. Claude Code's "
|
||||||
|
"per-turn reminders) are merged into the leading system prompt. "
|
||||||
|
"Each new one changes the prompt prefix, so the rest of the "
|
||||||
|
"conversation misses the prefix cache. Pass a --chat-template "
|
||||||
|
"that accepts non-leading system messages to avoid this."
|
||||||
|
)
|
||||||
|
return merge
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _detect_merge_inline_system(chat_template: str | None) -> bool:
|
def _detect_merge_inline_system(chat_template: str | None) -> bool:
|
||||||
"""Auto-detect whether the chat template requires system-first ordering.
|
"""Auto-detect whether the chat template requires system-first ordering.
|
||||||
|
|||||||
@@ -57,9 +57,7 @@ class ServingRender(BaseServing):
|
|||||||
self.tool_server = tool_server
|
self.tool_server = tool_server
|
||||||
|
|
||||||
self._merge_inline_system = (
|
self._merge_inline_system = (
|
||||||
AnthropicServingMessages._detect_merge_inline_system(
|
AnthropicServingMessages._should_merge_inline_system(online_renderer)
|
||||||
online_renderer.chat_template
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
self._placeholder_metadata_parser: MultiModalDataParser | None = None
|
self._placeholder_metadata_parser: MultiModalDataParser | None = None
|
||||||
|
|||||||
Reference in New Issue
Block a user