[Bugfix][Frontend] Detect Anthropic inline-system merge against the resolved chat template (#58754)

Signed-off-by: Robert Shaw <robshaw@redhat.com>
Co-authored-by: Robert Shaw <robshaw@redhat.com>
This commit is contained in:
Robert Shaw
2026-09-26 14:00:56 +00:00
committed by GitHub
co-authored by Robert Shaw
parent d64f277c54
commit dfab504333
6 changed files with 83 additions and 13 deletions
@@ -1383,6 +1383,11 @@ Q35_TEMPLATE = (
"{%- endfor %}"
)
PERMISSIVE_TEMPLATE = (
"{%- for message in messages %}{{- message.role }}: {{ message.content }}\n"
"{%- endfor %}"
)
class TestDetectMergeInlineSystem:
"""Verify _detect_merge_inline_system auto-detection.
@@ -1402,11 +1407,7 @@ class TestDetectMergeInlineSystem:
def test_no_restriction_no_merge(self):
"""Template without restriction accepts mid-conversation system."""
assert (
AnthropicServingMessages._detect_merge_inline_system(
"{%- for message in messages %}"
"{{- message.role }}: {{ message.content }}\n"
"{%- endfor %}"
)
AnthropicServingMessages._detect_merge_inline_system(PERMISSIVE_TEMPLATE)
is False
)
@@ -1414,6 +1415,38 @@ class TestDetectMergeInlineSystem:
"""No chat_template → conservative default: merge."""
assert AnthropicServingMessages._detect_merge_inline_system(None) is True
@pytest.mark.parametrize(
("resolved", "expected"),
[(PERMISSIVE_TEMPLATE, False), (Q35_TEMPLATE, True)],
)
def test_probes_resolved_template_without_cli_override(
self, monkeypatch, resolved, expected
):
"""Without --chat-template, probe the tokenizer's template (#58727).
Merging defeats prefix caching, so it is warned about at startup.
"""
import vllm.entrypoints.anthropic.serving as serving_mod
from vllm.renderers.hf import HfRenderer
monkeypatch.setattr(
serving_mod, "resolve_chat_template", lambda *a, **kw: resolved
)
mock_logger = MagicMock()
monkeypatch.setattr(serving_mod, "logger", mock_logger)
renderer = MagicMock(spec=HfRenderer)
renderer.tokenizer = MagicMock()
online_renderer = SimpleNamespace(
renderer=renderer,
chat_template=None,
model_config=None,
)
assert (
AnthropicServingMessages._should_merge_inline_system(online_renderer)
is expected
)
assert mock_logger.warning.called is expected
# ======================================================================
# Full (non-streaming) response conversion: messages_full_converter
@@ -45,6 +45,8 @@ class MockModelConfig:
trust_remote_code = False
tokenizer_mode = "auto"
max_model_len = 100
revision = None
code_revision = None
tokenizer_revision = None
multimodal_config = MultiModalConfig()
hf_config = MockHFConfig()
@@ -56,6 +56,8 @@ class MockModelConfig:
trust_remote_code = False
tokenizer_mode = "auto"
max_model_len = 100
revision = None
code_revision = None
tokenizer_revision = None
multimodal_config = MultiModalConfig()
hf_config = MockHFConfig()
@@ -651,9 +651,9 @@ async def test_messages_render_system_and_multi_turn(client):
async def test_messages_render_merges_inline_system(client):
"""Inline system messages merge into the leading system block.
Without a --chat-template arg the /v1/messages server path detects
merge_inline_system=True, so render must produce the same tokens as
the manually pre-merged request.
The model's chat template rejects non-leading system messages, so the
/v1/messages server path detects merge_inline_system=True and render must
produce the same tokens as the manually pre-merged request.
"""
inline = await client.post(
"/v1/messages/render",