[Bugfix][Frontend] Detect Anthropic inline-system merge against the resolved chat template (#58754)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
This commit is contained in:
co-authored by
Robert Shaw
parent
d64f277c54
commit
dfab504333
@@ -1383,6 +1383,11 @@ Q35_TEMPLATE = (
|
||||
"{%- endfor %}"
|
||||
)
|
||||
|
||||
PERMISSIVE_TEMPLATE = (
|
||||
"{%- for message in messages %}{{- message.role }}: {{ message.content }}\n"
|
||||
"{%- endfor %}"
|
||||
)
|
||||
|
||||
|
||||
class TestDetectMergeInlineSystem:
|
||||
"""Verify _detect_merge_inline_system auto-detection.
|
||||
@@ -1402,11 +1407,7 @@ class TestDetectMergeInlineSystem:
|
||||
def test_no_restriction_no_merge(self):
|
||||
"""Template without restriction accepts mid-conversation system."""
|
||||
assert (
|
||||
AnthropicServingMessages._detect_merge_inline_system(
|
||||
"{%- for message in messages %}"
|
||||
"{{- message.role }}: {{ message.content }}\n"
|
||||
"{%- endfor %}"
|
||||
)
|
||||
AnthropicServingMessages._detect_merge_inline_system(PERMISSIVE_TEMPLATE)
|
||||
is False
|
||||
)
|
||||
|
||||
@@ -1414,6 +1415,38 @@ class TestDetectMergeInlineSystem:
|
||||
"""No chat_template → conservative default: merge."""
|
||||
assert AnthropicServingMessages._detect_merge_inline_system(None) is True
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("resolved", "expected"),
|
||||
[(PERMISSIVE_TEMPLATE, False), (Q35_TEMPLATE, True)],
|
||||
)
|
||||
def test_probes_resolved_template_without_cli_override(
|
||||
self, monkeypatch, resolved, expected
|
||||
):
|
||||
"""Without --chat-template, probe the tokenizer's template (#58727).
|
||||
|
||||
Merging defeats prefix caching, so it is warned about at startup.
|
||||
"""
|
||||
import vllm.entrypoints.anthropic.serving as serving_mod
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
|
||||
monkeypatch.setattr(
|
||||
serving_mod, "resolve_chat_template", lambda *a, **kw: resolved
|
||||
)
|
||||
mock_logger = MagicMock()
|
||||
monkeypatch.setattr(serving_mod, "logger", mock_logger)
|
||||
renderer = MagicMock(spec=HfRenderer)
|
||||
renderer.tokenizer = MagicMock()
|
||||
online_renderer = SimpleNamespace(
|
||||
renderer=renderer,
|
||||
chat_template=None,
|
||||
model_config=None,
|
||||
)
|
||||
assert (
|
||||
AnthropicServingMessages._should_merge_inline_system(online_renderer)
|
||||
is expected
|
||||
)
|
||||
assert mock_logger.warning.called is expected
|
||||
|
||||
|
||||
# ======================================================================
|
||||
# Full (non-streaming) response conversion: messages_full_converter
|
||||
|
||||
@@ -45,6 +45,8 @@ class MockModelConfig:
|
||||
trust_remote_code = False
|
||||
tokenizer_mode = "auto"
|
||||
max_model_len = 100
|
||||
revision = None
|
||||
code_revision = None
|
||||
tokenizer_revision = None
|
||||
multimodal_config = MultiModalConfig()
|
||||
hf_config = MockHFConfig()
|
||||
|
||||
@@ -56,6 +56,8 @@ class MockModelConfig:
|
||||
trust_remote_code = False
|
||||
tokenizer_mode = "auto"
|
||||
max_model_len = 100
|
||||
revision = None
|
||||
code_revision = None
|
||||
tokenizer_revision = None
|
||||
multimodal_config = MultiModalConfig()
|
||||
hf_config = MockHFConfig()
|
||||
|
||||
@@ -651,9 +651,9 @@ async def test_messages_render_system_and_multi_turn(client):
|
||||
async def test_messages_render_merges_inline_system(client):
|
||||
"""Inline system messages merge into the leading system block.
|
||||
|
||||
Without a --chat-template arg the /v1/messages server path detects
|
||||
merge_inline_system=True, so render must produce the same tokens as
|
||||
the manually pre-merged request.
|
||||
The model's chat template rejects non-leading system messages, so the
|
||||
/v1/messages server path detects merge_inline_system=True and render must
|
||||
produce the same tokens as the manually pre-merged request.
|
||||
"""
|
||||
inline = await client.post(
|
||||
"/v1/messages/render",
|
||||
|
||||
Reference in New Issue
Block a user