[Bugfix][Frontend] Strip x-anthropic-billing-header billing header from /v1/chatcompletions (#59419)
Signed-off-by: Maroon Ayoub <mayoub@redhat.com> Signed-off-by: Robert Shaw <robertgshaw2@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Robert Shaw <robertgshaw2@gmail.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
Robert Shaw
Robert Shaw
parent
f3b77eff64
commit
5d9214d6b2
@@ -0,0 +1,77 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Claude Code's attribution header in chat completions system messages."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.chat_utils import _parse_chat_message_content
|
||||
|
||||
HEADER = (
|
||||
"x-anthropic-billing-header: cc_version=2.1.220.300; "
|
||||
"cc_entrypoint=claude-desktop; cch=7adab;"
|
||||
)
|
||||
|
||||
|
||||
def _parse(message, content_format="openai"):
|
||||
tracker = MagicMock()
|
||||
tracker.create_parser.return_value.mm_placeholder_storage.return_value = {}
|
||||
return _parse_chat_message_content(
|
||||
message, tracker, content_format, interleave_strings=False
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
"header",
|
||||
[
|
||||
HEADER,
|
||||
"x-anthropic-billing-header: cc_version=2.1.286.99b; cc_entrypoint=sdk-cli;",
|
||||
],
|
||||
)
|
||||
def test_system_header_part_is_dropped(header):
|
||||
(msg,) = _parse(
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{"type": "text", "text": header},
|
||||
{"type": "text", "text": "You are a Claude agent."},
|
||||
],
|
||||
}
|
||||
)
|
||||
assert msg["content"] == [{"type": "text", "text": "You are a Claude agent."}]
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_system_header_line_is_stripped_from_string():
|
||||
(msg,) = _parse(
|
||||
{"role": "system", "content": HEADER + "\nYou are a Claude agent."},
|
||||
content_format="string",
|
||||
)
|
||||
assert msg["content"] == "You are a Claude agent."
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
def test_requests_differing_only_in_cch_parse_identically():
|
||||
def system(cch):
|
||||
return {
|
||||
"role": "system",
|
||||
"content": HEADER.replace("7adab", cch) + "\nYou are a Claude agent.",
|
||||
}
|
||||
|
||||
assert _parse(system("7adab")) == _parse(system("986dd"))
|
||||
|
||||
|
||||
@pytest.mark.skip_global_cleanup
|
||||
@pytest.mark.parametrize(
|
||||
"message",
|
||||
[
|
||||
{"role": "system", "content": "You are a Claude agent."},
|
||||
{"role": "system", "content": "Quote: " + HEADER},
|
||||
{"role": "user", "content": HEADER + "Hello"},
|
||||
],
|
||||
)
|
||||
def test_other_content_is_unchanged(message):
|
||||
(msg,) = _parse(message, content_format="string")
|
||||
assert msg["content"] == message["content"]
|
||||
@@ -2035,6 +2035,41 @@ _AssistantParser = partial(cast, ChatCompletionAssistantMessageParam)
|
||||
_ToolParser = partial(cast, ChatCompletionToolMessageParam)
|
||||
|
||||
|
||||
_CLAUDE_CODE_BILLING_HEADER = "x-anthropic-billing-header"
|
||||
|
||||
|
||||
def _strip_claude_code_billing_header(
|
||||
parts: Iterable[ChatCompletionContentPartParam],
|
||||
) -> list[ChatCompletionContentPartParam]:
|
||||
"""Drop Claude Code's attribution header line from system text parts.
|
||||
|
||||
Its ``cch`` value changes on every request, so a prompt that keeps it
|
||||
misses the prefix cache from the header on. Gateways that translate
|
||||
Anthropic requests to chat completions keep it; ``/v1/messages`` drops
|
||||
the same block.
|
||||
"""
|
||||
out: list[ChatCompletionContentPartParam] = []
|
||||
for part in parts:
|
||||
if isinstance(part, str):
|
||||
text: object = part
|
||||
elif isinstance(part, dict) and part.get("type") == "text":
|
||||
text = part.get("text")
|
||||
else:
|
||||
text = None
|
||||
if not (isinstance(text, str) and text.startswith(_CLAUDE_CODE_BILLING_HEADER)):
|
||||
out.append(part)
|
||||
continue
|
||||
rest = text.partition("\n")[2]
|
||||
if not rest:
|
||||
continue
|
||||
out.append(
|
||||
cast(ChatCompletionContentPartParam, rest)
|
||||
if isinstance(part, str)
|
||||
else cast(ChatCompletionContentPartParam, {**part, "text": rest})
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _parse_chat_message_content(
|
||||
message: ChatCompletionMessageParam,
|
||||
mm_tracker: BaseMultiModalItemTracker,
|
||||
@@ -2050,6 +2085,8 @@ def _parse_chat_message_content(
|
||||
content = []
|
||||
elif isinstance(content, str):
|
||||
content = [ChatCompletionContentPartTextParam(type="text", text=content)]
|
||||
if role == "system":
|
||||
content = _strip_claude_code_billing_header(content) # type: ignore[arg-type]
|
||||
result = _parse_chat_message_content_parts(
|
||||
role,
|
||||
content, # type: ignore
|
||||
|
||||
Reference in New Issue
Block a user