[Frontend] Add cache_salt support to Anthropic Messages API (#49498)
Signed-off-by: aeon-x <talexcao@gmail.com>
This commit is contained in:
@@ -1385,3 +1385,26 @@ class TestMessagesFullConverter:
|
|||||||
assert len(result.content) == 1
|
assert len(result.content) == 1
|
||||||
assert result.content[0].type == "text"
|
assert result.content[0].type == "text"
|
||||||
assert result.content[0].text == ""
|
assert result.content[0].text == ""
|
||||||
|
|
||||||
|
|
||||||
|
# ======================================================================
|
||||||
|
# cache_salt pass-through (Issue #46688)
|
||||||
|
# ======================================================================
|
||||||
|
|
||||||
|
|
||||||
|
class TestCacheSalt:
|
||||||
|
def test_cache_salt_passed_through(self):
|
||||||
|
"""cache_salt on the Anthropic request reaches the converted
|
||||||
|
ChatCompletionRequest so prefix-cache isolation works via /v1/messages."""
|
||||||
|
request = _make_request(
|
||||||
|
[{"role": "user", "content": "Hello"}],
|
||||||
|
cache_salt="tenant-abc-secret-salt",
|
||||||
|
)
|
||||||
|
result = _convert(request)
|
||||||
|
assert result.cache_salt == "tenant-abc-secret-salt"
|
||||||
|
|
||||||
|
def test_cache_salt_defaults_to_none(self):
|
||||||
|
"""Omitting cache_salt leaves it unset (unchanged default behavior)."""
|
||||||
|
request = _make_request([{"role": "user", "content": "Hello"}])
|
||||||
|
result = _convert(request)
|
||||||
|
assert result.cache_salt is None
|
||||||
|
|||||||
@@ -133,6 +133,17 @@ class AnthropicMessagesRequest(BaseModel):
|
|||||||
top_p: float | None = None
|
top_p: float | None = None
|
||||||
|
|
||||||
# vLLM-specific fields that are not in Anthropic spec
|
# vLLM-specific fields that are not in Anthropic spec
|
||||||
|
cache_salt: str | None = Field(
|
||||||
|
default=None,
|
||||||
|
description=(
|
||||||
|
"If specified, the prefix cache will be salted with the provided "
|
||||||
|
"string to prevent an attacker to guess prompts in multi-user "
|
||||||
|
"environments. The salt should be random, protected from "
|
||||||
|
"access by 3rd parties, and long enough to be "
|
||||||
|
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||||
|
"to 256 bit)."
|
||||||
|
),
|
||||||
|
)
|
||||||
kv_transfer_params: dict[str, Any] | None = Field(
|
kv_transfer_params: dict[str, Any] | None = Field(
|
||||||
default=None,
|
default=None,
|
||||||
description="KVTransfer parameters used for disaggregated serving.",
|
description="KVTransfer parameters used for disaggregated serving.",
|
||||||
|
|||||||
@@ -486,6 +486,7 @@ class AnthropicServingMessages(OpenAIServingChat):
|
|||||||
temperature=anthropic_request.temperature,
|
temperature=anthropic_request.temperature,
|
||||||
top_p=anthropic_request.top_p,
|
top_p=anthropic_request.top_p,
|
||||||
top_k=anthropic_request.top_k,
|
top_k=anthropic_request.top_k,
|
||||||
|
cache_salt=anthropic_request.cache_salt,
|
||||||
kv_transfer_params=anthropic_request.kv_transfer_params,
|
kv_transfer_params=anthropic_request.kv_transfer_params,
|
||||||
ec_transfer_params=anthropic_request.ec_transfer_params,
|
ec_transfer_params=anthropic_request.ec_transfer_params,
|
||||||
chat_template_kwargs=anthropic_request.chat_template_kwargs,
|
chat_template_kwargs=anthropic_request.chat_template_kwargs,
|
||||||
|
|||||||
Reference in New Issue
Block a user