[Frontend] Add cache salt support to generative scoring (#60841)
Signed-off-by: Taewoong Kim <ktw2172@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
90ba2f34b5
commit
75e3b17c05
@@ -62,3 +62,10 @@ Here, each item is appended to the query to form prompts like `"Is this city the
|
||||
"usage": {"prompt_tokens": 45, "total_tokens": 48, "completion_tokens": 3}
|
||||
}
|
||||
```
|
||||
|
||||
## Cache salting
|
||||
|
||||
The optional `cache_salt` request field is applied to every item's prompt. With
|
||||
prefix caching enabled, requests with different salts do not share cached prefix
|
||||
blocks. See [Cache Salting](../../usage/security.md#cache-salting) for how to choose
|
||||
and manage salts.
|
||||
|
||||
@@ -572,7 +572,7 @@ An attacker sharing the same backend can measure differences in Time to First To
|
||||
|
||||
vLLM accepts an optional `cache_salt` parameter on requests. The salt is mixed into the hash of the first KV cache block, so only requests carrying the same salt can share cached prefix blocks. See [Automatic Prefix Caching](../design/prefix_caching.md) for the implementation details.
|
||||
|
||||
`cache_salt` is accepted by the OpenAI-compatible chat completions, completions, responses, and pooling (embeddings, classification, scoring) endpoints, the Anthropic `/v1/messages` endpoint, and the structured decisions `/v1/systemone` endpoint.
|
||||
`cache_salt` is accepted by the OpenAI-compatible chat completions, completions, responses, and pooling (embeddings, classification, scoring) endpoints, the Anthropic `/v1/messages` endpoint, the structured decisions `/v1/systemone` endpoint, and the generative scoring `/generative_scoring` endpoint.
|
||||
|
||||
#### Usage with the OpenAI Python client
|
||||
|
||||
|
||||
@@ -139,6 +139,7 @@ class TestProtocolModels:
|
||||
assert req_str.apply_softmax is True # default
|
||||
assert req_str.item_first is False # default
|
||||
assert req_str.add_special_tokens is True # default
|
||||
assert req_str.cache_salt is None
|
||||
|
||||
# Test request with pre-tokenized inputs and custom options
|
||||
req_tok = GenerativeScoringRequest(
|
||||
@@ -296,16 +297,26 @@ class TestGeneration:
|
||||
"""Tests for the full generation flow with mocked engine."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_generation(self):
|
||||
"""Test successful score generation returns valid response."""
|
||||
@pytest.mark.parametrize("cache_salt", [None, "tenant-a"])
|
||||
@pytest.mark.parametrize(
|
||||
"query,items",
|
||||
[
|
||||
("Is this city the capital of France? ", ["Paris", "London"]),
|
||||
([100, 101], [[200, 201], [300, 301]]),
|
||||
],
|
||||
ids=["text", "token_ids"],
|
||||
)
|
||||
async def test_successful_generation(self, cache_salt, query, items):
|
||||
"""Test scoring succeeds and preserves cache_salt for every item."""
|
||||
mock_engine = _create_mock_engine()
|
||||
serving = _create_serving(mock_engine)
|
||||
|
||||
mock_logprobs = {1234: -0.5, 5678: -2.0, 100: -3.0}
|
||||
mock_output = _create_mock_request_output(mock_logprobs)
|
||||
state = SimpleNamespace(sampling_params=None)
|
||||
state = SimpleNamespace(sampling_params=None, prompts=[])
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
state.prompts.append(args[0])
|
||||
state.sampling_params = args[1]
|
||||
yield mock_output
|
||||
|
||||
@@ -313,15 +324,22 @@ class TestGeneration:
|
||||
|
||||
request = GenerativeScoringRequest(
|
||||
model=MODEL_NAME,
|
||||
query="Is Paris the capital?",
|
||||
items=["Yes", "No"],
|
||||
query=query,
|
||||
items=items,
|
||||
label_token_ids=[1234, 5678],
|
||||
cache_salt=cache_salt,
|
||||
)
|
||||
result = await serving.create_generative_scoring(request, None)
|
||||
|
||||
assert isinstance(result, GenerativeScoringResponse)
|
||||
assert state.sampling_params is not None
|
||||
assert state.sampling_params.watermarking is False
|
||||
assert len(state.prompts) == len(items)
|
||||
for prompt in state.prompts:
|
||||
if cache_salt is None:
|
||||
assert "cache_salt" not in prompt
|
||||
else:
|
||||
assert prompt["cache_salt"] == cache_salt
|
||||
assert len(result.data) == 2
|
||||
for item_result in result.data:
|
||||
assert 0.0 <= item_result.score <= 1.0
|
||||
|
||||
@@ -13,6 +13,9 @@ request models whose before-validators were missing the same guard.
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.generate.generative_scoring.serving import (
|
||||
GenerativeScoringRequest,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.protocol import (
|
||||
StructuredDecisionRequest,
|
||||
)
|
||||
@@ -31,6 +34,7 @@ from vllm.exceptions import VLLMValidationError
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
REQUEST_MODELS = [
|
||||
GenerativeScoringRequest,
|
||||
CompletionRequest,
|
||||
ResponsesRequest,
|
||||
EmbeddingChatRequest,
|
||||
@@ -77,6 +81,14 @@ def test_tokenize_chat_request_still_validates_dict_bodies():
|
||||
|
||||
|
||||
CACHE_SALT_MODELS = [
|
||||
(
|
||||
GenerativeScoringRequest,
|
||||
{
|
||||
"query": "Is this city the capital of France?",
|
||||
"items": ["Paris"],
|
||||
"label_token_ids": [9454, 2753],
|
||||
},
|
||||
),
|
||||
(
|
||||
StructuredDecisionRequest,
|
||||
{
|
||||
|
||||
@@ -12,12 +12,13 @@ import asyncio
|
||||
import math
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Literal
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import Field
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
|
||||
from vllm.entrypoints.generate.label_reads import next_token_label_reads
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
@@ -59,6 +60,7 @@ class GenerativeScoringRequest(OpenAIBaseModel):
|
||||
the full vocab for those ids (False).
|
||||
item_first: If True, prepend items to query. Otherwise append items to query.
|
||||
add_special_tokens: Whether to add special tokens when tokenizing.
|
||||
cache_salt: Optional salt for prefix caching.
|
||||
|
||||
"""
|
||||
|
||||
@@ -91,6 +93,17 @@ class GenerativeScoringRequest(OpenAIBaseModel):
|
||||
default=True,
|
||||
description="Whether to add special tokens when tokenizing.",
|
||||
)
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
priority: int = Field(
|
||||
default=0,
|
||||
ge=-(2**63),
|
||||
@@ -104,6 +117,13 @@ class GenerativeScoringRequest(OpenAIBaseModel):
|
||||
description="The request_id related to this request.",
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_cache_salt_support(cls, data: Any) -> Any:
|
||||
if isinstance(data, dict):
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
|
||||
class GenerativeScoringItemResult(OpenAIBaseModel):
|
||||
"""Result for a single item in the generative scoring response.
|
||||
@@ -395,7 +415,9 @@ class ServingGenerativeScoring(BaseServing):
|
||||
if len(prompt_token_ids) > max_prompt_len:
|
||||
prompt_token_ids = prompt_token_ids[:max_prompt_len]
|
||||
|
||||
engine_inputs.append(tokens_input(prompt_token_ids))
|
||||
engine_inputs.append(
|
||||
tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
|
||||
)
|
||||
prompt_token_counts.append(len(prompt_token_ids))
|
||||
|
||||
return engine_inputs, prompt_token_counts
|
||||
|
||||
Reference in New Issue
Block a user