[Frontend] Add cache salt support to generative scoring (#60841)

Signed-off-by: Taewoong Kim <ktw2172@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Taewoong Kim
2026-10-09 16:07:50 +00:00
committed by GitHub
co-authored by Cyrus Leung
parent 90ba2f34b5
commit 75e3b17c05
5 changed files with 68 additions and 9 deletions
@@ -62,3 +62,10 @@ Here, each item is appended to the query to form prompts like `"Is this city the
"usage": {"prompt_tokens": 45, "total_tokens": 48, "completion_tokens": 3}
}
```
## Cache salting
The optional `cache_salt` request field is applied to every item's prompt. With
prefix caching enabled, requests with different salts do not share cached prefix
blocks. See [Cache Salting](../../usage/security.md#cache-salting) for how to choose
and manage salts.
+1 -1
View File
@@ -572,7 +572,7 @@ An attacker sharing the same backend can measure differences in Time to First To
vLLM accepts an optional `cache_salt` parameter on requests. The salt is mixed into the hash of the first KV cache block, so only requests carrying the same salt can share cached prefix blocks. See [Automatic Prefix Caching](../design/prefix_caching.md) for the implementation details.
`cache_salt` is accepted by the OpenAI-compatible chat completions, completions, responses, and pooling (embeddings, classification, scoring) endpoints, the Anthropic `/v1/messages` endpoint, and the structured decisions `/v1/systemone` endpoint.
`cache_salt` is accepted by the OpenAI-compatible chat completions, completions, responses, and pooling (embeddings, classification, scoring) endpoints, the Anthropic `/v1/messages` endpoint, the structured decisions `/v1/systemone` endpoint, and the generative scoring `/generative_scoring` endpoint.
#### Usage with the OpenAI Python client
@@ -139,6 +139,7 @@ class TestProtocolModels:
assert req_str.apply_softmax is True # default
assert req_str.item_first is False # default
assert req_str.add_special_tokens is True # default
assert req_str.cache_salt is None
# Test request with pre-tokenized inputs and custom options
req_tok = GenerativeScoringRequest(
@@ -296,16 +297,26 @@ class TestGeneration:
"""Tests for the full generation flow with mocked engine."""
@pytest.mark.asyncio
async def test_successful_generation(self):
"""Test successful score generation returns valid response."""
@pytest.mark.parametrize("cache_salt", [None, "tenant-a"])
@pytest.mark.parametrize(
"query,items",
[
("Is this city the capital of France? ", ["Paris", "London"]),
([100, 101], [[200, 201], [300, 301]]),
],
ids=["text", "token_ids"],
)
async def test_successful_generation(self, cache_salt, query, items):
"""Test scoring succeeds and preserves cache_salt for every item."""
mock_engine = _create_mock_engine()
serving = _create_serving(mock_engine)
mock_logprobs = {1234: -0.5, 5678: -2.0, 100: -3.0}
mock_output = _create_mock_request_output(mock_logprobs)
state = SimpleNamespace(sampling_params=None)
state = SimpleNamespace(sampling_params=None, prompts=[])
async def mock_generate(*args, **kwargs):
state.prompts.append(args[0])
state.sampling_params = args[1]
yield mock_output
@@ -313,15 +324,22 @@ class TestGeneration:
request = GenerativeScoringRequest(
model=MODEL_NAME,
query="Is Paris the capital?",
items=["Yes", "No"],
query=query,
items=items,
label_token_ids=[1234, 5678],
cache_salt=cache_salt,
)
result = await serving.create_generative_scoring(request, None)
assert isinstance(result, GenerativeScoringResponse)
assert state.sampling_params is not None
assert state.sampling_params.watermarking is False
assert len(state.prompts) == len(items)
for prompt in state.prompts:
if cache_salt is None:
assert "cache_salt" not in prompt
else:
assert prompt["cache_salt"] == cache_salt
assert len(result.data) == 2
for item_result in result.data:
assert 0.0 <= item_result.score <= 1.0
@@ -13,6 +13,9 @@ request models whose before-validators were missing the same guard.
import pytest
from pydantic import ValidationError
from vllm.entrypoints.generate.generative_scoring.serving import (
GenerativeScoringRequest,
)
from vllm.entrypoints.generate.structured_decisions.protocol import (
StructuredDecisionRequest,
)
@@ -31,6 +34,7 @@ from vllm.exceptions import VLLMValidationError
pytestmark = pytest.mark.skip_global_cleanup
REQUEST_MODELS = [
GenerativeScoringRequest,
CompletionRequest,
ResponsesRequest,
EmbeddingChatRequest,
@@ -77,6 +81,14 @@ def test_tokenize_chat_request_still_validates_dict_bodies():
CACHE_SALT_MODELS = [
(
GenerativeScoringRequest,
{
"query": "Is this city the capital of France?",
"items": ["Paris"],
"label_token_ids": [9454, 2753],
},
),
(
StructuredDecisionRequest,
{
@@ -12,12 +12,13 @@ import asyncio
import math
import time
from collections.abc import Mapping
from typing import Literal
from typing import Any, Literal
from fastapi import Request
from pydantic import Field
from pydantic import Field, model_validator
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
from vllm.entrypoints.generate.label_reads import next_token_label_reads
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import (
@@ -59,6 +60,7 @@ class GenerativeScoringRequest(OpenAIBaseModel):
the full vocab for those ids (False).
item_first: If True, prepend items to query. Otherwise append items to query.
add_special_tokens: Whether to add special tokens when tokenizing.
cache_salt: Optional salt for prefix caching.
"""
@@ -91,6 +93,17 @@ class GenerativeScoringRequest(OpenAIBaseModel):
default=True,
description="Whether to add special tokens when tokenizing.",
)
cache_salt: str | None = Field(
default=None,
description=(
"If specified, the prefix cache will be salted with the provided "
"string to prevent an attacker to guess prompts in multi-user "
"environments. The salt should be random, protected from "
"access by 3rd parties, and long enough to be "
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
"to 256 bit)."
),
)
priority: int = Field(
default=0,
ge=-(2**63),
@@ -104,6 +117,13 @@ class GenerativeScoringRequest(OpenAIBaseModel):
description="The request_id related to this request.",
)
@model_validator(mode="before")
@classmethod
def check_cache_salt_support(cls, data: Any) -> Any:
if isinstance(data, dict):
validate_cache_salt(data.get("cache_salt"))
return data
class GenerativeScoringItemResult(OpenAIBaseModel):
"""Result for a single item in the generative scoring response.
@@ -395,7 +415,9 @@ class ServingGenerativeScoring(BaseServing):
if len(prompt_token_ids) > max_prompt_len:
prompt_token_ids = prompt_token_ids[:max_prompt_len]
engine_inputs.append(tokens_input(prompt_token_ids))
engine_inputs.append(
tokens_input(prompt_token_ids, cache_salt=request.cache_salt)
)
prompt_token_counts.append(len(prompt_token_ids))
return engine_inputs, prompt_token_counts