From 75e3b17c055c609f02c64cd4ad3f8692b8170d8e Mon Sep 17 00:00:00 2001 From: Taewoong Kim <116135174+ultramancode@users.noreply.github.com> Date: Sat, 10 Oct 2026 01:07:50 +0900 Subject: [PATCH] [Frontend] Add cache salt support to generative scoring (#60841) Signed-off-by: Taewoong Kim Co-authored-by: Cyrus Leung --- .../online_serving/generative_scoring.md | 7 +++++ docs/usage/security.md | 2 +- .../test_generative_scoring.py | 28 +++++++++++++++---- .../test_non_object_body_validation.py | 12 ++++++++ .../generate/generative_scoring/serving.py | 28 +++++++++++++++++-- 5 files changed, 68 insertions(+), 9 deletions(-) diff --git a/docs/serving/online_serving/generative_scoring.md b/docs/serving/online_serving/generative_scoring.md index d0c578f210..eebe1f965e 100644 --- a/docs/serving/online_serving/generative_scoring.md +++ b/docs/serving/online_serving/generative_scoring.md @@ -62,3 +62,10 @@ Here, each item is appended to the query to form prompts like `"Is this city the "usage": {"prompt_tokens": 45, "total_tokens": 48, "completion_tokens": 3} } ``` + +## Cache salting + +The optional `cache_salt` request field is applied to every item's prompt. With +prefix caching enabled, requests with different salts do not share cached prefix +blocks. See [Cache Salting](../../usage/security.md#cache-salting) for how to choose +and manage salts. diff --git a/docs/usage/security.md b/docs/usage/security.md index a74ed1d6eb..ab5aa96bec 100644 --- a/docs/usage/security.md +++ b/docs/usage/security.md @@ -572,7 +572,7 @@ An attacker sharing the same backend can measure differences in Time to First To vLLM accepts an optional `cache_salt` parameter on requests. The salt is mixed into the hash of the first KV cache block, so only requests carrying the same salt can share cached prefix blocks. See [Automatic Prefix Caching](../design/prefix_caching.md) for the implementation details. -`cache_salt` is accepted by the OpenAI-compatible chat completions, completions, responses, and pooling (embeddings, classification, scoring) endpoints, the Anthropic `/v1/messages` endpoint, and the structured decisions `/v1/systemone` endpoint. +`cache_salt` is accepted by the OpenAI-compatible chat completions, completions, responses, and pooling (embeddings, classification, scoring) endpoints, the Anthropic `/v1/messages` endpoint, the structured decisions `/v1/systemone` endpoint, and the generative scoring `/generative_scoring` endpoint. #### Usage with the OpenAI Python client diff --git a/tests/entrypoints/generate/generative_scoring/test_generative_scoring.py b/tests/entrypoints/generate/generative_scoring/test_generative_scoring.py index 69be10b19d..ea7c81ea26 100644 --- a/tests/entrypoints/generate/generative_scoring/test_generative_scoring.py +++ b/tests/entrypoints/generate/generative_scoring/test_generative_scoring.py @@ -139,6 +139,7 @@ class TestProtocolModels: assert req_str.apply_softmax is True # default assert req_str.item_first is False # default assert req_str.add_special_tokens is True # default + assert req_str.cache_salt is None # Test request with pre-tokenized inputs and custom options req_tok = GenerativeScoringRequest( @@ -296,16 +297,26 @@ class TestGeneration: """Tests for the full generation flow with mocked engine.""" @pytest.mark.asyncio - async def test_successful_generation(self): - """Test successful score generation returns valid response.""" + @pytest.mark.parametrize("cache_salt", [None, "tenant-a"]) + @pytest.mark.parametrize( + "query,items", + [ + ("Is this city the capital of France? ", ["Paris", "London"]), + ([100, 101], [[200, 201], [300, 301]]), + ], + ids=["text", "token_ids"], + ) + async def test_successful_generation(self, cache_salt, query, items): + """Test scoring succeeds and preserves cache_salt for every item.""" mock_engine = _create_mock_engine() serving = _create_serving(mock_engine) mock_logprobs = {1234: -0.5, 5678: -2.0, 100: -3.0} mock_output = _create_mock_request_output(mock_logprobs) - state = SimpleNamespace(sampling_params=None) + state = SimpleNamespace(sampling_params=None, prompts=[]) async def mock_generate(*args, **kwargs): + state.prompts.append(args[0]) state.sampling_params = args[1] yield mock_output @@ -313,15 +324,22 @@ class TestGeneration: request = GenerativeScoringRequest( model=MODEL_NAME, - query="Is Paris the capital?", - items=["Yes", "No"], + query=query, + items=items, label_token_ids=[1234, 5678], + cache_salt=cache_salt, ) result = await serving.create_generative_scoring(request, None) assert isinstance(result, GenerativeScoringResponse) assert state.sampling_params is not None assert state.sampling_params.watermarking is False + assert len(state.prompts) == len(items) + for prompt in state.prompts: + if cache_salt is None: + assert "cache_salt" not in prompt + else: + assert prompt["cache_salt"] == cache_salt assert len(result.data) == 2 for item_result in result.data: assert 0.0 <= item_result.score <= 1.0 diff --git a/tests/entrypoints/unit_tests/test_non_object_body_validation.py b/tests/entrypoints/unit_tests/test_non_object_body_validation.py index dd0e3eafd0..2440c880ce 100644 --- a/tests/entrypoints/unit_tests/test_non_object_body_validation.py +++ b/tests/entrypoints/unit_tests/test_non_object_body_validation.py @@ -13,6 +13,9 @@ request models whose before-validators were missing the same guard. import pytest from pydantic import ValidationError +from vllm.entrypoints.generate.generative_scoring.serving import ( + GenerativeScoringRequest, +) from vllm.entrypoints.generate.structured_decisions.protocol import ( StructuredDecisionRequest, ) @@ -31,6 +34,7 @@ from vllm.exceptions import VLLMValidationError pytestmark = pytest.mark.skip_global_cleanup REQUEST_MODELS = [ + GenerativeScoringRequest, CompletionRequest, ResponsesRequest, EmbeddingChatRequest, @@ -77,6 +81,14 @@ def test_tokenize_chat_request_still_validates_dict_bodies(): CACHE_SALT_MODELS = [ + ( + GenerativeScoringRequest, + { + "query": "Is this city the capital of France?", + "items": ["Paris"], + "label_token_ids": [9454, 2753], + }, + ), ( StructuredDecisionRequest, { diff --git a/vllm/entrypoints/generate/generative_scoring/serving.py b/vllm/entrypoints/generate/generative_scoring/serving.py index bd46158edc..74dbfd0759 100644 --- a/vllm/entrypoints/generate/generative_scoring/serving.py +++ b/vllm/entrypoints/generate/generative_scoring/serving.py @@ -12,12 +12,13 @@ import asyncio import math import time from collections.abc import Mapping -from typing import Literal +from typing import Any, Literal from fastapi import Request -from pydantic import Field +from pydantic import Field, model_validator from vllm.engine.protocol import EngineClient +from vllm.entrypoints.generate.base.protocol import validate_cache_salt from vllm.entrypoints.generate.label_reads import next_token_label_reads from vllm.entrypoints.openai.models.serving import OpenAIServingModels from vllm.entrypoints.serve.engine.protocol import ( @@ -59,6 +60,7 @@ class GenerativeScoringRequest(OpenAIBaseModel): the full vocab for those ids (False). item_first: If True, prepend items to query. Otherwise append items to query. add_special_tokens: Whether to add special tokens when tokenizing. + cache_salt: Optional salt for prefix caching. """ @@ -91,6 +93,17 @@ class GenerativeScoringRequest(OpenAIBaseModel): default=True, description="Whether to add special tokens when tokenizing.", ) + cache_salt: str | None = Field( + default=None, + description=( + "If specified, the prefix cache will be salted with the provided " + "string to prevent an attacker to guess prompts in multi-user " + "environments. The salt should be random, protected from " + "access by 3rd parties, and long enough to be " + "unpredictable (e.g., 43 characters base64-encoded, corresponding " + "to 256 bit)." + ), + ) priority: int = Field( default=0, ge=-(2**63), @@ -104,6 +117,13 @@ class GenerativeScoringRequest(OpenAIBaseModel): description="The request_id related to this request.", ) + @model_validator(mode="before") + @classmethod + def check_cache_salt_support(cls, data: Any) -> Any: + if isinstance(data, dict): + validate_cache_salt(data.get("cache_salt")) + return data + class GenerativeScoringItemResult(OpenAIBaseModel): """Result for a single item in the generative scoring response. @@ -395,7 +415,9 @@ class ServingGenerativeScoring(BaseServing): if len(prompt_token_ids) > max_prompt_len: prompt_token_ids = prompt_token_ids[:max_prompt_len] - engine_inputs.append(tokens_input(prompt_token_ids)) + engine_inputs.append( + tokens_input(prompt_token_ids, cache_salt=request.cache_salt) + ) prompt_token_counts.append(len(prompt_token_ids)) return engine_inputs, prompt_token_counts