[Frontend] Add cache salt support to generative scoring (#60841)

Signed-off-by: Taewoong Kim <ktw2172@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Taewoong Kim
2026-10-09 16:07:50 +00:00
committed by GitHub
co-authored by Cyrus Leung
parent 90ba2f34b5
commit 75e3b17c05
5 changed files with 68 additions and 9 deletions
@@ -139,6 +139,7 @@ class TestProtocolModels:
assert req_str.apply_softmax is True # default
assert req_str.item_first is False # default
assert req_str.add_special_tokens is True # default
assert req_str.cache_salt is None
# Test request with pre-tokenized inputs and custom options
req_tok = GenerativeScoringRequest(
@@ -296,16 +297,26 @@ class TestGeneration:
"""Tests for the full generation flow with mocked engine."""
@pytest.mark.asyncio
async def test_successful_generation(self):
"""Test successful score generation returns valid response."""
@pytest.mark.parametrize("cache_salt", [None, "tenant-a"])
@pytest.mark.parametrize(
"query,items",
[
("Is this city the capital of France? ", ["Paris", "London"]),
([100, 101], [[200, 201], [300, 301]]),
],
ids=["text", "token_ids"],
)
async def test_successful_generation(self, cache_salt, query, items):
"""Test scoring succeeds and preserves cache_salt for every item."""
mock_engine = _create_mock_engine()
serving = _create_serving(mock_engine)
mock_logprobs = {1234: -0.5, 5678: -2.0, 100: -3.0}
mock_output = _create_mock_request_output(mock_logprobs)
state = SimpleNamespace(sampling_params=None)
state = SimpleNamespace(sampling_params=None, prompts=[])
async def mock_generate(*args, **kwargs):
state.prompts.append(args[0])
state.sampling_params = args[1]
yield mock_output
@@ -313,15 +324,22 @@ class TestGeneration:
request = GenerativeScoringRequest(
model=MODEL_NAME,
query="Is Paris the capital?",
items=["Yes", "No"],
query=query,
items=items,
label_token_ids=[1234, 5678],
cache_salt=cache_salt,
)
result = await serving.create_generative_scoring(request, None)
assert isinstance(result, GenerativeScoringResponse)
assert state.sampling_params is not None
assert state.sampling_params.watermarking is False
assert len(state.prompts) == len(items)
for prompt in state.prompts:
if cache_salt is None:
assert "cache_salt" not in prompt
else:
assert prompt["cache_salt"] == cache_salt
assert len(result.data) == 2
for item_result in result.data:
assert 0.0 <= item_result.score <= 1.0
@@ -13,6 +13,9 @@ request models whose before-validators were missing the same guard.
import pytest
from pydantic import ValidationError
from vllm.entrypoints.generate.generative_scoring.serving import (
GenerativeScoringRequest,
)
from vllm.entrypoints.generate.structured_decisions.protocol import (
StructuredDecisionRequest,
)
@@ -31,6 +34,7 @@ from vllm.exceptions import VLLMValidationError
pytestmark = pytest.mark.skip_global_cleanup
REQUEST_MODELS = [
GenerativeScoringRequest,
CompletionRequest,
ResponsesRequest,
EmbeddingChatRequest,
@@ -77,6 +81,14 @@ def test_tokenize_chat_request_still_validates_dict_bodies():
CACHE_SALT_MODELS = [
(
GenerativeScoringRequest,
{
"query": "Is this city the capital of France?",
"items": ["Paris"],
"label_token_ids": [9454, 2753],
},
),
(
StructuredDecisionRequest,
{