[Frontend] Add cache salt support to generative scoring (#60841)
Signed-off-by: Taewoong Kim <ktw2172@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
90ba2f34b5
commit
75e3b17c05
@@ -139,6 +139,7 @@ class TestProtocolModels:
|
||||
assert req_str.apply_softmax is True # default
|
||||
assert req_str.item_first is False # default
|
||||
assert req_str.add_special_tokens is True # default
|
||||
assert req_str.cache_salt is None
|
||||
|
||||
# Test request with pre-tokenized inputs and custom options
|
||||
req_tok = GenerativeScoringRequest(
|
||||
@@ -296,16 +297,26 @@ class TestGeneration:
|
||||
"""Tests for the full generation flow with mocked engine."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_generation(self):
|
||||
"""Test successful score generation returns valid response."""
|
||||
@pytest.mark.parametrize("cache_salt", [None, "tenant-a"])
|
||||
@pytest.mark.parametrize(
|
||||
"query,items",
|
||||
[
|
||||
("Is this city the capital of France? ", ["Paris", "London"]),
|
||||
([100, 101], [[200, 201], [300, 301]]),
|
||||
],
|
||||
ids=["text", "token_ids"],
|
||||
)
|
||||
async def test_successful_generation(self, cache_salt, query, items):
|
||||
"""Test scoring succeeds and preserves cache_salt for every item."""
|
||||
mock_engine = _create_mock_engine()
|
||||
serving = _create_serving(mock_engine)
|
||||
|
||||
mock_logprobs = {1234: -0.5, 5678: -2.0, 100: -3.0}
|
||||
mock_output = _create_mock_request_output(mock_logprobs)
|
||||
state = SimpleNamespace(sampling_params=None)
|
||||
state = SimpleNamespace(sampling_params=None, prompts=[])
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
state.prompts.append(args[0])
|
||||
state.sampling_params = args[1]
|
||||
yield mock_output
|
||||
|
||||
@@ -313,15 +324,22 @@ class TestGeneration:
|
||||
|
||||
request = GenerativeScoringRequest(
|
||||
model=MODEL_NAME,
|
||||
query="Is Paris the capital?",
|
||||
items=["Yes", "No"],
|
||||
query=query,
|
||||
items=items,
|
||||
label_token_ids=[1234, 5678],
|
||||
cache_salt=cache_salt,
|
||||
)
|
||||
result = await serving.create_generative_scoring(request, None)
|
||||
|
||||
assert isinstance(result, GenerativeScoringResponse)
|
||||
assert state.sampling_params is not None
|
||||
assert state.sampling_params.watermarking is False
|
||||
assert len(state.prompts) == len(items)
|
||||
for prompt in state.prompts:
|
||||
if cache_salt is None:
|
||||
assert "cache_salt" not in prompt
|
||||
else:
|
||||
assert prompt["cache_salt"] == cache_salt
|
||||
assert len(result.data) == 2
|
||||
for item_result in result.data:
|
||||
assert 0.0 <= item_result.score <= 1.0
|
||||
|
||||
@@ -13,6 +13,9 @@ request models whose before-validators were missing the same guard.
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.generate.generative_scoring.serving import (
|
||||
GenerativeScoringRequest,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.protocol import (
|
||||
StructuredDecisionRequest,
|
||||
)
|
||||
@@ -31,6 +34,7 @@ from vllm.exceptions import VLLMValidationError
|
||||
pytestmark = pytest.mark.skip_global_cleanup
|
||||
|
||||
REQUEST_MODELS = [
|
||||
GenerativeScoringRequest,
|
||||
CompletionRequest,
|
||||
ResponsesRequest,
|
||||
EmbeddingChatRequest,
|
||||
@@ -77,6 +81,14 @@ def test_tokenize_chat_request_still_validates_dict_bodies():
|
||||
|
||||
|
||||
CACHE_SALT_MODELS = [
|
||||
(
|
||||
GenerativeScoringRequest,
|
||||
{
|
||||
"query": "Is this city the capital of France?",
|
||||
"items": ["Paris"],
|
||||
"label_token_ids": [9454, 2753],
|
||||
},
|
||||
),
|
||||
(
|
||||
StructuredDecisionRequest,
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user