[Frontend] Add cache salt and tracing to structured decisions (#60786)
Signed-off-by: Taewoong Kim <ktw2172@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -2,24 +2,46 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.structured_decisions.api_router import (
|
||||
register_structured_decisions_api_router,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.protocol import (
|
||||
StructuredDecisionRequest,
|
||||
StructuredDecisionResponse,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.question_types import LABELS
|
||||
from vllm.entrypoints.generate.structured_decisions.serving import state_text
|
||||
from vllm.entrypoints.generate.structured_decisions.serving import (
|
||||
ServingStructuredDecisions,
|
||||
state_text,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.strategies import (
|
||||
NextTokenStrategy,
|
||||
ReadContext,
|
||||
reply_label_ids,
|
||||
select_read_strategy,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.exception_handling.register import (
|
||||
init_exception_handler,
|
||||
)
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
TRACE_HEADERS = {
|
||||
"traceparent": "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
|
||||
"tracestate": "vendor=value",
|
||||
}
|
||||
|
||||
|
||||
def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
|
||||
@@ -69,9 +91,9 @@ def test_structured_state_keeps_unicode_in_prompt():
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen():
|
||||
from transformers import AutoTokenizer
|
||||
from vllm.tokenizers import get_tokenizer
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
|
||||
tokenizer = get_tokenizer("Qwen/Qwen3-0.6B")
|
||||
prompt_ids = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": "state"}],
|
||||
add_generation_prompt=True,
|
||||
@@ -93,3 +115,108 @@ def test_labels_start_the_reply(qwen):
|
||||
# After a colon, Qwen writes ":A" as one token, so "A" is not one token.
|
||||
with pytest.raises(ValueError, match="not one distinct token"):
|
||||
reply_label_ids(tokenizer, tokenizer.encode("team:"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def serving_and_engine(qwen):
|
||||
tokenizer, _ = qwen
|
||||
model_config = ModelConfig(model="Qwen/Qwen3-0.6B", dtype="float32")
|
||||
renderer = HfRenderer(
|
||||
SimpleNamespace(
|
||||
model_config=model_config,
|
||||
parallel_config=SimpleNamespace(_api_process_rank=0),
|
||||
),
|
||||
tokenizer,
|
||||
)
|
||||
engine = MagicMock(spec=EngineClient)
|
||||
engine.errored = False
|
||||
engine.model_config = model_config
|
||||
engine.renderer = renderer
|
||||
engine.input_processor = MagicMock()
|
||||
online_renderer = OnlineRenderer(
|
||||
model_config,
|
||||
renderer,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
async def generate(prompt, params, request_id, **kwargs):
|
||||
label_ids = params.logprob_token_ids
|
||||
yield RequestOutput(
|
||||
request_id=request_id,
|
||||
prompt=None,
|
||||
prompt_token_ids=prompt["prompt_token_ids"],
|
||||
prompt_logprobs=None,
|
||||
outputs=[
|
||||
CompletionOutput(
|
||||
index=0,
|
||||
text="A",
|
||||
token_ids=[label_ids[0]],
|
||||
cumulative_logprob=None,
|
||||
logprobs=[{token_id: Logprob(-1.0) for token_id in label_ids}],
|
||||
finish_reason="length",
|
||||
)
|
||||
],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
engine.generate.side_effect = generate
|
||||
strategy = NextTokenStrategy(ReadContext(engine, online_renderer, None, "auto", {}))
|
||||
models = OpenAIServingModels(
|
||||
engine, [BaseModelPath(name=model_config.model, model_path=model_config.model)]
|
||||
)
|
||||
yield ServingStructuredDecisions(models, strategy), engine
|
||||
renderer.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"cache_salt,tracing_enabled,request_headers",
|
||||
[
|
||||
("tenant-a", True, {"authorization": "Bearer test", **TRACE_HEADERS}),
|
||||
("tenant-b", False, {"authorization": "Bearer test", **TRACE_HEADERS}),
|
||||
(None, True, {"authorization": "Bearer test", **TRACE_HEADERS}),
|
||||
(None, True, {}),
|
||||
("tenant-direct", True, None),
|
||||
],
|
||||
)
|
||||
async def test_request_metadata_reaches_every_question(
|
||||
serving_and_engine, cache_salt, tracing_enabled, request_headers
|
||||
):
|
||||
serving, engine = serving_and_engine
|
||||
engine.is_tracing_enabled.return_value = tracing_enabled
|
||||
raw_request = None
|
||||
if request_headers is not None:
|
||||
raw_request = Request(
|
||||
{
|
||||
"type": "http",
|
||||
"headers": [
|
||||
(k.encode(), v.encode()) for k, v in request_headers.items()
|
||||
],
|
||||
}
|
||||
)
|
||||
request = StructuredDecisionRequest(
|
||||
state={"team": "billing", "language": "English"},
|
||||
questions={
|
||||
"team": {
|
||||
"type": "choice",
|
||||
"instructions": "Which team does the message name?",
|
||||
"criteria": {"billing": "payments", "shipping": "deliveries"},
|
||||
},
|
||||
"lang": {
|
||||
"type": "choice",
|
||||
"instructions": "Which language does the message name?",
|
||||
"criteria": {"English": "English text", "French": "French text"},
|
||||
},
|
||||
},
|
||||
cache_salt=cache_salt,
|
||||
)
|
||||
response = await serving.create_decision(request, raw_request)
|
||||
assert isinstance(response, StructuredDecisionResponse)
|
||||
assert engine.generate.call_count == 2
|
||||
for call in engine.generate.call_args_list:
|
||||
assert call.args[0].get("cache_salt") == cache_salt
|
||||
assert call.kwargs["trace_headers"] == (
|
||||
TRACE_HEADERS if tracing_enabled and request_headers else None
|
||||
)
|
||||
|
||||
@@ -13,6 +13,9 @@ request models whose before-validators were missing the same guard.
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.protocol import (
|
||||
StructuredDecisionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionRequest
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
|
||||
@@ -74,6 +77,18 @@ def test_tokenize_chat_request_still_validates_dict_bodies():
|
||||
|
||||
|
||||
CACHE_SALT_MODELS = [
|
||||
(
|
||||
StructuredDecisionRequest,
|
||||
{
|
||||
"state": "hello",
|
||||
"questions": {
|
||||
"lang": {
|
||||
"type": "choice",
|
||||
"criteria": {"English": "Text written in English."},
|
||||
}
|
||||
},
|
||||
},
|
||||
),
|
||||
(CompletionRequest, {"prompt": "hello"}),
|
||||
(ChatCompletionRequest, {"messages": [{"role": "user", "content": "hello"}]}),
|
||||
(ResponsesRequest, {"input": "hello"}),
|
||||
|
||||
Reference in New Issue
Block a user