[Frontend] Add cache salt and tracing to structured decisions (#60786)

Signed-off-by: Taewoong Kim <ktw2172@gmail.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
Taewoong Kim
2026-10-09 08:53:50 +00:00
committed by GitHub
co-authored by Chauncey
parent a6f43e57d6
commit 18c19eaf23
7 changed files with 207 additions and 10 deletions
@@ -2,24 +2,46 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
import pytest
from fastapi import FastAPI
from fastapi import FastAPI, Request
from fastapi.testclient import TestClient
from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.structured_decisions.api_router import (
register_structured_decisions_api_router,
)
from vllm.entrypoints.generate.structured_decisions.protocol import (
StructuredDecisionRequest,
StructuredDecisionResponse,
)
from vllm.entrypoints.generate.structured_decisions.question_types import LABELS
from vllm.entrypoints.generate.structured_decisions.serving import state_text
from vllm.entrypoints.generate.structured_decisions.serving import (
ServingStructuredDecisions,
state_text,
)
from vllm.entrypoints.generate.structured_decisions.strategies import (
NextTokenStrategy,
ReadContext,
reply_label_ids,
select_read_strategy,
)
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.exception_handling.register import (
init_exception_handler,
)
from vllm.logprobs import Logprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
TRACE_HEADERS = {
"traceparent": "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01",
"tracestate": "vendor=value",
}
def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
@@ -69,9 +91,9 @@ def test_structured_state_keeps_unicode_in_prompt():
@pytest.fixture(scope="module")
def qwen():
from transformers import AutoTokenizer
from vllm.tokenizers import get_tokenizer
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
tokenizer = get_tokenizer("Qwen/Qwen3-0.6B")
prompt_ids = tokenizer.apply_chat_template(
[{"role": "user", "content": "state"}],
add_generation_prompt=True,
@@ -93,3 +115,108 @@ def test_labels_start_the_reply(qwen):
# After a colon, Qwen writes ":A" as one token, so "A" is not one token.
with pytest.raises(ValueError, match="not one distinct token"):
reply_label_ids(tokenizer, tokenizer.encode("team:"))
@pytest.fixture
def serving_and_engine(qwen):
tokenizer, _ = qwen
model_config = ModelConfig(model="Qwen/Qwen3-0.6B", dtype="float32")
renderer = HfRenderer(
SimpleNamespace(
model_config=model_config,
parallel_config=SimpleNamespace(_api_process_rank=0),
),
tokenizer,
)
engine = MagicMock(spec=EngineClient)
engine.errored = False
engine.model_config = model_config
engine.renderer = renderer
engine.input_processor = MagicMock()
online_renderer = OnlineRenderer(
model_config,
renderer,
request_logger=None,
chat_template=None,
chat_template_content_format="auto",
)
async def generate(prompt, params, request_id, **kwargs):
label_ids = params.logprob_token_ids
yield RequestOutput(
request_id=request_id,
prompt=None,
prompt_token_ids=prompt["prompt_token_ids"],
prompt_logprobs=None,
outputs=[
CompletionOutput(
index=0,
text="A",
token_ids=[label_ids[0]],
cumulative_logprob=None,
logprobs=[{token_id: Logprob(-1.0) for token_id in label_ids}],
finish_reason="length",
)
],
finished=True,
)
engine.generate.side_effect = generate
strategy = NextTokenStrategy(ReadContext(engine, online_renderer, None, "auto", {}))
models = OpenAIServingModels(
engine, [BaseModelPath(name=model_config.model, model_path=model_config.model)]
)
yield ServingStructuredDecisions(models, strategy), engine
renderer.shutdown()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"cache_salt,tracing_enabled,request_headers",
[
("tenant-a", True, {"authorization": "Bearer test", **TRACE_HEADERS}),
("tenant-b", False, {"authorization": "Bearer test", **TRACE_HEADERS}),
(None, True, {"authorization": "Bearer test", **TRACE_HEADERS}),
(None, True, {}),
("tenant-direct", True, None),
],
)
async def test_request_metadata_reaches_every_question(
serving_and_engine, cache_salt, tracing_enabled, request_headers
):
serving, engine = serving_and_engine
engine.is_tracing_enabled.return_value = tracing_enabled
raw_request = None
if request_headers is not None:
raw_request = Request(
{
"type": "http",
"headers": [
(k.encode(), v.encode()) for k, v in request_headers.items()
],
}
)
request = StructuredDecisionRequest(
state={"team": "billing", "language": "English"},
questions={
"team": {
"type": "choice",
"instructions": "Which team does the message name?",
"criteria": {"billing": "payments", "shipping": "deliveries"},
},
"lang": {
"type": "choice",
"instructions": "Which language does the message name?",
"criteria": {"English": "English text", "French": "French text"},
},
},
cache_salt=cache_salt,
)
response = await serving.create_decision(request, raw_request)
assert isinstance(response, StructuredDecisionResponse)
assert engine.generate.call_count == 2
for call in engine.generate.call_args_list:
assert call.args[0].get("cache_salt") == cache_salt
assert call.kwargs["trace_headers"] == (
TRACE_HEADERS if tracing_enabled and request_headers else None
)