[Frontend] Add cache salt and tracing to openai decisions (#61550)

Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Chauncey
2026-10-14 00:48:44 -07:00
committed by GitHub
co-authored by Cyrus Leung
parent 4dc5add6ea
commit e1726d9142
2 changed files with 28 additions and 0 deletions
@@ -6,6 +6,7 @@ from typing import Annotated, Any, Literal
from pydantic import ConfigDict, Field, StrictBool, StrictStr, model_validator
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
ShortText = Annotated[StrictStr, Field(max_length=1048576)]
@@ -96,6 +97,26 @@ class DecisionRequest(DecisionModel):
input: InputText | Annotated[list[DecisionInputMessage], Field(max_length=131072)]
questions: list[DecisionQuestion] = Field(min_length=1, max_length=200)
safety_identifier: Annotated[StrictStr, Field(max_length=128)] | None = None
cache_salt: str | None = Field(
default=None,
min_length=1,
max_length=1024,
description=(
"If specified, the prefix cache will be salted with the provided "
"string to prevent an attacker to guess prompts in multi-user "
"environments. The salt should be random, protected from "
"access by 3rd parties, and long enough to be "
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
"to 256 bit)."
),
)
@model_validator(mode="before")
@classmethod
def check_cache_salt_support(cls, data: Any) -> Any:
if isinstance(data, dict):
validate_cache_salt(data.get("cache_salt"))
return data
class PredicateAnswer(DecisionModel):
@@ -59,6 +59,11 @@ class OpenAIServingDecisions(BaseServingDecisions):
]
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
engine.check_admission(len(questions))
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
text = input_text(request.input)
self._log_inputs(request_id, text, None, lora_request)
reads = await self.strategy.read(
@@ -69,6 +74,8 @@ class OpenAIServingDecisions(BaseServingDecisions):
chat_template_kwargs=None,
lora_request=lora_request,
priority=0,
cache_salt=request.cache_salt,
trace_headers=trace_headers,
)
except StructuredDecisionError as exc:
return self.create_error_response(exc)