[Frontend] Add cache salt and tracing to openai decisions (#61550)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -6,6 +6,7 @@ from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import ConfigDict, Field, StrictBool, StrictStr, model_validator
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
|
||||
ShortText = Annotated[StrictStr, Field(max_length=1048576)]
|
||||
@@ -96,6 +97,26 @@ class DecisionRequest(DecisionModel):
|
||||
input: InputText | Annotated[list[DecisionInputMessage], Field(max_length=131072)]
|
||||
questions: list[DecisionQuestion] = Field(min_length=1, max_length=200)
|
||||
safety_identifier: Annotated[StrictStr, Field(max_length=128)] | None = None
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=1024,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_cache_salt_support(cls, data: Any) -> Any:
|
||||
if isinstance(data, dict):
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
|
||||
class PredicateAnswer(DecisionModel):
|
||||
|
||||
@@ -59,6 +59,11 @@ class OpenAIServingDecisions(BaseServingDecisions):
|
||||
]
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
engine.check_admission(len(questions))
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
text = input_text(request.input)
|
||||
self._log_inputs(request_id, text, None, lora_request)
|
||||
reads = await self.strategy.read(
|
||||
@@ -69,6 +74,8 @@ class OpenAIServingDecisions(BaseServingDecisions):
|
||||
chat_template_kwargs=None,
|
||||
lora_request=lora_request,
|
||||
priority=0,
|
||||
cache_salt=request.cache_salt,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
except StructuredDecisionError as exc:
|
||||
return self.create_error_response(exc)
|
||||
|
||||
Reference in New Issue
Block a user