diff --git a/vllm/entrypoints/openai/decisions/protocol.py b/vllm/entrypoints/openai/decisions/protocol.py index bdbb772dac..13f3bb30c0 100644 --- a/vllm/entrypoints/openai/decisions/protocol.py +++ b/vllm/entrypoints/openai/decisions/protocol.py @@ -6,6 +6,7 @@ from typing import Annotated, Any, Literal from pydantic import ConfigDict, Field, StrictBool, StrictStr, model_validator +from vllm.entrypoints.generate.base.protocol import validate_cache_salt from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel ShortText = Annotated[StrictStr, Field(max_length=1048576)] @@ -96,6 +97,26 @@ class DecisionRequest(DecisionModel): input: InputText | Annotated[list[DecisionInputMessage], Field(max_length=131072)] questions: list[DecisionQuestion] = Field(min_length=1, max_length=200) safety_identifier: Annotated[StrictStr, Field(max_length=128)] | None = None + cache_salt: str | None = Field( + default=None, + min_length=1, + max_length=1024, + description=( + "If specified, the prefix cache will be salted with the provided " + "string to prevent an attacker to guess prompts in multi-user " + "environments. The salt should be random, protected from " + "access by 3rd parties, and long enough to be " + "unpredictable (e.g., 43 characters base64-encoded, corresponding " + "to 256 bit)." + ), + ) + + @model_validator(mode="before") + @classmethod + def check_cache_salt_support(cls, data: Any) -> Any: + if isinstance(data, dict): + validate_cache_salt(data.get("cache_salt")) + return data class PredicateAnswer(DecisionModel): diff --git a/vllm/entrypoints/openai/decisions/serving.py b/vllm/entrypoints/openai/decisions/serving.py index ac7b3253a0..a9b5f56d4f 100644 --- a/vllm/entrypoints/openai/decisions/serving.py +++ b/vllm/entrypoints/openai/decisions/serving.py @@ -59,6 +59,11 @@ class OpenAIServingDecisions(BaseServingDecisions): ] lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type] engine.check_admission(len(questions)) + trace_headers = ( + None + if raw_request is None + else await self._get_trace_headers(raw_request.headers) + ) text = input_text(request.input) self._log_inputs(request_id, text, None, lora_request) reads = await self.strategy.read( @@ -69,6 +74,8 @@ class OpenAIServingDecisions(BaseServingDecisions): chat_template_kwargs=None, lora_request=lora_request, priority=0, + cache_salt=request.cache_salt, + trace_headers=trace_headers, ) except StructuredDecisionError as exc: return self.create_error_response(exc)