[Frontend] Add cache salt and tracing to structured decisions (#60786)
Signed-off-by: Taewoong Kim <ktw2172@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
@@ -5,10 +5,11 @@
|
||||
import time
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.utils import random_uuid
|
||||
@@ -30,9 +31,27 @@ class StructuredDecisionRequest(OpenAIBaseModel):
|
||||
default=None, description="Context placed ahead of the questions."
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
|
||||
request_id: str = Field(default_factory=random_uuid)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_cache_salt_support(cls, data: Any) -> Any:
|
||||
if isinstance(data, dict):
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
|
||||
class DecisionUsage(OpenAIBaseModel):
|
||||
input_tokens: int
|
||||
@@ -65,6 +84,7 @@ class ReadPromptRequest(OpenAIBaseModel):
|
||||
would follow a thought rather than start the reply."""
|
||||
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
cache_salt: str | None = None
|
||||
|
||||
def build_chat_params(
|
||||
self,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
@@ -11,6 +12,11 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
from vllm.tracing import (
|
||||
contains_trace_headers,
|
||||
extract_trace_headers,
|
||||
log_tracing_disabled_warning,
|
||||
)
|
||||
|
||||
from .protocol import (
|
||||
DecisionUsage,
|
||||
@@ -71,6 +77,16 @@ class ServingStructuredDecisions(BaseServing):
|
||||
self.strategy = strategy
|
||||
self.limits = strategy.limits()
|
||||
|
||||
async def _get_trace_headers(
|
||||
self, headers: Mapping[str, str]
|
||||
) -> Mapping[str, str] | None:
|
||||
if not contains_trace_headers(headers):
|
||||
return None
|
||||
if not await self.strategy.context.engine_client.is_tracing_enabled():
|
||||
log_tracing_disabled_warning()
|
||||
return None
|
||||
return extract_trace_headers(headers)
|
||||
|
||||
async def create_decision(
|
||||
self,
|
||||
request: StructuredDecisionRequest,
|
||||
@@ -88,6 +104,11 @@ class ServingStructuredDecisions(BaseServing):
|
||||
questions = parse_questions(request, self.limits)
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
engine_client.check_admission(len(questions))
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
reads = await self.strategy.read(
|
||||
questions,
|
||||
request.instructions,
|
||||
@@ -96,6 +117,8 @@ class ServingStructuredDecisions(BaseServing):
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
lora_request=lora_request,
|
||||
priority=request.priority,
|
||||
cache_salt=request.cache_salt,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
except StructuredDecisionError as e:
|
||||
return self.create_error_response(e)
|
||||
|
||||
@@ -7,7 +7,7 @@ Only models in NEXT_TOKEN_ARCHITECTURES are currently supported.
|
||||
|
||||
import math
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
@@ -74,6 +74,8 @@ class ReadStrategy(ABC):
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
lora_request: LoRARequest | None,
|
||||
priority: int,
|
||||
cache_salt: str | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
) -> list[QuestionRead]:
|
||||
"""One read per question, in order."""
|
||||
|
||||
@@ -136,13 +138,15 @@ class NextTokenStrategy(ReadStrategy):
|
||||
return DecisionLimits(max_questions=64, max_options=len(LABELS))
|
||||
|
||||
def _read_request(
|
||||
self, chat_template_kwargs: dict[str, Any] | None
|
||||
self, chat_template_kwargs: dict[str, Any] | None, cache_salt: str | None = None
|
||||
) -> ReadPromptRequest:
|
||||
if (chat_template_kwargs or {}).get("enable_thinking"):
|
||||
raise StructuredDecisionError(
|
||||
"a read is the reply's first token, so thinking must be off"
|
||||
)
|
||||
return ReadPromptRequest(chat_template_kwargs=chat_template_kwargs)
|
||||
return ReadPromptRequest(
|
||||
chat_template_kwargs=chat_template_kwargs, cache_salt=cache_salt
|
||||
)
|
||||
|
||||
async def _render(
|
||||
self, read_request: ReadPromptRequest, messages: list[dict[str, Any]]
|
||||
@@ -170,9 +174,11 @@ class NextTokenStrategy(ReadStrategy):
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
lora_request: LoRARequest | None,
|
||||
priority: int,
|
||||
cache_salt: str | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
) -> list[QuestionRead]:
|
||||
ctx = self.context
|
||||
read_request = self._read_request(chat_template_kwargs)
|
||||
read_request = self._read_request(chat_template_kwargs, cache_salt)
|
||||
|
||||
slots, engine_inputs = [], []
|
||||
for q in questions:
|
||||
@@ -197,6 +203,7 @@ class NextTokenStrategy(ReadStrategy):
|
||||
],
|
||||
request_id,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user