[Security] Fix chat template resource-exhaustion DoS (GHSA-4hhp-h66f-… (#50300)
Signed-off-by: jperezde <jperezde@redhat.com> Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
co-authored by
Cursor
parent
fc2c801ad9
commit
cff08b461e
@@ -334,6 +334,16 @@ class OpenAIServingResponses(GenerateBaseServing):
|
||||
if maybe_validation_error is not None:
|
||||
return maybe_validation_error
|
||||
|
||||
maybe_template_error = self.online_renderer.validate_chat_template(
|
||||
request_chat_template=None,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
trust_request_chat_template=(
|
||||
self.online_renderer.trust_request_chat_template
|
||||
),
|
||||
)
|
||||
if maybe_template_error is not None:
|
||||
return maybe_template_error
|
||||
|
||||
self._preflight()
|
||||
|
||||
if request.store and not self.enable_store:
|
||||
|
||||
@@ -25,6 +25,7 @@ if TYPE_CHECKING:
|
||||
CUDA_VISIBLE_DEVICES: str | None = None
|
||||
VLLM_ENGINE_ITERATION_TIMEOUT_S: int = 60
|
||||
VLLM_ENGINE_READY_TIMEOUT_S: int = 600
|
||||
VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: float = 30.0
|
||||
VLLM_API_KEY: str | None = None
|
||||
VLLM_DEBUG_LOG_API_SERVER_RESPONSE: bool = False
|
||||
S3_ACCESS_KEY_ID: str | None = None
|
||||
@@ -812,6 +813,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_ENGINE_READY_TIMEOUT_S": lambda: int(
|
||||
os.environ.get("VLLM_ENGINE_READY_TIMEOUT_S", "600")
|
||||
),
|
||||
# Maximum wall-clock seconds allowed for a single chat template render.
|
||||
# Set to 0 to disable the timeout.
|
||||
"VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT": lambda: float(
|
||||
os.environ.get("VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT", "30")
|
||||
),
|
||||
# API key for vLLM API server
|
||||
"VLLM_API_KEY": lambda: os.environ.get("VLLM_API_KEY", None),
|
||||
# Whether to log responses from API Server for debugging
|
||||
|
||||
+27
-2
@@ -5,7 +5,7 @@ import time
|
||||
import weakref
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from concurrent.futures import Executor, Future, ThreadPoolExecutor
|
||||
from contextlib import ExitStack
|
||||
from dataclasses import replace
|
||||
from functools import cached_property
|
||||
@@ -77,6 +77,31 @@ logger = init_logger(__name__)
|
||||
_T = TypeVar("_T", bound=TokenizerLike, default=TokenizerLike)
|
||||
|
||||
|
||||
class _SwappableExecutor(Executor):
|
||||
"""Executor whose inner pool can be replaced without changing identity.
|
||||
|
||||
``make_async`` captures the executor object at wrap time. Replacing
|
||||
``self._executor`` with a new ``ThreadPoolExecutor`` would leave those
|
||||
wrappers (tokenize, decode, pooling, derender) bound to a shutdown pool.
|
||||
"""
|
||||
|
||||
def __init__(self, max_workers: int) -> None:
|
||||
super().__init__()
|
||||
self._max_workers = max_workers
|
||||
self._inner = ThreadPoolExecutor(max_workers=max_workers)
|
||||
|
||||
def submit(self, fn, /, *args, **kwargs):
|
||||
return self._inner.submit(fn, *args, **kwargs)
|
||||
|
||||
def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None:
|
||||
self._inner.shutdown(wait=wait, cancel_futures=cancel_futures)
|
||||
|
||||
def replace_inner(self) -> None:
|
||||
old = self._inner
|
||||
old.shutdown(wait=False)
|
||||
self._inner = ThreadPoolExecutor(max_workers=self._max_workers)
|
||||
|
||||
|
||||
class BaseRenderer(ABC, Generic[_T]):
|
||||
def __init__(self, config: "VllmConfig", tokenizer: _T | None) -> None:
|
||||
super().__init__()
|
||||
@@ -115,7 +140,7 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
# multimodal processor receives a deep-copied tokenizer (see #36557)
|
||||
# so it is safe to run tokenization and MM preprocessing concurrently.
|
||||
pool_workers = config.model_config.renderer_num_workers
|
||||
self._executor = ThreadPoolExecutor(max_workers=pool_workers)
|
||||
self._executor = _SwappableExecutor(max_workers=pool_workers)
|
||||
self._resources.callback(self._executor.shutdown, wait=False)
|
||||
|
||||
# Separate single-worker executor so tokenization never queues behind
|
||||
|
||||
+47
-5
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import inspect
|
||||
import itertools
|
||||
@@ -20,6 +21,7 @@ import jinja2.sandbox
|
||||
import torch
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm import envs
|
||||
from vllm.entrypoints.chat_utils import (
|
||||
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
|
||||
ChatTemplateResolutionError,
|
||||
@@ -969,6 +971,44 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
self.tokenizer, config.model_config.renderer_num_workers + 1
|
||||
)
|
||||
|
||||
def _replace_executor(self) -> None:
|
||||
"""Replace the inner render pool after a stuck worker.
|
||||
|
||||
Called after a render timeout so that the stuck thread (which CPython
|
||||
cannot interrupt) does not permanently block all subsequent renders.
|
||||
The executor *object* is kept so ``make_async`` wrappers bound at
|
||||
init (tokenize, decode, pooling, derender) keep submitting work.
|
||||
The old inner pool is abandoned with wait=False — its thread will
|
||||
eventually finish or be reaped on process exit.
|
||||
"""
|
||||
self._executor.replace_inner()
|
||||
logger.warning(
|
||||
"Chat template render timed out — executor pool replaced "
|
||||
"(%d worker(s)). The stuck thread may continue consuming "
|
||||
"CPU until the process exits.",
|
||||
self.model_config.renderer_num_workers,
|
||||
)
|
||||
|
||||
async def _render_with_timeout(self, coro):
|
||||
"""Run a template-rendering coroutine with a timeout.
|
||||
|
||||
On timeout, replaces the executor so subsequent renders are not
|
||||
permanently blocked by the stuck thread.
|
||||
"""
|
||||
timeout = envs.VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT
|
||||
if timeout <= 0:
|
||||
return await coro
|
||||
try:
|
||||
return await asyncio.wait_for(coro, timeout=timeout)
|
||||
except asyncio.TimeoutError:
|
||||
self._replace_executor()
|
||||
raise TimeoutError(
|
||||
f"Chat template rendering timed out after "
|
||||
f"{timeout}s. This may indicate a malicious or "
|
||||
f"excessively complex template. Adjust "
|
||||
f"VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT to override."
|
||||
) from None
|
||||
|
||||
def _can_produce_offsets(self) -> bool:
|
||||
# Only fast tokenizers expose offset_mapping.
|
||||
return self.tokenizer is not None and self.tokenizer.is_fast
|
||||
@@ -1124,11 +1164,13 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
|
||||
chat_template_kwargs["tokenize"] = True
|
||||
|
||||
prompt_raw = await self._apply_chat_template_async(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**chat_template_kwargs,
|
||||
prompt_raw = await self._render_with_timeout(
|
||||
self._apply_chat_template_async(
|
||||
model_config,
|
||||
tokenizer,
|
||||
conversation,
|
||||
**chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
|
||||
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5
|
||||
|
||||
Reference in New Issue
Block a user