[Security] Fix chat template resource-exhaustion DoS (GHSA-4hhp-h66f-… (#50300)

Signed-off-by: jperezde <jperezde@redhat.com>
Signed-off-by: Juan Pérez de Algaba <jperezde@redhat.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Juan Pérez de Algaba
2026-09-30 08:32:00 -07:00
committed by GitHub
co-authored by Cursor
parent fc2c801ad9
commit cff08b461e
8 changed files with 230 additions and 9 deletions
@@ -334,6 +334,16 @@ class OpenAIServingResponses(GenerateBaseServing):
if maybe_validation_error is not None:
return maybe_validation_error
maybe_template_error = self.online_renderer.validate_chat_template(
request_chat_template=None,
chat_template_kwargs=request.chat_template_kwargs,
trust_request_chat_template=(
self.online_renderer.trust_request_chat_template
),
)
if maybe_template_error is not None:
return maybe_template_error
self._preflight()
if request.store and not self.enable_store:
+6
View File
@@ -25,6 +25,7 @@ if TYPE_CHECKING:
CUDA_VISIBLE_DEVICES: str | None = None
VLLM_ENGINE_ITERATION_TIMEOUT_S: int = 60
VLLM_ENGINE_READY_TIMEOUT_S: int = 600
VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT: float = 30.0
VLLM_API_KEY: str | None = None
VLLM_DEBUG_LOG_API_SERVER_RESPONSE: bool = False
S3_ACCESS_KEY_ID: str | None = None
@@ -812,6 +813,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_ENGINE_READY_TIMEOUT_S": lambda: int(
os.environ.get("VLLM_ENGINE_READY_TIMEOUT_S", "600")
),
# Maximum wall-clock seconds allowed for a single chat template render.
# Set to 0 to disable the timeout.
"VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT": lambda: float(
os.environ.get("VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT", "30")
),
# API key for vLLM API server
"VLLM_API_KEY": lambda: os.environ.get("VLLM_API_KEY", None),
# Whether to log responses from API Server for debugging
+27 -2
View File
@@ -5,7 +5,7 @@ import time
import weakref
from abc import ABC, abstractmethod
from collections.abc import Mapping, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from concurrent.futures import Executor, Future, ThreadPoolExecutor
from contextlib import ExitStack
from dataclasses import replace
from functools import cached_property
@@ -77,6 +77,31 @@ logger = init_logger(__name__)
_T = TypeVar("_T", bound=TokenizerLike, default=TokenizerLike)
class _SwappableExecutor(Executor):
"""Executor whose inner pool can be replaced without changing identity.
``make_async`` captures the executor object at wrap time. Replacing
``self._executor`` with a new ``ThreadPoolExecutor`` would leave those
wrappers (tokenize, decode, pooling, derender) bound to a shutdown pool.
"""
def __init__(self, max_workers: int) -> None:
super().__init__()
self._max_workers = max_workers
self._inner = ThreadPoolExecutor(max_workers=max_workers)
def submit(self, fn, /, *args, **kwargs):
return self._inner.submit(fn, *args, **kwargs)
def shutdown(self, wait: bool = True, *, cancel_futures: bool = False) -> None:
self._inner.shutdown(wait=wait, cancel_futures=cancel_futures)
def replace_inner(self) -> None:
old = self._inner
old.shutdown(wait=False)
self._inner = ThreadPoolExecutor(max_workers=self._max_workers)
class BaseRenderer(ABC, Generic[_T]):
def __init__(self, config: "VllmConfig", tokenizer: _T | None) -> None:
super().__init__()
@@ -115,7 +140,7 @@ class BaseRenderer(ABC, Generic[_T]):
# multimodal processor receives a deep-copied tokenizer (see #36557)
# so it is safe to run tokenization and MM preprocessing concurrently.
pool_workers = config.model_config.renderer_num_workers
self._executor = ThreadPoolExecutor(max_workers=pool_workers)
self._executor = _SwappableExecutor(max_workers=pool_workers)
self._resources.callback(self._executor.shutdown, wait=False)
# Separate single-worker executor so tokenization never queues behind
+47 -5
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import asyncio
import copy
import inspect
import itertools
@@ -20,6 +21,7 @@ import jinja2.sandbox
import torch
from typing_extensions import override
from vllm import envs
from vllm.entrypoints.chat_utils import (
PROMPT_EMBEDS_PLACEHOLDER_TOKEN,
ChatTemplateResolutionError,
@@ -969,6 +971,44 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
self.tokenizer, config.model_config.renderer_num_workers + 1
)
def _replace_executor(self) -> None:
"""Replace the inner render pool after a stuck worker.
Called after a render timeout so that the stuck thread (which CPython
cannot interrupt) does not permanently block all subsequent renders.
The executor *object* is kept so ``make_async`` wrappers bound at
init (tokenize, decode, pooling, derender) keep submitting work.
The old inner pool is abandoned with wait=False — its thread will
eventually finish or be reaped on process exit.
"""
self._executor.replace_inner()
logger.warning(
"Chat template render timed out — executor pool replaced "
"(%d worker(s)). The stuck thread may continue consuming "
"CPU until the process exits.",
self.model_config.renderer_num_workers,
)
async def _render_with_timeout(self, coro):
"""Run a template-rendering coroutine with a timeout.
On timeout, replaces the executor so subsequent renders are not
permanently blocked by the stuck thread.
"""
timeout = envs.VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT
if timeout <= 0:
return await coro
try:
return await asyncio.wait_for(coro, timeout=timeout)
except asyncio.TimeoutError:
self._replace_executor()
raise TimeoutError(
f"Chat template rendering timed out after "
f"{timeout}s. This may indicate a malicious or "
f"excessively complex template. Adjust "
f"VLLM_CHAT_TEMPLATE_RENDER_TIMEOUT to override."
) from None
def _can_produce_offsets(self) -> bool:
# Only fast tokenizers expose offset_mapping.
return self.tokenizer is not None and self.tokenizer.is_fast
@@ -1124,11 +1164,13 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
logger.warning_once(_TOKENIZE_OVERRIDE_WARNING)
chat_template_kwargs["tokenize"] = True
prompt_raw = await self._apply_chat_template_async(
model_config,
tokenizer,
conversation,
**chat_template_kwargs,
prompt_raw = await self._render_with_timeout(
self._apply_chat_template_async(
model_config,
tokenizer,
conversation,
**chat_template_kwargs,
)
)
# NOTE: use_unified_vision_chunk is currently specific to Kimi-K2.5