fix(security): block request-level GPU video backend selection withou… (#47259)
Signed-off-by: jperezde <jperezde@redhat.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
90ce3a09be
commit
ba22152096
@@ -16,7 +16,11 @@ from vllm.assets.video import (
|
||||
video_to_pil_images_list,
|
||||
)
|
||||
from vllm.multimodal.media import ImageMediaIO, VideoMediaIO
|
||||
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY, VideoLoader
|
||||
from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
VideoLoader,
|
||||
)
|
||||
|
||||
from ..utils import cosine_similarity, create_video_from_image, normalize_image
|
||||
|
||||
@@ -357,3 +361,95 @@ def test_load_base64_jpeg_raises_on_zero_num_frames():
|
||||
|
||||
with pytest.raises(ValueError, match="num_frames must be greater than 0 or -1"):
|
||||
videoio.load_base64("video/jpeg", data)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# GPU video backend policy tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMergeKwargsGpuBackendPolicy:
|
||||
"""Verify that merge_kwargs blocks request-level GPU backend selection
|
||||
when the static (engine-level) config did not configure that backend."""
|
||||
|
||||
def test_pynvvideocodec_requires_gpu(self):
|
||||
assert VIDEO_LOADER_REGISTRY.backend_requires_gpu(PYNVVIDEOCODEC_VIDEO_BACKEND)
|
||||
|
||||
def test_strips_video_backend_pynv_when_not_static(self):
|
||||
result = VideoMediaIO.merge_kwargs(
|
||||
default_kwargs=None,
|
||||
runtime_kwargs={"video_backend": "pynvvideocodec"},
|
||||
)
|
||||
assert "video_backend" not in result
|
||||
|
||||
def test_strips_backend_pynv_when_not_static(self):
|
||||
result = VideoMediaIO.merge_kwargs(
|
||||
default_kwargs={"num_frames": 16},
|
||||
runtime_kwargs={"backend": "pynvvideocodec"},
|
||||
)
|
||||
assert result.get("backend") != "pynvvideocodec"
|
||||
|
||||
def test_preserves_video_backend_pynv_when_static(self):
|
||||
result = VideoMediaIO.merge_kwargs(
|
||||
default_kwargs={"video_backend": "pynvvideocodec"},
|
||||
runtime_kwargs={"video_backend": "pynvvideocodec", "num_frames": 8},
|
||||
)
|
||||
assert result["video_backend"] == "pynvvideocodec"
|
||||
assert result["num_frames"] == 8
|
||||
|
||||
def test_preserves_backend_pynv_when_static(self):
|
||||
result = VideoMediaIO.merge_kwargs(
|
||||
default_kwargs={"backend": "pynvvideocodec"},
|
||||
runtime_kwargs={"backend": "pynvvideocodec"},
|
||||
)
|
||||
assert result["backend"] == "pynvvideocodec"
|
||||
|
||||
@pytest.mark.parametrize("backend", ["opencv", "pyav", "torchcodec"])
|
||||
def test_software_video_backend_passes_through(self, backend: str):
|
||||
result = VideoMediaIO.merge_kwargs(
|
||||
default_kwargs=None,
|
||||
runtime_kwargs={"video_backend": backend},
|
||||
)
|
||||
assert result["video_backend"] == backend
|
||||
|
||||
@pytest.mark.parametrize("backend", ["opencv", "pyav"])
|
||||
def test_software_codec_backend_passes_through(self, backend: str):
|
||||
result = VideoMediaIO.merge_kwargs(
|
||||
default_kwargs=None,
|
||||
runtime_kwargs={"backend": backend},
|
||||
)
|
||||
assert result["backend"] == backend
|
||||
|
||||
def test_strips_both_keys_independently(self):
|
||||
result = VideoMediaIO.merge_kwargs(
|
||||
default_kwargs=None,
|
||||
runtime_kwargs={
|
||||
"video_backend": "pynvvideocodec",
|
||||
"backend": "pynvvideocodec",
|
||||
"num_frames": 4,
|
||||
},
|
||||
)
|
||||
assert "video_backend" not in result
|
||||
assert result.get("backend") != "pynvvideocodec"
|
||||
assert result["num_frames"] == 4
|
||||
|
||||
def test_other_kwargs_preserved_when_gpu_backend_stripped(self):
|
||||
result = VideoMediaIO.merge_kwargs(
|
||||
default_kwargs={"fps": 2},
|
||||
runtime_kwargs={
|
||||
"video_backend": "pynvvideocodec",
|
||||
"num_frames": 16,
|
||||
},
|
||||
)
|
||||
assert "video_backend" not in result
|
||||
assert result["num_frames"] == 16
|
||||
|
||||
def test_static_pynv_with_different_runtime_gpu_backend(self):
|
||||
"""If static sets pynv via video_backend but runtime tries to set it
|
||||
via the codec-level 'backend' key (without a static match), strip it."""
|
||||
result = VideoMediaIO.merge_kwargs(
|
||||
default_kwargs={"video_backend": "pynvvideocodec"},
|
||||
runtime_kwargs={"backend": "pynvvideocodec"},
|
||||
)
|
||||
assert result.get("backend") != "pynvvideocodec"
|
||||
assert result["video_backend"] == "pynvvideocodec"
|
||||
|
||||
@@ -10,11 +10,14 @@ import pybase64
|
||||
from PIL import Image
|
||||
|
||||
from vllm import envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..video import VIDEO_LOADER_REGISTRY
|
||||
from .base import MediaIO
|
||||
from .image import ImageMediaIO
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
|
||||
"""Configuration values can be user-provided either by --media-io-kwargs or
|
||||
@@ -28,6 +31,24 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
|
||||
default_kwargs: dict[str, Any] | None,
|
||||
runtime_kwargs: dict[str, Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
if runtime_kwargs:
|
||||
# Block request-level selection of GPU video backends that
|
||||
# were not configured (and VRAM-reserved) at startup.
|
||||
for key in ("video_backend", "backend"):
|
||||
requested = runtime_kwargs.get(key)
|
||||
if requested and VIDEO_LOADER_REGISTRY.backend_requires_gpu(requested):
|
||||
static_val = (default_kwargs or {}).get(key)
|
||||
if static_val != requested:
|
||||
logger.warning_once(
|
||||
"Stripping request-level %s=%r: GPU video "
|
||||
"backend not configured at startup.",
|
||||
key,
|
||||
requested,
|
||||
)
|
||||
runtime_kwargs = {
|
||||
k: v for k, v in runtime_kwargs.items() if k != key
|
||||
}
|
||||
|
||||
merged = super().merge_kwargs(default_kwargs, runtime_kwargs)
|
||||
# fps and num_frames interact with each other, so if either is
|
||||
# overridden at request time, wipe the other from defaults to
|
||||
|
||||
@@ -39,6 +39,7 @@ class VideoLoaderRegistry(ExtensionManager):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.processor2backend: dict[str, str] = {}
|
||||
self._requires_gpu: dict[str, bool] = {}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_registered_video_processors(
|
||||
@@ -62,11 +63,13 @@ class VideoLoaderRegistry(ExtensionManager):
|
||||
name: str,
|
||||
*,
|
||||
video_processor: str | tuple[str, ...] | None = None,
|
||||
requires_gpu: bool = False,
|
||||
):
|
||||
processors = self._normalize_registered_video_processors(video_processor)
|
||||
|
||||
def wrap(cls_to_register):
|
||||
self.name2class[name] = cls_to_register
|
||||
self._requires_gpu[name] = requires_gpu
|
||||
for processor_name in processors:
|
||||
self.processor2backend[processor_name] = name
|
||||
return cls_to_register
|
||||
@@ -82,6 +85,9 @@ class VideoLoaderRegistry(ExtensionManager):
|
||||
|
||||
return self.processor2backend.get(video_processor)
|
||||
|
||||
def backend_requires_gpu(self, name: str) -> bool:
|
||||
return self._requires_gpu.get(name, False)
|
||||
|
||||
|
||||
def get_video_loader_backend_for_processor(
|
||||
video_processor: str | None,
|
||||
@@ -909,7 +915,7 @@ class VideoBackend(
|
||||
)
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register(PYNVVIDEOCODEC_VIDEO_BACKEND)
|
||||
@VIDEO_LOADER_REGISTRY.register(PYNVVIDEOCODEC_VIDEO_BACKEND, requires_gpu=True)
|
||||
class PyNvVideoCodecVideoBackend(VideoBackend):
|
||||
"""Hardware-accelerated video backend using PyNvVideoCodec.
|
||||
|
||||
|
||||
@@ -55,7 +55,7 @@ from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
|
||||
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper
|
||||
@@ -584,15 +584,15 @@ class Worker(WorkerBase):
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _uses_pynvvideocodec_video_backend(mm_config) -> bool:
|
||||
def _uses_gpu_video_backend(mm_config) -> bool:
|
||||
video_kwargs = mm_config.media_io_kwargs.get("video", {})
|
||||
video_loader_backend = (
|
||||
video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
|
||||
)
|
||||
codec_backend = video_kwargs.get("backend")
|
||||
return (
|
||||
video_loader_backend == PYNVVIDEOCODEC_VIDEO_BACKEND
|
||||
or codec_backend == PYNVVIDEOCODEC_VIDEO_BACKEND
|
||||
return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or (
|
||||
codec_backend is not None
|
||||
and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend)
|
||||
)
|
||||
|
||||
def _reserve_mm_ipc_gpu_memory(self, available_kv_cache_memory_bytes: int) -> int:
|
||||
@@ -623,7 +623,7 @@ class Worker(WorkerBase):
|
||||
)
|
||||
decoder_reserved_bytes = (
|
||||
num_api_servers * per_server_decoder_bytes
|
||||
if self._uses_pynvvideocodec_video_backend(mm_config)
|
||||
if self._uses_gpu_video_backend(mm_config)
|
||||
else 0
|
||||
)
|
||||
reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes
|
||||
|
||||
Reference in New Issue
Block a user