diff --git a/tests/multimodal/media/test_video.py b/tests/multimodal/media/test_video.py index e4b3afff08..671abd7b07 100644 --- a/tests/multimodal/media/test_video.py +++ b/tests/multimodal/media/test_video.py @@ -16,7 +16,11 @@ from vllm.assets.video import ( video_to_pil_images_list, ) from vllm.multimodal.media import ImageMediaIO, VideoMediaIO -from vllm.multimodal.video import VIDEO_LOADER_REGISTRY, VideoLoader +from vllm.multimodal.video import ( + PYNVVIDEOCODEC_VIDEO_BACKEND, + VIDEO_LOADER_REGISTRY, + VideoLoader, +) from ..utils import cosine_similarity, create_video_from_image, normalize_image @@ -357,3 +361,95 @@ def test_load_base64_jpeg_raises_on_zero_num_frames(): with pytest.raises(ValueError, match="num_frames must be greater than 0 or -1"): videoio.load_base64("video/jpeg", data) + + +# --------------------------------------------------------------------------- +# GPU video backend policy tests +# --------------------------------------------------------------------------- + + +class TestMergeKwargsGpuBackendPolicy: + """Verify that merge_kwargs blocks request-level GPU backend selection + when the static (engine-level) config did not configure that backend.""" + + def test_pynvvideocodec_requires_gpu(self): + assert VIDEO_LOADER_REGISTRY.backend_requires_gpu(PYNVVIDEOCODEC_VIDEO_BACKEND) + + def test_strips_video_backend_pynv_when_not_static(self): + result = VideoMediaIO.merge_kwargs( + default_kwargs=None, + runtime_kwargs={"video_backend": "pynvvideocodec"}, + ) + assert "video_backend" not in result + + def test_strips_backend_pynv_when_not_static(self): + result = VideoMediaIO.merge_kwargs( + default_kwargs={"num_frames": 16}, + runtime_kwargs={"backend": "pynvvideocodec"}, + ) + assert result.get("backend") != "pynvvideocodec" + + def test_preserves_video_backend_pynv_when_static(self): + result = VideoMediaIO.merge_kwargs( + default_kwargs={"video_backend": "pynvvideocodec"}, + runtime_kwargs={"video_backend": "pynvvideocodec", "num_frames": 8}, + ) + assert result["video_backend"] == "pynvvideocodec" + assert result["num_frames"] == 8 + + def test_preserves_backend_pynv_when_static(self): + result = VideoMediaIO.merge_kwargs( + default_kwargs={"backend": "pynvvideocodec"}, + runtime_kwargs={"backend": "pynvvideocodec"}, + ) + assert result["backend"] == "pynvvideocodec" + + @pytest.mark.parametrize("backend", ["opencv", "pyav", "torchcodec"]) + def test_software_video_backend_passes_through(self, backend: str): + result = VideoMediaIO.merge_kwargs( + default_kwargs=None, + runtime_kwargs={"video_backend": backend}, + ) + assert result["video_backend"] == backend + + @pytest.mark.parametrize("backend", ["opencv", "pyav"]) + def test_software_codec_backend_passes_through(self, backend: str): + result = VideoMediaIO.merge_kwargs( + default_kwargs=None, + runtime_kwargs={"backend": backend}, + ) + assert result["backend"] == backend + + def test_strips_both_keys_independently(self): + result = VideoMediaIO.merge_kwargs( + default_kwargs=None, + runtime_kwargs={ + "video_backend": "pynvvideocodec", + "backend": "pynvvideocodec", + "num_frames": 4, + }, + ) + assert "video_backend" not in result + assert result.get("backend") != "pynvvideocodec" + assert result["num_frames"] == 4 + + def test_other_kwargs_preserved_when_gpu_backend_stripped(self): + result = VideoMediaIO.merge_kwargs( + default_kwargs={"fps": 2}, + runtime_kwargs={ + "video_backend": "pynvvideocodec", + "num_frames": 16, + }, + ) + assert "video_backend" not in result + assert result["num_frames"] == 16 + + def test_static_pynv_with_different_runtime_gpu_backend(self): + """If static sets pynv via video_backend but runtime tries to set it + via the codec-level 'backend' key (without a static match), strip it.""" + result = VideoMediaIO.merge_kwargs( + default_kwargs={"video_backend": "pynvvideocodec"}, + runtime_kwargs={"backend": "pynvvideocodec"}, + ) + assert result.get("backend") != "pynvvideocodec" + assert result["video_backend"] == "pynvvideocodec" diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py index 404f5a0e7c..45ea4c2fdf 100644 --- a/vllm/multimodal/media/video.py +++ b/vllm/multimodal/media/video.py @@ -10,11 +10,14 @@ import pybase64 from PIL import Image from vllm import envs +from vllm.logger import init_logger from ..video import VIDEO_LOADER_REGISTRY from .base import MediaIO from .image import ImageMediaIO +logger = init_logger(__name__) + class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]): """Configuration values can be user-provided either by --media-io-kwargs or @@ -28,6 +31,24 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]): default_kwargs: dict[str, Any] | None, runtime_kwargs: dict[str, Any] | None, ) -> dict[str, Any]: + if runtime_kwargs: + # Block request-level selection of GPU video backends that + # were not configured (and VRAM-reserved) at startup. + for key in ("video_backend", "backend"): + requested = runtime_kwargs.get(key) + if requested and VIDEO_LOADER_REGISTRY.backend_requires_gpu(requested): + static_val = (default_kwargs or {}).get(key) + if static_val != requested: + logger.warning_once( + "Stripping request-level %s=%r: GPU video " + "backend not configured at startup.", + key, + requested, + ) + runtime_kwargs = { + k: v for k, v in runtime_kwargs.items() if k != key + } + merged = super().merge_kwargs(default_kwargs, runtime_kwargs) # fps and num_frames interact with each other, so if either is # overridden at request time, wipe the other from defaults to diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index 725e33e3f8..33363ed403 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -39,6 +39,7 @@ class VideoLoaderRegistry(ExtensionManager): def __init__(self) -> None: super().__init__() self.processor2backend: dict[str, str] = {} + self._requires_gpu: dict[str, bool] = {} @staticmethod def _normalize_registered_video_processors( @@ -62,11 +63,13 @@ class VideoLoaderRegistry(ExtensionManager): name: str, *, video_processor: str | tuple[str, ...] | None = None, + requires_gpu: bool = False, ): processors = self._normalize_registered_video_processors(video_processor) def wrap(cls_to_register): self.name2class[name] = cls_to_register + self._requires_gpu[name] = requires_gpu for processor_name in processors: self.processor2backend[processor_name] = name return cls_to_register @@ -82,6 +85,9 @@ class VideoLoaderRegistry(ExtensionManager): return self.processor2backend.get(video_processor) + def backend_requires_gpu(self, name: str) -> bool: + return self._requires_gpu.get(name, False) + def get_video_loader_backend_for_processor( video_processor: str | None, @@ -909,7 +915,7 @@ class VideoBackend( ) -@VIDEO_LOADER_REGISTRY.register(PYNVVIDEOCODEC_VIDEO_BACKEND) +@VIDEO_LOADER_REGISTRY.register(PYNVVIDEOCODEC_VIDEO_BACKEND, requires_gpu=True) class PyNvVideoCodecVideoBackend(VideoBackend): """Hardware-accelerated video backend using PyNvVideoCodec. diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 28efee3dee..03433ed752 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -55,7 +55,7 @@ from vllm.multimodal.video import ( PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES, PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES, PYNVVIDEOCODEC_MAX_RETAINED_DECODERS, - PYNVVIDEOCODEC_VIDEO_BACKEND, + VIDEO_LOADER_REGISTRY, ) from vllm.platforms import current_platform from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper @@ -584,15 +584,15 @@ class Worker(WorkerBase): ) @staticmethod - def _uses_pynvvideocodec_video_backend(mm_config) -> bool: + def _uses_gpu_video_backend(mm_config) -> bool: video_kwargs = mm_config.media_io_kwargs.get("video", {}) video_loader_backend = ( video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND ) codec_backend = video_kwargs.get("backend") - return ( - video_loader_backend == PYNVVIDEOCODEC_VIDEO_BACKEND - or codec_backend == PYNVVIDEOCODEC_VIDEO_BACKEND + return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or ( + codec_backend is not None + and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend) ) def _reserve_mm_ipc_gpu_memory(self, available_kv_cache_memory_bytes: int) -> int: @@ -623,7 +623,7 @@ class Worker(WorkerBase): ) decoder_reserved_bytes = ( num_api_servers * per_server_decoder_bytes - if self._uses_pynvvideocodec_video_backend(mm_config) + if self._uses_gpu_video_backend(mm_config) else 0 ) reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes