From 94682b79f40239369bcefdbebee683a6bed90c85 Mon Sep 17 00:00:00 2001 From: Brandon Pelfrey Date: Fri, 24 Jul 2026 23:31:41 -0700 Subject: [PATCH] [multimodal] Make PyNvVideoCodec decoder concurrency configurable (#49753) Signed-off-by: Brandon Pelfrey Co-authored-by: OpenAI Codex --- docs/features/multimodal_inputs.md | 25 +++++++++-- tests/multimodal/media/test_video.py | 17 ++++++++ tests/multimodal/test_gpu_ipc_memory.py | 35 +++++++++++++-- tests/multimodal/test_video.py | 38 +++++++++++++++-- vllm/multimodal/gpu_ipc_memory.py | 57 +++++++++++++++++++++---- vllm/multimodal/media/video.py | 4 ++ vllm/multimodal/video.py | 36 ++++++++++++++-- 7 files changed, 189 insertions(+), 23 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index df33cea054..7f6c1aee76 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -818,16 +818,18 @@ Full example: [examples/generate/multimodal/openai_chat_completion_client_for_mu #### Video Decoding Backend -vLLM decodes video bytes into frames using a selectable decoding backend. Three +vLLM decodes video bytes into frames using a selectable decoding backend. Five backends are supported: - `opencv` (default): OpenCV-based decoder. - `pyav`: PyAV decoder. - `torchcodec`: TorchCodec (PyTorch-native) decoder. +- `pynvvideocodec`: NVIDIA NVDEC-based decoder. +- `deepstream`: NVIDIA DeepStream NVDEC-based decoder. -All three backends are ultimately backed by FFmpeg. `torchcodec` lets -you choose which FFmpeg version is used while `opencv` and `pyav` rely on -whichever FFmpeg build they were linked against. +The CPU backends are backed by FFmpeg. `torchcodec` lets you choose which FFmpeg +version is used while `opencv` and `pyav` rely on whichever FFmpeg build they +were linked against. Select the backend by passing the `backend` parameter via `--media-io-kwargs`: @@ -854,6 +856,21 @@ vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \ --media-io-kwargs '{"video": {"backend": "torchcodec", "seek_mode": "approximate", "num_ffmpeg_threads": 4}}' ``` +**PyNvVideoCodec-specific parameters:** + +- `hw_decoders`: Maximum number of concurrent hardware decoder slots retained + by each API server process. It must be a positive integer and defaults to `2`, + which is the recommended starting point for concurrent video workloads. + Because vLLM reserves GPU memory for these slots at startup, this value cannot + be overridden per request. Benchmark before increasing it because each + additional slot increases the GPU memory reservation. + +```bash +# Example: explicitly use the recommended 2 hardware decoders +vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \ + --media-io-kwargs '{"video": {"backend": "pynvvideocodec", "hw_decoders": 2}}' +``` + #### Video Frame Recovery For improved robustness when processing potentially corrupted or truncated video files, vLLM supports optional frame recovery using a dynamic window forward-scan approach. When enabled, if a target frame fails to load during sequential reading, the next successfully grabbed frame (before the next target frame) will be used in its place. diff --git a/tests/multimodal/media/test_video.py b/tests/multimodal/media/test_video.py index 671abd7b07..a5595121b2 100644 --- a/tests/multimodal/media/test_video.py +++ b/tests/multimodal/media/test_video.py @@ -404,6 +404,23 @@ class TestMergeKwargsGpuBackendPolicy: ) assert result["backend"] == "pynvvideocodec" + def test_strips_request_level_hw_decoders_when_not_static(self): + result = VideoMediaIO.merge_kwargs( + default_kwargs={"video_backend": "pynvvideocodec"}, + runtime_kwargs={"hw_decoders": 4}, + ) + assert "hw_decoders" not in result + + def test_prevents_request_level_hw_decoders_override(self): + result = VideoMediaIO.merge_kwargs( + default_kwargs={ + "video_backend": "pynvvideocodec", + "hw_decoders": 2, + }, + runtime_kwargs={"hw_decoders": 4}, + ) + assert result["hw_decoders"] == 2 + @pytest.mark.parametrize("backend", ["opencv", "pyav", "torchcodec"]) def test_software_video_backend_passes_through(self, backend: str): result = VideoMediaIO.merge_kwargs( diff --git a/tests/multimodal/test_gpu_ipc_memory.py b/tests/multimodal/test_gpu_ipc_memory.py index 5c0f263af9..c98fd7673c 100644 --- a/tests/multimodal/test_gpu_ipc_memory.py +++ b/tests/multimodal/test_gpu_ipc_memory.py @@ -18,7 +18,6 @@ from vllm.multimodal.gpu_ipc_memory import ( from vllm.multimodal.video import ( PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES, PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES, - PYNVVIDEOCODEC_MAX_RETAINED_DECODERS, PYNVVIDEOCODEC_VIDEO_BACKEND, ) from vllm.utils.mem_constants import GiB_bytes @@ -28,17 +27,26 @@ def _mm_config( *, mm_ipc_gpu_memory_gb: float = 0, video_backend: str | None = None, + hw_decoders: int | None = None, ) -> MultiModalConfig: - video_kwargs = {} if video_backend is None else {"video_backend": video_backend} + video_kwargs: dict[str, object] = ( + {} if video_backend is None else {"video_backend": video_backend} + ) + if hw_decoders is not None: + video_kwargs["hw_decoders"] = hw_decoders + return MultiModalConfig( mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb, media_io_kwargs={"video": video_kwargs} if video_kwargs else {}, ) -def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int: +def _pynvvideocodec_decoder_budget( + api_process_count: int = 1, + hw_decoders: int = 2, +) -> int: return api_process_count * ( - PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * hw_decoders + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES ) @@ -240,3 +248,22 @@ def test_reserve_mm_ipc_gpu_memory_scales_decoder_budget_by_api_servers( _mm_config(), api_process_count=3, ) == available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3) + + +def test_reserve_mm_ipc_gpu_memory_uses_configured_hw_decoders( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + multimodal_config_module.envs, + "VLLM_VIDEO_LOADER_BACKEND", + "opencv", + ) + available_bytes = 4 * GiB_bytes + mm_config = _mm_config( + video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND, + hw_decoders=3, + ) + + assert reserve_mm_ipc_gpu_memory(available_bytes, mm_config) == ( + available_bytes - _pynvvideocodec_decoder_budget(hw_decoders=3) + ) diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index e10d6e1338..f07a6d888e 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -16,7 +16,6 @@ from transformers.video_utils import VideoMetadata from vllm.assets.base import get_vllm_public_assets from vllm.multimodal.video import ( PYNVVIDEOCODEC_DECODER_CACHE_SIZE, - PYNVVIDEOCODEC_MAX_RETAINED_DECODERS, PYNVVIDEOCODEC_VIDEO_BACKEND, VIDEO_LOADER_REGISTRY, DynamicVideoBackend, @@ -26,6 +25,7 @@ from vllm.multimodal.video import ( PyNvVideoCodecVideoBackend, Qwen2VLVideoBackend, Qwen3VLVideoBackend, + VideoBackend, VideoLoader, VideoSourceMetadata, VideoTargetMetadata, @@ -199,7 +199,11 @@ def test_pynvvideocodec_codec_uses_dynamic_sampling_strategy( assert metadata["frames_indices"] == [0, 9] -def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatch): +@pytest.mark.parametrize("hw_decoders", [1, 3]) +def test_pynvvideocodec_decoder_slots_are_bounded( + monkeypatch: pytest.MonkeyPatch, + hw_decoders: int, +): class FakeSlot: pass @@ -207,10 +211,13 @@ def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatc old_slots = PyNvVideoCodecVideoBackend._decoder_slots old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond + old_max_slots = PyNvVideoCodecVideoBackend._max_decoder_slots try: PyNvVideoCodecVideoBackend._decoder_slots = [] PyNvVideoCodecVideoBackend._active_decoder_slots = 0 PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition() + PyNvVideoCodecVideoBackend._max_decoder_slots = None + PyNvVideoCodecVideoBackend._configure_decoder_slots(hw_decoders) def fake_create_slot(cls): nonlocal create_count @@ -229,7 +236,7 @@ def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatc with ExitStack() as stack: retained_slots = [ stack.enter_context(PyNvVideoCodecVideoBackend._borrow_decoder_slot()) - for _ in range(PYNVVIDEOCODEC_MAX_RETAINED_DECODERS) + for _ in range(hw_decoders) ] def borrow_extra_slot(): @@ -246,11 +253,34 @@ def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatc assert not thread.is_alive() assert seen_slots[0] in retained_slots - assert create_count == PYNVVIDEOCODEC_MAX_RETAINED_DECODERS + assert create_count == hw_decoders finally: PyNvVideoCodecVideoBackend._decoder_slots = old_slots PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond + PyNvVideoCodecVideoBackend._max_decoder_slots = old_max_slots + + +def test_pynvvideocodec_decoder_slots_are_configured_once( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(PyNvVideoCodecVideoBackend, "_max_decoder_slots", None) + + PyNvVideoCodecVideoBackend._configure_decoder_slots(2) + PyNvVideoCodecVideoBackend._configure_decoder_slots(2) + + with pytest.raises(RuntimeError, match="already configured as 2, got 3"): + PyNvVideoCodecVideoBackend._configure_decoder_slots(3) + + +@pytest.mark.parametrize("hw_decoders", [0, -1, 1.5, True, "2"]) +def test_pynvvideocodec_rejects_invalid_hw_decoders(hw_decoders: object): + with pytest.raises(ValueError, match="hw_decoders must be a positive integer"): + VideoBackend.load_bytes( + b"fake video", + backend=PYNVVIDEOCODEC_VIDEO_BACKEND, + hw_decoders=hw_decoders, # type: ignore[arg-type] + ) def test_pynvvideocodec_decoder_slot_retains_simple_decoder(): diff --git a/vllm/multimodal/gpu_ipc_memory.py b/vllm/multimodal/gpu_ipc_memory.py index 02317490a8..58cb988880 100644 --- a/vllm/multimodal/gpu_ipc_memory.py +++ b/vllm/multimodal/gpu_ipc_memory.py @@ -157,19 +157,43 @@ def reserve_mm_ipc_gpu_memory( mm_config: "MultiModalConfig | None", api_process_count: int = 1, ) -> int: - """Carve frontend multimodal GPU memory out of the KV cache. + """Return KV-cache memory remaining after frontend multimodal reservations. - Raw decoded frames are bounded by ``mm_ipc_gpu_memory_gb`` and acquired by - the frontend semaphore. Some decoders also keep persistent surfaces around; - reserve a fixed upper bound for those when a GPU backend is configured. + The reservation covers: + + * The total ``mm_ipc_gpu_memory_gb`` budget for transient decoded-frame + buffers. This budget is divided among API processes, so it is not + multiplied by ``api_process_count``. + * For GPU video backends, a fixed upper bound for each API process's + retained decoder surfaces and CUDA context. The PyNvVideoCodec surface + reservation scales with its configured ``hw_decoders`` value, and the + entire decoder reservation scales with ``api_process_count`` because + these resources are not shared between processes. + + Args: + available_kv_cache_memory_bytes: KV-cache capacity before reserving + memory for frontend multimodal processing. + mm_config: Multimodal configuration, or ``None`` when multimodal + processing is disabled. + api_process_count: Number of frontend API processes sharing the GPU. + Values below one are treated as one. + + Returns: + KV-cache capacity after subtracting the frontend reservation. + + Raises: + ValueError: If the reservation leaves no memory for the KV cache. """ if mm_config is None: return available_kv_cache_memory_bytes + from vllm import envs from vllm.multimodal.video import ( PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES, PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES, - PYNVVIDEOCODEC_MAX_RETAINED_DECODERS, + PYNVVIDEOCODEC_DEFAULT_HW_DECODERS, + PYNVVIDEOCODEC_VIDEO_BACKEND, + validate_pynvvideocodec_hw_decoders, ) raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes) @@ -177,8 +201,24 @@ def reserve_mm_ipc_gpu_memory( # context on the GPU, outside the worker memory pool. Reserve that footprint # per process so gpu_memory_utilization bounds total GPU usage across them. num_api_servers = max(1, api_process_count) + video_kwargs = mm_config.media_io_kwargs.get("video", {}) + video_loader_backend = ( + video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND + ) + codec_backend = video_kwargs.get("backend") + uses_pynvvideocodec = ( + video_loader_backend == PYNVVIDEOCODEC_VIDEO_BACKEND + or codec_backend == PYNVVIDEOCODEC_VIDEO_BACKEND + ) + hw_decoders = ( + validate_pynvvideocodec_hw_decoders( + video_kwargs.get("hw_decoders", PYNVVIDEOCODEC_DEFAULT_HW_DECODERS) + ) + if uses_pynvvideocodec + else 1 + ) per_server_decoder_bytes = ( - PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * hw_decoders + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES ) decoder_reserved_bytes = ( @@ -198,8 +238,9 @@ def reserve_mm_ipc_gpu_memory( f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, " f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), " f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is " - "available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a " - "different video backend, or increase gpu_memory_utilization." + "available for the KV cache. Reduce mm_ipc_gpu_memory_gb or " + "hw_decoders, use a different video backend, or increase " + "gpu_memory_utilization." ) logger.info_once( "Reserving %s GiB of GPU memory for frontend multimodal decoding " diff --git a/vllm/multimodal/media/video.py b/vllm/multimodal/media/video.py index 45ea4c2fdf..978cf06c23 100644 --- a/vllm/multimodal/media/video.py +++ b/vllm/multimodal/media/video.py @@ -32,6 +32,10 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]): runtime_kwargs: dict[str, Any] | None, ) -> dict[str, Any]: if runtime_kwargs: + # Decoder GPU memory is reserved from the startup value. + runtime_kwargs = dict(runtime_kwargs) + runtime_kwargs.pop("hw_decoders", None) + # Block request-level selection of GPU video backends that # were not configured (and VRAM-reserved) at startup. for key in ("video_backend", "backend"): diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index d289972341..3e0969f1ad 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -210,15 +210,25 @@ class VideoLoader: VIDEO_LOADER_REGISTRY = VideoLoaderRegistry() PYNVVIDEOCODEC_VIDEO_BACKEND: Literal["pynvvideocodec"] = "pynvvideocodec" -# Fixed upper bound reserved for persistent PyNvVideoCodec decoder surfaces. +# Per-decoder upper bound reserved for persistent PyNvVideoCodec surfaces. PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES = 128 * MiB_bytes PYNVVIDEOCODEC_DECODER_CACHE_SIZE = 2 -PYNVVIDEOCODEC_MAX_RETAINED_DECODERS = 1 +PYNVVIDEOCODEC_DEFAULT_HW_DECODERS = 2 # Per-API-server CUDA context and driver allocation, measured with # PyNvVideoCodec 2.0.4 on H100. PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES = int(1.8 * 1024 * MiB_bytes) +def validate_pynvvideocodec_hw_decoders(hw_decoders: object) -> int: + if ( + isinstance(hw_decoders, bool) + or not isinstance(hw_decoders, int) + or hw_decoders < 1 + ): + raise ValueError("hw_decoders must be a positive integer") + return hw_decoders + + class PyNvVideoCodecDecoderSlot: """A retained PyNv decoder slot and its CUDA stream. @@ -628,6 +638,7 @@ class PyNvVideoCodecVideoBackendMixin: _decoder_slots: ClassVar[list[PyNvVideoCodecDecoderSlot]] = [] _active_decoder_slots: ClassVar[int] = 0 _decoder_slot_cond: ClassVar[threading.Condition] = threading.Condition() + _max_decoder_slots: ClassVar[int | None] = None _DEVICE_INDEX: ClassVar[int] = 0 @classmethod @@ -651,6 +662,18 @@ class PyNvVideoCodecVideoBackendMixin: return PyNvVideoCodecDecoderSlot(torch.cuda.Stream(device=cls._DEVICE_INDEX)) + @classmethod + def _configure_decoder_slots(cls, hw_decoders: object) -> None: + hw_decoders = validate_pynvvideocodec_hw_decoders(hw_decoders) + with cls._decoder_slot_cond: + if cls._max_decoder_slots is None: + cls._max_decoder_slots = hw_decoders + elif cls._max_decoder_slots != hw_decoders: + raise RuntimeError( + "PyNvVideoCodec decoder count is already configured as " + f"{cls._max_decoder_slots}, got {hw_decoders}" + ) + @staticmethod @contextmanager def _torch_stream_context(stream): @@ -669,11 +692,14 @@ class PyNvVideoCodecVideoBackendMixin: def _borrow_decoder_slot(cls): create_slot = False with cls._decoder_slot_cond: + max_decoder_slots = cls._max_decoder_slots + if max_decoder_slots is None: + raise RuntimeError("PyNvVideoCodec decoder slots are not configured") while True: if cls._decoder_slots: slot = cls._decoder_slots.pop() break - if cls._active_decoder_slots < PYNVVIDEOCODEC_MAX_RETAINED_DECODERS: + if cls._active_decoder_slots < max_decoder_slots: cls._active_decoder_slots += 1 create_slot = True break @@ -999,6 +1025,7 @@ class VideoBackend( ] = "opencv", num_ffmpeg_threads: int = 0, seek_mode: Literal["exact", "approximate"] = "exact", + hw_decoders: int = PYNVVIDEOCODEC_DEFAULT_HW_DECODERS, **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: """Load sampled frames from raw video bytes. @@ -1025,6 +1052,8 @@ class VideoBackend( at the cost of relying on the file's metadata. See https://meta-pytorch.org/torchcodec/stable/generated_examples/decoding/approximate_mode.html for details. + hw_decoders: Maximum number of concurrent PyNvVideoCodec decoder + slots. Defaults to 2 and must be a positive integer. Returns: Tuple of ``(frames_array, metadata_dict)``. @@ -1088,6 +1117,7 @@ class VideoBackend( "frame_recovery is not supported for " f"`{PYNVVIDEOCODEC_VIDEO_BACKEND}` backend" ) + cls._configure_decoder_slots(hw_decoders) frames, source, frame_idx, valid = cls.decode_frames_pynvvideocodec( data, target,