[multimodal] Make PyNvVideoCodec decoder concurrency configurable (#49753)

Signed-off-by: Brandon Pelfrey <bpelfrey@nvidia.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>

docs/features/multimodal_inputs.md

@@ -818,16 +818,18 @@ Full example: [examples/generate/multimodal/openai_chat_completion_client_for_mu
 
 #### Video Decoding Backend
 
-vLLM decodes video bytes into frames using a selectable decoding backend. Three
+vLLM decodes video bytes into frames using a selectable decoding backend. Five
 backends are supported:
 
 - `opencv` (default): OpenCV-based decoder.
 - `pyav`: PyAV decoder.
 - `torchcodec`: TorchCodec (PyTorch-native) decoder.
+- `pynvvideocodec`: NVIDIA NVDEC-based decoder.
+- `deepstream`: NVIDIA DeepStream NVDEC-based decoder.
 
-All three backends are ultimately backed by FFmpeg. `torchcodec` lets
-you choose which FFmpeg version is used while `opencv` and `pyav` rely on
-whichever FFmpeg build they were linked against.
+The CPU backends are backed by FFmpeg. `torchcodec` lets you choose which FFmpeg
+version is used while `opencv` and `pyav` rely on whichever FFmpeg build they
+were linked against.
 
 Select the backend by passing the `backend` parameter via `--media-io-kwargs`:
 
@@ -854,6 +856,21 @@ vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \
   --media-io-kwargs '{"video": {"backend": "torchcodec", "seek_mode": "approximate", "num_ffmpeg_threads": 4}}'
 ```
 
+**PyNvVideoCodec-specific parameters:**
+
+- `hw_decoders`: Maximum number of concurrent hardware decoder slots retained
+  by each API server process. It must be a positive integer and defaults to `2`,
+  which is the recommended starting point for concurrent video workloads.
+  Because vLLM reserves GPU memory for these slots at startup, this value cannot
+  be overridden per request. Benchmark before increasing it because each
+  additional slot increases the GPU memory reservation.
+
+```bash
+# Example: explicitly use the recommended 2 hardware decoders
+vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \
+  --media-io-kwargs '{"video": {"backend": "pynvvideocodec", "hw_decoders": 2}}'
+```
+
 #### Video Frame Recovery
 
 For improved robustness when processing potentially corrupted or truncated video files, vLLM supports optional frame recovery using a dynamic window forward-scan approach. When enabled, if a target frame fails to load during sequential reading, the next successfully grabbed frame (before the next target frame) will be used in its place.

tests/multimodal/media/test_video.py

@@ -404,6 +404,23 @@ class TestMergeKwargsGpuBackendPolicy:
         )
         assert result["backend"] == "pynvvideocodec"
 
+    def test_strips_request_level_hw_decoders_when_not_static(self):
+        result = VideoMediaIO.merge_kwargs(
+            default_kwargs={"video_backend": "pynvvideocodec"},
+            runtime_kwargs={"hw_decoders": 4},
+        )
+        assert "hw_decoders" not in result
+
+    def test_prevents_request_level_hw_decoders_override(self):
+        result = VideoMediaIO.merge_kwargs(
+            default_kwargs={
+                "video_backend": "pynvvideocodec",
+                "hw_decoders": 2,
+            },
+            runtime_kwargs={"hw_decoders": 4},
+        )
+        assert result["hw_decoders"] == 2
+
     @pytest.mark.parametrize("backend", ["opencv", "pyav", "torchcodec"])
     def test_software_video_backend_passes_through(self, backend: str):
         result = VideoMediaIO.merge_kwargs(

tests/multimodal/test_gpu_ipc_memory.py

@@ -18,7 +18,6 @@ from vllm.multimodal.gpu_ipc_memory import (
 from vllm.multimodal.video import (
     PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
     PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
-    PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
     PYNVVIDEOCODEC_VIDEO_BACKEND,
 )
 from vllm.utils.mem_constants import GiB_bytes
@@ -28,17 +27,26 @@ def _mm_config(
     *,
     mm_ipc_gpu_memory_gb: float = 0,
     video_backend: str | None = None,
+    hw_decoders: int | None = None,
 ) -> MultiModalConfig:
-    video_kwargs = {} if video_backend is None else {"video_backend": video_backend}
+    video_kwargs: dict[str, object] = (
+        {} if video_backend is None else {"video_backend": video_backend}
+    )
+    if hw_decoders is not None:
+        video_kwargs["hw_decoders"] = hw_decoders
+
     return MultiModalConfig(
         mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
         media_io_kwargs={"video": video_kwargs} if video_kwargs else {},
     )
 
 
-def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:
+def _pynvvideocodec_decoder_budget(
+    api_process_count: int = 1,
+    hw_decoders: int = 2,
+) -> int:
     return api_process_count * (
-        PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
+        PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * hw_decoders
         + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
     )
 
@@ -240,3 +248,22 @@ def test_reserve_mm_ipc_gpu_memory_scales_decoder_budget_by_api_servers(
         _mm_config(),
         api_process_count=3,
     ) == available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)
+
+
+def test_reserve_mm_ipc_gpu_memory_uses_configured_hw_decoders(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    monkeypatch.setattr(
+        multimodal_config_module.envs,
+        "VLLM_VIDEO_LOADER_BACKEND",
+        "opencv",
+    )
+    available_bytes = 4 * GiB_bytes
+    mm_config = _mm_config(
+        video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
+        hw_decoders=3,
+    )
+
+    assert reserve_mm_ipc_gpu_memory(available_bytes, mm_config) == (
+        available_bytes - _pynvvideocodec_decoder_budget(hw_decoders=3)
+    )

tests/multimodal/test_video.py

@@ -16,7 +16,6 @@ from transformers.video_utils import VideoMetadata
 from vllm.assets.base import get_vllm_public_assets
 from vllm.multimodal.video import (
     PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
-    PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
     PYNVVIDEOCODEC_VIDEO_BACKEND,
     VIDEO_LOADER_REGISTRY,
     DynamicVideoBackend,
@@ -26,6 +25,7 @@ from vllm.multimodal.video import (
     PyNvVideoCodecVideoBackend,
     Qwen2VLVideoBackend,
     Qwen3VLVideoBackend,
+    VideoBackend,
     VideoLoader,
     VideoSourceMetadata,
     VideoTargetMetadata,
@@ -199,7 +199,11 @@ def test_pynvvideocodec_codec_uses_dynamic_sampling_strategy(
     assert metadata["frames_indices"] == [0, 9]
 
 
-def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatch):
+@pytest.mark.parametrize("hw_decoders", [1, 3])
+def test_pynvvideocodec_decoder_slots_are_bounded(
+    monkeypatch: pytest.MonkeyPatch,
+    hw_decoders: int,
+):
     class FakeSlot:
         pass
 
@@ -207,10 +211,13 @@ def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatc
     old_slots = PyNvVideoCodecVideoBackend._decoder_slots
     old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots
     old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond
+    old_max_slots = PyNvVideoCodecVideoBackend._max_decoder_slots
     try:
         PyNvVideoCodecVideoBackend._decoder_slots = []
         PyNvVideoCodecVideoBackend._active_decoder_slots = 0
         PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition()
+        PyNvVideoCodecVideoBackend._max_decoder_slots = None
+        PyNvVideoCodecVideoBackend._configure_decoder_slots(hw_decoders)
 
         def fake_create_slot(cls):
             nonlocal create_count
@@ -229,7 +236,7 @@ def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatc
         with ExitStack() as stack:
             retained_slots = [
                 stack.enter_context(PyNvVideoCodecVideoBackend._borrow_decoder_slot())
-                for _ in range(PYNVVIDEOCODEC_MAX_RETAINED_DECODERS)
+                for _ in range(hw_decoders)
             ]
 
             def borrow_extra_slot():
@@ -246,11 +253,34 @@ def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatc
         assert not thread.is_alive()
 
         assert seen_slots[0] in retained_slots
-        assert create_count == PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
+        assert create_count == hw_decoders
     finally:
         PyNvVideoCodecVideoBackend._decoder_slots = old_slots
         PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots
         PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond
+        PyNvVideoCodecVideoBackend._max_decoder_slots = old_max_slots
+
+
+def test_pynvvideocodec_decoder_slots_are_configured_once(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    monkeypatch.setattr(PyNvVideoCodecVideoBackend, "_max_decoder_slots", None)
+
+    PyNvVideoCodecVideoBackend._configure_decoder_slots(2)
+    PyNvVideoCodecVideoBackend._configure_decoder_slots(2)
+
+    with pytest.raises(RuntimeError, match="already configured as 2, got 3"):
+        PyNvVideoCodecVideoBackend._configure_decoder_slots(3)
+
+
+@pytest.mark.parametrize("hw_decoders", [0, -1, 1.5, True, "2"])
+def test_pynvvideocodec_rejects_invalid_hw_decoders(hw_decoders: object):
+    with pytest.raises(ValueError, match="hw_decoders must be a positive integer"):
+        VideoBackend.load_bytes(
+            b"fake video",
+            backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
+            hw_decoders=hw_decoders,  # type: ignore[arg-type]
+        )
 
 
 def test_pynvvideocodec_decoder_slot_retains_simple_decoder():

vllm/multimodal/gpu_ipc_memory.py

@@ -157,19 +157,43 @@ def reserve_mm_ipc_gpu_memory(
     mm_config: "MultiModalConfig | None",
     api_process_count: int = 1,
 ) -> int:
-    """Carve frontend multimodal GPU memory out of the KV cache.
+    """Return KV-cache memory remaining after frontend multimodal reservations.
 
-    Raw decoded frames are bounded by ``mm_ipc_gpu_memory_gb`` and acquired by
-    the frontend semaphore. Some decoders also keep persistent surfaces around;
-    reserve a fixed upper bound for those when a GPU backend is configured.
+    The reservation covers:
+
+    * The total ``mm_ipc_gpu_memory_gb`` budget for transient decoded-frame
+      buffers. This budget is divided among API processes, so it is not
+      multiplied by ``api_process_count``.
+    * For GPU video backends, a fixed upper bound for each API process's
+      retained decoder surfaces and CUDA context. The PyNvVideoCodec surface
+      reservation scales with its configured ``hw_decoders`` value, and the
+      entire decoder reservation scales with ``api_process_count`` because
+      these resources are not shared between processes.
+
+    Args:
+        available_kv_cache_memory_bytes: KV-cache capacity before reserving
+            memory for frontend multimodal processing.
+        mm_config: Multimodal configuration, or ``None`` when multimodal
+            processing is disabled.
+        api_process_count: Number of frontend API processes sharing the GPU.
+            Values below one are treated as one.
+
+    Returns:
+        KV-cache capacity after subtracting the frontend reservation.
+
+    Raises:
+        ValueError: If the reservation leaves no memory for the KV cache.
     """
     if mm_config is None:
         return available_kv_cache_memory_bytes
 
+    from vllm import envs
     from vllm.multimodal.video import (
         PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
         PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
-        PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
+        PYNVVIDEOCODEC_DEFAULT_HW_DECODERS,
+        PYNVVIDEOCODEC_VIDEO_BACKEND,
+        validate_pynvvideocodec_hw_decoders,
     )
 
     raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes)
@@ -177,8 +201,24 @@ def reserve_mm_ipc_gpu_memory(
     # context on the GPU, outside the worker memory pool. Reserve that footprint
     # per process so gpu_memory_utilization bounds total GPU usage across them.
     num_api_servers = max(1, api_process_count)
+    video_kwargs = mm_config.media_io_kwargs.get("video", {})
+    video_loader_backend = (
+        video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
+    )
+    codec_backend = video_kwargs.get("backend")
+    uses_pynvvideocodec = (
+        video_loader_backend == PYNVVIDEOCODEC_VIDEO_BACKEND
+        or codec_backend == PYNVVIDEOCODEC_VIDEO_BACKEND
+    )
+    hw_decoders = (
+        validate_pynvvideocodec_hw_decoders(
+            video_kwargs.get("hw_decoders", PYNVVIDEOCODEC_DEFAULT_HW_DECODERS)
+        )
+        if uses_pynvvideocodec
+        else 1
+    )
     per_server_decoder_bytes = (
-        PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
+        PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * hw_decoders
         + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
     )
     decoder_reserved_bytes = (
@@ -198,8 +238,9 @@ def reserve_mm_ipc_gpu_memory(
             f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, "
             f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), "
             f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is "
-            "available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a "
-            "different video backend, or increase gpu_memory_utilization."
+            "available for the KV cache. Reduce mm_ipc_gpu_memory_gb or "
+            "hw_decoders, use a different video backend, or increase "
+            "gpu_memory_utilization."
         )
     logger.info_once(
         "Reserving %s GiB of GPU memory for frontend multimodal decoding "

vllm/multimodal/media/video.py

@@ -32,6 +32,10 @@ class VideoMediaIO(MediaIO[tuple[npt.NDArray, dict[str, Any]]]):
         runtime_kwargs: dict[str, Any] | None,
     ) -> dict[str, Any]:
         if runtime_kwargs:
+            # Decoder GPU memory is reserved from the startup value.
+            runtime_kwargs = dict(runtime_kwargs)
+            runtime_kwargs.pop("hw_decoders", None)
+
             # Block request-level selection of GPU video backends that
             # were not configured (and VRAM-reserved) at startup.
             for key in ("video_backend", "backend"):

vllm/multimodal/video.py

@@ -210,15 +210,25 @@ class VideoLoader:
 VIDEO_LOADER_REGISTRY = VideoLoaderRegistry()
 
 PYNVVIDEOCODEC_VIDEO_BACKEND: Literal["pynvvideocodec"] = "pynvvideocodec"
-# Fixed upper bound reserved for persistent PyNvVideoCodec decoder surfaces.
+# Per-decoder upper bound reserved for persistent PyNvVideoCodec surfaces.
 PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES = 128 * MiB_bytes
 PYNVVIDEOCODEC_DECODER_CACHE_SIZE = 2
-PYNVVIDEOCODEC_MAX_RETAINED_DECODERS = 1
+PYNVVIDEOCODEC_DEFAULT_HW_DECODERS = 2
 # Per-API-server CUDA context and driver allocation, measured with
 # PyNvVideoCodec 2.0.4 on H100.
 PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES = int(1.8 * 1024 * MiB_bytes)
 
 
+def validate_pynvvideocodec_hw_decoders(hw_decoders: object) -> int:
+    if (
+        isinstance(hw_decoders, bool)
+        or not isinstance(hw_decoders, int)
+        or hw_decoders < 1
+    ):
+        raise ValueError("hw_decoders must be a positive integer")
+    return hw_decoders
+
+
 class PyNvVideoCodecDecoderSlot:
     """A retained PyNv decoder slot and its CUDA stream.
 
@@ -628,6 +638,7 @@ class PyNvVideoCodecVideoBackendMixin:
     _decoder_slots: ClassVar[list[PyNvVideoCodecDecoderSlot]] = []
     _active_decoder_slots: ClassVar[int] = 0
     _decoder_slot_cond: ClassVar[threading.Condition] = threading.Condition()
+    _max_decoder_slots: ClassVar[int | None] = None
     _DEVICE_INDEX: ClassVar[int] = 0
 
     @classmethod
@@ -651,6 +662,18 @@ class PyNvVideoCodecVideoBackendMixin:
 
         return PyNvVideoCodecDecoderSlot(torch.cuda.Stream(device=cls._DEVICE_INDEX))
 
+    @classmethod
+    def _configure_decoder_slots(cls, hw_decoders: object) -> None:
+        hw_decoders = validate_pynvvideocodec_hw_decoders(hw_decoders)
+        with cls._decoder_slot_cond:
+            if cls._max_decoder_slots is None:
+                cls._max_decoder_slots = hw_decoders
+            elif cls._max_decoder_slots != hw_decoders:
+                raise RuntimeError(
+                    "PyNvVideoCodec decoder count is already configured as "
+                    f"{cls._max_decoder_slots}, got {hw_decoders}"
+                )
+
     @staticmethod
     @contextmanager
     def _torch_stream_context(stream):
@@ -669,11 +692,14 @@ class PyNvVideoCodecVideoBackendMixin:
     def _borrow_decoder_slot(cls):
         create_slot = False
         with cls._decoder_slot_cond:
+            max_decoder_slots = cls._max_decoder_slots
+            if max_decoder_slots is None:
+                raise RuntimeError("PyNvVideoCodec decoder slots are not configured")
             while True:
                 if cls._decoder_slots:
                     slot = cls._decoder_slots.pop()
                     break
-                if cls._active_decoder_slots < PYNVVIDEOCODEC_MAX_RETAINED_DECODERS:
+                if cls._active_decoder_slots < max_decoder_slots:
                     cls._active_decoder_slots += 1
                     create_slot = True
                     break
@@ -999,6 +1025,7 @@ class VideoBackend(
         ] = "opencv",
         num_ffmpeg_threads: int = 0,
         seek_mode: Literal["exact", "approximate"] = "exact",
+        hw_decoders: int = PYNVVIDEOCODEC_DEFAULT_HW_DECODERS,
         **kwargs,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         """Load sampled frames from raw video bytes.
@@ -1025,6 +1052,8 @@ class VideoBackend(
                 at the cost of relying on the file's metadata. See
                 https://meta-pytorch.org/torchcodec/stable/generated_examples/decoding/approximate_mode.html
                 for details.
+            hw_decoders: Maximum number of concurrent PyNvVideoCodec decoder
+                slots. Defaults to 2 and must be a positive integer.
 
         Returns:
             Tuple of ``(frames_array, metadata_dict)``.
@@ -1088,6 +1117,7 @@ class VideoBackend(
                     "frame_recovery is not supported for "
                     f"`{PYNVVIDEOCODEC_VIDEO_BACKEND}` backend"
                 )
+            cls._configure_decoder_slots(hw_decoders)
             frames, source, frame_idx, valid = cls.decode_frames_pynvvideocodec(
                 data,
                 target,