Add TorchCodec as a video decoding backend (#46609)

Signed-off-by: Nicolas Hug <contact@nicolas-hug.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>

docs/features/multimodal_inputs.md

@@ -816,6 +816,44 @@ Full example: [examples/generate/multimodal/openai_chat_completion_client_for_mu
     export VLLM_VIDEO_FETCH_TIMEOUT=<timeout>
     ```
 
+#### Video Decoding Backend
+
+vLLM decodes video bytes into frames using a selectable decoding backend. Three
+backends are supported:
+
+- `opencv` (default): OpenCV-based decoder.
+- `pyav`: PyAV decoder.
+- `torchcodec`: TorchCodec (PyTorch-native) decoder.
+
+All three backends are ultimately backed by FFmpeg. `torchcodec` lets
+you choose which FFmpeg version is used while `opencv` and `pyav` rely on
+whichever FFmpeg build they were linked against.
+
+Select the backend by passing the `backend` parameter via `--media-io-kwargs`:
+
+```bash
+vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \
+  --media-io-kwargs '{"video": {"backend": "torchcodec"}}'
+```
+
+**TorchCodec-specific parameters:**
+
+The following parameters only apply to the `torchcodec` backend:
+
+- `num_ffmpeg_threads`: Number of FFmpeg decoding threads. `0` (default) relies
+  on the FFmpeg default, which is `min(cpu_count + 1, 16)`. This allows you to
+  control thread over-subscription.
+- `seek_mode`: Seek mode for the decoder. `"exact"` (default) guarantees
+  frame-accurate sampling by scanning the file when the decoder is created.
+  `"approximate"` skips that scan for faster decoder creation, at the cost of
+  relying on the file's metadata (which may yield less accurate seeking).
+
+```bash
+# Example: TorchCodec with approximate seek mode and 4 FFmpeg threads
+vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \
+  --media-io-kwargs '{"video": {"backend": "torchcodec", "seek_mode": "approximate", "num_ffmpeg_threads": 4}}'
+```
+
 #### Video Frame Recovery
 
 For improved robustness when processing potentially corrupted or truncated video files, vLLM supports optional frame recovery using a dynamic window forward-scan approach. When enabled, if a target frame fails to load during sequential reading, the next successfully grabbed frame (before the next target frame) will be used in its place.

requirements/cpu.txt

@@ -15,6 +15,9 @@ torchaudio; platform_machine != "s390x" and platform_machine != "riscv64"
 # required for the image processor of phi3v, this must be updated alongside torch
 torchvision; platform_machine != "s390x"  and platform_machine != "riscv64"
 
+# required for the torchcodec video decoding backend
+torchcodec >= 0.14; platform_machine != "s390x" and platform_machine != "riscv64" and platform_machine != "ppc64le"
+
 # Intel Extension for PyTorch, only for x86_64 CPUs
 intel-openmp==2024.2.1; platform_machine == "x86_64"
 

requirements/cuda.txt

@@ -8,6 +8,7 @@ torch==2.11.0
 torchaudio==2.11.0
 # These must be updated alongside torch
 torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
+torchcodec >= 0.14
 PyNvVideoCodec==2.1.0
 # FlashInfer should be updated together with the Dockerfile
 flashinfer-python==0.6.13

requirements/test/cpu.txt

@@ -1134,6 +1134,8 @@ torchaudio==2.11.0+cpu
     #   -r requirements/test/cuda.in
     #   encodec
     #   vocos
+torchcodec==0.14.0+cpu
+    # via -r requirements/test/cuda.in
 torchvision==0.26.0+cpu
     # via
     #   -r requirements/test/cuda.in

requirements/test/cuda.in

@@ -13,6 +13,7 @@ pytest-cov
 # testing utils
 albumentations # required for Nemotron Parse in test_common.py
 av  # required for audio_in_video tests
+torchcodec >= 0.14 # required for torchcodec video backend tests
 backoff # required for phi4mm test
 blobfile # required for kimi-vl test
 httpx

requirements/test/cuda.txt

@@ -1233,6 +1233,10 @@ torchaudio==2.11.0+cu130
     #   -r requirements/test/cuda.in
     #   encodec
     #   vocos
+torchcodec==0.14.0+cu130
+    # via
+    #   -c requirements/cuda.txt
+    #   -r requirements/test/cuda.in
 torchvision==0.26.0+cu130
     # via
     #   -c requirements/cuda.txt

requirements/xpu.txt

@@ -15,6 +15,7 @@ numba == 0.65.0 # Required for N-gram speculative decoding
 torch==2.12.0
 torchaudio
 torchvision
+torchcodec >= 0.14 # Required for the torchcodec video decoding backend
 
 auto_round_lib>=0.14.0
 vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.10.1/vllm_xpu_kernels-0.1.10.1-cp38-abi3-manylinux_2_28_x86_64.whl

tests/multimodal/test_video.py

@@ -768,6 +768,118 @@ def test_pyav_backend_returns_target_frames_not_keyframes():
         )
 
 
+# ============================================================================
+# TorchCodec Backend Tests
+# ============================================================================
+
+
+def test_torchcodec_backend_loads_frames(
+    dummy_video_path, monkeypatch: pytest.MonkeyPatch
+):
+    """Test that the torchcodec codec backend can load frames."""
+    pytest.importorskip("torchcodec")
+    with monkeypatch.context() as m:
+        m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
+
+        with open(dummy_video_path, "rb") as f:
+            video_data = f.read()
+
+        loader = VIDEO_LOADER_REGISTRY.load("opencv")
+        frames, metadata = loader.load_bytes(
+            video_data, num_frames=8, backend="torchcodec"
+        )
+
+        assert frames.ndim == 4
+        assert frames.shape[3] == 3  # RGB
+        assert frames.shape[0] == 8
+        assert frames.shape[0] == len(metadata["frames_indices"])
+        assert metadata["video_backend"] == "torchcodec"
+        assert "total_num_frames" in metadata
+        assert "fps" in metadata
+        assert "duration" in metadata
+
+
+def test_torchcodec_dynamic_backend_loads_frames(
+    dummy_video_path, monkeypatch: pytest.MonkeyPatch
+):
+    """Test that the torchcodec codec with dynamic sampling can load frames."""
+    pytest.importorskip("torchcodec")
+    with monkeypatch.context() as m:
+        m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic")
+
+        with open(dummy_video_path, "rb") as f:
+            video_data = f.read()
+
+        loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
+        frames, metadata = loader.load_bytes(
+            video_data, fps=2, max_duration=10, backend="torchcodec"
+        )
+
+        assert frames.ndim == 4
+        assert frames.shape[3] == 3  # RGB
+        assert frames.shape[0] > 0
+        assert frames.shape[0] == len(metadata["frames_indices"])
+        assert metadata["video_backend"] == "torchcodec_dynamic"
+
+
+def test_torchcodec_backend_rejects_frame_recovery(dummy_video_path):
+    """frame_recovery is OpenCV-only; torchcodec must reject it."""
+    pytest.importorskip("torchcodec")
+    with open(dummy_video_path, "rb") as f:
+        video_data = f.read()
+
+    loader = VIDEO_LOADER_REGISTRY.load("opencv")
+    with pytest.raises(AssertionError):
+        loader.load_bytes(
+            video_data, num_frames=8, backend="torchcodec", frame_recovery=True
+        )
+
+
+def test_torchcodec_backend_returns_target_frames_not_keyframes():
+    """Regression test: torchcodec must return the requested frames, not the
+    GOP keyframe they seek back to.
+
+    Mirrors ``test_pyav_backend_returns_target_frames_not_keyframes``: a long
+    GOP (single keyframe at frame 0) with a per-frame green-channel marker.
+    With ``seek_mode="exact"`` torchcodec resolves each index to the exact
+    frame, so the returned markers must be distinct, ordered, and match the
+    requested indices.
+    """
+    pytest.importorskip("torchcodec")
+    num_frames = 50
+    num_sampled = 4
+    height, width = 64, 64
+
+    video_bytes = create_long_gop_video(
+        num_frames=num_frames, width=width, height=height
+    )
+
+    loader = VIDEO_LOADER_REGISTRY.load("opencv")
+    frames, metadata = loader.load_bytes(
+        video_bytes, num_frames=num_sampled, backend="torchcodec"
+    )
+    assert frames.shape == (num_sampled, height, width, 3)
+
+    requested = list(metadata["frames_indices"])
+    assert len(requested) == num_sampled
+
+    actual = [int(f[height // 2, width // 2, 1]) for f in frames]
+
+    assert len(set(actual)) == num_sampled, (
+        f"torchcodec returned only {len(set(actual))} distinct frames for "
+        f"{num_sampled} requested indices: markers={actual}, "
+        f"requested={requested}. Keyframe-snap regression."
+    )
+
+    assert actual == sorted(actual), f"Returned frames out of order: markers={actual}"
+
+    for marker, want_idx in zip(actual, requested):
+        assert abs(marker - want_idx) <= 10, (
+            f"Frame mismatch: requested index {want_idx}, "
+            f"got marker {marker} (tolerance ±10)"
+        )
+
+
 @pytest.mark.parametrize(
     "loader_key, kwargs, expected_num_frames",
     [
@@ -854,6 +966,42 @@ def test_pyav_backend_returns_target_frames_not_keyframes():
             120,
             id="glm46v-pyav-60s",
         ),
+        # uniform sampling + torchcodec codec (same frame counts as opencv)
+        pytest.param(
+            "opencv",
+            {"num_frames": 32, "backend": "torchcodec"},
+            32,
+            id="torchcodec-num_frames",
+        ),
+        pytest.param(
+            "opencv", {"fps": 2, "backend": "torchcodec"}, 120, id="torchcodec-fps"
+        ),
+        pytest.param(
+            "opencv",
+            {"num_frames": 500, "fps": 2, "backend": "torchcodec"},
+            120,
+            id="torchcodec-num_frames_wins_fps",
+        ),
+        # dynamic sampling + torchcodec codec
+        pytest.param(
+            "opencv_dynamic",
+            {"fps": 1, "max_duration": 60, "backend": "torchcodec"},
+            60,
+            id="torchcodec_dynamic-within_max_duration",
+        ),
+        pytest.param(
+            "opencv_dynamic",
+            {"fps": 2, "max_duration": 30, "backend": "torchcodec"},
+            60,
+            id="torchcodec_dynamic-exceeds_max_duration",
+        ),
+        # glm46v dynamic FPS + torchcodec codec
+        pytest.param(
+            "glm46v",
+            {"backend": "torchcodec"},
+            120,
+            id="glm46v-torchcodec-60s",
+        ),
     ],
 )
 def test_video_loader_frames_sampling(
@@ -864,6 +1012,8 @@ def test_video_loader_frames_sampling(
     expected_num_frames: int,
 ):
     """Test video loader frames sampling functionality."""
+    if kwargs.get("backend") == "torchcodec":
+        pytest.importorskip("torchcodec")
     monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", loader_key)
     loader = VIDEO_LOADER_REGISTRY.load(loader_key)
 

vllm/multimodal/video.py

@@ -31,6 +31,13 @@ try:
 except ImportError:
     av = PlaceholderModule("av")  # type: ignore[assignment]
 
+try:
+    from torchcodec.decoders import VideoDecoder
+except ImportError:
+    VideoDecoder = PlaceholderModule("torchcodec").placeholder_attr(  # type: ignore[assignment]
+        "decoders.VideoDecoder"
+    )
+
 
 logger = init_logger(__name__)
 
@@ -568,6 +575,53 @@ class PyAVVideoBackendMixin:
         return np.stack(frames_list), valid_indices
 
 
+class TorchCodecVideoBackendMixin:
+    """TorchCodec (FFmpeg-backed, PyTorch-native) codec utilities.
+
+    Builds a :class:`~torchcodec.decoders.VideoDecoder` over the in-memory
+    bytes and extracts the sampled indices with a single batched
+    ``get_frames_at`` call, while releasing the GIL during decode.
+    """
+
+    @staticmethod
+    def make_torchcodec_decoder(
+        data: bytes,
+        *,
+        num_ffmpeg_threads: int = 0,
+        seek_mode: Literal["exact", "approximate"] = "exact",
+    ) -> "VideoDecoder":
+        # NHWC matches the (num_frames, H, W, 3) uint8 RGB layout the rest
+        # of the pipeline expects, avoiding a transpose.
+        return VideoDecoder(
+            data,
+            dimension_order="NHWC",
+            num_ffmpeg_threads=num_ffmpeg_threads,
+            seek_mode=seek_mode,
+        )
+
+    @staticmethod
+    def get_torchcodec_metadata(decoder: "VideoDecoder") -> VideoSourceMetadata:
+        md = decoder.metadata
+        total_frames = md.num_frames or 0
+        fps = float(md.average_fps) if md.average_fps else 0.0
+        duration = float(md.duration_seconds) if md.duration_seconds else 0.0
+        if total_frames == 0 and duration > 0 and fps > 0:
+            total_frames = int(duration * fps)
+        return VideoSourceMetadata(total_frames, fps, duration)
+
+    @staticmethod
+    def decode_torchcodec_frames(
+        decoder: "VideoDecoder",
+        frame_indices: list[int],
+    ) -> tuple[npt.NDArray, list[int]]:
+        """Decode the requested indices in one batched, index-exact call."""
+        if not frame_indices:
+            return np.empty((0,), dtype=np.uint8), []
+        # Note: torchcodec releases the GIL for the entire call
+        batch = decoder.get_frames_at(frame_indices)
+        return batch.data.numpy(), list(frame_indices)
+
+
 class PyNvVideoCodecVideoBackendMixin:
     """PyNvVideoCodec utilities for GPU-backed frame decode."""
 
@@ -777,14 +831,15 @@ class VideoBackend(
     VideoLoader,
     OpenCVVideoBackendMixin,
     PyAVVideoBackendMixin,
+    TorchCodecVideoBackendMixin,
     PyNvVideoCodecVideoBackendMixin,
 ):
     """Uniform-sampling video backend.
 
     Samples ``num_frames`` uniformly across the video (or one frame every
     ``1/fps`` seconds, whichever produces fewer frames). The decoding codec
-    is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``, or
-    ``"pynvvideocodec"``), which can be passed through
+    is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``,
+    ``"torchcodec"`` or ``"pynvvideocodec"``), which can be passed through
     ``--media-io-kwargs``. Defaults to ``"opencv"``.
     """
 
@@ -830,7 +885,9 @@ class VideoBackend(
         max_duration: int = 300,
         frame_recovery: bool = False,
         *,
-        backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
+        backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
+        num_ffmpeg_threads: int = 0,
+        seek_mode: Literal["exact", "approximate"] = "exact",
         **kwargs,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         """Load sampled frames from raw video bytes.
@@ -843,8 +900,20 @@ class VideoBackend(
                 dynamic subclass; ignored here.
             frame_recovery: Enable forward-scan recovery for failed frames.
                 Only honored by the OpenCV codec.
-            backend: Decoding codec — ``"opencv"``, ``"pyav"``, or
-                ``"pynvvideocodec"``.
+            backend: Decoding codec — ``"opencv"``, ``"pyav"``,
+                ``"torchcodec"`` or ``"pynvvideocodec"``.
+            num_ffmpeg_threads: Number of FFmpeg decoding threads, only used by
+                TorchCodec: ``0`` (default) relies on the FFmpeg default value
+                which is ``min(cpu_count + 1, 16)``.
+                OpenCV will always use ``min(cpu_count, 16)`` while pyav will
+                always use ``min(cpu_count, (height + 15) / 16)``.
+            seek_mode: Seek mode for the TorchCodec decoder, only used by
+                TorchCodec: ``"exact"`` (default) guarantees frame-accurate
+                sampling by scanning the file on creation, while
+                ``"approximate"`` skips that scan for faster decoder creation
+                at the cost of relying on the file's metadata. See
+                https://meta-pytorch.org/torchcodec/stable/generated_examples/decoding/approximate_mode.html
+                for details.
 
         Returns:
             Tuple of ``(frames_array, metadata_dict)``.
@@ -883,6 +952,24 @@ class VideoBackend(
                 frames, valid = cls.decode_frames(
                     container, frame_idx, source.original_fps, source.duration
                 )
+        elif backend == "torchcodec":
+            assert not frame_recovery, (
+                "frame_recovery is only available for `opencv` backend"
+            )
+            decoder = cls.make_torchcodec_decoder(
+                data,
+                num_ffmpeg_threads=num_ffmpeg_threads,
+                seek_mode=seek_mode,
+            )
+            _check_frame_pixel_limit(
+                decoder.metadata.width or 0,
+                decoder.metadata.height or 0,
+            )
+            source = cls._prepare_source(cls.get_torchcodec_metadata(decoder))
+            frame_idx = cls.compute_frames_index_to_sample(
+                source=source, target=target, **kwargs
+            )
+            frames, valid = cls.decode_torchcodec_frames(decoder, frame_idx)
         elif backend == PYNVVIDEOCODEC_VIDEO_BACKEND:
             if frame_recovery:
                 raise ValueError(
@@ -897,7 +984,8 @@ class VideoBackend(
         else:
             raise ValueError(
                 f"Unknown video codec backend {backend!r}; "
-                "valid options: 'opencv', 'pyav', 'pynvvideocodec'."
+                "valid options: 'opencv', 'pyav', 'torchcodec', "
+                "'pynvvideocodec'."
             )
 
         if len(valid) < len(frame_idx):
@@ -984,7 +1072,7 @@ class Qwen3VLVideoBackend(VideoBackend):
         max_duration: int = 300,
         frame_recovery: bool = False,
         *,
-        backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
+        backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
         **kwargs,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         return super().load_bytes(
@@ -1063,7 +1151,7 @@ class Qwen2VLVideoBackend(VideoBackend):
         max_duration: int = 300,
         frame_recovery: bool = False,
         *,
-        backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
+        backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
         **kwargs,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         return super().load_bytes(
@@ -1155,7 +1243,7 @@ class DynamicVideoBackend(VideoBackend):
         max_duration: int = 300,
         frame_recovery: bool = False,
         *,
-        backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
+        backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
         **kwargs,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         return super().load_bytes(
@@ -1280,7 +1368,7 @@ class GLM46VVideoBackend(VideoBackend):
         max_duration: int = 300,
         frame_recovery: bool = False,
         *,
-        backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
+        backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
         **kwargs,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         return super().load_bytes(
@@ -1378,7 +1466,7 @@ class GLMGAVideoBackend(VideoBackend):
         max_duration: int = 300,
         frame_recovery: bool = False,
         *,
-        backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
+        backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
         **kwargs,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         frames, metadata = super().load_bytes(
@@ -1701,7 +1789,7 @@ class NemotronVLVideoBackend(VideoBackend):
         max_duration: int = 300,
         frame_recovery: bool = False,
         *,
-        backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
+        backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv",
         **kwargs,
     ) -> tuple[npt.NDArray, dict[str, Any]]:
         frames, metadata = super().load_bytes(