From 700e882eab7599ef59c42eb11d26d9879044d059 Mon Sep 17 00:00:00 2001 From: Nicolas Hug Date: Tue, 7 Jul 2026 03:58:51 +0100 Subject: [PATCH] Add TorchCodec as a video decoding backend (#46609) Signed-off-by: Nicolas Hug Co-authored-by: Claude Co-authored-by: Isotr0py --- docs/features/multimodal_inputs.md | 38 ++++++++ requirements/cpu.txt | 3 + requirements/cuda.txt | 1 + requirements/test/cpu.txt | 2 + requirements/test/cuda.in | 1 + requirements/test/cuda.txt | 4 + requirements/xpu.txt | 1 + tests/multimodal/test_video.py | 150 +++++++++++++++++++++++++++++ vllm/multimodal/video.py | 112 ++++++++++++++++++--- 9 files changed, 300 insertions(+), 12 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 847743dfff..e44596626f 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -816,6 +816,44 @@ Full example: [examples/generate/multimodal/openai_chat_completion_client_for_mu export VLLM_VIDEO_FETCH_TIMEOUT= ``` +#### Video Decoding Backend + +vLLM decodes video bytes into frames using a selectable decoding backend. Three +backends are supported: + +- `opencv` (default): OpenCV-based decoder. +- `pyav`: PyAV decoder. +- `torchcodec`: TorchCodec (PyTorch-native) decoder. + +All three backends are ultimately backed by FFmpeg. `torchcodec` lets +you choose which FFmpeg version is used while `opencv` and `pyav` rely on +whichever FFmpeg build they were linked against. + +Select the backend by passing the `backend` parameter via `--media-io-kwargs`: + +```bash +vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \ + --media-io-kwargs '{"video": {"backend": "torchcodec"}}' +``` + +**TorchCodec-specific parameters:** + +The following parameters only apply to the `torchcodec` backend: + +- `num_ffmpeg_threads`: Number of FFmpeg decoding threads. `0` (default) relies + on the FFmpeg default, which is `min(cpu_count + 1, 16)`. This allows you to + control thread over-subscription. +- `seek_mode`: Seek mode for the decoder. `"exact"` (default) guarantees + frame-accurate sampling by scanning the file when the decoder is created. + `"approximate"` skips that scan for faster decoder creation, at the cost of + relying on the file's metadata (which may yield less accurate seeking). + +```bash +# Example: TorchCodec with approximate seek mode and 4 FFmpeg threads +vllm serve Qwen/Qwen3-VL-30B-A3B-Instruct \ + --media-io-kwargs '{"video": {"backend": "torchcodec", "seek_mode": "approximate", "num_ffmpeg_threads": 4}}' +``` + #### Video Frame Recovery For improved robustness when processing potentially corrupted or truncated video files, vLLM supports optional frame recovery using a dynamic window forward-scan approach. When enabled, if a target frame fails to load during sequential reading, the next successfully grabbed frame (before the next target frame) will be used in its place. diff --git a/requirements/cpu.txt b/requirements/cpu.txt index 21e0fbc77b..c0b98d22c9 100644 --- a/requirements/cpu.txt +++ b/requirements/cpu.txt @@ -15,6 +15,9 @@ torchaudio; platform_machine != "s390x" and platform_machine != "riscv64" # required for the image processor of phi3v, this must be updated alongside torch torchvision; platform_machine != "s390x" and platform_machine != "riscv64" +# required for the torchcodec video decoding backend +torchcodec >= 0.14; platform_machine != "s390x" and platform_machine != "riscv64" and platform_machine != "ppc64le" + # Intel Extension for PyTorch, only for x86_64 CPUs intel-openmp==2024.2.1; platform_machine == "x86_64" diff --git a/requirements/cuda.txt b/requirements/cuda.txt index f89854f632..1d90c7ef40 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -8,6 +8,7 @@ torch==2.11.0 torchaudio==2.11.0 # These must be updated alongside torch torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version +torchcodec >= 0.14 PyNvVideoCodec==2.1.0 # FlashInfer should be updated together with the Dockerfile flashinfer-python==0.6.13 diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index 6cc3a3782b..5ae281c21d 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -1134,6 +1134,8 @@ torchaudio==2.11.0+cpu # -r requirements/test/cuda.in # encodec # vocos +torchcodec==0.14.0+cpu + # via -r requirements/test/cuda.in torchvision==0.26.0+cpu # via # -r requirements/test/cuda.in diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index 9a6e46712c..c5518ecaa4 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -13,6 +13,7 @@ pytest-cov # testing utils albumentations # required for Nemotron Parse in test_common.py av # required for audio_in_video tests +torchcodec >= 0.14 # required for torchcodec video backend tests backoff # required for phi4mm test blobfile # required for kimi-vl test httpx diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index 3f7e4a5b5f..53aaa78f95 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -1233,6 +1233,10 @@ torchaudio==2.11.0+cu130 # -r requirements/test/cuda.in # encodec # vocos +torchcodec==0.14.0+cu130 + # via + # -c requirements/cuda.txt + # -r requirements/test/cuda.in torchvision==0.26.0+cu130 # via # -c requirements/cuda.txt diff --git a/requirements/xpu.txt b/requirements/xpu.txt index f06f033d83..68b8eb1301 100644 --- a/requirements/xpu.txt +++ b/requirements/xpu.txt @@ -15,6 +15,7 @@ numba == 0.65.0 # Required for N-gram speculative decoding torch==2.12.0 torchaudio torchvision +torchcodec >= 0.14 # Required for the torchcodec video decoding backend auto_round_lib>=0.14.0 vllm_xpu_kernels @ https://github.com/vllm-project/vllm-xpu-kernels/releases/download/v0.1.10.1/vllm_xpu_kernels-0.1.10.1-cp38-abi3-manylinux_2_28_x86_64.whl diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 6fccc926a2..6aeb7dc486 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -768,6 +768,118 @@ def test_pyav_backend_returns_target_frames_not_keyframes(): ) +# ============================================================================ +# TorchCodec Backend Tests +# ============================================================================ + + +def test_torchcodec_backend_loads_frames( + dummy_video_path, monkeypatch: pytest.MonkeyPatch +): + """Test that the torchcodec codec backend can load frames.""" + pytest.importorskip("torchcodec") + with monkeypatch.context() as m: + m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv") + + with open(dummy_video_path, "rb") as f: + video_data = f.read() + + loader = VIDEO_LOADER_REGISTRY.load("opencv") + frames, metadata = loader.load_bytes( + video_data, num_frames=8, backend="torchcodec" + ) + + assert frames.ndim == 4 + assert frames.shape[3] == 3 # RGB + assert frames.shape[0] == 8 + assert frames.shape[0] == len(metadata["frames_indices"]) + assert metadata["video_backend"] == "torchcodec" + assert "total_num_frames" in metadata + assert "fps" in metadata + assert "duration" in metadata + + +def test_torchcodec_dynamic_backend_loads_frames( + dummy_video_path, monkeypatch: pytest.MonkeyPatch +): + """Test that the torchcodec codec with dynamic sampling can load frames.""" + pytest.importorskip("torchcodec") + with monkeypatch.context() as m: + m.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv_dynamic") + + with open(dummy_video_path, "rb") as f: + video_data = f.read() + + loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic") + frames, metadata = loader.load_bytes( + video_data, fps=2, max_duration=10, backend="torchcodec" + ) + + assert frames.ndim == 4 + assert frames.shape[3] == 3 # RGB + assert frames.shape[0] > 0 + assert frames.shape[0] == len(metadata["frames_indices"]) + assert metadata["video_backend"] == "torchcodec_dynamic" + + +def test_torchcodec_backend_rejects_frame_recovery(dummy_video_path): + """frame_recovery is OpenCV-only; torchcodec must reject it.""" + pytest.importorskip("torchcodec") + with open(dummy_video_path, "rb") as f: + video_data = f.read() + + loader = VIDEO_LOADER_REGISTRY.load("opencv") + with pytest.raises(AssertionError): + loader.load_bytes( + video_data, num_frames=8, backend="torchcodec", frame_recovery=True + ) + + +def test_torchcodec_backend_returns_target_frames_not_keyframes(): + """Regression test: torchcodec must return the requested frames, not the + GOP keyframe they seek back to. + + Mirrors ``test_pyav_backend_returns_target_frames_not_keyframes``: a long + GOP (single keyframe at frame 0) with a per-frame green-channel marker. + With ``seek_mode="exact"`` torchcodec resolves each index to the exact + frame, so the returned markers must be distinct, ordered, and match the + requested indices. + """ + pytest.importorskip("torchcodec") + num_frames = 50 + num_sampled = 4 + height, width = 64, 64 + + video_bytes = create_long_gop_video( + num_frames=num_frames, width=width, height=height + ) + + loader = VIDEO_LOADER_REGISTRY.load("opencv") + frames, metadata = loader.load_bytes( + video_bytes, num_frames=num_sampled, backend="torchcodec" + ) + assert frames.shape == (num_sampled, height, width, 3) + + requested = list(metadata["frames_indices"]) + assert len(requested) == num_sampled + + actual = [int(f[height // 2, width // 2, 1]) for f in frames] + + assert len(set(actual)) == num_sampled, ( + f"torchcodec returned only {len(set(actual))} distinct frames for " + f"{num_sampled} requested indices: markers={actual}, " + f"requested={requested}. Keyframe-snap regression." + ) + + assert actual == sorted(actual), f"Returned frames out of order: markers={actual}" + + for marker, want_idx in zip(actual, requested): + assert abs(marker - want_idx) <= 10, ( + f"Frame mismatch: requested index {want_idx}, " + f"got marker {marker} (tolerance ±10)" + ) + + @pytest.mark.parametrize( "loader_key, kwargs, expected_num_frames", [ @@ -854,6 +966,42 @@ def test_pyav_backend_returns_target_frames_not_keyframes(): 120, id="glm46v-pyav-60s", ), + # uniform sampling + torchcodec codec (same frame counts as opencv) + pytest.param( + "opencv", + {"num_frames": 32, "backend": "torchcodec"}, + 32, + id="torchcodec-num_frames", + ), + pytest.param( + "opencv", {"fps": 2, "backend": "torchcodec"}, 120, id="torchcodec-fps" + ), + pytest.param( + "opencv", + {"num_frames": 500, "fps": 2, "backend": "torchcodec"}, + 120, + id="torchcodec-num_frames_wins_fps", + ), + # dynamic sampling + torchcodec codec + pytest.param( + "opencv_dynamic", + {"fps": 1, "max_duration": 60, "backend": "torchcodec"}, + 60, + id="torchcodec_dynamic-within_max_duration", + ), + pytest.param( + "opencv_dynamic", + {"fps": 2, "max_duration": 30, "backend": "torchcodec"}, + 60, + id="torchcodec_dynamic-exceeds_max_duration", + ), + # glm46v dynamic FPS + torchcodec codec + pytest.param( + "glm46v", + {"backend": "torchcodec"}, + 120, + id="glm46v-torchcodec-60s", + ), ], ) def test_video_loader_frames_sampling( @@ -864,6 +1012,8 @@ def test_video_loader_frames_sampling( expected_num_frames: int, ): """Test video loader frames sampling functionality.""" + if kwargs.get("backend") == "torchcodec": + pytest.importorskip("torchcodec") monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", loader_key) loader = VIDEO_LOADER_REGISTRY.load(loader_key) diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index 33363ed403..874745e714 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -31,6 +31,13 @@ try: except ImportError: av = PlaceholderModule("av") # type: ignore[assignment] +try: + from torchcodec.decoders import VideoDecoder +except ImportError: + VideoDecoder = PlaceholderModule("torchcodec").placeholder_attr( # type: ignore[assignment] + "decoders.VideoDecoder" + ) + logger = init_logger(__name__) @@ -568,6 +575,53 @@ class PyAVVideoBackendMixin: return np.stack(frames_list), valid_indices +class TorchCodecVideoBackendMixin: + """TorchCodec (FFmpeg-backed, PyTorch-native) codec utilities. + + Builds a :class:`~torchcodec.decoders.VideoDecoder` over the in-memory + bytes and extracts the sampled indices with a single batched + ``get_frames_at`` call, while releasing the GIL during decode. + """ + + @staticmethod + def make_torchcodec_decoder( + data: bytes, + *, + num_ffmpeg_threads: int = 0, + seek_mode: Literal["exact", "approximate"] = "exact", + ) -> "VideoDecoder": + # NHWC matches the (num_frames, H, W, 3) uint8 RGB layout the rest + # of the pipeline expects, avoiding a transpose. + return VideoDecoder( + data, + dimension_order="NHWC", + num_ffmpeg_threads=num_ffmpeg_threads, + seek_mode=seek_mode, + ) + + @staticmethod + def get_torchcodec_metadata(decoder: "VideoDecoder") -> VideoSourceMetadata: + md = decoder.metadata + total_frames = md.num_frames or 0 + fps = float(md.average_fps) if md.average_fps else 0.0 + duration = float(md.duration_seconds) if md.duration_seconds else 0.0 + if total_frames == 0 and duration > 0 and fps > 0: + total_frames = int(duration * fps) + return VideoSourceMetadata(total_frames, fps, duration) + + @staticmethod + def decode_torchcodec_frames( + decoder: "VideoDecoder", + frame_indices: list[int], + ) -> tuple[npt.NDArray, list[int]]: + """Decode the requested indices in one batched, index-exact call.""" + if not frame_indices: + return np.empty((0,), dtype=np.uint8), [] + # Note: torchcodec releases the GIL for the entire call + batch = decoder.get_frames_at(frame_indices) + return batch.data.numpy(), list(frame_indices) + + class PyNvVideoCodecVideoBackendMixin: """PyNvVideoCodec utilities for GPU-backed frame decode.""" @@ -777,14 +831,15 @@ class VideoBackend( VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin, + TorchCodecVideoBackendMixin, PyNvVideoCodecVideoBackendMixin, ): """Uniform-sampling video backend. Samples ``num_frames`` uniformly across the video (or one frame every ``1/fps`` seconds, whichever produces fewer frames). The decoding codec - is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``, or - ``"pynvvideocodec"``), which can be passed through + is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``, + ``"torchcodec"`` or ``"pynvvideocodec"``), which can be passed through ``--media-io-kwargs``. Defaults to ``"opencv"``. """ @@ -830,7 +885,9 @@ class VideoBackend( max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", + backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv", + num_ffmpeg_threads: int = 0, + seek_mode: Literal["exact", "approximate"] = "exact", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: """Load sampled frames from raw video bytes. @@ -843,8 +900,20 @@ class VideoBackend( dynamic subclass; ignored here. frame_recovery: Enable forward-scan recovery for failed frames. Only honored by the OpenCV codec. - backend: Decoding codec — ``"opencv"``, ``"pyav"``, or - ``"pynvvideocodec"``. + backend: Decoding codec — ``"opencv"``, ``"pyav"``, + ``"torchcodec"`` or ``"pynvvideocodec"``. + num_ffmpeg_threads: Number of FFmpeg decoding threads, only used by + TorchCodec: ``0`` (default) relies on the FFmpeg default value + which is ``min(cpu_count + 1, 16)``. + OpenCV will always use ``min(cpu_count, 16)`` while pyav will + always use ``min(cpu_count, (height + 15) / 16)``. + seek_mode: Seek mode for the TorchCodec decoder, only used by + TorchCodec: ``"exact"`` (default) guarantees frame-accurate + sampling by scanning the file on creation, while + ``"approximate"`` skips that scan for faster decoder creation + at the cost of relying on the file's metadata. See + https://meta-pytorch.org/torchcodec/stable/generated_examples/decoding/approximate_mode.html + for details. Returns: Tuple of ``(frames_array, metadata_dict)``. @@ -883,6 +952,24 @@ class VideoBackend( frames, valid = cls.decode_frames( container, frame_idx, source.original_fps, source.duration ) + elif backend == "torchcodec": + assert not frame_recovery, ( + "frame_recovery is only available for `opencv` backend" + ) + decoder = cls.make_torchcodec_decoder( + data, + num_ffmpeg_threads=num_ffmpeg_threads, + seek_mode=seek_mode, + ) + _check_frame_pixel_limit( + decoder.metadata.width or 0, + decoder.metadata.height or 0, + ) + source = cls._prepare_source(cls.get_torchcodec_metadata(decoder)) + frame_idx = cls.compute_frames_index_to_sample( + source=source, target=target, **kwargs + ) + frames, valid = cls.decode_torchcodec_frames(decoder, frame_idx) elif backend == PYNVVIDEOCODEC_VIDEO_BACKEND: if frame_recovery: raise ValueError( @@ -897,7 +984,8 @@ class VideoBackend( else: raise ValueError( f"Unknown video codec backend {backend!r}; " - "valid options: 'opencv', 'pyav', 'pynvvideocodec'." + "valid options: 'opencv', 'pyav', 'torchcodec', " + "'pynvvideocodec'." ) if len(valid) < len(frame_idx): @@ -984,7 +1072,7 @@ class Qwen3VLVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", + backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( @@ -1063,7 +1151,7 @@ class Qwen2VLVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", + backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( @@ -1155,7 +1243,7 @@ class DynamicVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", + backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( @@ -1280,7 +1368,7 @@ class GLM46VVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", + backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( @@ -1378,7 +1466,7 @@ class GLMGAVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", + backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: frames, metadata = super().load_bytes( @@ -1701,7 +1789,7 @@ class NemotronVLVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", + backend: Literal["opencv", "pyav", "torchcodec", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: frames, metadata = super().load_bytes(