[Multimodal] Use GPU NVDEC for EPD encoder-only instance video media IO (#53675)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-09-11 22:20:11 +08:00
committed by GitHub
parent 5392fbca2a
commit 22258a26bc
12 changed files with 373 additions and 24 deletions
+44
View File
@@ -532,6 +532,50 @@ class TestMergeKwargsGpuBackendPolicy:
def test_unknown_backend_not_treated_as_gpu(self):
assert not VIDEO_LOADER_REGISTRY.backend_requires_gpu("totally_unknown")
def test_strips_request_level_device(self):
"""The decode device is a startup-only knob: a request must not move
decoding onto the GPU when the startup config did not opt in, nor off
it when it did."""
result = VideoMediaIO.merge_kwargs(
default_kwargs={"backend": "torchcodec"},
runtime_kwargs={"device": "cuda"},
)
assert "device" not in result
result = VideoMediaIO.merge_kwargs(
default_kwargs={"backend": "torchcodec", "device": "cuda"},
runtime_kwargs={"device": "cpu", "num_frames": 8},
)
assert result["device"] == "cuda"
assert result["num_frames"] == 8
@pytest.mark.parametrize(
"default_kwargs",
[
{"backend": "torchcodec", "device": "cuda", "seek_mode": "approximate"},
{"backend": "pynvvideocodec", "hw_decoders": 2},
],
)
def test_switching_backend_drops_stale_codec_options(default_kwargs):
"""Codec-specific options from the static config must not leak into a
different codec backend selected per-request, where they would fail the
new backend's option validation."""
result = VideoMediaIO.merge_kwargs(
default_kwargs={**default_kwargs, "num_frames": 8},
runtime_kwargs={"backend": "opencv"},
)
assert result == {"backend": "opencv", "num_frames": 8}
def test_same_backend_keeps_codec_options():
result = VideoMediaIO.merge_kwargs(
default_kwargs={"backend": "torchcodec", "device": "cuda"},
runtime_kwargs={"backend": "torchcodec", "num_frames": 8},
)
assert result["device"] == "cuda"
assert result["num_frames"] == 8
@pytest.mark.parametrize("layout", ["nhwc", "nchw"])
def test_pynvvc_frames_normalized_to_nhwc(layout: str):
+27
View File
@@ -137,6 +137,33 @@ def test_hash_collision_video_num_frames():
)
def test_hash_video_tensor_frames():
"""Videos holding tensor frames (e.g. NVDEC-decoded) hash like
array-framed ones, from the original bytes without a D2H copy."""
source = b"x" * 100
def item_for_hash(frames):
metadata = {
"total_num_frames": 2,
"fps": 2.0,
"duration": 1.0,
"video_backend": "torchcodec",
"frames_indices": [0, 1],
"do_sample_frames": False,
}
video = MediaWithBytes((frames, metadata), source)
items = MultiModalDataParser()._parse_video_data([video])
return items.get_all_items_for_hash()[0]
np_frames = np.zeros((2, 8, 8, 3), dtype=np.uint8)
torch_frames = torch.zeros((2, 8, 8, 3), dtype=torch.uint8)
hasher = MultiModalHasher
assert hasher.hash_kwargs("blake3", video=item_for_hash(np_frames)) == (
hasher.hash_kwargs("blake3", video=item_for_hash(torch_frames))
)
def test_hash_non_contiguous_array():
arr = np.arange(24).reshape(4, 6).T
assert not arr.flags.c_contiguous
+22
View File
@@ -56,6 +56,28 @@ def test_frame_size_hwc_chw(frame):
assert items.get_frame_size(0) == (W, H)
def test_video_with_metadata_tensor_passthrough():
"""Tensor frames pass through unchanged regardless of device: HF video
processors accept tensors, and device-resident frames (e.g. NVDEC-decoded)
must not be copied back to host."""
frames = torch.zeros((4, H, W, 3), dtype=torch.uint8)
video, metadata = MultiModalDataParser()._get_video_with_metadata(frames)
assert video is frames
assert metadata is None
@pytest.mark.skipif(not torch.cuda.is_available(), reason="Requires CUDA")
def test_video_with_metadata_keeps_device_tensor():
"""Device-resident frames (e.g. NVDEC-decoded) pass through as tensors,
so a device-side HF processor can consume them without a D2H copy."""
frames = torch.zeros((4, H, W, 3), dtype=torch.uint8, device="cuda")
video, metadata = MultiModalDataParser()._get_video_with_metadata(frames)
assert video is frames
assert metadata is None
@pytest.mark.parametrize(
"frames",
[
+37 -2
View File
@@ -11,6 +11,7 @@ from pathlib import Path
import numpy as np
import numpy.typing as npt
import pytest
import torch
from transformers import AutoVideoProcessor
from transformers.video_utils import VideoMetadata
@@ -173,9 +174,14 @@ def test_decode_video_imports_only_selected_backend(
[
(
"torchcodec",
{"min_frames": 4, "num_ffmpeg_threads": 2, "seek_mode": "approximate"},
{
"min_frames": 4,
"num_ffmpeg_threads": 2,
"seek_mode": "approximate",
"device": "cuda",
},
{"min_frames": 4},
{"num_ffmpeg_threads": 2, "seek_mode": "approximate"},
{"num_ffmpeg_threads": 2, "seek_mode": "approximate", "device": "cuda"},
),
(
"deepstream",
@@ -205,6 +211,11 @@ def test_video_backend_rejects_options_for_another_decoder():
):
resolve_video_backend_kwargs("opencv", {"num_ffmpeg_threads": 2})
with pytest.raises(
ValueError, match="device is not supported by the 'opencv' backend"
):
resolve_video_backend_kwargs("opencv", {"device": "cuda"})
@pytest.mark.parametrize(
("backend", "error"),
@@ -1205,6 +1216,30 @@ def test_torchcodec_backend_rejects_frame_recovery(dummy_video_path):
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_torchcodec_backend_cuda_decodes_on_gpu(dummy_video_path):
"""With device="cuda", torchcodec decodes via NVDEC and keeps the frames
on the GPU instead of returning a host-side numpy array."""
pytest.importorskip("torchcodec")
with open(dummy_video_path, "rb") as f:
video_data = f.read()
loader = VIDEO_LOADER_REGISTRY.load("opencv")
frames, metadata = loader.load_bytes(
video_data, num_frames=8, backend="torchcodec", device="cuda"
)
assert isinstance(frames, torch.Tensor)
assert frames.device.type == "cuda"
assert frames.dtype == torch.uint8
assert frames.ndim == 4
assert frames.shape[3] == 3 # RGB
assert frames.shape[0] == 8
assert frames.shape[0] == len(metadata["frames_indices"])
assert metadata["video_backend"] == "torchcodec"
def test_torchcodec_backend_returns_target_frames_not_keyframes():
"""Regression test: torchcodec must return the requested frames, not the
GOP keyframe they seek back to.