[Multimodal] Use GPU NVDEC for EPD encoder-only instance video media IO (#53675)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-09-11 22:20:11 +08:00
committed by GitHub
parent 5392fbca2a
commit 22258a26bc
12 changed files with 373 additions and 24 deletions
+9 -7
View File
@@ -36,6 +36,7 @@ from .inputs import (
VideoItem,
)
from .media import MediaWithBytes
from .video import DecodedFrames
_T = TypeVar("_T")
_I = TypeVar("_I")
@@ -653,7 +654,7 @@ class MultiModalDataParser:
def _get_video_with_metadata(
self,
video: VideoItem,
) -> tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any] | None]:
) -> tuple[DecodedFrames | MediaWithBytes[DecodedFrames], dict[str, Any] | None]:
if isinstance(video, MediaWithBytes):
new_video, metadata = self._get_video_with_metadata(video.media)
return MediaWithBytes(new_video, video.original_bytes), metadata
@@ -661,10 +662,11 @@ class MultiModalDataParser:
return video
if isinstance(video, list):
return np.array(video), None
if isinstance(video, np.ndarray):
if isinstance(video, (np.ndarray, torch.Tensor)):
# Tensors pass through untouched: HF video processors accept them
# directly, and device tensors (e.g. NVDEC-decoded frames) must
# stay on-device to avoid a D2H round-trip.
return video, None
if isinstance(video, torch.Tensor):
return video.numpy(), None
assert_never(video)
@@ -769,9 +771,9 @@ class MultiModalDataParser:
data_items = data # type: ignore[assignment]
new_videos = list[
np.ndarray
| MediaWithBytes[np.ndarray]
| tuple[np.ndarray | MediaWithBytes[np.ndarray], dict[str, Any]]
DecodedFrames
| MediaWithBytes[DecodedFrames]
| tuple[DecodedFrames | MediaWithBytes[DecodedFrames], dict[str, Any]]
| None
]()
metadata_lst: list[dict[str, Any] | None] = []