134 lines
4.5 KiB
Python
134 lines
4.5 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from typing import Literal
|
|
|
|
import numpy as np
|
|
import numpy.typing as npt
|
|
import torch
|
|
|
|
from vllm.logger import init_logger
|
|
from vllm.platforms import current_platform
|
|
from vllm.utils.import_utils import PlaceholderModule, check_torchcodec_available
|
|
|
|
from .base import (
|
|
VideoSourceMetadata,
|
|
VideoTargetMetadata,
|
|
check_frame_pixel_limit,
|
|
)
|
|
|
|
try:
|
|
from torchcodec.decoders import VideoDecoder
|
|
except (ImportError, RuntimeError):
|
|
VideoDecoder = PlaceholderModule("torchcodec").placeholder_attr( # type: ignore[assignment]
|
|
"decoders.VideoDecoder"
|
|
)
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
def decode_torchcodec(
|
|
loader_cls,
|
|
data: bytes,
|
|
target: VideoTargetMetadata,
|
|
sampling_kwargs: dict,
|
|
*,
|
|
num_ffmpeg_threads: int = 0,
|
|
seek_mode: Literal["exact", "approximate"] = "exact",
|
|
device: str = "cpu",
|
|
) -> tuple[npt.NDArray | torch.Tensor, VideoSourceMetadata, list[int], list[int]]:
|
|
check_torchcodec_available()
|
|
decoder = TorchCodecVideoBackendMixin.make_torchcodec_decoder(
|
|
data,
|
|
num_ffmpeg_threads=num_ffmpeg_threads,
|
|
seek_mode=seek_mode,
|
|
device=device,
|
|
)
|
|
check_frame_pixel_limit(
|
|
decoder.metadata.width or 0,
|
|
decoder.metadata.height or 0,
|
|
)
|
|
source = loader_cls._prepare_source(
|
|
TorchCodecVideoBackendMixin.get_torchcodec_metadata(decoder)
|
|
)
|
|
frame_idx = loader_cls.compute_frames_index_to_sample(
|
|
source=source, target=target, **sampling_kwargs
|
|
)
|
|
frames, valid = TorchCodecVideoBackendMixin.decode_torchcodec_frames(
|
|
decoder, frame_idx, device=device
|
|
)
|
|
return frames, source, frame_idx, valid
|
|
|
|
|
|
class TorchCodecVideoBackendMixin:
|
|
"""TorchCodec (FFmpeg-backed, PyTorch-native) codec utilities.
|
|
|
|
Builds a :class:`~torchcodec.decoders.VideoDecoder` over the in-memory
|
|
bytes and extracts the sampled indices with a single batched
|
|
``get_frames_at`` call, while releasing the GIL during decode.
|
|
"""
|
|
|
|
@staticmethod
|
|
def make_torchcodec_decoder(
|
|
data: bytes,
|
|
*,
|
|
num_ffmpeg_threads: int = 0,
|
|
seek_mode: Literal["exact", "approximate"] = "exact",
|
|
device: str = "cpu",
|
|
) -> "VideoDecoder":
|
|
torch_device = torch.device(device)
|
|
if torch_device.type == "cuda" and not current_platform.is_cuda():
|
|
raise ValueError(
|
|
f"torchcodec video decoding on device {device!r} requires "
|
|
"a CUDA-capable platform."
|
|
)
|
|
elif torch_device.type not in ("cpu", "cuda"):
|
|
raise ValueError(
|
|
f"torchcodec video decoding only supports 'cpu' and 'cuda' "
|
|
f"devices, got {device!r}."
|
|
)
|
|
# NHWC matches the (num_frames, H, W, 3) uint8 RGB layout the rest
|
|
# of the pipeline expects, avoiding a transpose.
|
|
return VideoDecoder(
|
|
data,
|
|
dimension_order="NHWC",
|
|
num_ffmpeg_threads=num_ffmpeg_threads,
|
|
seek_mode=seek_mode,
|
|
device=device,
|
|
)
|
|
|
|
@staticmethod
|
|
def get_torchcodec_metadata(decoder: "VideoDecoder") -> VideoSourceMetadata:
|
|
md = decoder.metadata
|
|
total_frames = md.num_frames or 0
|
|
fps = float(md.average_fps) if md.average_fps else 0.0
|
|
duration = float(md.duration_seconds) if md.duration_seconds else 0.0
|
|
if total_frames == 0 and duration > 0 and fps > 0:
|
|
total_frames = int(duration * fps)
|
|
return VideoSourceMetadata(total_frames, fps, duration)
|
|
|
|
@staticmethod
|
|
def decode_torchcodec_frames(
|
|
decoder: "VideoDecoder",
|
|
frame_indices: list[int],
|
|
*,
|
|
device: str = "cpu",
|
|
) -> tuple[npt.NDArray | torch.Tensor, list[int]]:
|
|
"""Decode the requested indices in one batched, index-exact call."""
|
|
if not frame_indices:
|
|
return np.empty((0,), dtype=np.uint8), []
|
|
# Note: torchcodec releases the GIL for the entire call
|
|
batch = decoder.get_frames_at(frame_indices)
|
|
frames = batch.data
|
|
if (
|
|
torch.device(device).type != "cpu"
|
|
and (status := getattr(decoder, "cpu_fallback", None)) is not None
|
|
and bool(status)
|
|
):
|
|
logger.warning_once(
|
|
"torchcodec could not use NVDEC for this video and "
|
|
"decoded on CPU instead; check codec support and "
|
|
"libnvcuvid."
|
|
)
|
|
return frames, list(frame_indices)
|