[Bugfix][Multimodal] Recover PyNvVideoCodec decoder slots after failures (#59597)

Signed-off-by: Daniel Afrimi <dafrimi@nvidia.com>
Signed-off-by: danielafrimi <45691845+danielafrimi@users.noreply.github.com>
Signed-off-by: Daniel Afrimi <dafrimi@oci-hsg-cs-001-login-02.cm.cluster>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Daniel Afrimi <dafrimi@oci-hsg-cs-001-login-02.cm.cluster>
Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com>
This commit is contained in:
danielafrimi
2026-10-15 09:17:19 +03:00
committed by GitHub
co-authored by OpenAI Codex Daniel Afrimi tomeras91
parent c5f8adc0b1
commit a9fe30e9f6
8 changed files with 912 additions and 88 deletions
+9 -2
View File
@@ -196,6 +196,7 @@ def reserve_mm_ipc_gpu_memory(
from vllm.multimodal.video_decoders.pynvvideocodec import (
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
get_pynvvideocodec_decoder_gpu_memory_bytes,
validate_pynvvideocodec_hw_decoders,
)
@@ -220,9 +221,15 @@ def reserve_mm_ipc_gpu_memory(
if uses_pynvvideocodec
else 1
)
decoder_bytes = (
get_pynvvideocodec_decoder_gpu_memory_bytes(
video_kwargs.get("max_width"), video_kwargs.get("max_height")
)
if uses_pynvvideocodec
else PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES
)
per_server_decoder_bytes = (
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * hw_decoders
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
decoder_bytes * hw_decoders + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
)
decoder_reserved_bytes = (
num_api_servers * per_server_decoder_bytes
+2
View File
@@ -48,6 +48,8 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[DecodedFrames, dict[str, Any]]]]
# Decoder GPU memory is reserved from the startup value.
runtime_kwargs = dict(runtime_kwargs)
runtime_kwargs.pop("hw_decoders", None)
runtime_kwargs.pop("max_width", None)
runtime_kwargs.pop("max_height", None)
runtime_kwargs.pop("pool_size", None)
# The decode device determines which device the decoded frames
# (and thus the processor pipeline) live on; startup-only.
+6
View File
@@ -282,6 +282,12 @@ class VideoBackend(VideoLoader):
- ``hw_decoders`` (PyNvVideoCodec): maximum number of
concurrent decoder slots. Defaults to 2 and must be a
positive integer.
- ``max_width`` / ``max_height`` (PyNvVideoCodec): optional
maximum input dimensions configured for each decoder. Both
must be positive integers and set together. Values are
clamped to each codec's hardware limits. By default, decoder
dimensions also cover the input's opposite orientation when
that fits the reserved memory and hardware limits.
- ``pool_size`` / ``timeout_sec`` (DeepStream): decoder pool
size and pool acquisition timeout in seconds.
@@ -23,6 +23,8 @@ _BACKEND_OPTION_DEFAULTS: dict[str, dict[str, Any]] = {
},
PYNVVIDEOCODEC_VIDEO_BACKEND: {
"hw_decoders": PYNVVIDEOCODEC_DEFAULT_HW_DECODERS,
"max_width": None,
"max_height": None,
},
"deepstream": {
"pool_size": None,
+288 -55
View File
@@ -30,12 +30,19 @@ def decode_pynvvideocodec(
sampling_kwargs: dict,
*,
hw_decoders: int = PYNVVIDEOCODEC_DEFAULT_HW_DECODERS,
max_width: int | None = None,
max_height: int | None = None,
) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]:
max_width, max_height = validate_pynvvideocodec_max_dimensions(
max_width, max_height
)
PyNvVideoCodecVideoBackendMixin._configure_decoder_slots(hw_decoders)
return PyNvVideoCodecVideoBackendMixin.decode_frames_pynvvideocodec(
loader_cls,
data,
target,
max_width=max_width,
max_height=max_height,
**sampling_kwargs,
)
@@ -48,9 +55,11 @@ class PyNvVideoCodecSourceMetadata(NamedTuple):
height: int
# Per-decoder upper bound reserved for persistent PyNvVideoCodec surfaces.
# Per-decoder baseline reserved for persistent PyNvVideoCodec surfaces.
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES = 128 * MiB_bytes
PYNVVIDEOCODEC_DECODER_CACHE_SIZE = 2
# Bound native frame surfaces retained by each batch decode call.
PYNVVIDEOCODEC_DECODE_BATCH_SIZE = 32
# Per-API-server CUDA context and driver allocation, measured with
# PyNvVideoCodec 2.0.4 on H100.
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES = int(1.8 * 1024 * MiB_bytes)
@@ -66,6 +75,43 @@ def validate_pynvvideocodec_hw_decoders(hw_decoders: object) -> int:
return hw_decoders
def validate_pynvvideocodec_max_dimensions(
max_width: object,
max_height: object,
) -> tuple[int | None, int | None]:
if max_width is None and max_height is None:
return None, None
if max_width is None or max_height is None:
raise ValueError("max_width and max_height must be set together")
if isinstance(max_width, bool) or not isinstance(max_width, int) or max_width < 1:
raise ValueError("max_width must be a positive integer")
if (
isinstance(max_height, bool)
or not isinstance(max_height, int)
or max_height < 1
):
raise ValueError("max_height must be a positive integer")
return max_width, max_height
def get_pynvvideocodec_decoder_gpu_memory_bytes(
max_width: object,
max_height: object,
) -> int:
max_width, max_height = validate_pynvvideocodec_max_dimensions(
max_width, max_height
)
if max_width is None or max_height is None:
return PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES
configured_surface_bytes = _decoder_surface_bytes(max_width, max_height)
return max(PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES, configured_surface_bytes)
def _decoder_surface_bytes(width: int, height: int) -> int:
return width * height * 3 * PYNVVIDEOCODEC_DECODER_CACHE_SIZE
def _pynvvideocodec_exception_types(nvc) -> tuple[type[Exception], ...]:
return tuple(
exception_type
@@ -90,22 +136,54 @@ class PyNvVideoCodecDecoderSlot:
existing decoder at each new source instead of paying a fresh
``SimpleDecoder`` construction per request. Construction (CUVID parser +
decoder + surface-pool allocation) is the dominant per-request cost, so
reconfiguring is far cheaper. A single decoder serves both metadata
(``len``/``get_stream_metadata``) and frame decode -- no separate
metadata decoder.
reconfiguring is far cheaper. Decoder dimensions are bounded at construction
so resolution changes do not replace its native frame pool. A single decoder
serves both metadata (``len``/``get_stream_metadata``) and frame decode -- no
separate metadata decoder.
"""
def __init__(self, stream) -> None:
self.stream = stream
self.decoder = None
self.source_path: str | None = None
self.max_width = 0
self.max_height = 0
def invalidate(self) -> None:
decoder = self.decoder
self.decoder = None
self.source_path = None
self.max_width = 0
self.max_height = 0
if decoder is not None:
with suppress(Exception):
decoder.stop()
def _construct(self, file_path: str, nvc, device_index: int) -> None:
def reset(self) -> None:
"""Rebuild this slot after draining only its CUDA stream."""
with suppress(Exception):
self.stream.synchronize()
self.invalidate()
def recover(self) -> None:
"""Release decoder resources and drain work after a failed request."""
self.reset()
with suppress(Exception):
import torch
with torch.accelerator.device_index(self.stream.device.index):
torch.accelerator.synchronize()
torch.accelerator.empty_cache()
def _construct(
self,
file_path: str,
nvc,
device_index: int,
max_width: int,
max_height: int,
) -> None:
decoder = nvc.SimpleDecoder(
file_path,
output_color_type=nvc.OutputColorType.RGB,
@@ -114,20 +192,36 @@ class PyNvVideoCodecDecoderSlot:
gpu_id=device_index,
cuda_stream=self.stream.cuda_stream,
decoder_cache_size=PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
max_width=max_width,
max_height=max_height,
)
self.decoder = decoder
self.source_path = file_path
self.max_width = max_width
self.max_height = max_height
def get_decoder(self, file_path: str, nvc, device_index: int):
def get_decoder(
self,
file_path: str,
nvc,
device_index: int,
max_width: int,
max_height: int,
):
if self.decoder is None:
self._construct(file_path, nvc, device_index)
self._construct(file_path, nvc, device_index, max_width, max_height)
elif self.source_path != file_path:
try:
self.decoder.reconfigure_decoder(file_path)
self.source_path = file_path
except Exception:
# reconfigure unsupported/unsafe for this source -> rebuild.
self._construct(file_path, nvc, device_index)
if max_width > self.max_width or max_height > self.max_height:
self.reset()
self._construct(file_path, nvc, device_index, max_width, max_height)
else:
try:
self.decoder.reconfigure_decoder(file_path)
self.source_path = file_path
except Exception:
# reconfigure unsupported/unsafe for this source -> rebuild.
self.reset()
self._construct(file_path, nvc, device_index, max_width, max_height)
return self.decoder
@@ -222,9 +316,12 @@ class PyNvVideoCodecVideoBackendMixin:
borrow_succeeded = True
finally:
if not borrow_succeeded:
slot.invalidate()
slot.recover()
with pool.cond:
pool.slots.append(slot)
if borrow_succeeded:
pool.slots.append(slot)
else:
pool.active -= 1
pool.cond.notify()
@staticmethod
@@ -235,19 +332,86 @@ class PyNvVideoCodecVideoBackendMixin:
return value
return default
@classmethod
def _validate_decoder_caps(
cls,
file_path: str,
nvc,
max_width: int | None = None,
max_height: int | None = None,
) -> tuple[int, int]:
demuxer = nvc.CreateDemuxer(file_path)
width = demuxer.Width()
height = demuxer.Height()
caps = nvc.GetDecoderCaps(
cls._DEVICE_INDEX,
demuxer.GetNvCodecId(),
demuxer.ChromaFormat(),
demuxer.BitDepth(),
)
width_min = int(caps["width_min"])
width_max = int(caps["width_max"])
height_min = int(caps["height_min"])
height_max = int(caps["height_max"])
mb_num_max = int(caps["mb_num_max"])
macroblock_count = ((width + 15) // 16) * ((height + 15) // 16)
if (
not caps["supported"]
or width < width_min
or width > width_max
or height < height_min
or height > height_max
or macroblock_count > mb_num_max
):
raise ValueError("Invalid or unsupported video file.")
if max_width is None or max_height is None:
max_dimension = max(width, height)
orientation_mb_count = ((max_dimension + 15) // 16) ** 2
if (
max_dimension <= width_max
and max_dimension <= height_max
and orientation_mb_count <= mb_num_max
and _decoder_surface_bytes(max_dimension, max_dimension)
<= PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES
):
return max_dimension, max_dimension
return width, height
decoder_max_width = min(max_width, width_max)
decoder_max_height = min(max_height, height_max)
decoder_mb_count = ((decoder_max_width + 15) // 16) * (
(decoder_max_height + 15) // 16
)
if decoder_mb_count > mb_num_max:
decoder_max_width = width
decoder_max_height = height
if width > decoder_max_width or height > decoder_max_height:
raise ValueError("Invalid or unsupported video file.")
return decoder_max_width, decoder_max_height
@classmethod
def _read_source_metadata(
cls,
file_path: str,
nvc,
max_width: int,
max_height: int,
) -> PyNvVideoCodecSourceMetadata:
with cls._borrow_decoder_slot() as decoder_slot:
with cls._torch_stream_context(decoder_slot.stream):
decoder = decoder_slot.get_decoder(
file_path, nvc, device_index=cls._DEVICE_INDEX
file_path,
nvc,
device_index=cls._DEVICE_INDEX,
max_width=max_width,
max_height=max_height,
)
metadata = decoder.get_stream_metadata()
total_frames_num = len(decoder)
try:
metadata = decoder.get_stream_metadata()
total_frames_num = len(decoder)
finally:
del decoder
width = int(cls._metadata_value(metadata, "width", default=0))
height = int(cls._metadata_value(metadata, "height", default=0))
original_fps = float(
@@ -280,6 +444,8 @@ class PyNvVideoCodecVideoBackendMixin:
file_path: str,
frame_idx: list[int],
nvc,
max_width: int,
max_height: int,
) -> npt.NDArray:
import torch
@@ -289,45 +455,79 @@ class PyNvVideoCodecVideoBackendMixin:
with cls._borrow_decoder_slot() as decoder_slot:
stream = decoder_slot.stream
with cls._torch_stream_context(stream):
try:
decoder = decoder_slot.get_decoder(
file_path, nvc, device_index=cls._DEVICE_INDEX
decoder = decoder_slot.get_decoder(
file_path,
nvc,
device_index=cls._DEVICE_INDEX,
max_width=max_width,
max_height=max_height,
)
host_frames = None
decoded_count = 0
for start in range(0, len(frame_idx), PYNVVIDEOCODEC_DECODE_BATCH_SIZE):
chunk_indices = frame_idx[
start : start + PYNVVIDEOCODEC_DECODE_BATCH_SIZE
]
invalid_video = False
try:
decoded_frames = decoder.get_batch_frames_by_index(
chunk_indices
)
except Exception as exc:
decoder = None
if not isinstance(
exc,
_pynvvideocodec_exception_types(nvc) + (IndexError,),
):
raise
invalid_video = True
if invalid_video:
raise ValueError("Invalid or unsupported video file.")
torch_frames = [
torch.from_dlpack(frame) for frame in decoded_frames
]
if not torch_frames:
stream.synchronize()
break
device_frames = _pynvvc_frames_to_nhwc(torch.stack(torch_frames))
if device_frames.ndim != 4:
raise ValueError(
"PyNvVideoCodec returned frames with unexpected shape "
f"{tuple(device_frames.shape)}"
)
if host_frames is None:
host_frames = torch.empty(
(len(frame_idx), *device_frames.shape[1:]),
dtype=device_frames.dtype,
device="cpu",
pin_memory=True,
)
elif tuple(device_frames.shape[1:]) != tuple(host_frames.shape[1:]):
raise ValueError(
"PyNvVideoCodec returned frames with inconsistent shapes"
)
next_count = decoded_count + len(device_frames)
host_frames[decoded_count:next_count].copy_(
device_frames, non_blocking=True
)
decoded_frames = decoder.get_batch_frames_by_index(frame_idx)
except Exception as exc:
if not isinstance(
exc,
_pynvvideocodec_exception_types(nvc) + (IndexError,),
):
raise
raise ValueError("Invalid or unsupported video file.") from exc
if len(decoded_frames) < len(frame_idx):
stream.synchronize()
decoded_count = next_count
del decoded_frames, torch_frames, device_frames
if decoded_count < start + len(chunk_indices):
break
if decoded_count < len(frame_idx):
logger.warning(
"pynvvideocodec video loading: expected %d frames but got %d.",
len(frame_idx),
len(decoded_frames),
decoded_count,
)
torch_frames = [torch.from_dlpack(frame) for frame in decoded_frames]
if not torch_frames:
if host_frames is None:
return np.empty((0,), dtype=np.uint8)
device_frames = torch.stack(torch_frames)
if device_frames.ndim != 4:
raise ValueError(
"PyNvVideoCodec returned frames with unexpected shape "
f"{tuple(device_frames.shape)}"
)
device_frames = _pynvvc_frames_to_nhwc(device_frames)
host_frames = torch.empty(
device_frames.shape,
dtype=device_frames.dtype,
device="cpu",
pin_memory=True,
)
host_frames.copy_(device_frames, non_blocking=True)
stream.synchronize()
host_array = host_frames.numpy()
del decoded_frames, torch_frames, device_frames
return host_array
return host_frames[:decoded_count].numpy()
@classmethod
def decode_frames_pynvvideocodec(
@@ -335,6 +535,8 @@ class PyNvVideoCodecVideoBackendMixin:
loader_cls,
data: bytes,
target: VideoTargetMetadata,
max_width: int | None = None,
max_height: int | None = None,
**kwargs,
) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]:
import PyNvVideoCodec as nvc
@@ -346,12 +548,31 @@ class PyNvVideoCodecVideoBackendMixin:
with os.fdopen(temp_fd, "wb") as temp_file:
temp_file.write(data)
invalid_video = False
try:
gpu_source = cls._read_source_metadata(temp_path, nvc)
decoder_max_width, decoder_max_height = cls._validate_decoder_caps(
temp_path,
nvc,
max_width=max_width,
max_height=max_height,
)
except Exception as exc:
if not isinstance(exc, _pynvvideocodec_exception_types(nvc)):
raise
raise ValueError("Invalid or unsupported video file.") from exc
invalid_video = True
if invalid_video:
raise ValueError("Invalid or unsupported video file.")
try:
gpu_source = cls._read_source_metadata(
temp_path, nvc, decoder_max_width, decoder_max_height
)
except Exception as exc:
if not isinstance(exc, _pynvvideocodec_exception_types(nvc)):
raise
invalid_video = True
if invalid_video:
raise ValueError("Invalid or unsupported video file.")
check_frame_pixel_limit(gpu_source.width, gpu_source.height)
source = loader_cls._prepare_source(gpu_source.source)
frame_idx = loader_cls.compute_frames_index_to_sample(
@@ -360,10 +581,22 @@ class PyNvVideoCodecVideoBackendMixin:
raw_frame_bytes = len(frame_idx) * gpu_source.height * gpu_source.width * 3
pool = get_mm_gpu_ipc_pool()
if pool is None or raw_frame_bytes == 0:
frames = cls._decode_to_pinned_host(temp_path, frame_idx, nvc)
frames = cls._decode_to_pinned_host(
temp_path,
frame_idx,
nvc,
decoder_max_width,
decoder_max_height,
)
else:
with pool.acquire(raw_frame_bytes):
frames = cls._decode_to_pinned_host(temp_path, frame_idx, nvc)
frames = cls._decode_to_pinned_host(
temp_path,
frame_idx,
nvc,
decoder_max_width,
decoder_max_height,
)
finally:
with suppress(FileNotFoundError):
os.unlink(temp_path)