[Multimodal] Reorganize video decoder backends (#49155)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-18 15:06:18 +00:00
committed by GitHub
parent 689be2bcd3
commit 3bb9c18f0c
12 changed files with 1416 additions and 1097 deletions
+7 -3
View File
@@ -21,8 +21,10 @@ from vllm.multimodal.media import ImageMediaIO, VideoMediaIO
from vllm.multimodal.video import (
PYNVVIDEOCODEC_VIDEO_BACKEND,
VIDEO_LOADER_REGISTRY,
PyNvVideoCodecVideoBackend,
VideoLoader,
)
from vllm.multimodal.video_decoders.pynvvideocodec import (
PyNvVideoCodecVideoBackendMixin,
_pynvvc_frames_to_nhwc,
)
@@ -381,13 +383,15 @@ def test_pynvvideocodec_unrelated_error_propagates(
raise original_error
monkeypatch.setattr(
PyNvVideoCodecVideoBackend,
PyNvVideoCodecVideoBackendMixin,
"_read_source_metadata",
classmethod(raise_unrelated_error),
)
with pytest.raises(RuntimeError) as exc_info:
PyNvVideoCodecVideoBackend.decode_frames_pynvvideocodec(b"video", None)
PyNvVideoCodecVideoBackendMixin.decode_frames_pynvvideocodec(
None, b"video", None
)
assert exc_info.value is original_error
+7 -28
View File
@@ -6,7 +6,6 @@ import time
import pytest
import vllm.config.multimodal as multimodal_config_module
from vllm.config.multimodal import MultiModalConfig
from vllm.multimodal.gpu_ipc_memory import (
MultiModalGPUMemoryPool,
@@ -15,10 +14,10 @@ from vllm.multimodal.gpu_ipc_memory import (
reserve_mm_ipc_gpu_memory,
set_mm_gpu_ipc_pool,
)
from vllm.multimodal.video import (
from vllm.multimodal.video_decoders import PYNVVIDEOCODEC_VIDEO_BACKEND
from vllm.multimodal.video_decoders.pynvvideocodec import (
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
from vllm.utils.mem_constants import GiB_bytes
@@ -186,11 +185,7 @@ def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
monkeypatch: pytest.MonkeyPatch,
video_backend: str | None,
):
monkeypatch.setattr(
multimodal_config_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
"opencv",
)
monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
mm_config = _mm_config(
mm_ipc_gpu_memory_gb=0.25,
video_backend=video_backend,
@@ -202,11 +197,7 @@ def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
multimodal_config_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
"opencv",
)
monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
mm_config = _mm_config(
mm_ipc_gpu_memory_gb=0.25,
video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
@@ -221,11 +212,7 @@ def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
multimodal_config_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", PYNVVIDEOCODEC_VIDEO_BACKEND)
available_bytes = 4 * GiB_bytes
assert reserve_mm_ipc_gpu_memory(available_bytes, _mm_config()) == (
@@ -236,11 +223,7 @@ def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
def test_reserve_mm_ipc_gpu_memory_scales_decoder_budget_by_api_servers(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
multimodal_config_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", PYNVVIDEOCODEC_VIDEO_BACKEND)
available_bytes = 8 * GiB_bytes
assert reserve_mm_ipc_gpu_memory(
@@ -253,11 +236,7 @@ def test_reserve_mm_ipc_gpu_memory_scales_decoder_budget_by_api_servers(
def test_reserve_mm_ipc_gpu_memory_uses_configured_hw_decoders(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
multimodal_config_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
"opencv",
)
monkeypatch.setenv("VLLM_VIDEO_LOADER_BACKEND", "opencv")
available_bytes = 4 * GiB_bytes
mm_config = _mm_config(
video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
+155 -18
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
import subprocess
import sys
import threading
from contextlib import ExitStack, contextmanager
@@ -16,24 +17,26 @@ from transformers.video_utils import VideoMetadata
from vllm.assets.base import get_vllm_public_assets
from vllm.models.minimax_m3.common.mm_preprocess import MiniMaxM3VideoBackend
from vllm.multimodal.video import (
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
PYNVVIDEOCODEC_VIDEO_BACKEND,
VIDEO_LOADER_REGISTRY,
DynamicVideoBackend,
GLM46VVideoBackend,
Molmo2VideoBackend,
PyNvVideoCodecDecoderSlot,
PyNvVideoCodecVideoBackend,
PyNvVideoCodecVideoBackendMixin,
Qwen2VLVideoBackend,
Qwen3VLVideoBackend,
VideoBackend,
VideoLoader,
VideoSourceMetadata,
VideoTargetMetadata,
_pynv_decoder_pool,
get_video_loader_backend_for_processor,
)
from vllm.multimodal.video_decoders import decode_video, resolve_video_backend_kwargs
from vllm.multimodal.video_decoders.pynvvideocodec import (
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
PyNvVideoCodecDecoderSlot,
PyNvVideoCodecVideoBackendMixin,
_pynv_decoder_pool,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
@@ -99,6 +102,126 @@ def test_video_loader_type_doesnt_exist():
VIDEO_LOADER_REGISTRY.load("non_existing_video_loader")
def test_video_decoder_backends_are_lazy_imported():
code = """
import sys
import vllm.multimodal.video # noqa: F401
backend_modules = {
f"vllm.multimodal.video_decoders.{backend}"
for backend in ("opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream")
}
loaded = sorted(backend_modules & sys.modules.keys())
assert not loaded, loaded
"""
result = subprocess.run(
[sys.executable, "-c", code],
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
@pytest.mark.parametrize(
"backend",
["opencv", "pyav", "torchcodec", "pynvvideocodec", "deepstream"],
)
def test_decode_video_imports_only_selected_backend(
backend: str,
monkeypatch: pytest.MonkeyPatch,
):
imports = []
decoded = object()
def fake_decoder(*args, **kwargs):
return decoded
class FakeBackendModule:
pass
setattr(FakeBackendModule, f"decode_{backend}", fake_decoder)
def fake_import_module(name: str, package: str):
imports.append((name, package))
return FakeBackendModule
monkeypatch.setattr(
"vllm.multimodal.video_decoders.import_module", fake_import_module
)
result = decode_video(
backend,
loader_cls=None,
data=b"",
target=VideoTargetMetadata(-1, -1, 300),
sampling_kwargs={},
backend_kwargs={},
frame_recovery=False,
)
assert result is decoded
assert imports == [(f".{backend}", "vllm.multimodal.video_decoders")]
@pytest.mark.parametrize(
("backend", "kwargs", "expected_sampling", "expected_backend"),
[
(
"torchcodec",
{"min_frames": 4, "num_ffmpeg_threads": 2, "seek_mode": "approximate"},
{"min_frames": 4},
{"num_ffmpeg_threads": 2, "seek_mode": "approximate"},
),
(
"deepstream",
{"max_frames": 16, "pool_size": 3, "timeout_sec": 10.0},
{"max_frames": 16},
{"pool_size": 3, "timeout_sec": 10.0},
),
],
)
def test_video_backend_kwargs_are_separated_from_sampling_kwargs(
backend: str,
kwargs: dict,
expected_sampling: dict,
expected_backend: dict,
):
original_kwargs = dict(kwargs)
sampling_kwargs, backend_kwargs = resolve_video_backend_kwargs(backend, kwargs)
assert sampling_kwargs == expected_sampling
assert backend_kwargs == expected_backend
assert kwargs == original_kwargs
def test_video_backend_rejects_options_for_another_decoder():
with pytest.raises(
ValueError, match="num_ffmpeg_threads is not supported by the 'pyav' backend"
):
resolve_video_backend_kwargs("pyav", {"num_ffmpeg_threads": 2})
@pytest.mark.parametrize(
("backend", "error"),
[
("pyav", AssertionError),
(PYNVVIDEOCODEC_VIDEO_BACKEND, ValueError),
],
)
def test_video_decoder_spec_validates_frame_recovery(
backend: str, error: type[Exception]
):
with pytest.raises(error, match="frame_recovery is not supported"):
decode_video(
backend,
loader_cls=None,
data=b"",
target=VideoTargetMetadata(-1, -1, 300),
sampling_kwargs={},
backend_kwargs={},
frame_recovery=True,
)
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_pynvvideocodec_backend_accounts_raw_decoded_frames(
monkeypatch: pytest.MonkeyPatch,
@@ -145,7 +268,9 @@ def test_pynvvideocodec_backend_accounts_raw_decoded_frames(
"vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool
)
monkeypatch.setattr(
PyNvVideoCodecVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode)
PyNvVideoCodecVideoBackendMixin,
"_decode_to_pinned_host",
classmethod(fake_decode),
)
loader = VIDEO_LOADER_REGISTRY.load(PYNVVIDEOCODEC_VIDEO_BACKEND)
@@ -205,7 +330,9 @@ def test_pynvvideocodec_codec_uses_dynamic_sampling_strategy(
"vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool
)
monkeypatch.setattr(
DynamicVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode)
PyNvVideoCodecVideoBackendMixin,
"_decode_to_pinned_host",
classmethod(fake_decode),
)
loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
@@ -273,7 +400,7 @@ def test_pynvvideocodec_decoder_slots_are_bounded(
create_count = 0
with _fresh_decoder_pool():
PyNvVideoCodecVideoBackend._configure_decoder_slots(hw_decoders)
PyNvVideoCodecVideoBackendMixin._configure_decoder_slots(hw_decoders)
def fake_create_slot(cls):
nonlocal create_count
@@ -281,7 +408,7 @@ def test_pynvvideocodec_decoder_slots_are_bounded(
return FakeSlot()
monkeypatch.setattr(
PyNvVideoCodecVideoBackend,
PyNvVideoCodecVideoBackendMixin,
"_create_decoder_slot",
classmethod(fake_create_slot),
)
@@ -291,12 +418,16 @@ def test_pynvvideocodec_decoder_slots_are_bounded(
with ExitStack() as stack:
retained_slots = [
stack.enter_context(PyNvVideoCodecVideoBackend._borrow_decoder_slot())
stack.enter_context(
PyNvVideoCodecVideoBackendMixin._borrow_decoder_slot()
)
for _ in range(hw_decoders)
]
def borrow_extra_slot():
with PyNvVideoCodecVideoBackend._borrow_decoder_slot() as extra_slot:
with (
PyNvVideoCodecVideoBackendMixin._borrow_decoder_slot()
) as extra_slot:
seen_slots.append(extra_slot)
borrowed.set()
@@ -317,11 +448,11 @@ def test_pynvvideocodec_decoder_slots_are_configured_once(
):
monkeypatch.setattr(_pynv_decoder_pool, "max_slots", None)
PyNvVideoCodecVideoBackend._configure_decoder_slots(2)
PyNvVideoCodecVideoBackend._configure_decoder_slots(2)
PyNvVideoCodecVideoBackendMixin._configure_decoder_slots(2)
PyNvVideoCodecVideoBackendMixin._configure_decoder_slots(2)
with pytest.raises(RuntimeError, match="already configured as 2, got 3"):
PyNvVideoCodecVideoBackend._configure_decoder_slots(3)
PyNvVideoCodecVideoBackendMixin._configure_decoder_slots(3)
def test_pynvvideocodec_failed_rebuild_invalidates_decoder_slot():
@@ -367,7 +498,7 @@ def test_pynvvideocodec_failed_rebuild_invalidates_decoder_slot():
with (
pytest.raises(RuntimeError, match="construct failed"),
PyNvVideoCodecVideoBackend._borrow_decoder_slot() as borrowed,
PyNvVideoCodecVideoBackendMixin._borrow_decoder_slot() as borrowed,
):
assert borrowed is slot
borrowed.get_decoder(
@@ -456,6 +587,12 @@ def test_pynvvideocodec_cross_subclass_shares_single_pool():
counters via ClassVar shadowing.
"""
class MixinSubclassA(PyNvVideoCodecVideoBackendMixin):
pass
class MixinSubclassB(PyNvVideoCodecVideoBackendMixin):
pass
class FakeSlot:
pass
@@ -475,8 +612,8 @@ def test_pynvvideocodec_cross_subclass_shares_single_pool():
)
try:
with ExitStack() as stack:
stack.enter_context(VideoBackend._borrow_decoder_slot())
stack.enter_context(Qwen3VLVideoBackend._borrow_decoder_slot())
stack.enter_context(MixinSubclassA._borrow_decoder_slot())
stack.enter_context(MixinSubclassB._borrow_decoder_slot())
assert pool.active == 2
blocked = threading.Event()
@@ -484,7 +621,7 @@ def test_pynvvideocodec_cross_subclass_shares_single_pool():
def try_borrow():
blocked.set()
with Qwen2VLVideoBackend._borrow_decoder_slot():
with MixinSubclassB._borrow_decoder_slot():
acquired.set()
t = threading.Thread(target=try_borrow)