From af16446bf39de047ab57649c933063cf1cbf1e50 Mon Sep 17 00:00:00 2001 From: Brandon Pelfrey Date: Fri, 26 Jun 2026 17:32:51 -0700 Subject: [PATCH] Vram semaphore infra (#44465) Signed-off-by: Brandon Pelfrey Co-authored-by: Roger Wang --- requirements/cuda.txt | 1 + tests/multimodal/test_gpu_ipc_memory.py | 145 ++++++++++ tests/multimodal/test_video.py | 234 ++++++++++++++++ tests/v1/worker/test_gpu_worker.py | 116 ++++++++ vllm/config/model.py | 4 + vllm/config/multimodal.py | 10 + vllm/engine/arg_utils.py | 6 + vllm/multimodal/gpu_ipc_memory.py | 147 ++++++++++ vllm/multimodal/video.py | 344 +++++++++++++++++++++++- vllm/renderers/base.py | 11 + vllm/v1/worker/gpu_worker.py | 84 +++++- 11 files changed, 1087 insertions(+), 15 deletions(-) create mode 100644 tests/multimodal/test_gpu_ipc_memory.py create mode 100644 tests/v1/worker/test_gpu_worker.py create mode 100644 vllm/multimodal/gpu_ipc_memory.py diff --git a/requirements/cuda.txt b/requirements/cuda.txt index 19a1f63dd9..124dae4846 100644 --- a/requirements/cuda.txt +++ b/requirements/cuda.txt @@ -8,6 +8,7 @@ torch==2.11.0 torchaudio==2.11.0 # These must be updated alongside torch torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version +PyNvVideoCodec==2.0.4 # FlashInfer should be updated together with the Dockerfile flashinfer-python==0.6.12 flashinfer-cubin==0.6.12 diff --git a/tests/multimodal/test_gpu_ipc_memory.py b/tests/multimodal/test_gpu_ipc_memory.py new file mode 100644 index 0000000000..bc6bf031fe --- /dev/null +++ b/tests/multimodal/test_gpu_ipc_memory.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import threading +import time + +import pytest + +from vllm.multimodal.gpu_ipc_memory import ( + MultiModalGPUMemoryPool, + get_mm_gpu_ipc_pool, + maybe_init_mm_gpu_ipc_pool, + set_mm_gpu_ipc_pool, +) +from vllm.utils.mem_constants import GiB_bytes + + +def test_acquire_release_accounting(): + pool = MultiModalGPUMemoryPool(total_bytes=100) + assert pool.available_bytes == 100 + + lease = pool.acquire(40) + assert pool.available_bytes == 60 + + lease.release() + assert pool.available_bytes == 100 + + +def test_acquire_too_large_raises(): + pool = MultiModalGPUMemoryPool(total_bytes=100) + with pytest.raises(ValueError): + pool.acquire(101) + # Nothing should have been reserved. + assert pool.available_bytes == 100 + + +def test_negative_acquire_raises(): + pool = MultiModalGPUMemoryPool(total_bytes=100) + with pytest.raises(ValueError): + pool.acquire(-1) + + +def test_double_release_is_noop(): + pool = MultiModalGPUMemoryPool(total_bytes=100) + lease = pool.acquire(50) + lease.release() + assert pool.available_bytes == 100 + # Releasing again must not inflate the pool past its capacity. + lease.release() + assert pool.available_bytes == 100 + + +def test_context_manager_releases_on_exception(): + pool = MultiModalGPUMemoryPool(total_bytes=100) + with pytest.raises(RuntimeError), pool.acquire(50): + assert pool.available_bytes == 50 + raise RuntimeError("boom") + assert pool.available_bytes == 100 + + +def test_acquire_blocks_until_release(): + pool = MultiModalGPUMemoryPool(total_bytes=100) + first = pool.acquire(80) + + acquired = threading.Event() + + def waiter(): + # Needs 50 bytes but only 20 are free; must block until `first` + # is released. + with pool.acquire(50): + acquired.set() + + t = threading.Thread(target=waiter) + t.start() + + # The waiter cannot proceed yet. + assert not acquired.wait(timeout=0.2) + + # Releasing the first lease frees enough budget to unblock the waiter. + first.release() + assert acquired.wait(timeout=2.0) + t.join(timeout=2.0) + assert not t.is_alive() + assert pool.available_bytes == 100 + + +def test_concurrent_acquires_serialize(): + pool = MultiModalGPUMemoryPool(total_bytes=100) + # Each task needs 60 bytes, so only one can hold the budget at a time. + in_section = [] + max_concurrent = 0 + lock = threading.Lock() + + def task(): + nonlocal max_concurrent + with pool.acquire(60): + with lock: + in_section.append(1) + max_concurrent = max(max_concurrent, len(in_section)) + time.sleep(0.05) + with lock: + in_section.pop() + + threads = [threading.Thread(target=task) for _ in range(4)] + for t in threads: + t.start() + for t in threads: + t.join(timeout=5.0) + assert not t.is_alive() + + assert max_concurrent == 1 + assert pool.available_bytes == 100 + + +def test_zero_total_bytes_rejected(): + with pytest.raises(ValueError): + MultiModalGPUMemoryPool(total_bytes=0) + + +def test_global_pool_accessor(): + try: + assert maybe_init_mm_gpu_ipc_pool(0) is None + assert get_mm_gpu_ipc_pool() is None + + pool = maybe_init_mm_gpu_ipc_pool(2) + assert pool is not None + assert get_mm_gpu_ipc_pool() is pool + assert pool.total_bytes == 2 * GiB_bytes + finally: + set_mm_gpu_ipc_pool(None) + + +def test_global_pool_splits_budget_across_api_processes(): + try: + pool = maybe_init_mm_gpu_ipc_pool(2, api_process_count=4) + assert pool is not None + assert get_mm_gpu_ipc_pool() is pool + assert pool.total_bytes == GiB_bytes // 2 + finally: + set_mm_gpu_ipc_pool(None) + + +def test_global_pool_rejects_invalid_api_process_count(): + with pytest.raises(ValueError): + maybe_init_mm_gpu_ipc_pool(2, api_process_count=0) diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 694eb392c4..6fccc926a2 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -1,6 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project + import itertools +import sys +import threading +from contextlib import ExitStack, contextmanager from pathlib import Path import numpy as np @@ -11,10 +15,15 @@ from transformers.video_utils import VideoMetadata from vllm.assets.base import get_vllm_public_assets from vllm.multimodal.video import ( + PYNVVIDEOCODEC_DECODER_CACHE_SIZE, + PYNVVIDEOCODEC_MAX_RETAINED_DECODERS, + PYNVVIDEOCODEC_VIDEO_BACKEND, VIDEO_LOADER_REGISTRY, DynamicVideoBackend, GLM46VVideoBackend, Molmo2VideoBackend, + PyNvVideoCodecDecoderSlot, + PyNvVideoCodecVideoBackend, Qwen2VLVideoBackend, Qwen3VLVideoBackend, VideoLoader, @@ -22,6 +31,7 @@ from vllm.multimodal.video import ( VideoTargetMetadata, get_video_loader_backend_for_processor, ) +from vllm.platforms import current_platform from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config from .utils import create_long_gop_video, create_video_from_image @@ -65,6 +75,230 @@ def test_video_loader_type_doesnt_exist(): VIDEO_LOADER_REGISTRY.load("non_existing_video_loader") +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +def test_pynvvideocodec_backend_accounts_raw_decoded_frames( + monkeypatch: pytest.MonkeyPatch, +): + decoder_cache_sizes = [] + + class FakeMetadata: + width = 10 + height = 20 + average_fps = 5.0 + duration = 2.0 + + class FakeDecoder: + def __init__(self, *args, **kwargs): + decoder_cache_sizes.append(kwargs["decoder_cache_size"]) + + def __len__(self): + return 10 + + def get_stream_metadata(self): + return FakeMetadata() + + class FakeNvc: + class OutputColorType: + RGB = "rgb" + + SimpleDecoder = FakeDecoder + + class RecordingPool: + def __init__(self): + self.acquired: list[int] = [] + + @contextmanager + def acquire(self, size: int): + self.acquired.append(size) + yield + + def fake_decode(cls, file_path: str, frame_idx: list[int], nvc): + return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8) + + pool = RecordingPool() + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc) + monkeypatch.setattr( + "vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool + ) + monkeypatch.setattr( + PyNvVideoCodecVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode) + ) + + loader = VIDEO_LOADER_REGISTRY.load(PYNVVIDEOCODEC_VIDEO_BACKEND) + frames, metadata = loader.load_bytes(b"fake video", num_frames=4) + + assert frames.shape == (4, 20, 10, 3) + assert pool.acquired == [4 * 20 * 10 * 3] + assert decoder_cache_sizes == [PYNVVIDEOCODEC_DECODER_CACHE_SIZE] + assert metadata["video_backend"] == PYNVVIDEOCODEC_VIDEO_BACKEND + assert metadata["frames_indices"] == [0, 3, 6, 9] + + +@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA") +def test_pynvvideocodec_codec_uses_dynamic_sampling_strategy( + monkeypatch: pytest.MonkeyPatch, +): + decoded_indices = [] + + class FakeMetadata: + width = 10 + height = 20 + average_fps = 5.0 + duration = 2.0 + + class FakeDecoder: + def __init__(self, *args, **kwargs): + pass + + def __len__(self): + return 10 + + def get_stream_metadata(self): + return FakeMetadata() + + class FakeNvc: + class OutputColorType: + RGB = "rgb" + + SimpleDecoder = FakeDecoder + + class RecordingPool: + def __init__(self): + self.acquired: list[int] = [] + + @contextmanager + def acquire(self, size: int): + self.acquired.append(size) + yield + + def fake_decode(cls, file_path: str, frame_idx: list[int], nvc): + decoded_indices.append(frame_idx) + return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8) + + pool = RecordingPool() + monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc) + monkeypatch.setattr( + "vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool + ) + monkeypatch.setattr( + DynamicVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode) + ) + + loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic") + frames, metadata = loader.load_bytes( + b"fake video", + fps=2, + max_duration=1, + backend=PYNVVIDEOCODEC_VIDEO_BACKEND, + ) + + assert frames.shape == (2, 20, 10, 3) + assert decoded_indices == [[0, 9]] + assert pool.acquired == [2 * 20 * 10 * 3] + assert metadata["video_backend"] == f"{PYNVVIDEOCODEC_VIDEO_BACKEND}_dynamic" + assert metadata["frames_indices"] == [0, 9] + + +def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatch): + class FakeSlot: + pass + + create_count = 0 + old_slots = PyNvVideoCodecVideoBackend._decoder_slots + old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots + old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond + try: + PyNvVideoCodecVideoBackend._decoder_slots = [] + PyNvVideoCodecVideoBackend._active_decoder_slots = 0 + PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition() + + def fake_create_slot(cls): + nonlocal create_count + create_count += 1 + return FakeSlot() + + monkeypatch.setattr( + PyNvVideoCodecVideoBackend, + "_create_decoder_slot", + classmethod(fake_create_slot), + ) + + borrowed = threading.Event() + seen_slots = [] + + with ExitStack() as stack: + retained_slots = [ + stack.enter_context(PyNvVideoCodecVideoBackend._borrow_decoder_slot()) + for _ in range(PYNVVIDEOCODEC_MAX_RETAINED_DECODERS) + ] + + def borrow_extra_slot(): + with PyNvVideoCodecVideoBackend._borrow_decoder_slot() as extra_slot: + seen_slots.append(extra_slot) + borrowed.set() + + thread = threading.Thread(target=borrow_extra_slot) + thread.start() + assert not borrowed.wait(timeout=0.2) + + assert borrowed.wait(timeout=2.0) + thread.join(timeout=2.0) + assert not thread.is_alive() + + assert seen_slots[0] in retained_slots + assert create_count == PYNVVIDEOCODEC_MAX_RETAINED_DECODERS + finally: + PyNvVideoCodecVideoBackend._decoder_slots = old_slots + PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots + PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond + + +def test_pynvvideocodec_decoder_slot_retains_simple_decoder(): + events: list[tuple[object, ...]] = [] + + class FakeStream: + cuda_stream = "cuda-stream" + + class FakeDecoder: + def __init__(self, file_path: str, **kwargs): + events.append( + ( + "create", + file_path, + kwargs["gpu_id"], + kwargs["cuda_stream"], + kwargs["decoder_cache_size"], + ) + ) + + def reconfigure_decoder(self, file_path: str): + events.append(("reconfigure", file_path)) + + class FakeNvc: + class OutputColorType: + RGB = "rgb" + + SimpleDecoder = FakeDecoder + + slot = PyNvVideoCodecDecoderSlot(FakeStream()) + + decoder = slot.get_decoder("first.mp4", FakeNvc, device_index=7) + assert slot.get_decoder("first.mp4", FakeNvc, device_index=7) is decoder + assert slot.get_decoder("second.mp4", FakeNvc, device_index=7) is decoder + + assert events == [ + ( + "create", + "first.mp4", + 7, + "cuda-stream", + PYNVVIDEOCODEC_DECODER_CACHE_SIZE, + ), + ("reconfigure", "second.mp4"), + ] + assert slot.source_path == "second.mp4" + + # ============================================================================ # Video Processor → Video Loader Tests (via model repo) # ============================================================================ diff --git a/tests/v1/worker/test_gpu_worker.py b/tests/v1/worker/test_gpu_worker.py new file mode 100644 index 0000000000..31be4a8402 --- /dev/null +++ b/tests/v1/worker/test_gpu_worker.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from types import SimpleNamespace + +import pytest + +import vllm.v1.worker.gpu_worker as gpu_worker_module +from vllm.multimodal.video import ( + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES, + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES, + PYNVVIDEOCODEC_MAX_RETAINED_DECODERS, + PYNVVIDEOCODEC_VIDEO_BACKEND, +) +from vllm.utils.mem_constants import GiB_bytes +from vllm.v1.worker.gpu_worker import Worker + + +def _worker_with_mm_config( + mm_config: SimpleNamespace, + *, + api_process_count: int = 1, +) -> Worker: + worker = object.__new__(Worker) + worker.model_config = SimpleNamespace(multimodal_config=mm_config) + worker.parallel_config = SimpleNamespace(_api_process_count=api_process_count) + return worker + + +def _mm_config( + *, + mm_ipc_gpu_memory_gb: float = 0, + video_backend: str | None = None, +) -> SimpleNamespace: + video_kwargs = {} if video_backend is None else {"video_backend": video_backend} + return SimpleNamespace( + mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb, + media_io_kwargs={"video": video_kwargs} if video_kwargs else {}, + ) + + +def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int: + return api_process_count * ( + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS + + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES + ) + + +@pytest.mark.parametrize("video_backend", [None, "opencv"]) +def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only( + monkeypatch: pytest.MonkeyPatch, + video_backend: str | None, +): + monkeypatch.setattr( + gpu_worker_module.envs, + "VLLM_VIDEO_LOADER_BACKEND", + "opencv", + ) + worker = _worker_with_mm_config( + _mm_config(mm_ipc_gpu_memory_gb=0.25, video_backend=video_backend) + ) + + assert worker._reserve_mm_ipc_gpu_memory(GiB_bytes) == int(0.75 * GiB_bytes) + + +def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + gpu_worker_module.envs, + "VLLM_VIDEO_LOADER_BACKEND", + "opencv", + ) + worker = _worker_with_mm_config( + _mm_config( + mm_ipc_gpu_memory_gb=0.25, + video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND, + ) + ) + available_bytes = 4 * GiB_bytes + + assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == ( + available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget() + ) + + +def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + gpu_worker_module.envs, + "VLLM_VIDEO_LOADER_BACKEND", + PYNVVIDEOCODEC_VIDEO_BACKEND, + ) + worker = _worker_with_mm_config(_mm_config()) + available_bytes = 4 * GiB_bytes + + assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == ( + available_bytes - _pynvvideocodec_decoder_budget() + ) + + +def test_reserve_mm_ipc_gpu_memory_scales_pynvvideocodec_budget_by_api_servers( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + gpu_worker_module.envs, + "VLLM_VIDEO_LOADER_BACKEND", + PYNVVIDEOCODEC_VIDEO_BACKEND, + ) + worker = _worker_with_mm_config(_mm_config(), api_process_count=3) + available_bytes = 8 * GiB_bytes + + assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == ( + available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3) + ) diff --git a/vllm/config/model.py b/vllm/config/model.py index c7736f985d..fecb26aa7e 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -351,6 +351,7 @@ class ModelConfig: skip_mm_profiling: InitVar[bool | None] = None video_pruning_rate: InitVar[float | None] = None mm_tensor_ipc: InitVar[MMTensorIPC] = None + mm_ipc_gpu_memory_gb: InitVar[float | None] = None def compute_hash(self) -> str: """ @@ -397,6 +398,7 @@ class ModelConfig: "mm_encoder_tp_mode", "interleave_mm_strings", "skip_mm_profiling", + "mm_ipc_gpu_memory_gb", } from vllm.config.utils import get_hash_factors, hash_factors @@ -477,6 +479,7 @@ class ModelConfig: skip_mm_profiling: bool | None, video_pruning_rate: float | None, mm_tensor_ipc: MMTensorIPC, + mm_ipc_gpu_memory_gb: float | None, ) -> None: # Keep set served_model_name before maybe_model_redirect(self.model) self.served_model_name = get_served_model_name( @@ -690,6 +693,7 @@ class ModelConfig: skip_mm_profiling=skip_mm_profiling, video_pruning_rate=video_pruning_rate, mm_tensor_ipc=mm_tensor_ipc, + mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb, ) mm_config_kwargs = { diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index 56333b1116..150d58cf1f 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -197,6 +197,16 @@ class MultiModalConfig: - "direct_rpc": Use msgspec serialization via RPC - "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC Defaults to "direct_rpc". """ + mm_ipc_gpu_memory_gb: float = Field(default=0, ge=0) + """Amount of GPU memory (in GiB) sequestered on the engine's device for + GPU-side multimodal work in the API-server (frontend) process, such as + hardware video decoding. + + This budget is carved out of the engine's KV-cache memory so the headroom + physically exists, and frontend GPU decode paths acquire from a blocking + byte-counting semaphore of this size before allocating on the device. + + Set to `0` (default) to disable frontend GPU multimodal memory gating.""" @field_validator("limit_per_prompt", mode="before") @classmethod diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 8cc219264f..efdd7696fd 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -582,6 +582,7 @@ class EngineArgs: skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc + mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb # LoRA fields enable_lora: bool = False max_loras: int = LoRAConfig.max_loras @@ -1294,6 +1295,10 @@ class EngineArgs: multimodal_group.add_argument( "--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"] ) + multimodal_group.add_argument( + "--mm-ipc-gpu-memory-gb", + **multimodal_kwargs["mm_ipc_gpu_memory_gb"], + ) # LoRA related configs lora_kwargs = get_kwargs(LoRAConfig) @@ -1660,6 +1665,7 @@ class EngineArgs: logits_processors=self.logits_processors, video_pruning_rate=self.video_pruning_rate, mm_tensor_ipc=self.mm_tensor_ipc, + mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb, io_processor_plugin=self.io_processor_plugin, renderer_num_workers=self.renderer_num_workers, ) diff --git a/vllm/multimodal/gpu_ipc_memory.py b/vllm/multimodal/gpu_ipc_memory.py new file mode 100644 index 0000000000..15b912064a --- /dev/null +++ b/vllm/multimodal/gpu_ipc_memory.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Admission control for frontend GPU-side multimodal work. + +When multimodal media is decoded on the GPU in the API-server (frontend) +process, the decoded buffers compete for the same device memory that the +engine reserves for weights, activations, and the KV cache. To keep the +frontend's GPU usage within a sequestered budget (see +``MultiModalConfig.mm_ipc_gpu_memory_gb``), decode paths acquire the number of +bytes they need from a process-global :class:`MultiModalGPUMemoryPool` before +allocating on the device and release them once the device memory is freed. + +The pool is a simple byte-counting semaphore: ``acquire`` blocks until enough +budget is free, so concurrent requests serialize rather than oversubscribe the +GPU. It lives only in the frontend process; the engine carves the matching +amount out of its KV-cache budget so the headroom physically exists. +""" + +import threading + +from vllm.logger import init_logger +from vllm.utils.mem_constants import GiB_bytes + +logger = init_logger(__name__) + + +class MultiModalGPUMemoryLease: + """A handle for bytes acquired from a :class:`MultiModalGPUMemoryPool`. + + Releasing is idempotent and the lease doubles as a context manager so the + budget is returned even if the decode raises. + """ + + def __init__(self, pool: "MultiModalGPUMemoryPool", lease_id: int, nbytes: int): + self.lease_id = lease_id + self.nbytes = nbytes + self._pool = pool + + def release(self) -> None: + self._pool._release(self) + + def __enter__(self) -> "MultiModalGPUMemoryLease": + return self + + def __exit__(self, *exc_info) -> None: + self.release() + + +class MultiModalGPUMemoryPool: + """Blocking byte-counting semaphore for frontend GPU multimodal memory. + + Thread-safe in both directions: ``acquire`` (blocking) and ``release`` are + typically called from the renderer's multimodal executor threads. + """ + + def __init__(self, total_bytes: int): + if total_bytes <= 0: + raise ValueError(f"total_bytes must be positive, got {total_bytes}") + self._total_bytes = total_bytes + self._available = total_bytes + self._cond = threading.Condition() + self._next_lease_id = 0 + # Outstanding lease ids, so a double release is a no-op. + self._outstanding: set[int] = set() + + @property + def total_bytes(self) -> int: + return self._total_bytes + + @property + def available_bytes(self) -> int: + with self._cond: + return self._available + + def acquire(self, nbytes: int) -> MultiModalGPUMemoryLease: + """Reserve ``nbytes``, blocking until that much budget is free. + + Raises ``ValueError`` if ``nbytes`` exceeds the pool's total capacity, + since such a request could never be satisfied. + """ + if nbytes < 0: + raise ValueError(f"Cannot acquire negative bytes: {nbytes}") + if nbytes > self._total_bytes: + raise ValueError( + f"Multimodal GPU decode requested {nbytes} bytes, which exceeds " + f"the total pool size of {self._total_bytes} bytes. Increase " + f"--mm-ipc-gpu-memory-gb or reduce the multimodal input size." + ) + with self._cond: + while self._available < nbytes: + self._cond.wait() + self._available -= nbytes + lease_id = self._next_lease_id + self._next_lease_id += 1 + self._outstanding.add(lease_id) + return MultiModalGPUMemoryLease(self, lease_id, nbytes) + + def _release(self, lease: MultiModalGPUMemoryLease) -> None: + with self._cond: + if lease.lease_id not in self._outstanding: + # Already released — idempotent. + return + self._outstanding.discard(lease.lease_id) + self._available += lease.nbytes + self._cond.notify_all() + + +_GLOBAL_POOL: MultiModalGPUMemoryPool | None = None + + +def set_mm_gpu_ipc_pool(pool: MultiModalGPUMemoryPool | None) -> None: + """Install the process-global pool (frontend process only).""" + global _GLOBAL_POOL + _GLOBAL_POOL = pool + + +def get_mm_gpu_ipc_pool() -> MultiModalGPUMemoryPool | None: + """Return the process-global pool, or ``None`` when gating is disabled.""" + return _GLOBAL_POOL + + +def maybe_init_mm_gpu_ipc_pool( + mm_ipc_gpu_memory_gb: float, + api_process_count: int = 1, +) -> MultiModalGPUMemoryPool | None: + """Create and install the global pool from the configured GiB budget. + + Returns ``None`` (and leaves gating disabled) when the budget is 0. When + multiple API-server processes share one engine, each process gets an equal + slice of the user-provided frontend budget. + """ + if mm_ipc_gpu_memory_gb <= 0: + set_mm_gpu_ipc_pool(None) + return None + if api_process_count <= 0: + raise ValueError(f"api_process_count must be positive, got {api_process_count}") + total_bytes = int(mm_ipc_gpu_memory_gb * GiB_bytes) // api_process_count + pool = MultiModalGPUMemoryPool(total_bytes) + set_mm_gpu_ipc_pool(pool) + logger.info( + "Initialized multimodal GPU IPC memory pool with %d bytes for this API " + "process (%.2f GiB total budget across %d API process(es)).", + total_bytes, + mm_ipc_gpu_memory_gb, + api_process_count, + ) + return pool diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index 700bd0802c..8cd4870026 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -1,7 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import math +import os +import tempfile +import threading from abc import abstractmethod +from contextlib import contextmanager, suppress from io import BytesIO from typing import Any, ClassVar, Literal, NamedTuple, cast @@ -11,6 +15,7 @@ import torch from vllm.logger import init_logger from vllm.utils.import_utils import PlaceholderModule +from vllm.utils.mem_constants import MiB_bytes from vllm.utils.registry import ExtensionManager try: @@ -130,6 +135,14 @@ class VideoSourceMetadata(NamedTuple): duration: float +class PyNvVideoCodecSourceMetadata(NamedTuple): + """Metadata needed before GPU video decode.""" + + source: VideoSourceMetadata + width: int + height: int + + class VideoLoader: @classmethod def compute_frames_index_to_sample( @@ -170,6 +183,57 @@ class VideoLoader: VIDEO_LOADER_REGISTRY = VideoLoaderRegistry() +PYNVVIDEOCODEC_VIDEO_BACKEND: Literal["pynvvideocodec"] = "pynvvideocodec" +# Fixed upper bound reserved for persistent PyNvVideoCodec decoder surfaces. +PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES = 128 * MiB_bytes +PYNVVIDEOCODEC_DECODER_CACHE_SIZE = 2 +PYNVVIDEOCODEC_MAX_RETAINED_DECODERS = 1 +# Per-API-server CUDA context and driver allocation, measured with +# PyNvVideoCodec 2.0.4 on H100. +PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES = int(1.8 * 1024 * MiB_bytes) + + +class PyNvVideoCodecDecoderSlot: + """A retained PyNv decoder slot and its CUDA stream. + + The decoder is reused across requests: ``reconfigure_decoder`` repoints the + existing decoder at each new source instead of paying a fresh + ``SimpleDecoder`` construction per request. Construction (CUVID parser + + decoder + surface-pool allocation) is the dominant per-request cost, so + reconfiguring is far cheaper. A single decoder serves both metadata + (``len``/``get_stream_metadata``) and frame decode -- no separate + metadata decoder. + """ + + def __init__(self, stream) -> None: + self.stream = stream + self.decoder = None + self.source_path: str | None = None + + def _construct(self, file_path: str, nvc, device_index: int) -> None: + self.decoder = nvc.SimpleDecoder( + file_path, + output_color_type=nvc.OutputColorType.RGB, + use_device_memory=True, + need_scanned_stream_metadata=True, + gpu_id=device_index, + cuda_stream=self.stream.cuda_stream, + decoder_cache_size=PYNVVIDEOCODEC_DECODER_CACHE_SIZE, + ) + self.source_path = file_path + + def get_decoder(self, file_path: str, nvc, device_index: int): + if self.decoder is None: + self._construct(file_path, nvc, device_index) + elif self.source_path != file_path: + try: + self.decoder.reconfigure_decoder(file_path) + self.source_path = file_path + except Exception: + # reconfigure unsupported/unsafe for this source -> rebuild. + self._construct(file_path, nvc, device_index) + return self.decoder + class OpenCVVideoBackendMixin: @staticmethod @@ -485,15 +549,223 @@ class PyAVVideoBackendMixin: return np.stack(frames_list), valid_indices +class PyNvVideoCodecVideoBackendMixin: + """PyNvVideoCodec utilities for GPU-backed frame decode.""" + + _decoder_slots: ClassVar[list[PyNvVideoCodecDecoderSlot]] = [] + _active_decoder_slots: ClassVar[int] = 0 + _decoder_slot_cond: ClassVar[threading.Condition] = threading.Condition() + _DEVICE_INDEX: ClassVar[int] = 0 + + @classmethod + @abstractmethod + def compute_frames_index_to_sample( + cls, + source: VideoSourceMetadata, + target: VideoTargetMetadata, + **kwargs, + ) -> list[int]: + raise NotImplementedError + + @classmethod + @abstractmethod + def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata: + raise NotImplementedError + + @classmethod + def _create_decoder_slot(cls) -> PyNvVideoCodecDecoderSlot: + import torch + + return PyNvVideoCodecDecoderSlot(torch.cuda.Stream(device=cls._DEVICE_INDEX)) + + @staticmethod + @contextmanager + def _torch_stream_context(stream): + import torch + + torch.accelerator.set_device_index(stream.device.index) + previous_stream = torch.accelerator.current_stream() + torch.accelerator.set_stream(stream) + try: + yield + finally: + torch.accelerator.set_stream(previous_stream) + + @classmethod + @contextmanager + def _borrow_decoder_slot(cls): + create_slot = False + with cls._decoder_slot_cond: + while True: + if cls._decoder_slots: + slot = cls._decoder_slots.pop() + break + if cls._active_decoder_slots < PYNVVIDEOCODEC_MAX_RETAINED_DECODERS: + cls._active_decoder_slots += 1 + create_slot = True + break + cls._decoder_slot_cond.wait() + + if create_slot: + try: + slot = cls._create_decoder_slot() + except Exception: + with cls._decoder_slot_cond: + cls._active_decoder_slots -= 1 + cls._decoder_slot_cond.notify() + raise + + try: + yield slot + finally: + with cls._decoder_slot_cond: + cls._decoder_slots.append(slot) + cls._decoder_slot_cond.notify() + + @staticmethod + def _metadata_value(metadata, *names: str, default=None): + for name in names: + value = getattr(metadata, name, None) + if value is not None: + return value + return default + + @classmethod + def _read_source_metadata( + cls, + file_path: str, + nvc, + ) -> PyNvVideoCodecSourceMetadata: + with cls._borrow_decoder_slot() as decoder_slot: + with cls._torch_stream_context(decoder_slot.stream): + decoder = decoder_slot.get_decoder( + file_path, nvc, device_index=cls._DEVICE_INDEX + ) + metadata = decoder.get_stream_metadata() + total_frames_num = len(decoder) + width = int(cls._metadata_value(metadata, "width", default=0)) + height = int(cls._metadata_value(metadata, "height", default=0)) + original_fps = float( + cls._metadata_value( + metadata, + "average_fps", + "avg_frame_rate", + "frame_rate", + "frameRate", + default=0.0, + ) + ) + duration = float( + cls._metadata_value(metadata, "duration", default=0.0) + or (total_frames_num / original_fps if original_fps > 0 else 0.0) + ) + if total_frames_num <= 0: + raise ValueError("Could not determine video frame count") + if width <= 0 or height <= 0: + raise ValueError("Could not determine video dimensions") + return PyNvVideoCodecSourceMetadata( + source=VideoSourceMetadata(total_frames_num, original_fps, duration), + width=width, + height=height, + ) + + @classmethod + def _decode_to_pinned_host( + cls, + file_path: str, + frame_idx: list[int], + nvc, + ) -> npt.NDArray: + import torch + + if not frame_idx: + return np.empty((0,), dtype=np.uint8) + + with cls._borrow_decoder_slot() as decoder_slot: + stream = decoder_slot.stream + with cls._torch_stream_context(stream): + decoder = decoder_slot.get_decoder( + file_path, nvc, device_index=cls._DEVICE_INDEX + ) + decoded_frames = decoder.get_batch_frames_by_index(frame_idx) + if len(decoded_frames) < len(frame_idx): + logger.warning( + "pynvvideocodec video loading: expected %d frames but got %d.", + len(frame_idx), + len(decoded_frames), + ) + torch_frames = [torch.from_dlpack(frame) for frame in decoded_frames] + if not torch_frames: + return np.empty((0,), dtype=np.uint8) + device_frames = torch.stack(torch_frames) + if device_frames.ndim != 4: + raise ValueError( + "PyNvVideoCodec returned frames with unexpected shape " + f"{tuple(device_frames.shape)}" + ) + device_frames = device_frames.permute(0, 3, 1, 2).contiguous() + host_frames = torch.empty( + device_frames.shape, + dtype=device_frames.dtype, + device="cpu", + pin_memory=True, + ) + host_frames.copy_(device_frames, non_blocking=True) + stream.synchronize() + host_array = host_frames.numpy() + del decoded_frames, torch_frames, device_frames + return host_array + + @classmethod + def decode_frames_pynvvideocodec( + cls, + data: bytes, + target: VideoTargetMetadata, + **kwargs, + ) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]: + import PyNvVideoCodec as nvc + + from vllm.multimodal.gpu_ipc_memory import get_mm_gpu_ipc_pool + + temp_fd, temp_path = tempfile.mkstemp(suffix=".mp4") + try: + with os.fdopen(temp_fd, "wb") as temp_file: + temp_file.write(data) + + gpu_source = cls._read_source_metadata(temp_path, nvc) + source = cls._prepare_source(gpu_source.source) + frame_idx = cls.compute_frames_index_to_sample( + source=source, target=target, **kwargs + ) + raw_frame_bytes = len(frame_idx) * gpu_source.height * gpu_source.width * 3 + pool = get_mm_gpu_ipc_pool() + if pool is None or raw_frame_bytes == 0: + frames = cls._decode_to_pinned_host(temp_path, frame_idx, nvc) + else: + with pool.acquire(raw_frame_bytes): + frames = cls._decode_to_pinned_host(temp_path, frame_idx, nvc) + finally: + with suppress(FileNotFoundError): + os.unlink(temp_path) + + valid_frame_indices = frame_idx[: int(frames.shape[0])] + return frames, source, frame_idx, valid_frame_indices + + @VIDEO_LOADER_REGISTRY.register("opencv") -class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin): +class VideoBackend( + VideoLoader, + OpenCVVideoBackendMixin, + PyAVVideoBackendMixin, + PyNvVideoCodecVideoBackendMixin, +): """Uniform-sampling video backend. Samples ``num_frames`` uniformly across the video (or one frame every ``1/fps`` seconds, whichever produces fewer frames). The decoding codec - is selected via the ``backend`` kwarg (``"opencv"`` or ``"pyav"``), - which can be passed through ``--media-io-kwargs``. Defaults to - ``"pyav"`` for concurrent decoding. + is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``, or + ``"pynvvideocodec"``), which can be passed through + ``--media-io-kwargs``. Defaults to ``"opencv"``. """ _sampling_suffix: ClassVar[str] = "" @@ -538,7 +810,7 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav"] = "opencv", + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: """Load sampled frames from raw video bytes. @@ -551,7 +823,8 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin): dynamic subclass; ignored here. frame_recovery: Enable forward-scan recovery for failed frames. Only honored by the OpenCV codec. - backend: Decoding codec — ``"opencv"`` or ``"pyav"`` . + backend: Decoding codec — ``"opencv"``, ``"pyav"``, or + ``"pynvvideocodec"``. Returns: Tuple of ``(frames_array, metadata_dict)``. @@ -584,10 +857,21 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin): frames, valid = cls.decode_frames( container, frame_idx, source.original_fps, source.duration ) + elif backend == PYNVVIDEOCODEC_VIDEO_BACKEND: + if frame_recovery: + raise ValueError( + "frame_recovery is not supported for " + f"`{PYNVVIDEOCODEC_VIDEO_BACKEND}` backend" + ) + frames, source, frame_idx, valid = cls.decode_frames_pynvvideocodec( + data, + target, + **kwargs, + ) else: raise ValueError( f"Unknown video codec backend {backend!r}; " - "valid options: 'opencv', 'pyav'." + "valid options: 'opencv', 'pyav', 'pynvvideocodec'." ) if len(valid) < len(frame_idx): @@ -605,6 +889,40 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin): ) +@VIDEO_LOADER_REGISTRY.register(PYNVVIDEOCODEC_VIDEO_BACKEND) +class PyNvVideoCodecVideoBackend(VideoBackend): + """Hardware-accelerated video backend using PyNvVideoCodec. + + The backend first opens the stream only to read metadata and compute the + sampled frame indices. It then acquires the raw decoded RGB byte count from + the process-local multimodal GPU memory pool before decoding the selected + frames into VRAM. Decoded frames are copied into pinned host memory before + the lease is released, so downstream preprocessing continues to receive a + CPU ``np.ndarray`` in NHWC RGB format. + """ + + @classmethod + def load_bytes( + cls, + data: bytes, + num_frames: int = -1, + fps: int = -1, + max_duration: int = 300, + frame_recovery: bool = False, + **kwargs, + ) -> tuple[npt.NDArray, dict[str, Any]]: + kwargs.pop("backend", None) + return super().load_bytes( + data, + num_frames=num_frames, + fps=fps, + max_duration=max_duration, + frame_recovery=frame_recovery, + backend=PYNVVIDEOCODEC_VIDEO_BACKEND, + **kwargs, + ) + + @VIDEO_LOADER_REGISTRY.register( "qwen3_vl", video_processor="Qwen3VLVideoProcessor", @@ -640,7 +958,7 @@ class Qwen3VLVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav"] = "opencv", + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( @@ -719,7 +1037,7 @@ class Qwen2VLVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav"] = "opencv", + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( @@ -811,7 +1129,7 @@ class DynamicVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav"] = "opencv", + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( @@ -936,7 +1254,7 @@ class GLM46VVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav"] = "opencv", + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: return super().load_bytes( @@ -1034,7 +1352,7 @@ class GLMGAVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav"] = "opencv", + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: frames, metadata = super().load_bytes( @@ -1353,7 +1671,7 @@ class NemotronVLVideoBackend(VideoBackend): max_duration: int = 300, frame_recovery: bool = False, *, - backend: Literal["opencv", "pyav"] = "opencv", + backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv", **kwargs, ) -> tuple[npt.NDArray, dict[str, Any]]: frames, metadata = super().load_bytes( diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 00cbec33d6..a0f2508ccc 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -29,6 +29,7 @@ from vllm.inputs import ( from vllm.logger import init_logger from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry from vllm.multimodal.cache import BaseMultiModalProcessorCache +from vllm.multimodal.gpu_ipc_memory import maybe_init_mm_gpu_ipc_pool from vllm.multimodal.parse import ( MultiModalDataItems, MultiModalUUIDItems, @@ -111,6 +112,16 @@ class BaseRenderer(ABC, Generic[_T]): safe_load_prompt_embeds, executor=self._executor ) if mm_registry.supports_multimodal_inputs(config.model_config): + # Install the process-global GPU memory pool used to gate + # frontend GPU-side multimodal decoding (no-op when the budget + # is 0). Lives in the API-server process only. + mm_config = config.model_config.multimodal_config + if mm_config is not None: + maybe_init_mm_gpu_ipc_pool( + mm_config.mm_ipc_gpu_memory_gb, + config.parallel_config._api_process_count, + ) + mm_processor_cache = mm_registry.processor_cache_from_config(config) with set_default_torch_num_threads(): diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 589a16576e..9afc235252 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -50,6 +50,12 @@ from vllm.distributed.weight_transfer import ( from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.model_executor.warmup.kernel_warmup import kernel_warmup +from vllm.multimodal.video import ( + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES, + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES, + PYNVVIDEOCODEC_MAX_RETAINED_DECODERS, + PYNVVIDEOCODEC_VIDEO_BACKEND, +) from vllm.platforms import current_platform from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper from vllm.sequence import IntermediateTensors @@ -428,7 +434,7 @@ class Worker(WorkerBase): "correspondingly." ) logger.info(msg) - return kv_cache_memory_bytes + return self._reserve_mm_ipc_gpu_memory(kv_cache_memory_bytes) # Execute a forward pass with dummy inputs to profile the memory usage # of the model. @@ -550,7 +556,81 @@ class Worker(WorkerBase): suggested_util, ) - return int(self.available_kv_cache_memory_bytes) + return self._reserve_mm_ipc_gpu_memory( + int(self.available_kv_cache_memory_bytes) + ) + + @staticmethod + def _uses_pynvvideocodec_video_backend(mm_config) -> bool: + video_kwargs = mm_config.media_io_kwargs.get("video", {}) + video_loader_backend = ( + video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND + ) + codec_backend = video_kwargs.get("backend") + return ( + video_loader_backend == PYNVVIDEOCODEC_VIDEO_BACKEND + or codec_backend == PYNVVIDEOCODEC_VIDEO_BACKEND + ) + + def _reserve_mm_ipc_gpu_memory(self, available_kv_cache_memory_bytes: int) -> int: + """Carve frontend multimodal GPU memory out of the KV cache. + + The frontend (API-server) process allocates GPU memory for hardware + multimodal decoding. Raw decoded frames are bounded by + ``mm_ipc_gpu_memory_gb`` and acquired by the frontend semaphore. Some + decoders also keep persistent surfaces around; reserve a fixed upper + bound for those when the corresponding backend is configured. + """ + mm_config = self.model_config.multimodal_config + if mm_config is None: + return available_kv_cache_memory_bytes + + raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes) + # Each api_server_count process runs its OWN decoder surfaces + NVDEC/CUVID + # CUDA context on the GPU, outside this (worker) memory pool. Reserve that + # per-server footprint x api_server_count so gpu_memory_utilization bounds + # TOTAL GPU usage across all API-server processes. Without the multiply, + # HW decode overshoots the budget by ~(api_server_count-1) x per-server and + # OOMs at high gmu, while SW decode (no per-server GPU allocation) does not. + num_api_servers = max(1, getattr(self.parallel_config, "_api_process_count", 1)) + per_server_decoder_bytes = ( + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES + * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS + + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES + ) + decoder_reserved_bytes = ( + num_api_servers * per_server_decoder_bytes + if self._uses_pynvvideocodec_video_backend(mm_config) + else 0 + ) + reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes + if reserved_bytes <= 0: + return available_kv_cache_memory_bytes + + remaining = available_kv_cache_memory_bytes - reserved_bytes + if remaining <= 0: + raise ValueError( + f"frontend multimodal GPU decoding reserves " + f"{format_gib(reserved_bytes)} GiB " + f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, " + f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), " + f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is " + "available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a " + "different video backend, or increase gpu_memory_utilization." + ) + logger.info_once( + "Reserving %s GiB of GPU memory for frontend multimodal decoding " + "(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context " + "across %d API server(s) @ %s GiB/server); " + "KV cache memory reduced to %s GiB.", + format_gib(reserved_bytes), + format_gib(raw_frame_reserved_bytes), + format_gib(decoder_reserved_bytes), + num_api_servers, + format_gib(per_server_decoder_bytes), + format_gib(remaining), + ) + return remaining def get_kv_connector_handshake_metadata( self,