Vram semaphore infra (#44465)

Signed-off-by: Brandon Pelfrey <bpelfrey@nvidia.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Brandon Pelfrey
2026-06-26 17:32:51 -07:00
committed by GitHub
co-authored by Roger Wang
parent 3f67477497
commit af16446bf3
11 changed files with 1087 additions and 15 deletions
+1
View File
@@ -8,6 +8,7 @@ torch==2.11.0
torchaudio==2.11.0
# These must be updated alongside torch
torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
PyNvVideoCodec==2.0.4
# FlashInfer should be updated together with the Dockerfile
flashinfer-python==0.6.12
flashinfer-cubin==0.6.12
+145
View File
@@ -0,0 +1,145 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
import time
import pytest
from vllm.multimodal.gpu_ipc_memory import (
MultiModalGPUMemoryPool,
get_mm_gpu_ipc_pool,
maybe_init_mm_gpu_ipc_pool,
set_mm_gpu_ipc_pool,
)
from vllm.utils.mem_constants import GiB_bytes
def test_acquire_release_accounting():
pool = MultiModalGPUMemoryPool(total_bytes=100)
assert pool.available_bytes == 100
lease = pool.acquire(40)
assert pool.available_bytes == 60
lease.release()
assert pool.available_bytes == 100
def test_acquire_too_large_raises():
pool = MultiModalGPUMemoryPool(total_bytes=100)
with pytest.raises(ValueError):
pool.acquire(101)
# Nothing should have been reserved.
assert pool.available_bytes == 100
def test_negative_acquire_raises():
pool = MultiModalGPUMemoryPool(total_bytes=100)
with pytest.raises(ValueError):
pool.acquire(-1)
def test_double_release_is_noop():
pool = MultiModalGPUMemoryPool(total_bytes=100)
lease = pool.acquire(50)
lease.release()
assert pool.available_bytes == 100
# Releasing again must not inflate the pool past its capacity.
lease.release()
assert pool.available_bytes == 100
def test_context_manager_releases_on_exception():
pool = MultiModalGPUMemoryPool(total_bytes=100)
with pytest.raises(RuntimeError), pool.acquire(50):
assert pool.available_bytes == 50
raise RuntimeError("boom")
assert pool.available_bytes == 100
def test_acquire_blocks_until_release():
pool = MultiModalGPUMemoryPool(total_bytes=100)
first = pool.acquire(80)
acquired = threading.Event()
def waiter():
# Needs 50 bytes but only 20 are free; must block until `first`
# is released.
with pool.acquire(50):
acquired.set()
t = threading.Thread(target=waiter)
t.start()
# The waiter cannot proceed yet.
assert not acquired.wait(timeout=0.2)
# Releasing the first lease frees enough budget to unblock the waiter.
first.release()
assert acquired.wait(timeout=2.0)
t.join(timeout=2.0)
assert not t.is_alive()
assert pool.available_bytes == 100
def test_concurrent_acquires_serialize():
pool = MultiModalGPUMemoryPool(total_bytes=100)
# Each task needs 60 bytes, so only one can hold the budget at a time.
in_section = []
max_concurrent = 0
lock = threading.Lock()
def task():
nonlocal max_concurrent
with pool.acquire(60):
with lock:
in_section.append(1)
max_concurrent = max(max_concurrent, len(in_section))
time.sleep(0.05)
with lock:
in_section.pop()
threads = [threading.Thread(target=task) for _ in range(4)]
for t in threads:
t.start()
for t in threads:
t.join(timeout=5.0)
assert not t.is_alive()
assert max_concurrent == 1
assert pool.available_bytes == 100
def test_zero_total_bytes_rejected():
with pytest.raises(ValueError):
MultiModalGPUMemoryPool(total_bytes=0)
def test_global_pool_accessor():
try:
assert maybe_init_mm_gpu_ipc_pool(0) is None
assert get_mm_gpu_ipc_pool() is None
pool = maybe_init_mm_gpu_ipc_pool(2)
assert pool is not None
assert get_mm_gpu_ipc_pool() is pool
assert pool.total_bytes == 2 * GiB_bytes
finally:
set_mm_gpu_ipc_pool(None)
def test_global_pool_splits_budget_across_api_processes():
try:
pool = maybe_init_mm_gpu_ipc_pool(2, api_process_count=4)
assert pool is not None
assert get_mm_gpu_ipc_pool() is pool
assert pool.total_bytes == GiB_bytes // 2
finally:
set_mm_gpu_ipc_pool(None)
def test_global_pool_rejects_invalid_api_process_count():
with pytest.raises(ValueError):
maybe_init_mm_gpu_ipc_pool(2, api_process_count=0)
+234
View File
@@ -1,6 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
import sys
import threading
from contextlib import ExitStack, contextmanager
from pathlib import Path
import numpy as np
@@ -11,10 +15,15 @@ from transformers.video_utils import VideoMetadata
from vllm.assets.base import get_vllm_public_assets
from vllm.multimodal.video import (
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
PYNVVIDEOCODEC_VIDEO_BACKEND,
VIDEO_LOADER_REGISTRY,
DynamicVideoBackend,
GLM46VVideoBackend,
Molmo2VideoBackend,
PyNvVideoCodecDecoderSlot,
PyNvVideoCodecVideoBackend,
Qwen2VLVideoBackend,
Qwen3VLVideoBackend,
VideoLoader,
@@ -22,6 +31,7 @@ from vllm.multimodal.video import (
VideoTargetMetadata,
get_video_loader_backend_for_processor,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
from .utils import create_long_gop_video, create_video_from_image
@@ -65,6 +75,230 @@ def test_video_loader_type_doesnt_exist():
VIDEO_LOADER_REGISTRY.load("non_existing_video_loader")
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_pynvvideocodec_backend_accounts_raw_decoded_frames(
monkeypatch: pytest.MonkeyPatch,
):
decoder_cache_sizes = []
class FakeMetadata:
width = 10
height = 20
average_fps = 5.0
duration = 2.0
class FakeDecoder:
def __init__(self, *args, **kwargs):
decoder_cache_sizes.append(kwargs["decoder_cache_size"])
def __len__(self):
return 10
def get_stream_metadata(self):
return FakeMetadata()
class FakeNvc:
class OutputColorType:
RGB = "rgb"
SimpleDecoder = FakeDecoder
class RecordingPool:
def __init__(self):
self.acquired: list[int] = []
@contextmanager
def acquire(self, size: int):
self.acquired.append(size)
yield
def fake_decode(cls, file_path: str, frame_idx: list[int], nvc):
return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8)
pool = RecordingPool()
monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc)
monkeypatch.setattr(
"vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool
)
monkeypatch.setattr(
PyNvVideoCodecVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode)
)
loader = VIDEO_LOADER_REGISTRY.load(PYNVVIDEOCODEC_VIDEO_BACKEND)
frames, metadata = loader.load_bytes(b"fake video", num_frames=4)
assert frames.shape == (4, 20, 10, 3)
assert pool.acquired == [4 * 20 * 10 * 3]
assert decoder_cache_sizes == [PYNVVIDEOCODEC_DECODER_CACHE_SIZE]
assert metadata["video_backend"] == PYNVVIDEOCODEC_VIDEO_BACKEND
assert metadata["frames_indices"] == [0, 3, 6, 9]
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_pynvvideocodec_codec_uses_dynamic_sampling_strategy(
monkeypatch: pytest.MonkeyPatch,
):
decoded_indices = []
class FakeMetadata:
width = 10
height = 20
average_fps = 5.0
duration = 2.0
class FakeDecoder:
def __init__(self, *args, **kwargs):
pass
def __len__(self):
return 10
def get_stream_metadata(self):
return FakeMetadata()
class FakeNvc:
class OutputColorType:
RGB = "rgb"
SimpleDecoder = FakeDecoder
class RecordingPool:
def __init__(self):
self.acquired: list[int] = []
@contextmanager
def acquire(self, size: int):
self.acquired.append(size)
yield
def fake_decode(cls, file_path: str, frame_idx: list[int], nvc):
decoded_indices.append(frame_idx)
return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8)
pool = RecordingPool()
monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc)
monkeypatch.setattr(
"vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool
)
monkeypatch.setattr(
DynamicVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode)
)
loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
frames, metadata = loader.load_bytes(
b"fake video",
fps=2,
max_duration=1,
backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
)
assert frames.shape == (2, 20, 10, 3)
assert decoded_indices == [[0, 9]]
assert pool.acquired == [2 * 20 * 10 * 3]
assert metadata["video_backend"] == f"{PYNVVIDEOCODEC_VIDEO_BACKEND}_dynamic"
assert metadata["frames_indices"] == [0, 9]
def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatch):
class FakeSlot:
pass
create_count = 0
old_slots = PyNvVideoCodecVideoBackend._decoder_slots
old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots
old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond
try:
PyNvVideoCodecVideoBackend._decoder_slots = []
PyNvVideoCodecVideoBackend._active_decoder_slots = 0
PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition()
def fake_create_slot(cls):
nonlocal create_count
create_count += 1
return FakeSlot()
monkeypatch.setattr(
PyNvVideoCodecVideoBackend,
"_create_decoder_slot",
classmethod(fake_create_slot),
)
borrowed = threading.Event()
seen_slots = []
with ExitStack() as stack:
retained_slots = [
stack.enter_context(PyNvVideoCodecVideoBackend._borrow_decoder_slot())
for _ in range(PYNVVIDEOCODEC_MAX_RETAINED_DECODERS)
]
def borrow_extra_slot():
with PyNvVideoCodecVideoBackend._borrow_decoder_slot() as extra_slot:
seen_slots.append(extra_slot)
borrowed.set()
thread = threading.Thread(target=borrow_extra_slot)
thread.start()
assert not borrowed.wait(timeout=0.2)
assert borrowed.wait(timeout=2.0)
thread.join(timeout=2.0)
assert not thread.is_alive()
assert seen_slots[0] in retained_slots
assert create_count == PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
finally:
PyNvVideoCodecVideoBackend._decoder_slots = old_slots
PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots
PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond
def test_pynvvideocodec_decoder_slot_retains_simple_decoder():
events: list[tuple[object, ...]] = []
class FakeStream:
cuda_stream = "cuda-stream"
class FakeDecoder:
def __init__(self, file_path: str, **kwargs):
events.append(
(
"create",
file_path,
kwargs["gpu_id"],
kwargs["cuda_stream"],
kwargs["decoder_cache_size"],
)
)
def reconfigure_decoder(self, file_path: str):
events.append(("reconfigure", file_path))
class FakeNvc:
class OutputColorType:
RGB = "rgb"
SimpleDecoder = FakeDecoder
slot = PyNvVideoCodecDecoderSlot(FakeStream())
decoder = slot.get_decoder("first.mp4", FakeNvc, device_index=7)
assert slot.get_decoder("first.mp4", FakeNvc, device_index=7) is decoder
assert slot.get_decoder("second.mp4", FakeNvc, device_index=7) is decoder
assert events == [
(
"create",
"first.mp4",
7,
"cuda-stream",
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
),
("reconfigure", "second.mp4"),
]
assert slot.source_path == "second.mp4"
# ============================================================================
# Video Processor → Video Loader Tests (via model repo)
# ============================================================================
+116
View File
@@ -0,0 +1,116 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import vllm.v1.worker.gpu_worker as gpu_worker_module
from vllm.multimodal.video import (
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
from vllm.utils.mem_constants import GiB_bytes
from vllm.v1.worker.gpu_worker import Worker
def _worker_with_mm_config(
mm_config: SimpleNamespace,
*,
api_process_count: int = 1,
) -> Worker:
worker = object.__new__(Worker)
worker.model_config = SimpleNamespace(multimodal_config=mm_config)
worker.parallel_config = SimpleNamespace(_api_process_count=api_process_count)
return worker
def _mm_config(
*,
mm_ipc_gpu_memory_gb: float = 0,
video_backend: str | None = None,
) -> SimpleNamespace:
video_kwargs = {} if video_backend is None else {"video_backend": video_backend}
return SimpleNamespace(
mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
media_io_kwargs={"video": video_kwargs} if video_kwargs else {},
)
def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:
return api_process_count * (
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
)
@pytest.mark.parametrize("video_backend", [None, "opencv"])
def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
monkeypatch: pytest.MonkeyPatch,
video_backend: str | None,
):
monkeypatch.setattr(
gpu_worker_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
"opencv",
)
worker = _worker_with_mm_config(
_mm_config(mm_ipc_gpu_memory_gb=0.25, video_backend=video_backend)
)
assert worker._reserve_mm_ipc_gpu_memory(GiB_bytes) == int(0.75 * GiB_bytes)
def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
gpu_worker_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
"opencv",
)
worker = _worker_with_mm_config(
_mm_config(
mm_ipc_gpu_memory_gb=0.25,
video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
)
)
available_bytes = 4 * GiB_bytes
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget()
)
def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
gpu_worker_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
worker = _worker_with_mm_config(_mm_config())
available_bytes = 4 * GiB_bytes
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
available_bytes - _pynvvideocodec_decoder_budget()
)
def test_reserve_mm_ipc_gpu_memory_scales_pynvvideocodec_budget_by_api_servers(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
gpu_worker_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
worker = _worker_with_mm_config(_mm_config(), api_process_count=3)
available_bytes = 8 * GiB_bytes
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)
)
+4
View File
@@ -351,6 +351,7 @@ class ModelConfig:
skip_mm_profiling: InitVar[bool | None] = None
video_pruning_rate: InitVar[float | None] = None
mm_tensor_ipc: InitVar[MMTensorIPC] = None
mm_ipc_gpu_memory_gb: InitVar[float | None] = None
def compute_hash(self) -> str:
"""
@@ -397,6 +398,7 @@ class ModelConfig:
"mm_encoder_tp_mode",
"interleave_mm_strings",
"skip_mm_profiling",
"mm_ipc_gpu_memory_gb",
}
from vllm.config.utils import get_hash_factors, hash_factors
@@ -477,6 +479,7 @@ class ModelConfig:
skip_mm_profiling: bool | None,
video_pruning_rate: float | None,
mm_tensor_ipc: MMTensorIPC,
mm_ipc_gpu_memory_gb: float | None,
) -> None:
# Keep set served_model_name before maybe_model_redirect(self.model)
self.served_model_name = get_served_model_name(
@@ -690,6 +693,7 @@ class ModelConfig:
skip_mm_profiling=skip_mm_profiling,
video_pruning_rate=video_pruning_rate,
mm_tensor_ipc=mm_tensor_ipc,
mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
)
mm_config_kwargs = {
+10
View File
@@ -197,6 +197,16 @@ class MultiModalConfig:
- "direct_rpc": Use msgspec serialization via RPC
- "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC
Defaults to "direct_rpc". """
mm_ipc_gpu_memory_gb: float = Field(default=0, ge=0)
"""Amount of GPU memory (in GiB) sequestered on the engine's device for
GPU-side multimodal work in the API-server (frontend) process, such as
hardware video decoding.
This budget is carved out of the engine's KV-cache memory so the headroom
physically exists, and frontend GPU decode paths acquire from a blocking
byte-counting semaphore of this size before allocating on the device.
Set to `0` (default) to disable frontend GPU multimodal memory gating."""
@field_validator("limit_per_prompt", mode="before")
@classmethod
+6
View File
@@ -582,6 +582,7 @@ class EngineArgs:
skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate
mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc
mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb
# LoRA fields
enable_lora: bool = False
max_loras: int = LoRAConfig.max_loras
@@ -1294,6 +1295,10 @@ class EngineArgs:
multimodal_group.add_argument(
"--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"]
)
multimodal_group.add_argument(
"--mm-ipc-gpu-memory-gb",
**multimodal_kwargs["mm_ipc_gpu_memory_gb"],
)
# LoRA related configs
lora_kwargs = get_kwargs(LoRAConfig)
@@ -1660,6 +1665,7 @@ class EngineArgs:
logits_processors=self.logits_processors,
video_pruning_rate=self.video_pruning_rate,
mm_tensor_ipc=self.mm_tensor_ipc,
mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb,
io_processor_plugin=self.io_processor_plugin,
renderer_num_workers=self.renderer_num_workers,
)
+147
View File
@@ -0,0 +1,147 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Admission control for frontend GPU-side multimodal work.
When multimodal media is decoded on the GPU in the API-server (frontend)
process, the decoded buffers compete for the same device memory that the
engine reserves for weights, activations, and the KV cache. To keep the
frontend's GPU usage within a sequestered budget (see
``MultiModalConfig.mm_ipc_gpu_memory_gb``), decode paths acquire the number of
bytes they need from a process-global :class:`MultiModalGPUMemoryPool` before
allocating on the device and release them once the device memory is freed.
The pool is a simple byte-counting semaphore: ``acquire`` blocks until enough
budget is free, so concurrent requests serialize rather than oversubscribe the
GPU. It lives only in the frontend process; the engine carves the matching
amount out of its KV-cache budget so the headroom physically exists.
"""
import threading
from vllm.logger import init_logger
from vllm.utils.mem_constants import GiB_bytes
logger = init_logger(__name__)
class MultiModalGPUMemoryLease:
"""A handle for bytes acquired from a :class:`MultiModalGPUMemoryPool`.
Releasing is idempotent and the lease doubles as a context manager so the
budget is returned even if the decode raises.
"""
def __init__(self, pool: "MultiModalGPUMemoryPool", lease_id: int, nbytes: int):
self.lease_id = lease_id
self.nbytes = nbytes
self._pool = pool
def release(self) -> None:
self._pool._release(self)
def __enter__(self) -> "MultiModalGPUMemoryLease":
return self
def __exit__(self, *exc_info) -> None:
self.release()
class MultiModalGPUMemoryPool:
"""Blocking byte-counting semaphore for frontend GPU multimodal memory.
Thread-safe in both directions: ``acquire`` (blocking) and ``release`` are
typically called from the renderer's multimodal executor threads.
"""
def __init__(self, total_bytes: int):
if total_bytes <= 0:
raise ValueError(f"total_bytes must be positive, got {total_bytes}")
self._total_bytes = total_bytes
self._available = total_bytes
self._cond = threading.Condition()
self._next_lease_id = 0
# Outstanding lease ids, so a double release is a no-op.
self._outstanding: set[int] = set()
@property
def total_bytes(self) -> int:
return self._total_bytes
@property
def available_bytes(self) -> int:
with self._cond:
return self._available
def acquire(self, nbytes: int) -> MultiModalGPUMemoryLease:
"""Reserve ``nbytes``, blocking until that much budget is free.
Raises ``ValueError`` if ``nbytes`` exceeds the pool's total capacity,
since such a request could never be satisfied.
"""
if nbytes < 0:
raise ValueError(f"Cannot acquire negative bytes: {nbytes}")
if nbytes > self._total_bytes:
raise ValueError(
f"Multimodal GPU decode requested {nbytes} bytes, which exceeds "
f"the total pool size of {self._total_bytes} bytes. Increase "
f"--mm-ipc-gpu-memory-gb or reduce the multimodal input size."
)
with self._cond:
while self._available < nbytes:
self._cond.wait()
self._available -= nbytes
lease_id = self._next_lease_id
self._next_lease_id += 1
self._outstanding.add(lease_id)
return MultiModalGPUMemoryLease(self, lease_id, nbytes)
def _release(self, lease: MultiModalGPUMemoryLease) -> None:
with self._cond:
if lease.lease_id not in self._outstanding:
# Already released — idempotent.
return
self._outstanding.discard(lease.lease_id)
self._available += lease.nbytes
self._cond.notify_all()
_GLOBAL_POOL: MultiModalGPUMemoryPool | None = None
def set_mm_gpu_ipc_pool(pool: MultiModalGPUMemoryPool | None) -> None:
"""Install the process-global pool (frontend process only)."""
global _GLOBAL_POOL
_GLOBAL_POOL = pool
def get_mm_gpu_ipc_pool() -> MultiModalGPUMemoryPool | None:
"""Return the process-global pool, or ``None`` when gating is disabled."""
return _GLOBAL_POOL
def maybe_init_mm_gpu_ipc_pool(
mm_ipc_gpu_memory_gb: float,
api_process_count: int = 1,
) -> MultiModalGPUMemoryPool | None:
"""Create and install the global pool from the configured GiB budget.
Returns ``None`` (and leaves gating disabled) when the budget is 0. When
multiple API-server processes share one engine, each process gets an equal
slice of the user-provided frontend budget.
"""
if mm_ipc_gpu_memory_gb <= 0:
set_mm_gpu_ipc_pool(None)
return None
if api_process_count <= 0:
raise ValueError(f"api_process_count must be positive, got {api_process_count}")
total_bytes = int(mm_ipc_gpu_memory_gb * GiB_bytes) // api_process_count
pool = MultiModalGPUMemoryPool(total_bytes)
set_mm_gpu_ipc_pool(pool)
logger.info(
"Initialized multimodal GPU IPC memory pool with %d bytes for this API "
"process (%.2f GiB total budget across %d API process(es)).",
total_bytes,
mm_ipc_gpu_memory_gb,
api_process_count,
)
return pool
+331 -13
View File
@@ -1,7 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
import os
import tempfile
import threading
from abc import abstractmethod
from contextlib import contextmanager, suppress
from io import BytesIO
from typing import Any, ClassVar, Literal, NamedTuple, cast
@@ -11,6 +15,7 @@ import torch
from vllm.logger import init_logger
from vllm.utils.import_utils import PlaceholderModule
from vllm.utils.mem_constants import MiB_bytes
from vllm.utils.registry import ExtensionManager
try:
@@ -130,6 +135,14 @@ class VideoSourceMetadata(NamedTuple):
duration: float
class PyNvVideoCodecSourceMetadata(NamedTuple):
"""Metadata needed before GPU video decode."""
source: VideoSourceMetadata
width: int
height: int
class VideoLoader:
@classmethod
def compute_frames_index_to_sample(
@@ -170,6 +183,57 @@ class VideoLoader:
VIDEO_LOADER_REGISTRY = VideoLoaderRegistry()
PYNVVIDEOCODEC_VIDEO_BACKEND: Literal["pynvvideocodec"] = "pynvvideocodec"
# Fixed upper bound reserved for persistent PyNvVideoCodec decoder surfaces.
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES = 128 * MiB_bytes
PYNVVIDEOCODEC_DECODER_CACHE_SIZE = 2
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS = 1
# Per-API-server CUDA context and driver allocation, measured with
# PyNvVideoCodec 2.0.4 on H100.
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES = int(1.8 * 1024 * MiB_bytes)
class PyNvVideoCodecDecoderSlot:
"""A retained PyNv decoder slot and its CUDA stream.
The decoder is reused across requests: ``reconfigure_decoder`` repoints the
existing decoder at each new source instead of paying a fresh
``SimpleDecoder`` construction per request. Construction (CUVID parser +
decoder + surface-pool allocation) is the dominant per-request cost, so
reconfiguring is far cheaper. A single decoder serves both metadata
(``len``/``get_stream_metadata``) and frame decode -- no separate
metadata decoder.
"""
def __init__(self, stream) -> None:
self.stream = stream
self.decoder = None
self.source_path: str | None = None
def _construct(self, file_path: str, nvc, device_index: int) -> None:
self.decoder = nvc.SimpleDecoder(
file_path,
output_color_type=nvc.OutputColorType.RGB,
use_device_memory=True,
need_scanned_stream_metadata=True,
gpu_id=device_index,
cuda_stream=self.stream.cuda_stream,
decoder_cache_size=PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
)
self.source_path = file_path
def get_decoder(self, file_path: str, nvc, device_index: int):
if self.decoder is None:
self._construct(file_path, nvc, device_index)
elif self.source_path != file_path:
try:
self.decoder.reconfigure_decoder(file_path)
self.source_path = file_path
except Exception:
# reconfigure unsupported/unsafe for this source -> rebuild.
self._construct(file_path, nvc, device_index)
return self.decoder
class OpenCVVideoBackendMixin:
@staticmethod
@@ -485,15 +549,223 @@ class PyAVVideoBackendMixin:
return np.stack(frames_list), valid_indices
class PyNvVideoCodecVideoBackendMixin:
"""PyNvVideoCodec utilities for GPU-backed frame decode."""
_decoder_slots: ClassVar[list[PyNvVideoCodecDecoderSlot]] = []
_active_decoder_slots: ClassVar[int] = 0
_decoder_slot_cond: ClassVar[threading.Condition] = threading.Condition()
_DEVICE_INDEX: ClassVar[int] = 0
@classmethod
@abstractmethod
def compute_frames_index_to_sample(
cls,
source: VideoSourceMetadata,
target: VideoTargetMetadata,
**kwargs,
) -> list[int]:
raise NotImplementedError
@classmethod
@abstractmethod
def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:
raise NotImplementedError
@classmethod
def _create_decoder_slot(cls) -> PyNvVideoCodecDecoderSlot:
import torch
return PyNvVideoCodecDecoderSlot(torch.cuda.Stream(device=cls._DEVICE_INDEX))
@staticmethod
@contextmanager
def _torch_stream_context(stream):
import torch
torch.accelerator.set_device_index(stream.device.index)
previous_stream = torch.accelerator.current_stream()
torch.accelerator.set_stream(stream)
try:
yield
finally:
torch.accelerator.set_stream(previous_stream)
@classmethod
@contextmanager
def _borrow_decoder_slot(cls):
create_slot = False
with cls._decoder_slot_cond:
while True:
if cls._decoder_slots:
slot = cls._decoder_slots.pop()
break
if cls._active_decoder_slots < PYNVVIDEOCODEC_MAX_RETAINED_DECODERS:
cls._active_decoder_slots += 1
create_slot = True
break
cls._decoder_slot_cond.wait()
if create_slot:
try:
slot = cls._create_decoder_slot()
except Exception:
with cls._decoder_slot_cond:
cls._active_decoder_slots -= 1
cls._decoder_slot_cond.notify()
raise
try:
yield slot
finally:
with cls._decoder_slot_cond:
cls._decoder_slots.append(slot)
cls._decoder_slot_cond.notify()
@staticmethod
def _metadata_value(metadata, *names: str, default=None):
for name in names:
value = getattr(metadata, name, None)
if value is not None:
return value
return default
@classmethod
def _read_source_metadata(
cls,
file_path: str,
nvc,
) -> PyNvVideoCodecSourceMetadata:
with cls._borrow_decoder_slot() as decoder_slot:
with cls._torch_stream_context(decoder_slot.stream):
decoder = decoder_slot.get_decoder(
file_path, nvc, device_index=cls._DEVICE_INDEX
)
metadata = decoder.get_stream_metadata()
total_frames_num = len(decoder)
width = int(cls._metadata_value(metadata, "width", default=0))
height = int(cls._metadata_value(metadata, "height", default=0))
original_fps = float(
cls._metadata_value(
metadata,
"average_fps",
"avg_frame_rate",
"frame_rate",
"frameRate",
default=0.0,
)
)
duration = float(
cls._metadata_value(metadata, "duration", default=0.0)
or (total_frames_num / original_fps if original_fps > 0 else 0.0)
)
if total_frames_num <= 0:
raise ValueError("Could not determine video frame count")
if width <= 0 or height <= 0:
raise ValueError("Could not determine video dimensions")
return PyNvVideoCodecSourceMetadata(
source=VideoSourceMetadata(total_frames_num, original_fps, duration),
width=width,
height=height,
)
@classmethod
def _decode_to_pinned_host(
cls,
file_path: str,
frame_idx: list[int],
nvc,
) -> npt.NDArray:
import torch
if not frame_idx:
return np.empty((0,), dtype=np.uint8)
with cls._borrow_decoder_slot() as decoder_slot:
stream = decoder_slot.stream
with cls._torch_stream_context(stream):
decoder = decoder_slot.get_decoder(
file_path, nvc, device_index=cls._DEVICE_INDEX
)
decoded_frames = decoder.get_batch_frames_by_index(frame_idx)
if len(decoded_frames) < len(frame_idx):
logger.warning(
"pynvvideocodec video loading: expected %d frames but got %d.",
len(frame_idx),
len(decoded_frames),
)
torch_frames = [torch.from_dlpack(frame) for frame in decoded_frames]
if not torch_frames:
return np.empty((0,), dtype=np.uint8)
device_frames = torch.stack(torch_frames)
if device_frames.ndim != 4:
raise ValueError(
"PyNvVideoCodec returned frames with unexpected shape "
f"{tuple(device_frames.shape)}"
)
device_frames = device_frames.permute(0, 3, 1, 2).contiguous()
host_frames = torch.empty(
device_frames.shape,
dtype=device_frames.dtype,
device="cpu",
pin_memory=True,
)
host_frames.copy_(device_frames, non_blocking=True)
stream.synchronize()
host_array = host_frames.numpy()
del decoded_frames, torch_frames, device_frames
return host_array
@classmethod
def decode_frames_pynvvideocodec(
cls,
data: bytes,
target: VideoTargetMetadata,
**kwargs,
) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]:
import PyNvVideoCodec as nvc
from vllm.multimodal.gpu_ipc_memory import get_mm_gpu_ipc_pool
temp_fd, temp_path = tempfile.mkstemp(suffix=".mp4")
try:
with os.fdopen(temp_fd, "wb") as temp_file:
temp_file.write(data)
gpu_source = cls._read_source_metadata(temp_path, nvc)
source = cls._prepare_source(gpu_source.source)
frame_idx = cls.compute_frames_index_to_sample(
source=source, target=target, **kwargs
)
raw_frame_bytes = len(frame_idx) * gpu_source.height * gpu_source.width * 3
pool = get_mm_gpu_ipc_pool()
if pool is None or raw_frame_bytes == 0:
frames = cls._decode_to_pinned_host(temp_path, frame_idx, nvc)
else:
with pool.acquire(raw_frame_bytes):
frames = cls._decode_to_pinned_host(temp_path, frame_idx, nvc)
finally:
with suppress(FileNotFoundError):
os.unlink(temp_path)
valid_frame_indices = frame_idx[: int(frames.shape[0])]
return frames, source, frame_idx, valid_frame_indices
@VIDEO_LOADER_REGISTRY.register("opencv")
class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
class VideoBackend(
VideoLoader,
OpenCVVideoBackendMixin,
PyAVVideoBackendMixin,
PyNvVideoCodecVideoBackendMixin,
):
"""Uniform-sampling video backend.
Samples ``num_frames`` uniformly across the video (or one frame every
``1/fps`` seconds, whichever produces fewer frames). The decoding codec
is selected via the ``backend`` kwarg (``"opencv"`` or ``"pyav"``),
which can be passed through ``--media-io-kwargs``. Defaults to
``"pyav"`` for concurrent decoding.
is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``, or
``"pynvvideocodec"``), which can be passed through
``--media-io-kwargs``. Defaults to ``"opencv"``.
"""
_sampling_suffix: ClassVar[str] = ""
@@ -538,7 +810,7 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal["opencv", "pyav"] = "opencv",
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
"""Load sampled frames from raw video bytes.
@@ -551,7 +823,8 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
dynamic subclass; ignored here.
frame_recovery: Enable forward-scan recovery for failed frames.
Only honored by the OpenCV codec.
backend: Decoding codec — ``"opencv"`` or ``"pyav"`` .
backend: Decoding codec — ``"opencv"``, ``"pyav"``, or
``"pynvvideocodec"``.
Returns:
Tuple of ``(frames_array, metadata_dict)``.
@@ -584,10 +857,21 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
frames, valid = cls.decode_frames(
container, frame_idx, source.original_fps, source.duration
)
elif backend == PYNVVIDEOCODEC_VIDEO_BACKEND:
if frame_recovery:
raise ValueError(
"frame_recovery is not supported for "
f"`{PYNVVIDEOCODEC_VIDEO_BACKEND}` backend"
)
frames, source, frame_idx, valid = cls.decode_frames_pynvvideocodec(
data,
target,
**kwargs,
)
else:
raise ValueError(
f"Unknown video codec backend {backend!r}; "
"valid options: 'opencv', 'pyav'."
"valid options: 'opencv', 'pyav', 'pynvvideocodec'."
)
if len(valid) < len(frame_idx):
@@ -605,6 +889,40 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
)
@VIDEO_LOADER_REGISTRY.register(PYNVVIDEOCODEC_VIDEO_BACKEND)
class PyNvVideoCodecVideoBackend(VideoBackend):
"""Hardware-accelerated video backend using PyNvVideoCodec.
The backend first opens the stream only to read metadata and compute the
sampled frame indices. It then acquires the raw decoded RGB byte count from
the process-local multimodal GPU memory pool before decoding the selected
frames into VRAM. Decoded frames are copied into pinned host memory before
the lease is released, so downstream preprocessing continues to receive a
CPU ``np.ndarray`` in NHWC RGB format.
"""
@classmethod
def load_bytes(
cls,
data: bytes,
num_frames: int = -1,
fps: int = -1,
max_duration: int = 300,
frame_recovery: bool = False,
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
kwargs.pop("backend", None)
return super().load_bytes(
data,
num_frames=num_frames,
fps=fps,
max_duration=max_duration,
frame_recovery=frame_recovery,
backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
**kwargs,
)
@VIDEO_LOADER_REGISTRY.register(
"qwen3_vl",
video_processor="Qwen3VLVideoProcessor",
@@ -640,7 +958,7 @@ class Qwen3VLVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal["opencv", "pyav"] = "opencv",
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
@@ -719,7 +1037,7 @@ class Qwen2VLVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal["opencv", "pyav"] = "opencv",
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
@@ -811,7 +1129,7 @@ class DynamicVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal["opencv", "pyav"] = "opencv",
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
@@ -936,7 +1254,7 @@ class GLM46VVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal["opencv", "pyav"] = "opencv",
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
return super().load_bytes(
@@ -1034,7 +1352,7 @@ class GLMGAVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal["opencv", "pyav"] = "opencv",
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
frames, metadata = super().load_bytes(
@@ -1353,7 +1671,7 @@ class NemotronVLVideoBackend(VideoBackend):
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: Literal["opencv", "pyav"] = "opencv",
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
**kwargs,
) -> tuple[npt.NDArray, dict[str, Any]]:
frames, metadata = super().load_bytes(
+11
View File
@@ -29,6 +29,7 @@ from vllm.inputs import (
from vllm.logger import init_logger
from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry
from vllm.multimodal.cache import BaseMultiModalProcessorCache
from vllm.multimodal.gpu_ipc_memory import maybe_init_mm_gpu_ipc_pool
from vllm.multimodal.parse import (
MultiModalDataItems,
MultiModalUUIDItems,
@@ -111,6 +112,16 @@ class BaseRenderer(ABC, Generic[_T]):
safe_load_prompt_embeds, executor=self._executor
)
if mm_registry.supports_multimodal_inputs(config.model_config):
# Install the process-global GPU memory pool used to gate
# frontend GPU-side multimodal decoding (no-op when the budget
# is 0). Lives in the API-server process only.
mm_config = config.model_config.multimodal_config
if mm_config is not None:
maybe_init_mm_gpu_ipc_pool(
mm_config.mm_ipc_gpu_memory_gb,
config.parallel_config._api_process_count,
)
mm_processor_cache = mm_registry.processor_cache_from_config(config)
with set_default_torch_num_threads():
+82 -2
View File
@@ -50,6 +50,12 @@ from vllm.distributed.weight_transfer import (
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
from vllm.multimodal.video import (
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
from vllm.platforms import current_platform
from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper
from vllm.sequence import IntermediateTensors
@@ -428,7 +434,7 @@ class Worker(WorkerBase):
"correspondingly."
)
logger.info(msg)
return kv_cache_memory_bytes
return self._reserve_mm_ipc_gpu_memory(kv_cache_memory_bytes)
# Execute a forward pass with dummy inputs to profile the memory usage
# of the model.
@@ -550,7 +556,81 @@ class Worker(WorkerBase):
suggested_util,
)
return int(self.available_kv_cache_memory_bytes)
return self._reserve_mm_ipc_gpu_memory(
int(self.available_kv_cache_memory_bytes)
)
@staticmethod
def _uses_pynvvideocodec_video_backend(mm_config) -> bool:
video_kwargs = mm_config.media_io_kwargs.get("video", {})
video_loader_backend = (
video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
)
codec_backend = video_kwargs.get("backend")
return (
video_loader_backend == PYNVVIDEOCODEC_VIDEO_BACKEND
or codec_backend == PYNVVIDEOCODEC_VIDEO_BACKEND
)
def _reserve_mm_ipc_gpu_memory(self, available_kv_cache_memory_bytes: int) -> int:
"""Carve frontend multimodal GPU memory out of the KV cache.
The frontend (API-server) process allocates GPU memory for hardware
multimodal decoding. Raw decoded frames are bounded by
``mm_ipc_gpu_memory_gb`` and acquired by the frontend semaphore. Some
decoders also keep persistent surfaces around; reserve a fixed upper
bound for those when the corresponding backend is configured.
"""
mm_config = self.model_config.multimodal_config
if mm_config is None:
return available_kv_cache_memory_bytes
raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes)
# Each api_server_count process runs its OWN decoder surfaces + NVDEC/CUVID
# CUDA context on the GPU, outside this (worker) memory pool. Reserve that
# per-server footprint x api_server_count so gpu_memory_utilization bounds
# TOTAL GPU usage across all API-server processes. Without the multiply,
# HW decode overshoots the budget by ~(api_server_count-1) x per-server and
# OOMs at high gmu, while SW decode (no per-server GPU allocation) does not.
num_api_servers = max(1, getattr(self.parallel_config, "_api_process_count", 1))
per_server_decoder_bytes = (
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES
* PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
)
decoder_reserved_bytes = (
num_api_servers * per_server_decoder_bytes
if self._uses_pynvvideocodec_video_backend(mm_config)
else 0
)
reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes
if reserved_bytes <= 0:
return available_kv_cache_memory_bytes
remaining = available_kv_cache_memory_bytes - reserved_bytes
if remaining <= 0:
raise ValueError(
f"frontend multimodal GPU decoding reserves "
f"{format_gib(reserved_bytes)} GiB "
f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, "
f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), "
f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is "
"available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a "
"different video backend, or increase gpu_memory_utilization."
)
logger.info_once(
"Reserving %s GiB of GPU memory for frontend multimodal decoding "
"(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context "
"across %d API server(s) @ %s GiB/server); "
"KV cache memory reduced to %s GiB.",
format_gib(reserved_bytes),
format_gib(raw_frame_reserved_bytes),
format_gib(decoder_reserved_bytes),
num_api_servers,
format_gib(per_server_decoder_bytes),
format_gib(remaining),
)
return remaining
def get_kv_connector_handshake_metadata(
self,