Vram semaphore infra (#44465)

Signed-off-by: Brandon Pelfrey <bpelfrey@nvidia.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Brandon Pelfrey
2026-06-26 17:32:51 -07:00
committed by GitHub
co-authored by Roger Wang
parent 3f67477497
commit af16446bf3
11 changed files with 1087 additions and 15 deletions
+145
View File
@@ -0,0 +1,145 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import threading
import time
import pytest
from vllm.multimodal.gpu_ipc_memory import (
MultiModalGPUMemoryPool,
get_mm_gpu_ipc_pool,
maybe_init_mm_gpu_ipc_pool,
set_mm_gpu_ipc_pool,
)
from vllm.utils.mem_constants import GiB_bytes
def test_acquire_release_accounting():
pool = MultiModalGPUMemoryPool(total_bytes=100)
assert pool.available_bytes == 100
lease = pool.acquire(40)
assert pool.available_bytes == 60
lease.release()
assert pool.available_bytes == 100
def test_acquire_too_large_raises():
pool = MultiModalGPUMemoryPool(total_bytes=100)
with pytest.raises(ValueError):
pool.acquire(101)
# Nothing should have been reserved.
assert pool.available_bytes == 100
def test_negative_acquire_raises():
pool = MultiModalGPUMemoryPool(total_bytes=100)
with pytest.raises(ValueError):
pool.acquire(-1)
def test_double_release_is_noop():
pool = MultiModalGPUMemoryPool(total_bytes=100)
lease = pool.acquire(50)
lease.release()
assert pool.available_bytes == 100
# Releasing again must not inflate the pool past its capacity.
lease.release()
assert pool.available_bytes == 100
def test_context_manager_releases_on_exception():
pool = MultiModalGPUMemoryPool(total_bytes=100)
with pytest.raises(RuntimeError), pool.acquire(50):
assert pool.available_bytes == 50
raise RuntimeError("boom")
assert pool.available_bytes == 100
def test_acquire_blocks_until_release():
pool = MultiModalGPUMemoryPool(total_bytes=100)
first = pool.acquire(80)
acquired = threading.Event()
def waiter():
# Needs 50 bytes but only 20 are free; must block until `first`
# is released.
with pool.acquire(50):
acquired.set()
t = threading.Thread(target=waiter)
t.start()
# The waiter cannot proceed yet.
assert not acquired.wait(timeout=0.2)
# Releasing the first lease frees enough budget to unblock the waiter.
first.release()
assert acquired.wait(timeout=2.0)
t.join(timeout=2.0)
assert not t.is_alive()
assert pool.available_bytes == 100
def test_concurrent_acquires_serialize():
pool = MultiModalGPUMemoryPool(total_bytes=100)
# Each task needs 60 bytes, so only one can hold the budget at a time.
in_section = []
max_concurrent = 0
lock = threading.Lock()
def task():
nonlocal max_concurrent
with pool.acquire(60):
with lock:
in_section.append(1)
max_concurrent = max(max_concurrent, len(in_section))
time.sleep(0.05)
with lock:
in_section.pop()
threads = [threading.Thread(target=task) for _ in range(4)]
for t in threads:
t.start()
for t in threads:
t.join(timeout=5.0)
assert not t.is_alive()
assert max_concurrent == 1
assert pool.available_bytes == 100
def test_zero_total_bytes_rejected():
with pytest.raises(ValueError):
MultiModalGPUMemoryPool(total_bytes=0)
def test_global_pool_accessor():
try:
assert maybe_init_mm_gpu_ipc_pool(0) is None
assert get_mm_gpu_ipc_pool() is None
pool = maybe_init_mm_gpu_ipc_pool(2)
assert pool is not None
assert get_mm_gpu_ipc_pool() is pool
assert pool.total_bytes == 2 * GiB_bytes
finally:
set_mm_gpu_ipc_pool(None)
def test_global_pool_splits_budget_across_api_processes():
try:
pool = maybe_init_mm_gpu_ipc_pool(2, api_process_count=4)
assert pool is not None
assert get_mm_gpu_ipc_pool() is pool
assert pool.total_bytes == GiB_bytes // 2
finally:
set_mm_gpu_ipc_pool(None)
def test_global_pool_rejects_invalid_api_process_count():
with pytest.raises(ValueError):
maybe_init_mm_gpu_ipc_pool(2, api_process_count=0)
+234
View File
@@ -1,6 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import itertools
import sys
import threading
from contextlib import ExitStack, contextmanager
from pathlib import Path
import numpy as np
@@ -11,10 +15,15 @@ from transformers.video_utils import VideoMetadata
from vllm.assets.base import get_vllm_public_assets
from vllm.multimodal.video import (
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
PYNVVIDEOCODEC_VIDEO_BACKEND,
VIDEO_LOADER_REGISTRY,
DynamicVideoBackend,
GLM46VVideoBackend,
Molmo2VideoBackend,
PyNvVideoCodecDecoderSlot,
PyNvVideoCodecVideoBackend,
Qwen2VLVideoBackend,
Qwen3VLVideoBackend,
VideoLoader,
@@ -22,6 +31,7 @@ from vllm.multimodal.video import (
VideoTargetMetadata,
get_video_loader_backend_for_processor,
)
from vllm.platforms import current_platform
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
from .utils import create_long_gop_video, create_video_from_image
@@ -65,6 +75,230 @@ def test_video_loader_type_doesnt_exist():
VIDEO_LOADER_REGISTRY.load("non_existing_video_loader")
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_pynvvideocodec_backend_accounts_raw_decoded_frames(
monkeypatch: pytest.MonkeyPatch,
):
decoder_cache_sizes = []
class FakeMetadata:
width = 10
height = 20
average_fps = 5.0
duration = 2.0
class FakeDecoder:
def __init__(self, *args, **kwargs):
decoder_cache_sizes.append(kwargs["decoder_cache_size"])
def __len__(self):
return 10
def get_stream_metadata(self):
return FakeMetadata()
class FakeNvc:
class OutputColorType:
RGB = "rgb"
SimpleDecoder = FakeDecoder
class RecordingPool:
def __init__(self):
self.acquired: list[int] = []
@contextmanager
def acquire(self, size: int):
self.acquired.append(size)
yield
def fake_decode(cls, file_path: str, frame_idx: list[int], nvc):
return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8)
pool = RecordingPool()
monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc)
monkeypatch.setattr(
"vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool
)
monkeypatch.setattr(
PyNvVideoCodecVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode)
)
loader = VIDEO_LOADER_REGISTRY.load(PYNVVIDEOCODEC_VIDEO_BACKEND)
frames, metadata = loader.load_bytes(b"fake video", num_frames=4)
assert frames.shape == (4, 20, 10, 3)
assert pool.acquired == [4 * 20 * 10 * 3]
assert decoder_cache_sizes == [PYNVVIDEOCODEC_DECODER_CACHE_SIZE]
assert metadata["video_backend"] == PYNVVIDEOCODEC_VIDEO_BACKEND
assert metadata["frames_indices"] == [0, 3, 6, 9]
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
def test_pynvvideocodec_codec_uses_dynamic_sampling_strategy(
monkeypatch: pytest.MonkeyPatch,
):
decoded_indices = []
class FakeMetadata:
width = 10
height = 20
average_fps = 5.0
duration = 2.0
class FakeDecoder:
def __init__(self, *args, **kwargs):
pass
def __len__(self):
return 10
def get_stream_metadata(self):
return FakeMetadata()
class FakeNvc:
class OutputColorType:
RGB = "rgb"
SimpleDecoder = FakeDecoder
class RecordingPool:
def __init__(self):
self.acquired: list[int] = []
@contextmanager
def acquire(self, size: int):
self.acquired.append(size)
yield
def fake_decode(cls, file_path: str, frame_idx: list[int], nvc):
decoded_indices.append(frame_idx)
return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8)
pool = RecordingPool()
monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc)
monkeypatch.setattr(
"vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool
)
monkeypatch.setattr(
DynamicVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode)
)
loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
frames, metadata = loader.load_bytes(
b"fake video",
fps=2,
max_duration=1,
backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
)
assert frames.shape == (2, 20, 10, 3)
assert decoded_indices == [[0, 9]]
assert pool.acquired == [2 * 20 * 10 * 3]
assert metadata["video_backend"] == f"{PYNVVIDEOCODEC_VIDEO_BACKEND}_dynamic"
assert metadata["frames_indices"] == [0, 9]
def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatch):
class FakeSlot:
pass
create_count = 0
old_slots = PyNvVideoCodecVideoBackend._decoder_slots
old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots
old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond
try:
PyNvVideoCodecVideoBackend._decoder_slots = []
PyNvVideoCodecVideoBackend._active_decoder_slots = 0
PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition()
def fake_create_slot(cls):
nonlocal create_count
create_count += 1
return FakeSlot()
monkeypatch.setattr(
PyNvVideoCodecVideoBackend,
"_create_decoder_slot",
classmethod(fake_create_slot),
)
borrowed = threading.Event()
seen_slots = []
with ExitStack() as stack:
retained_slots = [
stack.enter_context(PyNvVideoCodecVideoBackend._borrow_decoder_slot())
for _ in range(PYNVVIDEOCODEC_MAX_RETAINED_DECODERS)
]
def borrow_extra_slot():
with PyNvVideoCodecVideoBackend._borrow_decoder_slot() as extra_slot:
seen_slots.append(extra_slot)
borrowed.set()
thread = threading.Thread(target=borrow_extra_slot)
thread.start()
assert not borrowed.wait(timeout=0.2)
assert borrowed.wait(timeout=2.0)
thread.join(timeout=2.0)
assert not thread.is_alive()
assert seen_slots[0] in retained_slots
assert create_count == PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
finally:
PyNvVideoCodecVideoBackend._decoder_slots = old_slots
PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots
PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond
def test_pynvvideocodec_decoder_slot_retains_simple_decoder():
events: list[tuple[object, ...]] = []
class FakeStream:
cuda_stream = "cuda-stream"
class FakeDecoder:
def __init__(self, file_path: str, **kwargs):
events.append(
(
"create",
file_path,
kwargs["gpu_id"],
kwargs["cuda_stream"],
kwargs["decoder_cache_size"],
)
)
def reconfigure_decoder(self, file_path: str):
events.append(("reconfigure", file_path))
class FakeNvc:
class OutputColorType:
RGB = "rgb"
SimpleDecoder = FakeDecoder
slot = PyNvVideoCodecDecoderSlot(FakeStream())
decoder = slot.get_decoder("first.mp4", FakeNvc, device_index=7)
assert slot.get_decoder("first.mp4", FakeNvc, device_index=7) is decoder
assert slot.get_decoder("second.mp4", FakeNvc, device_index=7) is decoder
assert events == [
(
"create",
"first.mp4",
7,
"cuda-stream",
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
),
("reconfigure", "second.mp4"),
]
assert slot.source_path == "second.mp4"
# ============================================================================
# Video Processor → Video Loader Tests (via model repo)
# ============================================================================
+116
View File
@@ -0,0 +1,116 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
import pytest
import vllm.v1.worker.gpu_worker as gpu_worker_module
from vllm.multimodal.video import (
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
from vllm.utils.mem_constants import GiB_bytes
from vllm.v1.worker.gpu_worker import Worker
def _worker_with_mm_config(
mm_config: SimpleNamespace,
*,
api_process_count: int = 1,
) -> Worker:
worker = object.__new__(Worker)
worker.model_config = SimpleNamespace(multimodal_config=mm_config)
worker.parallel_config = SimpleNamespace(_api_process_count=api_process_count)
return worker
def _mm_config(
*,
mm_ipc_gpu_memory_gb: float = 0,
video_backend: str | None = None,
) -> SimpleNamespace:
video_kwargs = {} if video_backend is None else {"video_backend": video_backend}
return SimpleNamespace(
mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
media_io_kwargs={"video": video_kwargs} if video_kwargs else {},
)
def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:
return api_process_count * (
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
)
@pytest.mark.parametrize("video_backend", [None, "opencv"])
def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
monkeypatch: pytest.MonkeyPatch,
video_backend: str | None,
):
monkeypatch.setattr(
gpu_worker_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
"opencv",
)
worker = _worker_with_mm_config(
_mm_config(mm_ipc_gpu_memory_gb=0.25, video_backend=video_backend)
)
assert worker._reserve_mm_ipc_gpu_memory(GiB_bytes) == int(0.75 * GiB_bytes)
def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
gpu_worker_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
"opencv",
)
worker = _worker_with_mm_config(
_mm_config(
mm_ipc_gpu_memory_gb=0.25,
video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
)
)
available_bytes = 4 * GiB_bytes
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget()
)
def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
gpu_worker_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
worker = _worker_with_mm_config(_mm_config())
available_bytes = 4 * GiB_bytes
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
available_bytes - _pynvvideocodec_decoder_budget()
)
def test_reserve_mm_ipc_gpu_memory_scales_pynvvideocodec_budget_by_api_servers(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
gpu_worker_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
worker = _worker_with_mm_config(_mm_config(), api_process_count=3)
available_bytes = 8 * GiB_bytes
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)
)