Vram semaphore infra (#44465)
Signed-off-by: Brandon Pelfrey <bpelfrey@nvidia.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Roger Wang
parent
3f67477497
commit
af16446bf3
@@ -0,0 +1,145 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal.gpu_ipc_memory import (
|
||||
MultiModalGPUMemoryPool,
|
||||
get_mm_gpu_ipc_pool,
|
||||
maybe_init_mm_gpu_ipc_pool,
|
||||
set_mm_gpu_ipc_pool,
|
||||
)
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
|
||||
def test_acquire_release_accounting():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
lease = pool.acquire(40)
|
||||
assert pool.available_bytes == 60
|
||||
|
||||
lease.release()
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
|
||||
def test_acquire_too_large_raises():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
with pytest.raises(ValueError):
|
||||
pool.acquire(101)
|
||||
# Nothing should have been reserved.
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
|
||||
def test_negative_acquire_raises():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
with pytest.raises(ValueError):
|
||||
pool.acquire(-1)
|
||||
|
||||
|
||||
def test_double_release_is_noop():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
lease = pool.acquire(50)
|
||||
lease.release()
|
||||
assert pool.available_bytes == 100
|
||||
# Releasing again must not inflate the pool past its capacity.
|
||||
lease.release()
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
|
||||
def test_context_manager_releases_on_exception():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
with pytest.raises(RuntimeError), pool.acquire(50):
|
||||
assert pool.available_bytes == 50
|
||||
raise RuntimeError("boom")
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
|
||||
def test_acquire_blocks_until_release():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
first = pool.acquire(80)
|
||||
|
||||
acquired = threading.Event()
|
||||
|
||||
def waiter():
|
||||
# Needs 50 bytes but only 20 are free; must block until `first`
|
||||
# is released.
|
||||
with pool.acquire(50):
|
||||
acquired.set()
|
||||
|
||||
t = threading.Thread(target=waiter)
|
||||
t.start()
|
||||
|
||||
# The waiter cannot proceed yet.
|
||||
assert not acquired.wait(timeout=0.2)
|
||||
|
||||
# Releasing the first lease frees enough budget to unblock the waiter.
|
||||
first.release()
|
||||
assert acquired.wait(timeout=2.0)
|
||||
t.join(timeout=2.0)
|
||||
assert not t.is_alive()
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
|
||||
def test_concurrent_acquires_serialize():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
# Each task needs 60 bytes, so only one can hold the budget at a time.
|
||||
in_section = []
|
||||
max_concurrent = 0
|
||||
lock = threading.Lock()
|
||||
|
||||
def task():
|
||||
nonlocal max_concurrent
|
||||
with pool.acquire(60):
|
||||
with lock:
|
||||
in_section.append(1)
|
||||
max_concurrent = max(max_concurrent, len(in_section))
|
||||
time.sleep(0.05)
|
||||
with lock:
|
||||
in_section.pop()
|
||||
|
||||
threads = [threading.Thread(target=task) for _ in range(4)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join(timeout=5.0)
|
||||
assert not t.is_alive()
|
||||
|
||||
assert max_concurrent == 1
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
|
||||
def test_zero_total_bytes_rejected():
|
||||
with pytest.raises(ValueError):
|
||||
MultiModalGPUMemoryPool(total_bytes=0)
|
||||
|
||||
|
||||
def test_global_pool_accessor():
|
||||
try:
|
||||
assert maybe_init_mm_gpu_ipc_pool(0) is None
|
||||
assert get_mm_gpu_ipc_pool() is None
|
||||
|
||||
pool = maybe_init_mm_gpu_ipc_pool(2)
|
||||
assert pool is not None
|
||||
assert get_mm_gpu_ipc_pool() is pool
|
||||
assert pool.total_bytes == 2 * GiB_bytes
|
||||
finally:
|
||||
set_mm_gpu_ipc_pool(None)
|
||||
|
||||
|
||||
def test_global_pool_splits_budget_across_api_processes():
|
||||
try:
|
||||
pool = maybe_init_mm_gpu_ipc_pool(2, api_process_count=4)
|
||||
assert pool is not None
|
||||
assert get_mm_gpu_ipc_pool() is pool
|
||||
assert pool.total_bytes == GiB_bytes // 2
|
||||
finally:
|
||||
set_mm_gpu_ipc_pool(None)
|
||||
|
||||
|
||||
def test_global_pool_rejects_invalid_api_process_count():
|
||||
with pytest.raises(ValueError):
|
||||
maybe_init_mm_gpu_ipc_pool(2, api_process_count=0)
|
||||
@@ -1,6 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import itertools
|
||||
import sys
|
||||
import threading
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
@@ -11,10 +15,15 @@ from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.assets.base import get_vllm_public_assets
|
||||
from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
|
||||
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
DynamicVideoBackend,
|
||||
GLM46VVideoBackend,
|
||||
Molmo2VideoBackend,
|
||||
PyNvVideoCodecDecoderSlot,
|
||||
PyNvVideoCodecVideoBackend,
|
||||
Qwen2VLVideoBackend,
|
||||
Qwen3VLVideoBackend,
|
||||
VideoLoader,
|
||||
@@ -22,6 +31,7 @@ from vllm.multimodal.video import (
|
||||
VideoTargetMetadata,
|
||||
get_video_loader_backend_for_processor,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
|
||||
|
||||
from .utils import create_long_gop_video, create_video_from_image
|
||||
@@ -65,6 +75,230 @@ def test_video_loader_type_doesnt_exist():
|
||||
VIDEO_LOADER_REGISTRY.load("non_existing_video_loader")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
def test_pynvvideocodec_backend_accounts_raw_decoded_frames(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
decoder_cache_sizes = []
|
||||
|
||||
class FakeMetadata:
|
||||
width = 10
|
||||
height = 20
|
||||
average_fps = 5.0
|
||||
duration = 2.0
|
||||
|
||||
class FakeDecoder:
|
||||
def __init__(self, *args, **kwargs):
|
||||
decoder_cache_sizes.append(kwargs["decoder_cache_size"])
|
||||
|
||||
def __len__(self):
|
||||
return 10
|
||||
|
||||
def get_stream_metadata(self):
|
||||
return FakeMetadata()
|
||||
|
||||
class FakeNvc:
|
||||
class OutputColorType:
|
||||
RGB = "rgb"
|
||||
|
||||
SimpleDecoder = FakeDecoder
|
||||
|
||||
class RecordingPool:
|
||||
def __init__(self):
|
||||
self.acquired: list[int] = []
|
||||
|
||||
@contextmanager
|
||||
def acquire(self, size: int):
|
||||
self.acquired.append(size)
|
||||
yield
|
||||
|
||||
def fake_decode(cls, file_path: str, frame_idx: list[int], nvc):
|
||||
return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8)
|
||||
|
||||
pool = RecordingPool()
|
||||
monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc)
|
||||
monkeypatch.setattr(
|
||||
"vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
PyNvVideoCodecVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode)
|
||||
)
|
||||
|
||||
loader = VIDEO_LOADER_REGISTRY.load(PYNVVIDEOCODEC_VIDEO_BACKEND)
|
||||
frames, metadata = loader.load_bytes(b"fake video", num_frames=4)
|
||||
|
||||
assert frames.shape == (4, 20, 10, 3)
|
||||
assert pool.acquired == [4 * 20 * 10 * 3]
|
||||
assert decoder_cache_sizes == [PYNVVIDEOCODEC_DECODER_CACHE_SIZE]
|
||||
assert metadata["video_backend"] == PYNVVIDEOCODEC_VIDEO_BACKEND
|
||||
assert metadata["frames_indices"] == [0, 3, 6, 9]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
def test_pynvvideocodec_codec_uses_dynamic_sampling_strategy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
decoded_indices = []
|
||||
|
||||
class FakeMetadata:
|
||||
width = 10
|
||||
height = 20
|
||||
average_fps = 5.0
|
||||
duration = 2.0
|
||||
|
||||
class FakeDecoder:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def __len__(self):
|
||||
return 10
|
||||
|
||||
def get_stream_metadata(self):
|
||||
return FakeMetadata()
|
||||
|
||||
class FakeNvc:
|
||||
class OutputColorType:
|
||||
RGB = "rgb"
|
||||
|
||||
SimpleDecoder = FakeDecoder
|
||||
|
||||
class RecordingPool:
|
||||
def __init__(self):
|
||||
self.acquired: list[int] = []
|
||||
|
||||
@contextmanager
|
||||
def acquire(self, size: int):
|
||||
self.acquired.append(size)
|
||||
yield
|
||||
|
||||
def fake_decode(cls, file_path: str, frame_idx: list[int], nvc):
|
||||
decoded_indices.append(frame_idx)
|
||||
return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8)
|
||||
|
||||
pool = RecordingPool()
|
||||
monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc)
|
||||
monkeypatch.setattr(
|
||||
"vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DynamicVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode)
|
||||
)
|
||||
|
||||
loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
|
||||
frames, metadata = loader.load_bytes(
|
||||
b"fake video",
|
||||
fps=2,
|
||||
max_duration=1,
|
||||
backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
|
||||
assert frames.shape == (2, 20, 10, 3)
|
||||
assert decoded_indices == [[0, 9]]
|
||||
assert pool.acquired == [2 * 20 * 10 * 3]
|
||||
assert metadata["video_backend"] == f"{PYNVVIDEOCODEC_VIDEO_BACKEND}_dynamic"
|
||||
assert metadata["frames_indices"] == [0, 9]
|
||||
|
||||
|
||||
def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatch):
|
||||
class FakeSlot:
|
||||
pass
|
||||
|
||||
create_count = 0
|
||||
old_slots = PyNvVideoCodecVideoBackend._decoder_slots
|
||||
old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots
|
||||
old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond
|
||||
try:
|
||||
PyNvVideoCodecVideoBackend._decoder_slots = []
|
||||
PyNvVideoCodecVideoBackend._active_decoder_slots = 0
|
||||
PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition()
|
||||
|
||||
def fake_create_slot(cls):
|
||||
nonlocal create_count
|
||||
create_count += 1
|
||||
return FakeSlot()
|
||||
|
||||
monkeypatch.setattr(
|
||||
PyNvVideoCodecVideoBackend,
|
||||
"_create_decoder_slot",
|
||||
classmethod(fake_create_slot),
|
||||
)
|
||||
|
||||
borrowed = threading.Event()
|
||||
seen_slots = []
|
||||
|
||||
with ExitStack() as stack:
|
||||
retained_slots = [
|
||||
stack.enter_context(PyNvVideoCodecVideoBackend._borrow_decoder_slot())
|
||||
for _ in range(PYNVVIDEOCODEC_MAX_RETAINED_DECODERS)
|
||||
]
|
||||
|
||||
def borrow_extra_slot():
|
||||
with PyNvVideoCodecVideoBackend._borrow_decoder_slot() as extra_slot:
|
||||
seen_slots.append(extra_slot)
|
||||
borrowed.set()
|
||||
|
||||
thread = threading.Thread(target=borrow_extra_slot)
|
||||
thread.start()
|
||||
assert not borrowed.wait(timeout=0.2)
|
||||
|
||||
assert borrowed.wait(timeout=2.0)
|
||||
thread.join(timeout=2.0)
|
||||
assert not thread.is_alive()
|
||||
|
||||
assert seen_slots[0] in retained_slots
|
||||
assert create_count == PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
|
||||
finally:
|
||||
PyNvVideoCodecVideoBackend._decoder_slots = old_slots
|
||||
PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots
|
||||
PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond
|
||||
|
||||
|
||||
def test_pynvvideocodec_decoder_slot_retains_simple_decoder():
|
||||
events: list[tuple[object, ...]] = []
|
||||
|
||||
class FakeStream:
|
||||
cuda_stream = "cuda-stream"
|
||||
|
||||
class FakeDecoder:
|
||||
def __init__(self, file_path: str, **kwargs):
|
||||
events.append(
|
||||
(
|
||||
"create",
|
||||
file_path,
|
||||
kwargs["gpu_id"],
|
||||
kwargs["cuda_stream"],
|
||||
kwargs["decoder_cache_size"],
|
||||
)
|
||||
)
|
||||
|
||||
def reconfigure_decoder(self, file_path: str):
|
||||
events.append(("reconfigure", file_path))
|
||||
|
||||
class FakeNvc:
|
||||
class OutputColorType:
|
||||
RGB = "rgb"
|
||||
|
||||
SimpleDecoder = FakeDecoder
|
||||
|
||||
slot = PyNvVideoCodecDecoderSlot(FakeStream())
|
||||
|
||||
decoder = slot.get_decoder("first.mp4", FakeNvc, device_index=7)
|
||||
assert slot.get_decoder("first.mp4", FakeNvc, device_index=7) is decoder
|
||||
assert slot.get_decoder("second.mp4", FakeNvc, device_index=7) is decoder
|
||||
|
||||
assert events == [
|
||||
(
|
||||
"create",
|
||||
"first.mp4",
|
||||
7,
|
||||
"cuda-stream",
|
||||
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
|
||||
),
|
||||
("reconfigure", "second.mp4"),
|
||||
]
|
||||
assert slot.source_path == "second.mp4"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Video Processor → Video Loader Tests (via model repo)
|
||||
# ============================================================================
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.v1.worker.gpu_worker as gpu_worker_module
|
||||
from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
|
||||
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
|
||||
def _worker_with_mm_config(
|
||||
mm_config: SimpleNamespace,
|
||||
*,
|
||||
api_process_count: int = 1,
|
||||
) -> Worker:
|
||||
worker = object.__new__(Worker)
|
||||
worker.model_config = SimpleNamespace(multimodal_config=mm_config)
|
||||
worker.parallel_config = SimpleNamespace(_api_process_count=api_process_count)
|
||||
return worker
|
||||
|
||||
|
||||
def _mm_config(
|
||||
*,
|
||||
mm_ipc_gpu_memory_gb: float = 0,
|
||||
video_backend: str | None = None,
|
||||
) -> SimpleNamespace:
|
||||
video_kwargs = {} if video_backend is None else {"video_backend": video_backend}
|
||||
return SimpleNamespace(
|
||||
mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
|
||||
media_io_kwargs={"video": video_kwargs} if video_kwargs else {},
|
||||
)
|
||||
|
||||
|
||||
def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:
|
||||
return api_process_count * (
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
|
||||
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("video_backend", [None, "opencv"])
|
||||
def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
video_backend: str | None,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gpu_worker_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
"opencv",
|
||||
)
|
||||
worker = _worker_with_mm_config(
|
||||
_mm_config(mm_ipc_gpu_memory_gb=0.25, video_backend=video_backend)
|
||||
)
|
||||
|
||||
assert worker._reserve_mm_ipc_gpu_memory(GiB_bytes) == int(0.75 * GiB_bytes)
|
||||
|
||||
|
||||
def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gpu_worker_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
"opencv",
|
||||
)
|
||||
worker = _worker_with_mm_config(
|
||||
_mm_config(
|
||||
mm_ipc_gpu_memory_gb=0.25,
|
||||
video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
)
|
||||
available_bytes = 4 * GiB_bytes
|
||||
|
||||
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
|
||||
available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget()
|
||||
)
|
||||
|
||||
|
||||
def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gpu_worker_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
worker = _worker_with_mm_config(_mm_config())
|
||||
available_bytes = 4 * GiB_bytes
|
||||
|
||||
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
|
||||
available_bytes - _pynvvideocodec_decoder_budget()
|
||||
)
|
||||
|
||||
|
||||
def test_reserve_mm_ipc_gpu_memory_scales_pynvvideocodec_budget_by_api_servers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gpu_worker_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
worker = _worker_with_mm_config(_mm_config(), api_process_count=3)
|
||||
available_bytes = 8 * GiB_bytes
|
||||
|
||||
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
|
||||
available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)
|
||||
)
|
||||
Reference in New Issue
Block a user