Vram semaphore infra (#44465)
Signed-off-by: Brandon Pelfrey <bpelfrey@nvidia.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Roger Wang
parent
3f67477497
commit
af16446bf3
@@ -8,6 +8,7 @@ torch==2.11.0
|
||||
torchaudio==2.11.0
|
||||
# These must be updated alongside torch
|
||||
torchvision==0.26.0 # Required for phi3v processor. See https://github.com/pytorch/vision?tab=readme-ov-file#installation for corresponding version
|
||||
PyNvVideoCodec==2.0.4
|
||||
# FlashInfer should be updated together with the Dockerfile
|
||||
flashinfer-python==0.6.12
|
||||
flashinfer-cubin==0.6.12
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.multimodal.gpu_ipc_memory import (
|
||||
MultiModalGPUMemoryPool,
|
||||
get_mm_gpu_ipc_pool,
|
||||
maybe_init_mm_gpu_ipc_pool,
|
||||
set_mm_gpu_ipc_pool,
|
||||
)
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
|
||||
def test_acquire_release_accounting():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
lease = pool.acquire(40)
|
||||
assert pool.available_bytes == 60
|
||||
|
||||
lease.release()
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
|
||||
def test_acquire_too_large_raises():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
with pytest.raises(ValueError):
|
||||
pool.acquire(101)
|
||||
# Nothing should have been reserved.
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
|
||||
def test_negative_acquire_raises():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
with pytest.raises(ValueError):
|
||||
pool.acquire(-1)
|
||||
|
||||
|
||||
def test_double_release_is_noop():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
lease = pool.acquire(50)
|
||||
lease.release()
|
||||
assert pool.available_bytes == 100
|
||||
# Releasing again must not inflate the pool past its capacity.
|
||||
lease.release()
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
|
||||
def test_context_manager_releases_on_exception():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
with pytest.raises(RuntimeError), pool.acquire(50):
|
||||
assert pool.available_bytes == 50
|
||||
raise RuntimeError("boom")
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
|
||||
def test_acquire_blocks_until_release():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
first = pool.acquire(80)
|
||||
|
||||
acquired = threading.Event()
|
||||
|
||||
def waiter():
|
||||
# Needs 50 bytes but only 20 are free; must block until `first`
|
||||
# is released.
|
||||
with pool.acquire(50):
|
||||
acquired.set()
|
||||
|
||||
t = threading.Thread(target=waiter)
|
||||
t.start()
|
||||
|
||||
# The waiter cannot proceed yet.
|
||||
assert not acquired.wait(timeout=0.2)
|
||||
|
||||
# Releasing the first lease frees enough budget to unblock the waiter.
|
||||
first.release()
|
||||
assert acquired.wait(timeout=2.0)
|
||||
t.join(timeout=2.0)
|
||||
assert not t.is_alive()
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
|
||||
def test_concurrent_acquires_serialize():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
# Each task needs 60 bytes, so only one can hold the budget at a time.
|
||||
in_section = []
|
||||
max_concurrent = 0
|
||||
lock = threading.Lock()
|
||||
|
||||
def task():
|
||||
nonlocal max_concurrent
|
||||
with pool.acquire(60):
|
||||
with lock:
|
||||
in_section.append(1)
|
||||
max_concurrent = max(max_concurrent, len(in_section))
|
||||
time.sleep(0.05)
|
||||
with lock:
|
||||
in_section.pop()
|
||||
|
||||
threads = [threading.Thread(target=task) for _ in range(4)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join(timeout=5.0)
|
||||
assert not t.is_alive()
|
||||
|
||||
assert max_concurrent == 1
|
||||
assert pool.available_bytes == 100
|
||||
|
||||
|
||||
def test_zero_total_bytes_rejected():
|
||||
with pytest.raises(ValueError):
|
||||
MultiModalGPUMemoryPool(total_bytes=0)
|
||||
|
||||
|
||||
def test_global_pool_accessor():
|
||||
try:
|
||||
assert maybe_init_mm_gpu_ipc_pool(0) is None
|
||||
assert get_mm_gpu_ipc_pool() is None
|
||||
|
||||
pool = maybe_init_mm_gpu_ipc_pool(2)
|
||||
assert pool is not None
|
||||
assert get_mm_gpu_ipc_pool() is pool
|
||||
assert pool.total_bytes == 2 * GiB_bytes
|
||||
finally:
|
||||
set_mm_gpu_ipc_pool(None)
|
||||
|
||||
|
||||
def test_global_pool_splits_budget_across_api_processes():
|
||||
try:
|
||||
pool = maybe_init_mm_gpu_ipc_pool(2, api_process_count=4)
|
||||
assert pool is not None
|
||||
assert get_mm_gpu_ipc_pool() is pool
|
||||
assert pool.total_bytes == GiB_bytes // 2
|
||||
finally:
|
||||
set_mm_gpu_ipc_pool(None)
|
||||
|
||||
|
||||
def test_global_pool_rejects_invalid_api_process_count():
|
||||
with pytest.raises(ValueError):
|
||||
maybe_init_mm_gpu_ipc_pool(2, api_process_count=0)
|
||||
@@ -1,6 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import itertools
|
||||
import sys
|
||||
import threading
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
@@ -11,10 +15,15 @@ from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.assets.base import get_vllm_public_assets
|
||||
from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
|
||||
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
DynamicVideoBackend,
|
||||
GLM46VVideoBackend,
|
||||
Molmo2VideoBackend,
|
||||
PyNvVideoCodecDecoderSlot,
|
||||
PyNvVideoCodecVideoBackend,
|
||||
Qwen2VLVideoBackend,
|
||||
Qwen3VLVideoBackend,
|
||||
VideoLoader,
|
||||
@@ -22,6 +31,7 @@ from vllm.multimodal.video import (
|
||||
VideoTargetMetadata,
|
||||
get_video_loader_backend_for_processor,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.transformers_utils.processor import get_video_processor_cls_name_from_config
|
||||
|
||||
from .utils import create_long_gop_video, create_video_from_image
|
||||
@@ -65,6 +75,230 @@ def test_video_loader_type_doesnt_exist():
|
||||
VIDEO_LOADER_REGISTRY.load("non_existing_video_loader")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
def test_pynvvideocodec_backend_accounts_raw_decoded_frames(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
decoder_cache_sizes = []
|
||||
|
||||
class FakeMetadata:
|
||||
width = 10
|
||||
height = 20
|
||||
average_fps = 5.0
|
||||
duration = 2.0
|
||||
|
||||
class FakeDecoder:
|
||||
def __init__(self, *args, **kwargs):
|
||||
decoder_cache_sizes.append(kwargs["decoder_cache_size"])
|
||||
|
||||
def __len__(self):
|
||||
return 10
|
||||
|
||||
def get_stream_metadata(self):
|
||||
return FakeMetadata()
|
||||
|
||||
class FakeNvc:
|
||||
class OutputColorType:
|
||||
RGB = "rgb"
|
||||
|
||||
SimpleDecoder = FakeDecoder
|
||||
|
||||
class RecordingPool:
|
||||
def __init__(self):
|
||||
self.acquired: list[int] = []
|
||||
|
||||
@contextmanager
|
||||
def acquire(self, size: int):
|
||||
self.acquired.append(size)
|
||||
yield
|
||||
|
||||
def fake_decode(cls, file_path: str, frame_idx: list[int], nvc):
|
||||
return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8)
|
||||
|
||||
pool = RecordingPool()
|
||||
monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc)
|
||||
monkeypatch.setattr(
|
||||
"vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
PyNvVideoCodecVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode)
|
||||
)
|
||||
|
||||
loader = VIDEO_LOADER_REGISTRY.load(PYNVVIDEOCODEC_VIDEO_BACKEND)
|
||||
frames, metadata = loader.load_bytes(b"fake video", num_frames=4)
|
||||
|
||||
assert frames.shape == (4, 20, 10, 3)
|
||||
assert pool.acquired == [4 * 20 * 10 * 3]
|
||||
assert decoder_cache_sizes == [PYNVVIDEOCODEC_DECODER_CACHE_SIZE]
|
||||
assert metadata["video_backend"] == PYNVVIDEOCODEC_VIDEO_BACKEND
|
||||
assert metadata["frames_indices"] == [0, 3, 6, 9]
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
def test_pynvvideocodec_codec_uses_dynamic_sampling_strategy(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
decoded_indices = []
|
||||
|
||||
class FakeMetadata:
|
||||
width = 10
|
||||
height = 20
|
||||
average_fps = 5.0
|
||||
duration = 2.0
|
||||
|
||||
class FakeDecoder:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def __len__(self):
|
||||
return 10
|
||||
|
||||
def get_stream_metadata(self):
|
||||
return FakeMetadata()
|
||||
|
||||
class FakeNvc:
|
||||
class OutputColorType:
|
||||
RGB = "rgb"
|
||||
|
||||
SimpleDecoder = FakeDecoder
|
||||
|
||||
class RecordingPool:
|
||||
def __init__(self):
|
||||
self.acquired: list[int] = []
|
||||
|
||||
@contextmanager
|
||||
def acquire(self, size: int):
|
||||
self.acquired.append(size)
|
||||
yield
|
||||
|
||||
def fake_decode(cls, file_path: str, frame_idx: list[int], nvc):
|
||||
decoded_indices.append(frame_idx)
|
||||
return np.zeros((len(frame_idx), 20, 10, 3), dtype=np.uint8)
|
||||
|
||||
pool = RecordingPool()
|
||||
monkeypatch.setitem(sys.modules, "PyNvVideoCodec", FakeNvc)
|
||||
monkeypatch.setattr(
|
||||
"vllm.multimodal.gpu_ipc_memory.get_mm_gpu_ipc_pool", lambda: pool
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DynamicVideoBackend, "_decode_to_pinned_host", classmethod(fake_decode)
|
||||
)
|
||||
|
||||
loader = VIDEO_LOADER_REGISTRY.load("opencv_dynamic")
|
||||
frames, metadata = loader.load_bytes(
|
||||
b"fake video",
|
||||
fps=2,
|
||||
max_duration=1,
|
||||
backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
|
||||
assert frames.shape == (2, 20, 10, 3)
|
||||
assert decoded_indices == [[0, 9]]
|
||||
assert pool.acquired == [2 * 20 * 10 * 3]
|
||||
assert metadata["video_backend"] == f"{PYNVVIDEOCODEC_VIDEO_BACKEND}_dynamic"
|
||||
assert metadata["frames_indices"] == [0, 9]
|
||||
|
||||
|
||||
def test_pynvvideocodec_decoder_slots_are_bounded(monkeypatch: pytest.MonkeyPatch):
|
||||
class FakeSlot:
|
||||
pass
|
||||
|
||||
create_count = 0
|
||||
old_slots = PyNvVideoCodecVideoBackend._decoder_slots
|
||||
old_active_slots = PyNvVideoCodecVideoBackend._active_decoder_slots
|
||||
old_cond = PyNvVideoCodecVideoBackend._decoder_slot_cond
|
||||
try:
|
||||
PyNvVideoCodecVideoBackend._decoder_slots = []
|
||||
PyNvVideoCodecVideoBackend._active_decoder_slots = 0
|
||||
PyNvVideoCodecVideoBackend._decoder_slot_cond = threading.Condition()
|
||||
|
||||
def fake_create_slot(cls):
|
||||
nonlocal create_count
|
||||
create_count += 1
|
||||
return FakeSlot()
|
||||
|
||||
monkeypatch.setattr(
|
||||
PyNvVideoCodecVideoBackend,
|
||||
"_create_decoder_slot",
|
||||
classmethod(fake_create_slot),
|
||||
)
|
||||
|
||||
borrowed = threading.Event()
|
||||
seen_slots = []
|
||||
|
||||
with ExitStack() as stack:
|
||||
retained_slots = [
|
||||
stack.enter_context(PyNvVideoCodecVideoBackend._borrow_decoder_slot())
|
||||
for _ in range(PYNVVIDEOCODEC_MAX_RETAINED_DECODERS)
|
||||
]
|
||||
|
||||
def borrow_extra_slot():
|
||||
with PyNvVideoCodecVideoBackend._borrow_decoder_slot() as extra_slot:
|
||||
seen_slots.append(extra_slot)
|
||||
borrowed.set()
|
||||
|
||||
thread = threading.Thread(target=borrow_extra_slot)
|
||||
thread.start()
|
||||
assert not borrowed.wait(timeout=0.2)
|
||||
|
||||
assert borrowed.wait(timeout=2.0)
|
||||
thread.join(timeout=2.0)
|
||||
assert not thread.is_alive()
|
||||
|
||||
assert seen_slots[0] in retained_slots
|
||||
assert create_count == PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
|
||||
finally:
|
||||
PyNvVideoCodecVideoBackend._decoder_slots = old_slots
|
||||
PyNvVideoCodecVideoBackend._active_decoder_slots = old_active_slots
|
||||
PyNvVideoCodecVideoBackend._decoder_slot_cond = old_cond
|
||||
|
||||
|
||||
def test_pynvvideocodec_decoder_slot_retains_simple_decoder():
|
||||
events: list[tuple[object, ...]] = []
|
||||
|
||||
class FakeStream:
|
||||
cuda_stream = "cuda-stream"
|
||||
|
||||
class FakeDecoder:
|
||||
def __init__(self, file_path: str, **kwargs):
|
||||
events.append(
|
||||
(
|
||||
"create",
|
||||
file_path,
|
||||
kwargs["gpu_id"],
|
||||
kwargs["cuda_stream"],
|
||||
kwargs["decoder_cache_size"],
|
||||
)
|
||||
)
|
||||
|
||||
def reconfigure_decoder(self, file_path: str):
|
||||
events.append(("reconfigure", file_path))
|
||||
|
||||
class FakeNvc:
|
||||
class OutputColorType:
|
||||
RGB = "rgb"
|
||||
|
||||
SimpleDecoder = FakeDecoder
|
||||
|
||||
slot = PyNvVideoCodecDecoderSlot(FakeStream())
|
||||
|
||||
decoder = slot.get_decoder("first.mp4", FakeNvc, device_index=7)
|
||||
assert slot.get_decoder("first.mp4", FakeNvc, device_index=7) is decoder
|
||||
assert slot.get_decoder("second.mp4", FakeNvc, device_index=7) is decoder
|
||||
|
||||
assert events == [
|
||||
(
|
||||
"create",
|
||||
"first.mp4",
|
||||
7,
|
||||
"cuda-stream",
|
||||
PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
|
||||
),
|
||||
("reconfigure", "second.mp4"),
|
||||
]
|
||||
assert slot.source_path == "second.mp4"
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Video Processor → Video Loader Tests (via model repo)
|
||||
# ============================================================================
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.v1.worker.gpu_worker as gpu_worker_module
|
||||
from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
|
||||
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
|
||||
def _worker_with_mm_config(
|
||||
mm_config: SimpleNamespace,
|
||||
*,
|
||||
api_process_count: int = 1,
|
||||
) -> Worker:
|
||||
worker = object.__new__(Worker)
|
||||
worker.model_config = SimpleNamespace(multimodal_config=mm_config)
|
||||
worker.parallel_config = SimpleNamespace(_api_process_count=api_process_count)
|
||||
return worker
|
||||
|
||||
|
||||
def _mm_config(
|
||||
*,
|
||||
mm_ipc_gpu_memory_gb: float = 0,
|
||||
video_backend: str | None = None,
|
||||
) -> SimpleNamespace:
|
||||
video_kwargs = {} if video_backend is None else {"video_backend": video_backend}
|
||||
return SimpleNamespace(
|
||||
mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
|
||||
media_io_kwargs={"video": video_kwargs} if video_kwargs else {},
|
||||
)
|
||||
|
||||
|
||||
def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:
|
||||
return api_process_count * (
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
|
||||
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("video_backend", [None, "opencv"])
|
||||
def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
video_backend: str | None,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gpu_worker_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
"opencv",
|
||||
)
|
||||
worker = _worker_with_mm_config(
|
||||
_mm_config(mm_ipc_gpu_memory_gb=0.25, video_backend=video_backend)
|
||||
)
|
||||
|
||||
assert worker._reserve_mm_ipc_gpu_memory(GiB_bytes) == int(0.75 * GiB_bytes)
|
||||
|
||||
|
||||
def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gpu_worker_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
"opencv",
|
||||
)
|
||||
worker = _worker_with_mm_config(
|
||||
_mm_config(
|
||||
mm_ipc_gpu_memory_gb=0.25,
|
||||
video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
)
|
||||
available_bytes = 4 * GiB_bytes
|
||||
|
||||
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
|
||||
available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget()
|
||||
)
|
||||
|
||||
|
||||
def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gpu_worker_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
worker = _worker_with_mm_config(_mm_config())
|
||||
available_bytes = 4 * GiB_bytes
|
||||
|
||||
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
|
||||
available_bytes - _pynvvideocodec_decoder_budget()
|
||||
)
|
||||
|
||||
|
||||
def test_reserve_mm_ipc_gpu_memory_scales_pynvvideocodec_budget_by_api_servers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gpu_worker_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
worker = _worker_with_mm_config(_mm_config(), api_process_count=3)
|
||||
available_bytes = 8 * GiB_bytes
|
||||
|
||||
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
|
||||
available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)
|
||||
)
|
||||
@@ -351,6 +351,7 @@ class ModelConfig:
|
||||
skip_mm_profiling: InitVar[bool | None] = None
|
||||
video_pruning_rate: InitVar[float | None] = None
|
||||
mm_tensor_ipc: InitVar[MMTensorIPC] = None
|
||||
mm_ipc_gpu_memory_gb: InitVar[float | None] = None
|
||||
|
||||
def compute_hash(self) -> str:
|
||||
"""
|
||||
@@ -397,6 +398,7 @@ class ModelConfig:
|
||||
"mm_encoder_tp_mode",
|
||||
"interleave_mm_strings",
|
||||
"skip_mm_profiling",
|
||||
"mm_ipc_gpu_memory_gb",
|
||||
}
|
||||
|
||||
from vllm.config.utils import get_hash_factors, hash_factors
|
||||
@@ -477,6 +479,7 @@ class ModelConfig:
|
||||
skip_mm_profiling: bool | None,
|
||||
video_pruning_rate: float | None,
|
||||
mm_tensor_ipc: MMTensorIPC,
|
||||
mm_ipc_gpu_memory_gb: float | None,
|
||||
) -> None:
|
||||
# Keep set served_model_name before maybe_model_redirect(self.model)
|
||||
self.served_model_name = get_served_model_name(
|
||||
@@ -690,6 +693,7 @@ class ModelConfig:
|
||||
skip_mm_profiling=skip_mm_profiling,
|
||||
video_pruning_rate=video_pruning_rate,
|
||||
mm_tensor_ipc=mm_tensor_ipc,
|
||||
mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
|
||||
)
|
||||
|
||||
mm_config_kwargs = {
|
||||
|
||||
@@ -197,6 +197,16 @@ class MultiModalConfig:
|
||||
- "direct_rpc": Use msgspec serialization via RPC
|
||||
- "torch_shm": Use torch.multiprocessing shared memory for zero-copy IPC
|
||||
Defaults to "direct_rpc". """
|
||||
mm_ipc_gpu_memory_gb: float = Field(default=0, ge=0)
|
||||
"""Amount of GPU memory (in GiB) sequestered on the engine's device for
|
||||
GPU-side multimodal work in the API-server (frontend) process, such as
|
||||
hardware video decoding.
|
||||
|
||||
This budget is carved out of the engine's KV-cache memory so the headroom
|
||||
physically exists, and frontend GPU decode paths acquire from a blocking
|
||||
byte-counting semaphore of this size before allocating on the device.
|
||||
|
||||
Set to `0` (default) to disable frontend GPU multimodal memory gating."""
|
||||
|
||||
@field_validator("limit_per_prompt", mode="before")
|
||||
@classmethod
|
||||
|
||||
@@ -582,6 +582,7 @@ class EngineArgs:
|
||||
skip_mm_profiling: bool = MultiModalConfig.skip_mm_profiling
|
||||
video_pruning_rate: float | None = MultiModalConfig.video_pruning_rate
|
||||
mm_tensor_ipc: MMTensorIPC = MultiModalConfig.mm_tensor_ipc
|
||||
mm_ipc_gpu_memory_gb: float = MultiModalConfig.mm_ipc_gpu_memory_gb
|
||||
# LoRA fields
|
||||
enable_lora: bool = False
|
||||
max_loras: int = LoRAConfig.max_loras
|
||||
@@ -1294,6 +1295,10 @@ class EngineArgs:
|
||||
multimodal_group.add_argument(
|
||||
"--mm-tensor-ipc", **multimodal_kwargs["mm_tensor_ipc"]
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-ipc-gpu-memory-gb",
|
||||
**multimodal_kwargs["mm_ipc_gpu_memory_gb"],
|
||||
)
|
||||
|
||||
# LoRA related configs
|
||||
lora_kwargs = get_kwargs(LoRAConfig)
|
||||
@@ -1660,6 +1665,7 @@ class EngineArgs:
|
||||
logits_processors=self.logits_processors,
|
||||
video_pruning_rate=self.video_pruning_rate,
|
||||
mm_tensor_ipc=self.mm_tensor_ipc,
|
||||
mm_ipc_gpu_memory_gb=self.mm_ipc_gpu_memory_gb,
|
||||
io_processor_plugin=self.io_processor_plugin,
|
||||
renderer_num_workers=self.renderer_num_workers,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Admission control for frontend GPU-side multimodal work.
|
||||
|
||||
When multimodal media is decoded on the GPU in the API-server (frontend)
|
||||
process, the decoded buffers compete for the same device memory that the
|
||||
engine reserves for weights, activations, and the KV cache. To keep the
|
||||
frontend's GPU usage within a sequestered budget (see
|
||||
``MultiModalConfig.mm_ipc_gpu_memory_gb``), decode paths acquire the number of
|
||||
bytes they need from a process-global :class:`MultiModalGPUMemoryPool` before
|
||||
allocating on the device and release them once the device memory is freed.
|
||||
|
||||
The pool is a simple byte-counting semaphore: ``acquire`` blocks until enough
|
||||
budget is free, so concurrent requests serialize rather than oversubscribe the
|
||||
GPU. It lives only in the frontend process; the engine carves the matching
|
||||
amount out of its KV-cache budget so the headroom physically exists.
|
||||
"""
|
||||
|
||||
import threading
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MultiModalGPUMemoryLease:
|
||||
"""A handle for bytes acquired from a :class:`MultiModalGPUMemoryPool`.
|
||||
|
||||
Releasing is idempotent and the lease doubles as a context manager so the
|
||||
budget is returned even if the decode raises.
|
||||
"""
|
||||
|
||||
def __init__(self, pool: "MultiModalGPUMemoryPool", lease_id: int, nbytes: int):
|
||||
self.lease_id = lease_id
|
||||
self.nbytes = nbytes
|
||||
self._pool = pool
|
||||
|
||||
def release(self) -> None:
|
||||
self._pool._release(self)
|
||||
|
||||
def __enter__(self) -> "MultiModalGPUMemoryLease":
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info) -> None:
|
||||
self.release()
|
||||
|
||||
|
||||
class MultiModalGPUMemoryPool:
|
||||
"""Blocking byte-counting semaphore for frontend GPU multimodal memory.
|
||||
|
||||
Thread-safe in both directions: ``acquire`` (blocking) and ``release`` are
|
||||
typically called from the renderer's multimodal executor threads.
|
||||
"""
|
||||
|
||||
def __init__(self, total_bytes: int):
|
||||
if total_bytes <= 0:
|
||||
raise ValueError(f"total_bytes must be positive, got {total_bytes}")
|
||||
self._total_bytes = total_bytes
|
||||
self._available = total_bytes
|
||||
self._cond = threading.Condition()
|
||||
self._next_lease_id = 0
|
||||
# Outstanding lease ids, so a double release is a no-op.
|
||||
self._outstanding: set[int] = set()
|
||||
|
||||
@property
|
||||
def total_bytes(self) -> int:
|
||||
return self._total_bytes
|
||||
|
||||
@property
|
||||
def available_bytes(self) -> int:
|
||||
with self._cond:
|
||||
return self._available
|
||||
|
||||
def acquire(self, nbytes: int) -> MultiModalGPUMemoryLease:
|
||||
"""Reserve ``nbytes``, blocking until that much budget is free.
|
||||
|
||||
Raises ``ValueError`` if ``nbytes`` exceeds the pool's total capacity,
|
||||
since such a request could never be satisfied.
|
||||
"""
|
||||
if nbytes < 0:
|
||||
raise ValueError(f"Cannot acquire negative bytes: {nbytes}")
|
||||
if nbytes > self._total_bytes:
|
||||
raise ValueError(
|
||||
f"Multimodal GPU decode requested {nbytes} bytes, which exceeds "
|
||||
f"the total pool size of {self._total_bytes} bytes. Increase "
|
||||
f"--mm-ipc-gpu-memory-gb or reduce the multimodal input size."
|
||||
)
|
||||
with self._cond:
|
||||
while self._available < nbytes:
|
||||
self._cond.wait()
|
||||
self._available -= nbytes
|
||||
lease_id = self._next_lease_id
|
||||
self._next_lease_id += 1
|
||||
self._outstanding.add(lease_id)
|
||||
return MultiModalGPUMemoryLease(self, lease_id, nbytes)
|
||||
|
||||
def _release(self, lease: MultiModalGPUMemoryLease) -> None:
|
||||
with self._cond:
|
||||
if lease.lease_id not in self._outstanding:
|
||||
# Already released — idempotent.
|
||||
return
|
||||
self._outstanding.discard(lease.lease_id)
|
||||
self._available += lease.nbytes
|
||||
self._cond.notify_all()
|
||||
|
||||
|
||||
_GLOBAL_POOL: MultiModalGPUMemoryPool | None = None
|
||||
|
||||
|
||||
def set_mm_gpu_ipc_pool(pool: MultiModalGPUMemoryPool | None) -> None:
|
||||
"""Install the process-global pool (frontend process only)."""
|
||||
global _GLOBAL_POOL
|
||||
_GLOBAL_POOL = pool
|
||||
|
||||
|
||||
def get_mm_gpu_ipc_pool() -> MultiModalGPUMemoryPool | None:
|
||||
"""Return the process-global pool, or ``None`` when gating is disabled."""
|
||||
return _GLOBAL_POOL
|
||||
|
||||
|
||||
def maybe_init_mm_gpu_ipc_pool(
|
||||
mm_ipc_gpu_memory_gb: float,
|
||||
api_process_count: int = 1,
|
||||
) -> MultiModalGPUMemoryPool | None:
|
||||
"""Create and install the global pool from the configured GiB budget.
|
||||
|
||||
Returns ``None`` (and leaves gating disabled) when the budget is 0. When
|
||||
multiple API-server processes share one engine, each process gets an equal
|
||||
slice of the user-provided frontend budget.
|
||||
"""
|
||||
if mm_ipc_gpu_memory_gb <= 0:
|
||||
set_mm_gpu_ipc_pool(None)
|
||||
return None
|
||||
if api_process_count <= 0:
|
||||
raise ValueError(f"api_process_count must be positive, got {api_process_count}")
|
||||
total_bytes = int(mm_ipc_gpu_memory_gb * GiB_bytes) // api_process_count
|
||||
pool = MultiModalGPUMemoryPool(total_bytes)
|
||||
set_mm_gpu_ipc_pool(pool)
|
||||
logger.info(
|
||||
"Initialized multimodal GPU IPC memory pool with %d bytes for this API "
|
||||
"process (%.2f GiB total budget across %d API process(es)).",
|
||||
total_bytes,
|
||||
mm_ipc_gpu_memory_gb,
|
||||
api_process_count,
|
||||
)
|
||||
return pool
|
||||
+331
-13
@@ -1,7 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from abc import abstractmethod
|
||||
from contextlib import contextmanager, suppress
|
||||
from io import BytesIO
|
||||
from typing import Any, ClassVar, Literal, NamedTuple, cast
|
||||
|
||||
@@ -11,6 +15,7 @@ import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import PlaceholderModule
|
||||
from vllm.utils.mem_constants import MiB_bytes
|
||||
from vllm.utils.registry import ExtensionManager
|
||||
|
||||
try:
|
||||
@@ -130,6 +135,14 @@ class VideoSourceMetadata(NamedTuple):
|
||||
duration: float
|
||||
|
||||
|
||||
class PyNvVideoCodecSourceMetadata(NamedTuple):
|
||||
"""Metadata needed before GPU video decode."""
|
||||
|
||||
source: VideoSourceMetadata
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
class VideoLoader:
|
||||
@classmethod
|
||||
def compute_frames_index_to_sample(
|
||||
@@ -170,6 +183,57 @@ class VideoLoader:
|
||||
|
||||
VIDEO_LOADER_REGISTRY = VideoLoaderRegistry()
|
||||
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND: Literal["pynvvideocodec"] = "pynvvideocodec"
|
||||
# Fixed upper bound reserved for persistent PyNvVideoCodec decoder surfaces.
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES = 128 * MiB_bytes
|
||||
PYNVVIDEOCODEC_DECODER_CACHE_SIZE = 2
|
||||
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS = 1
|
||||
# Per-API-server CUDA context and driver allocation, measured with
|
||||
# PyNvVideoCodec 2.0.4 on H100.
|
||||
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES = int(1.8 * 1024 * MiB_bytes)
|
||||
|
||||
|
||||
class PyNvVideoCodecDecoderSlot:
|
||||
"""A retained PyNv decoder slot and its CUDA stream.
|
||||
|
||||
The decoder is reused across requests: ``reconfigure_decoder`` repoints the
|
||||
existing decoder at each new source instead of paying a fresh
|
||||
``SimpleDecoder`` construction per request. Construction (CUVID parser +
|
||||
decoder + surface-pool allocation) is the dominant per-request cost, so
|
||||
reconfiguring is far cheaper. A single decoder serves both metadata
|
||||
(``len``/``get_stream_metadata``) and frame decode -- no separate
|
||||
metadata decoder.
|
||||
"""
|
||||
|
||||
def __init__(self, stream) -> None:
|
||||
self.stream = stream
|
||||
self.decoder = None
|
||||
self.source_path: str | None = None
|
||||
|
||||
def _construct(self, file_path: str, nvc, device_index: int) -> None:
|
||||
self.decoder = nvc.SimpleDecoder(
|
||||
file_path,
|
||||
output_color_type=nvc.OutputColorType.RGB,
|
||||
use_device_memory=True,
|
||||
need_scanned_stream_metadata=True,
|
||||
gpu_id=device_index,
|
||||
cuda_stream=self.stream.cuda_stream,
|
||||
decoder_cache_size=PYNVVIDEOCODEC_DECODER_CACHE_SIZE,
|
||||
)
|
||||
self.source_path = file_path
|
||||
|
||||
def get_decoder(self, file_path: str, nvc, device_index: int):
|
||||
if self.decoder is None:
|
||||
self._construct(file_path, nvc, device_index)
|
||||
elif self.source_path != file_path:
|
||||
try:
|
||||
self.decoder.reconfigure_decoder(file_path)
|
||||
self.source_path = file_path
|
||||
except Exception:
|
||||
# reconfigure unsupported/unsafe for this source -> rebuild.
|
||||
self._construct(file_path, nvc, device_index)
|
||||
return self.decoder
|
||||
|
||||
|
||||
class OpenCVVideoBackendMixin:
|
||||
@staticmethod
|
||||
@@ -485,15 +549,223 @@ class PyAVVideoBackendMixin:
|
||||
return np.stack(frames_list), valid_indices
|
||||
|
||||
|
||||
class PyNvVideoCodecVideoBackendMixin:
|
||||
"""PyNvVideoCodec utilities for GPU-backed frame decode."""
|
||||
|
||||
_decoder_slots: ClassVar[list[PyNvVideoCodecDecoderSlot]] = []
|
||||
_active_decoder_slots: ClassVar[int] = 0
|
||||
_decoder_slot_cond: ClassVar[threading.Condition] = threading.Condition()
|
||||
_DEVICE_INDEX: ClassVar[int] = 0
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def compute_frames_index_to_sample(
|
||||
cls,
|
||||
source: VideoSourceMetadata,
|
||||
target: VideoTargetMetadata,
|
||||
**kwargs,
|
||||
) -> list[int]:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def _prepare_source(cls, source: VideoSourceMetadata) -> VideoSourceMetadata:
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def _create_decoder_slot(cls) -> PyNvVideoCodecDecoderSlot:
|
||||
import torch
|
||||
|
||||
return PyNvVideoCodecDecoderSlot(torch.cuda.Stream(device=cls._DEVICE_INDEX))
|
||||
|
||||
@staticmethod
|
||||
@contextmanager
|
||||
def _torch_stream_context(stream):
|
||||
import torch
|
||||
|
||||
torch.accelerator.set_device_index(stream.device.index)
|
||||
previous_stream = torch.accelerator.current_stream()
|
||||
torch.accelerator.set_stream(stream)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
torch.accelerator.set_stream(previous_stream)
|
||||
|
||||
@classmethod
|
||||
@contextmanager
|
||||
def _borrow_decoder_slot(cls):
|
||||
create_slot = False
|
||||
with cls._decoder_slot_cond:
|
||||
while True:
|
||||
if cls._decoder_slots:
|
||||
slot = cls._decoder_slots.pop()
|
||||
break
|
||||
if cls._active_decoder_slots < PYNVVIDEOCODEC_MAX_RETAINED_DECODERS:
|
||||
cls._active_decoder_slots += 1
|
||||
create_slot = True
|
||||
break
|
||||
cls._decoder_slot_cond.wait()
|
||||
|
||||
if create_slot:
|
||||
try:
|
||||
slot = cls._create_decoder_slot()
|
||||
except Exception:
|
||||
with cls._decoder_slot_cond:
|
||||
cls._active_decoder_slots -= 1
|
||||
cls._decoder_slot_cond.notify()
|
||||
raise
|
||||
|
||||
try:
|
||||
yield slot
|
||||
finally:
|
||||
with cls._decoder_slot_cond:
|
||||
cls._decoder_slots.append(slot)
|
||||
cls._decoder_slot_cond.notify()
|
||||
|
||||
@staticmethod
|
||||
def _metadata_value(metadata, *names: str, default=None):
|
||||
for name in names:
|
||||
value = getattr(metadata, name, None)
|
||||
if value is not None:
|
||||
return value
|
||||
return default
|
||||
|
||||
@classmethod
|
||||
def _read_source_metadata(
|
||||
cls,
|
||||
file_path: str,
|
||||
nvc,
|
||||
) -> PyNvVideoCodecSourceMetadata:
|
||||
with cls._borrow_decoder_slot() as decoder_slot:
|
||||
with cls._torch_stream_context(decoder_slot.stream):
|
||||
decoder = decoder_slot.get_decoder(
|
||||
file_path, nvc, device_index=cls._DEVICE_INDEX
|
||||
)
|
||||
metadata = decoder.get_stream_metadata()
|
||||
total_frames_num = len(decoder)
|
||||
width = int(cls._metadata_value(metadata, "width", default=0))
|
||||
height = int(cls._metadata_value(metadata, "height", default=0))
|
||||
original_fps = float(
|
||||
cls._metadata_value(
|
||||
metadata,
|
||||
"average_fps",
|
||||
"avg_frame_rate",
|
||||
"frame_rate",
|
||||
"frameRate",
|
||||
default=0.0,
|
||||
)
|
||||
)
|
||||
duration = float(
|
||||
cls._metadata_value(metadata, "duration", default=0.0)
|
||||
or (total_frames_num / original_fps if original_fps > 0 else 0.0)
|
||||
)
|
||||
if total_frames_num <= 0:
|
||||
raise ValueError("Could not determine video frame count")
|
||||
if width <= 0 or height <= 0:
|
||||
raise ValueError("Could not determine video dimensions")
|
||||
return PyNvVideoCodecSourceMetadata(
|
||||
source=VideoSourceMetadata(total_frames_num, original_fps, duration),
|
||||
width=width,
|
||||
height=height,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _decode_to_pinned_host(
|
||||
cls,
|
||||
file_path: str,
|
||||
frame_idx: list[int],
|
||||
nvc,
|
||||
) -> npt.NDArray:
|
||||
import torch
|
||||
|
||||
if not frame_idx:
|
||||
return np.empty((0,), dtype=np.uint8)
|
||||
|
||||
with cls._borrow_decoder_slot() as decoder_slot:
|
||||
stream = decoder_slot.stream
|
||||
with cls._torch_stream_context(stream):
|
||||
decoder = decoder_slot.get_decoder(
|
||||
file_path, nvc, device_index=cls._DEVICE_INDEX
|
||||
)
|
||||
decoded_frames = decoder.get_batch_frames_by_index(frame_idx)
|
||||
if len(decoded_frames) < len(frame_idx):
|
||||
logger.warning(
|
||||
"pynvvideocodec video loading: expected %d frames but got %d.",
|
||||
len(frame_idx),
|
||||
len(decoded_frames),
|
||||
)
|
||||
torch_frames = [torch.from_dlpack(frame) for frame in decoded_frames]
|
||||
if not torch_frames:
|
||||
return np.empty((0,), dtype=np.uint8)
|
||||
device_frames = torch.stack(torch_frames)
|
||||
if device_frames.ndim != 4:
|
||||
raise ValueError(
|
||||
"PyNvVideoCodec returned frames with unexpected shape "
|
||||
f"{tuple(device_frames.shape)}"
|
||||
)
|
||||
device_frames = device_frames.permute(0, 3, 1, 2).contiguous()
|
||||
host_frames = torch.empty(
|
||||
device_frames.shape,
|
||||
dtype=device_frames.dtype,
|
||||
device="cpu",
|
||||
pin_memory=True,
|
||||
)
|
||||
host_frames.copy_(device_frames, non_blocking=True)
|
||||
stream.synchronize()
|
||||
host_array = host_frames.numpy()
|
||||
del decoded_frames, torch_frames, device_frames
|
||||
return host_array
|
||||
|
||||
@classmethod
|
||||
def decode_frames_pynvvideocodec(
|
||||
cls,
|
||||
data: bytes,
|
||||
target: VideoTargetMetadata,
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, VideoSourceMetadata, list[int], list[int]]:
|
||||
import PyNvVideoCodec as nvc
|
||||
|
||||
from vllm.multimodal.gpu_ipc_memory import get_mm_gpu_ipc_pool
|
||||
|
||||
temp_fd, temp_path = tempfile.mkstemp(suffix=".mp4")
|
||||
try:
|
||||
with os.fdopen(temp_fd, "wb") as temp_file:
|
||||
temp_file.write(data)
|
||||
|
||||
gpu_source = cls._read_source_metadata(temp_path, nvc)
|
||||
source = cls._prepare_source(gpu_source.source)
|
||||
frame_idx = cls.compute_frames_index_to_sample(
|
||||
source=source, target=target, **kwargs
|
||||
)
|
||||
raw_frame_bytes = len(frame_idx) * gpu_source.height * gpu_source.width * 3
|
||||
pool = get_mm_gpu_ipc_pool()
|
||||
if pool is None or raw_frame_bytes == 0:
|
||||
frames = cls._decode_to_pinned_host(temp_path, frame_idx, nvc)
|
||||
else:
|
||||
with pool.acquire(raw_frame_bytes):
|
||||
frames = cls._decode_to_pinned_host(temp_path, frame_idx, nvc)
|
||||
finally:
|
||||
with suppress(FileNotFoundError):
|
||||
os.unlink(temp_path)
|
||||
|
||||
valid_frame_indices = frame_idx[: int(frames.shape[0])]
|
||||
return frames, source, frame_idx, valid_frame_indices
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register("opencv")
|
||||
class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
|
||||
class VideoBackend(
|
||||
VideoLoader,
|
||||
OpenCVVideoBackendMixin,
|
||||
PyAVVideoBackendMixin,
|
||||
PyNvVideoCodecVideoBackendMixin,
|
||||
):
|
||||
"""Uniform-sampling video backend.
|
||||
|
||||
Samples ``num_frames`` uniformly across the video (or one frame every
|
||||
``1/fps`` seconds, whichever produces fewer frames). The decoding codec
|
||||
is selected via the ``backend`` kwarg (``"opencv"`` or ``"pyav"``),
|
||||
which can be passed through ``--media-io-kwargs``. Defaults to
|
||||
``"pyav"`` for concurrent decoding.
|
||||
is selected via the ``backend`` kwarg (``"opencv"``, ``"pyav"``, or
|
||||
``"pynvvideocodec"``), which can be passed through
|
||||
``--media-io-kwargs``. Defaults to ``"opencv"``.
|
||||
"""
|
||||
|
||||
_sampling_suffix: ClassVar[str] = ""
|
||||
@@ -538,7 +810,7 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal["opencv", "pyav"] = "opencv",
|
||||
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
"""Load sampled frames from raw video bytes.
|
||||
@@ -551,7 +823,8 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
|
||||
dynamic subclass; ignored here.
|
||||
frame_recovery: Enable forward-scan recovery for failed frames.
|
||||
Only honored by the OpenCV codec.
|
||||
backend: Decoding codec — ``"opencv"`` or ``"pyav"`` .
|
||||
backend: Decoding codec — ``"opencv"``, ``"pyav"``, or
|
||||
``"pynvvideocodec"``.
|
||||
|
||||
Returns:
|
||||
Tuple of ``(frames_array, metadata_dict)``.
|
||||
@@ -584,10 +857,21 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
|
||||
frames, valid = cls.decode_frames(
|
||||
container, frame_idx, source.original_fps, source.duration
|
||||
)
|
||||
elif backend == PYNVVIDEOCODEC_VIDEO_BACKEND:
|
||||
if frame_recovery:
|
||||
raise ValueError(
|
||||
"frame_recovery is not supported for "
|
||||
f"`{PYNVVIDEOCODEC_VIDEO_BACKEND}` backend"
|
||||
)
|
||||
frames, source, frame_idx, valid = cls.decode_frames_pynvvideocodec(
|
||||
data,
|
||||
target,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Unknown video codec backend {backend!r}; "
|
||||
"valid options: 'opencv', 'pyav'."
|
||||
"valid options: 'opencv', 'pyav', 'pynvvideocodec'."
|
||||
)
|
||||
|
||||
if len(valid) < len(frame_idx):
|
||||
@@ -605,6 +889,40 @@ class VideoBackend(VideoLoader, OpenCVVideoBackendMixin, PyAVVideoBackendMixin):
|
||||
)
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register(PYNVVIDEOCODEC_VIDEO_BACKEND)
|
||||
class PyNvVideoCodecVideoBackend(VideoBackend):
|
||||
"""Hardware-accelerated video backend using PyNvVideoCodec.
|
||||
|
||||
The backend first opens the stream only to read metadata and compute the
|
||||
sampled frame indices. It then acquires the raw decoded RGB byte count from
|
||||
the process-local multimodal GPU memory pool before decoding the selected
|
||||
frames into VRAM. Decoded frames are copied into pinned host memory before
|
||||
the lease is released, so downstream preprocessing continues to receive a
|
||||
CPU ``np.ndarray`` in NHWC RGB format.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def load_bytes(
|
||||
cls,
|
||||
data: bytes,
|
||||
num_frames: int = -1,
|
||||
fps: int = -1,
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
kwargs.pop("backend", None)
|
||||
return super().load_bytes(
|
||||
data,
|
||||
num_frames=num_frames,
|
||||
fps=fps,
|
||||
max_duration=max_duration,
|
||||
frame_recovery=frame_recovery,
|
||||
backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register(
|
||||
"qwen3_vl",
|
||||
video_processor="Qwen3VLVideoProcessor",
|
||||
@@ -640,7 +958,7 @@ class Qwen3VLVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal["opencv", "pyav"] = "opencv",
|
||||
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
@@ -719,7 +1037,7 @@ class Qwen2VLVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal["opencv", "pyav"] = "opencv",
|
||||
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
@@ -811,7 +1129,7 @@ class DynamicVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal["opencv", "pyav"] = "opencv",
|
||||
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
@@ -936,7 +1254,7 @@ class GLM46VVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal["opencv", "pyav"] = "opencv",
|
||||
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
return super().load_bytes(
|
||||
@@ -1034,7 +1352,7 @@ class GLMGAVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal["opencv", "pyav"] = "opencv",
|
||||
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
frames, metadata = super().load_bytes(
|
||||
@@ -1353,7 +1671,7 @@ class NemotronVLVideoBackend(VideoBackend):
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: Literal["opencv", "pyav"] = "opencv",
|
||||
backend: Literal["opencv", "pyav", "pynvvideocodec"] = "opencv",
|
||||
**kwargs,
|
||||
) -> tuple[npt.NDArray, dict[str, Any]]:
|
||||
frames, metadata = super().load_bytes(
|
||||
|
||||
@@ -29,6 +29,7 @@ from vllm.inputs import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry
|
||||
from vllm.multimodal.cache import BaseMultiModalProcessorCache
|
||||
from vllm.multimodal.gpu_ipc_memory import maybe_init_mm_gpu_ipc_pool
|
||||
from vllm.multimodal.parse import (
|
||||
MultiModalDataItems,
|
||||
MultiModalUUIDItems,
|
||||
@@ -111,6 +112,16 @@ class BaseRenderer(ABC, Generic[_T]):
|
||||
safe_load_prompt_embeds, executor=self._executor
|
||||
)
|
||||
if mm_registry.supports_multimodal_inputs(config.model_config):
|
||||
# Install the process-global GPU memory pool used to gate
|
||||
# frontend GPU-side multimodal decoding (no-op when the budget
|
||||
# is 0). Lives in the API-server process only.
|
||||
mm_config = config.model_config.multimodal_config
|
||||
if mm_config is not None:
|
||||
maybe_init_mm_gpu_ipc_pool(
|
||||
mm_config.mm_ipc_gpu_memory_gb,
|
||||
config.parallel_config._api_process_count,
|
||||
)
|
||||
|
||||
mm_processor_cache = mm_registry.processor_cache_from_config(config)
|
||||
|
||||
with set_default_torch_num_threads():
|
||||
|
||||
@@ -50,6 +50,12 @@ from vllm.distributed.weight_transfer import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
|
||||
from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
|
||||
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -428,7 +434,7 @@ class Worker(WorkerBase):
|
||||
"correspondingly."
|
||||
)
|
||||
logger.info(msg)
|
||||
return kv_cache_memory_bytes
|
||||
return self._reserve_mm_ipc_gpu_memory(kv_cache_memory_bytes)
|
||||
|
||||
# Execute a forward pass with dummy inputs to profile the memory usage
|
||||
# of the model.
|
||||
@@ -550,7 +556,81 @@ class Worker(WorkerBase):
|
||||
suggested_util,
|
||||
)
|
||||
|
||||
return int(self.available_kv_cache_memory_bytes)
|
||||
return self._reserve_mm_ipc_gpu_memory(
|
||||
int(self.available_kv_cache_memory_bytes)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _uses_pynvvideocodec_video_backend(mm_config) -> bool:
|
||||
video_kwargs = mm_config.media_io_kwargs.get("video", {})
|
||||
video_loader_backend = (
|
||||
video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
|
||||
)
|
||||
codec_backend = video_kwargs.get("backend")
|
||||
return (
|
||||
video_loader_backend == PYNVVIDEOCODEC_VIDEO_BACKEND
|
||||
or codec_backend == PYNVVIDEOCODEC_VIDEO_BACKEND
|
||||
)
|
||||
|
||||
def _reserve_mm_ipc_gpu_memory(self, available_kv_cache_memory_bytes: int) -> int:
|
||||
"""Carve frontend multimodal GPU memory out of the KV cache.
|
||||
|
||||
The frontend (API-server) process allocates GPU memory for hardware
|
||||
multimodal decoding. Raw decoded frames are bounded by
|
||||
``mm_ipc_gpu_memory_gb`` and acquired by the frontend semaphore. Some
|
||||
decoders also keep persistent surfaces around; reserve a fixed upper
|
||||
bound for those when the corresponding backend is configured.
|
||||
"""
|
||||
mm_config = self.model_config.multimodal_config
|
||||
if mm_config is None:
|
||||
return available_kv_cache_memory_bytes
|
||||
|
||||
raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes)
|
||||
# Each api_server_count process runs its OWN decoder surfaces + NVDEC/CUVID
|
||||
# CUDA context on the GPU, outside this (worker) memory pool. Reserve that
|
||||
# per-server footprint x api_server_count so gpu_memory_utilization bounds
|
||||
# TOTAL GPU usage across all API-server processes. Without the multiply,
|
||||
# HW decode overshoots the budget by ~(api_server_count-1) x per-server and
|
||||
# OOMs at high gmu, while SW decode (no per-server GPU allocation) does not.
|
||||
num_api_servers = max(1, getattr(self.parallel_config, "_api_process_count", 1))
|
||||
per_server_decoder_bytes = (
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES
|
||||
* PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
|
||||
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
|
||||
)
|
||||
decoder_reserved_bytes = (
|
||||
num_api_servers * per_server_decoder_bytes
|
||||
if self._uses_pynvvideocodec_video_backend(mm_config)
|
||||
else 0
|
||||
)
|
||||
reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes
|
||||
if reserved_bytes <= 0:
|
||||
return available_kv_cache_memory_bytes
|
||||
|
||||
remaining = available_kv_cache_memory_bytes - reserved_bytes
|
||||
if remaining <= 0:
|
||||
raise ValueError(
|
||||
f"frontend multimodal GPU decoding reserves "
|
||||
f"{format_gib(reserved_bytes)} GiB "
|
||||
f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, "
|
||||
f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), "
|
||||
f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is "
|
||||
"available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a "
|
||||
"different video backend, or increase gpu_memory_utilization."
|
||||
)
|
||||
logger.info_once(
|
||||
"Reserving %s GiB of GPU memory for frontend multimodal decoding "
|
||||
"(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context "
|
||||
"across %d API server(s) @ %s GiB/server); "
|
||||
"KV cache memory reduced to %s GiB.",
|
||||
format_gib(reserved_bytes),
|
||||
format_gib(raw_frame_reserved_bytes),
|
||||
format_gib(decoder_reserved_bytes),
|
||||
num_api_servers,
|
||||
format_gib(per_server_decoder_bytes),
|
||||
format_gib(remaining),
|
||||
)
|
||||
return remaining
|
||||
|
||||
def get_kv_connector_handshake_metadata(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user