diff --git a/tests/config/test_multimodal_config.py b/tests/config/test_multimodal_config.py index 9720d84672..5260d7a40f 100644 --- a/tests/config/test_multimodal_config.py +++ b/tests/config/test_multimodal_config.py @@ -43,6 +43,15 @@ def test_language_model_only_affects_model_hash(): assert base_hash != lm_only_hash +@pytest.mark.parametrize("backend_arg", ["video_backend", "backend"]) +def test_use_gpu_video_backend_from_media_io_kwargs(backend_arg: str): + config = MultiModalConfig( + media_io_kwargs={"video": {backend_arg: "pynvvideocodec"}} + ) + + assert config.use_gpu_video_backend() + + def test_mm_encoder_fp8_scale_path_requires_fp8(): with pytest.raises(ValueError, match="mm_encoder_attn_dtype"): MultiModalConfig(mm_encoder_fp8_scale_path="/tmp/scales.json") diff --git a/tests/multimodal/test_gpu_ipc_memory.py b/tests/multimodal/test_gpu_ipc_memory.py index bc6bf031fe..5c0f263af9 100644 --- a/tests/multimodal/test_gpu_ipc_memory.py +++ b/tests/multimodal/test_gpu_ipc_memory.py @@ -6,15 +6,43 @@ import time import pytest +import vllm.config.multimodal as multimodal_config_module +from vllm.config.multimodal import MultiModalConfig from vllm.multimodal.gpu_ipc_memory import ( MultiModalGPUMemoryPool, get_mm_gpu_ipc_pool, maybe_init_mm_gpu_ipc_pool, + reserve_mm_ipc_gpu_memory, set_mm_gpu_ipc_pool, ) +from vllm.multimodal.video import ( + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES, + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES, + PYNVVIDEOCODEC_MAX_RETAINED_DECODERS, + PYNVVIDEOCODEC_VIDEO_BACKEND, +) from vllm.utils.mem_constants import GiB_bytes +def _mm_config( + *, + mm_ipc_gpu_memory_gb: float = 0, + video_backend: str | None = None, +) -> MultiModalConfig: + video_kwargs = {} if video_backend is None else {"video_backend": video_backend} + return MultiModalConfig( + mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb, + media_io_kwargs={"video": video_kwargs} if video_kwargs else {}, + ) + + +def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int: + return api_process_count * ( + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS + + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES + ) + + def test_acquire_release_accounting(): pool = MultiModalGPUMemoryPool(total_bytes=100) assert pool.available_bytes == 100 @@ -143,3 +171,72 @@ def test_global_pool_splits_budget_across_api_processes(): def test_global_pool_rejects_invalid_api_process_count(): with pytest.raises(ValueError): maybe_init_mm_gpu_ipc_pool(2, api_process_count=0) + + +@pytest.mark.parametrize("video_backend", [None, "opencv"]) +def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only( + monkeypatch: pytest.MonkeyPatch, + video_backend: str | None, +): + monkeypatch.setattr( + multimodal_config_module.envs, + "VLLM_VIDEO_LOADER_BACKEND", + "opencv", + ) + mm_config = _mm_config( + mm_ipc_gpu_memory_gb=0.25, + video_backend=video_backend, + ) + + assert reserve_mm_ipc_gpu_memory(GiB_bytes, mm_config) == int(0.75 * GiB_bytes) + + +def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + multimodal_config_module.envs, + "VLLM_VIDEO_LOADER_BACKEND", + "opencv", + ) + mm_config = _mm_config( + mm_ipc_gpu_memory_gb=0.25, + video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND, + ) + available_bytes = 4 * GiB_bytes + + assert reserve_mm_ipc_gpu_memory(available_bytes, mm_config) == ( + available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget() + ) + + +def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + multimodal_config_module.envs, + "VLLM_VIDEO_LOADER_BACKEND", + PYNVVIDEOCODEC_VIDEO_BACKEND, + ) + available_bytes = 4 * GiB_bytes + + assert reserve_mm_ipc_gpu_memory(available_bytes, _mm_config()) == ( + available_bytes - _pynvvideocodec_decoder_budget() + ) + + +def test_reserve_mm_ipc_gpu_memory_scales_decoder_budget_by_api_servers( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + multimodal_config_module.envs, + "VLLM_VIDEO_LOADER_BACKEND", + PYNVVIDEOCODEC_VIDEO_BACKEND, + ) + available_bytes = 8 * GiB_bytes + + assert reserve_mm_ipc_gpu_memory( + available_bytes, + _mm_config(), + api_process_count=3, + ) == available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3) diff --git a/tests/v1/worker/test_gpu_worker.py b/tests/v1/worker/test_gpu_worker.py index cdaa644b62..43232ca6be 100644 --- a/tests/v1/worker/test_gpu_worker.py +++ b/tests/v1/worker/test_gpu_worker.py @@ -6,122 +6,13 @@ from unittest.mock import patch import pytest -import vllm.v1.worker.gpu_worker as gpu_worker_module -from vllm.multimodal.video import ( - PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES, - PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES, - PYNVVIDEOCODEC_MAX_RETAINED_DECODERS, - PYNVVIDEOCODEC_VIDEO_BACKEND, -) from vllm.utils.mem_constants import GiB_bytes from vllm.v1.worker import startup_plan -from vllm.v1.worker.gpu_worker import Worker from vllm.v1.worker.startup_plan import ( maybe_apply_startup_plan, maybe_save_startup_plan, ) - -def _worker_with_mm_config( - mm_config: SimpleNamespace, - *, - api_process_count: int = 1, -) -> Worker: - worker = object.__new__(Worker) - worker.model_config = SimpleNamespace(multimodal_config=mm_config) - worker.parallel_config = SimpleNamespace(_api_process_count=api_process_count) - return worker - - -def _mm_config( - *, - mm_ipc_gpu_memory_gb: float = 0, - video_backend: str | None = None, -) -> SimpleNamespace: - video_kwargs = {} if video_backend is None else {"video_backend": video_backend} - return SimpleNamespace( - mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb, - media_io_kwargs={"video": video_kwargs} if video_kwargs else {}, - ) - - -def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int: - return api_process_count * ( - PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS - + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES - ) - - -@pytest.mark.parametrize("video_backend", [None, "opencv"]) -def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only( - monkeypatch: pytest.MonkeyPatch, - video_backend: str | None, -): - monkeypatch.setattr( - gpu_worker_module.envs, - "VLLM_VIDEO_LOADER_BACKEND", - "opencv", - ) - worker = _worker_with_mm_config( - _mm_config(mm_ipc_gpu_memory_gb=0.25, video_backend=video_backend) - ) - - assert worker._reserve_mm_ipc_gpu_memory(GiB_bytes) == int(0.75 * GiB_bytes) - - -def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget( - monkeypatch: pytest.MonkeyPatch, -): - monkeypatch.setattr( - gpu_worker_module.envs, - "VLLM_VIDEO_LOADER_BACKEND", - "opencv", - ) - worker = _worker_with_mm_config( - _mm_config( - mm_ipc_gpu_memory_gb=0.25, - video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND, - ) - ) - available_bytes = 4 * GiB_bytes - - assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == ( - available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget() - ) - - -def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend( - monkeypatch: pytest.MonkeyPatch, -): - monkeypatch.setattr( - gpu_worker_module.envs, - "VLLM_VIDEO_LOADER_BACKEND", - PYNVVIDEOCODEC_VIDEO_BACKEND, - ) - worker = _worker_with_mm_config(_mm_config()) - available_bytes = 4 * GiB_bytes - - assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == ( - available_bytes - _pynvvideocodec_decoder_budget() - ) - - -def test_reserve_mm_ipc_gpu_memory_scales_pynvvideocodec_budget_by_api_servers( - monkeypatch: pytest.MonkeyPatch, -): - monkeypatch.setattr( - gpu_worker_module.envs, - "VLLM_VIDEO_LOADER_BACKEND", - PYNVVIDEOCODEC_VIDEO_BACKEND, - ) - worker = _worker_with_mm_config(_mm_config(), api_process_count=3) - available_bytes = 8 * GiB_bytes - - assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == ( - available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3) - ) - - # Startup-plan persistence (vllm/v1/worker/startup_plan.py), applied and # saved by Worker.determine_available_memory / compile_or_warm_up_model. diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index 150d58cf1f..865615dff7 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -8,6 +8,7 @@ from typing import Any, Literal, TypeAlias, TypedDict, final from pydantic import ConfigDict, Field, field_validator, model_validator from pydantic.dataclasses import dataclass +import vllm.envs as envs from vllm.config.utils import config from vllm.utils.hashing import safe_hash from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -344,5 +345,19 @@ class MultiModalConfig: kwargs = self.mm_processor_kwargs or {} return kwargs | dict(inference_kwargs) + def use_gpu_video_backend(self) -> bool: + """Return whether the configured video loader or codec uses the GPU.""" + from vllm.multimodal.video import VIDEO_LOADER_REGISTRY + + video_kwargs = self.media_io_kwargs.get("video", {}) + video_loader_backend = ( + video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND + ) + codec_backend = video_kwargs.get("backend") + return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or ( + codec_backend is not None + and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend) + ) + def is_multimodal_pruning_enabled(self): return self.video_pruning_rate is not None and self.video_pruning_rate > 0 diff --git a/vllm/multimodal/gpu_ipc_memory.py b/vllm/multimodal/gpu_ipc_memory.py index 15b912064a..02317490a8 100644 --- a/vllm/multimodal/gpu_ipc_memory.py +++ b/vllm/multimodal/gpu_ipc_memory.py @@ -17,9 +17,14 @@ amount out of its KV-cache budget so the headroom physically exists. """ import threading +from typing import TYPE_CHECKING from vllm.logger import init_logger from vllm.utils.mem_constants import GiB_bytes +from vllm.utils.mem_utils import format_gib + +if TYPE_CHECKING: + from vllm.config.multimodal import MultiModalConfig logger = init_logger(__name__) @@ -145,3 +150,67 @@ def maybe_init_mm_gpu_ipc_pool( api_process_count, ) return pool + + +def reserve_mm_ipc_gpu_memory( + available_kv_cache_memory_bytes: int, + mm_config: "MultiModalConfig | None", + api_process_count: int = 1, +) -> int: + """Carve frontend multimodal GPU memory out of the KV cache. + + Raw decoded frames are bounded by ``mm_ipc_gpu_memory_gb`` and acquired by + the frontend semaphore. Some decoders also keep persistent surfaces around; + reserve a fixed upper bound for those when a GPU backend is configured. + """ + if mm_config is None: + return available_kv_cache_memory_bytes + + from vllm.multimodal.video import ( + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES, + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES, + PYNVVIDEOCODEC_MAX_RETAINED_DECODERS, + ) + + raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes) + # Each API server process runs its own decoder surfaces and NVDEC/CUVID CUDA + # context on the GPU, outside the worker memory pool. Reserve that footprint + # per process so gpu_memory_utilization bounds total GPU usage across them. + num_api_servers = max(1, api_process_count) + per_server_decoder_bytes = ( + PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS + + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES + ) + decoder_reserved_bytes = ( + num_api_servers * per_server_decoder_bytes + if mm_config.use_gpu_video_backend() + else 0 + ) + reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes + if reserved_bytes <= 0: + return available_kv_cache_memory_bytes + + remaining = available_kv_cache_memory_bytes - reserved_bytes + if remaining <= 0: + raise ValueError( + f"frontend multimodal GPU decoding reserves " + f"{format_gib(reserved_bytes)} GiB " + f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, " + f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), " + f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is " + "available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a " + "different video backend, or increase gpu_memory_utilization." + ) + logger.info_once( + "Reserving %s GiB of GPU memory for frontend multimodal decoding " + "(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context " + "across %d API server(s) @ %s GiB/server); " + "KV cache memory reduced to %s GiB.", + format_gib(reserved_bytes), + format_gib(raw_frame_reserved_bytes), + format_gib(decoder_reserved_bytes), + num_api_servers, + format_gib(per_server_decoder_bytes), + format_gib(remaining), + ) + return remaining diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 0c89b2fb13..ca63e0a117 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -51,12 +51,7 @@ from vllm.distributed.weight_transfer import ( from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.model_executor.warmup.kernel_warmup import kernel_warmup -from vllm.multimodal.video import ( - PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES, - PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES, - PYNVVIDEOCODEC_MAX_RETAINED_DECODERS, - VIDEO_LOADER_REGISTRY, -) +from vllm.multimodal.gpu_ipc_memory import reserve_mm_ipc_gpu_memory from vllm.platforms import current_platform from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper from vllm.sequence import IntermediateTensors @@ -478,7 +473,11 @@ class Worker(WorkerBase): "correspondingly." ) logger.info(msg) - return self._reserve_mm_ipc_gpu_memory(kv_cache_memory_bytes) + return reserve_mm_ipc_gpu_memory( + kv_cache_memory_bytes, + self.model_config.multimodal_config, + getattr(self.parallel_config, "_api_process_count", 1), + ) # Execute a forward pass with dummy inputs to profile the memory usage # of the model. @@ -589,82 +588,12 @@ class Worker(WorkerBase): suggested_util, ) - return self._reserve_mm_ipc_gpu_memory( - int(self.available_kv_cache_memory_bytes) + return reserve_mm_ipc_gpu_memory( + int(self.available_kv_cache_memory_bytes), + self.model_config.multimodal_config, + getattr(self.parallel_config, "_api_process_count", 1), ) - @staticmethod - def _uses_gpu_video_backend(mm_config) -> bool: - video_kwargs = mm_config.media_io_kwargs.get("video", {}) - video_loader_backend = ( - video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND - ) - codec_backend = video_kwargs.get("backend") - return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or ( - codec_backend is not None - and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend) - ) - - def _reserve_mm_ipc_gpu_memory(self, available_kv_cache_memory_bytes: int) -> int: - """Carve frontend multimodal GPU memory out of the KV cache. - - The frontend (API-server) process allocates GPU memory for hardware - multimodal decoding. Raw decoded frames are bounded by - ``mm_ipc_gpu_memory_gb`` and acquired by the frontend semaphore. Some - decoders also keep persistent surfaces around; reserve a fixed upper - bound for those when the corresponding backend is configured. - """ - mm_config = self.model_config.multimodal_config - if mm_config is None: - return available_kv_cache_memory_bytes - - raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes) - # Each api_server_count process runs its OWN decoder surfaces + NVDEC/CUVID - # CUDA context on the GPU, outside this (worker) memory pool. Reserve that - # per-server footprint x api_server_count so gpu_memory_utilization bounds - # TOTAL GPU usage across all API-server processes. Without the multiply, - # HW decode overshoots the budget by ~(api_server_count-1) x per-server and - # OOMs at high gmu, while SW decode (no per-server GPU allocation) does not. - num_api_servers = max(1, getattr(self.parallel_config, "_api_process_count", 1)) - per_server_decoder_bytes = ( - PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES - * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS - + PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES - ) - decoder_reserved_bytes = ( - num_api_servers * per_server_decoder_bytes - if self._uses_gpu_video_backend(mm_config) - else 0 - ) - reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes - if reserved_bytes <= 0: - return available_kv_cache_memory_bytes - - remaining = available_kv_cache_memory_bytes - reserved_bytes - if remaining <= 0: - raise ValueError( - f"frontend multimodal GPU decoding reserves " - f"{format_gib(reserved_bytes)} GiB " - f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, " - f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), " - f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is " - "available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a " - "different video backend, or increase gpu_memory_utilization." - ) - logger.info_once( - "Reserving %s GiB of GPU memory for frontend multimodal decoding " - "(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context " - "across %d API server(s) @ %s GiB/server); " - "KV cache memory reduced to %s GiB.", - format_gib(reserved_bytes), - format_gib(raw_frame_reserved_bytes), - format_gib(decoder_reserved_bytes), - num_api_servers, - format_gib(per_server_decoder_bytes), - format_gib(remaining), - ) - return remaining - def get_kv_connector_handshake_metadata( self, ) -> dict[tuple[int, int], KVConnectorHandshakeMetadata] | None: