[Misc] Move PyNvVideoCodec stuff out of gpu worker (#49322)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-07-22 10:11:56 +08:00
committed by GitHub
parent ba18929079
commit 1750e443f2
6 changed files with 200 additions and 190 deletions
+9
View File
@@ -43,6 +43,15 @@ def test_language_model_only_affects_model_hash():
assert base_hash != lm_only_hash
@pytest.mark.parametrize("backend_arg", ["video_backend", "backend"])
def test_use_gpu_video_backend_from_media_io_kwargs(backend_arg: str):
config = MultiModalConfig(
media_io_kwargs={"video": {backend_arg: "pynvvideocodec"}}
)
assert config.use_gpu_video_backend()
def test_mm_encoder_fp8_scale_path_requires_fp8():
with pytest.raises(ValueError, match="mm_encoder_attn_dtype"):
MultiModalConfig(mm_encoder_fp8_scale_path="/tmp/scales.json")
+97
View File
@@ -6,15 +6,43 @@ import time
import pytest
import vllm.config.multimodal as multimodal_config_module
from vllm.config.multimodal import MultiModalConfig
from vllm.multimodal.gpu_ipc_memory import (
MultiModalGPUMemoryPool,
get_mm_gpu_ipc_pool,
maybe_init_mm_gpu_ipc_pool,
reserve_mm_ipc_gpu_memory,
set_mm_gpu_ipc_pool,
)
from vllm.multimodal.video import (
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
from vllm.utils.mem_constants import GiB_bytes
def _mm_config(
*,
mm_ipc_gpu_memory_gb: float = 0,
video_backend: str | None = None,
) -> MultiModalConfig:
video_kwargs = {} if video_backend is None else {"video_backend": video_backend}
return MultiModalConfig(
mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
media_io_kwargs={"video": video_kwargs} if video_kwargs else {},
)
def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:
return api_process_count * (
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
)
def test_acquire_release_accounting():
pool = MultiModalGPUMemoryPool(total_bytes=100)
assert pool.available_bytes == 100
@@ -143,3 +171,72 @@ def test_global_pool_splits_budget_across_api_processes():
def test_global_pool_rejects_invalid_api_process_count():
with pytest.raises(ValueError):
maybe_init_mm_gpu_ipc_pool(2, api_process_count=0)
@pytest.mark.parametrize("video_backend", [None, "opencv"])
def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
monkeypatch: pytest.MonkeyPatch,
video_backend: str | None,
):
monkeypatch.setattr(
multimodal_config_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
"opencv",
)
mm_config = _mm_config(
mm_ipc_gpu_memory_gb=0.25,
video_backend=video_backend,
)
assert reserve_mm_ipc_gpu_memory(GiB_bytes, mm_config) == int(0.75 * GiB_bytes)
def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
multimodal_config_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
"opencv",
)
mm_config = _mm_config(
mm_ipc_gpu_memory_gb=0.25,
video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
)
available_bytes = 4 * GiB_bytes
assert reserve_mm_ipc_gpu_memory(available_bytes, mm_config) == (
available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget()
)
def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
multimodal_config_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
available_bytes = 4 * GiB_bytes
assert reserve_mm_ipc_gpu_memory(available_bytes, _mm_config()) == (
available_bytes - _pynvvideocodec_decoder_budget()
)
def test_reserve_mm_ipc_gpu_memory_scales_decoder_budget_by_api_servers(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
multimodal_config_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
available_bytes = 8 * GiB_bytes
assert reserve_mm_ipc_gpu_memory(
available_bytes,
_mm_config(),
api_process_count=3,
) == available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)
-109
View File
@@ -6,122 +6,13 @@ from unittest.mock import patch
import pytest
import vllm.v1.worker.gpu_worker as gpu_worker_module
from vllm.multimodal.video import (
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
from vllm.utils.mem_constants import GiB_bytes
from vllm.v1.worker import startup_plan
from vllm.v1.worker.gpu_worker import Worker
from vllm.v1.worker.startup_plan import (
maybe_apply_startup_plan,
maybe_save_startup_plan,
)
def _worker_with_mm_config(
mm_config: SimpleNamespace,
*,
api_process_count: int = 1,
) -> Worker:
worker = object.__new__(Worker)
worker.model_config = SimpleNamespace(multimodal_config=mm_config)
worker.parallel_config = SimpleNamespace(_api_process_count=api_process_count)
return worker
def _mm_config(
*,
mm_ipc_gpu_memory_gb: float = 0,
video_backend: str | None = None,
) -> SimpleNamespace:
video_kwargs = {} if video_backend is None else {"video_backend": video_backend}
return SimpleNamespace(
mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
media_io_kwargs={"video": video_kwargs} if video_kwargs else {},
)
def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:
return api_process_count * (
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
)
@pytest.mark.parametrize("video_backend", [None, "opencv"])
def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
monkeypatch: pytest.MonkeyPatch,
video_backend: str | None,
):
monkeypatch.setattr(
gpu_worker_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
"opencv",
)
worker = _worker_with_mm_config(
_mm_config(mm_ipc_gpu_memory_gb=0.25, video_backend=video_backend)
)
assert worker._reserve_mm_ipc_gpu_memory(GiB_bytes) == int(0.75 * GiB_bytes)
def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
gpu_worker_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
"opencv",
)
worker = _worker_with_mm_config(
_mm_config(
mm_ipc_gpu_memory_gb=0.25,
video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
)
)
available_bytes = 4 * GiB_bytes
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget()
)
def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
gpu_worker_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
worker = _worker_with_mm_config(_mm_config())
available_bytes = 4 * GiB_bytes
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
available_bytes - _pynvvideocodec_decoder_budget()
)
def test_reserve_mm_ipc_gpu_memory_scales_pynvvideocodec_budget_by_api_servers(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.setattr(
gpu_worker_module.envs,
"VLLM_VIDEO_LOADER_BACKEND",
PYNVVIDEOCODEC_VIDEO_BACKEND,
)
worker = _worker_with_mm_config(_mm_config(), api_process_count=3)
available_bytes = 8 * GiB_bytes
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)
)
# Startup-plan persistence (vllm/v1/worker/startup_plan.py), applied and
# saved by Worker.determine_available_memory / compile_or_warm_up_model.
+15
View File
@@ -8,6 +8,7 @@ from typing import Any, Literal, TypeAlias, TypedDict, final
from pydantic import ConfigDict, Field, field_validator, model_validator
from pydantic.dataclasses import dataclass
import vllm.envs as envs
from vllm.config.utils import config
from vllm.utils.hashing import safe_hash
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -344,5 +345,19 @@ class MultiModalConfig:
kwargs = self.mm_processor_kwargs or {}
return kwargs | dict(inference_kwargs)
def use_gpu_video_backend(self) -> bool:
"""Return whether the configured video loader or codec uses the GPU."""
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
video_kwargs = self.media_io_kwargs.get("video", {})
video_loader_backend = (
video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
)
codec_backend = video_kwargs.get("backend")
return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or (
codec_backend is not None
and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend)
)
def is_multimodal_pruning_enabled(self):
return self.video_pruning_rate is not None and self.video_pruning_rate > 0
+69
View File
@@ -17,9 +17,14 @@ amount out of its KV-cache budget so the headroom physically exists.
"""
import threading
from typing import TYPE_CHECKING
from vllm.logger import init_logger
from vllm.utils.mem_constants import GiB_bytes
from vllm.utils.mem_utils import format_gib
if TYPE_CHECKING:
from vllm.config.multimodal import MultiModalConfig
logger = init_logger(__name__)
@@ -145,3 +150,67 @@ def maybe_init_mm_gpu_ipc_pool(
api_process_count,
)
return pool
def reserve_mm_ipc_gpu_memory(
available_kv_cache_memory_bytes: int,
mm_config: "MultiModalConfig | None",
api_process_count: int = 1,
) -> int:
"""Carve frontend multimodal GPU memory out of the KV cache.
Raw decoded frames are bounded by ``mm_ipc_gpu_memory_gb`` and acquired by
the frontend semaphore. Some decoders also keep persistent surfaces around;
reserve a fixed upper bound for those when a GPU backend is configured.
"""
if mm_config is None:
return available_kv_cache_memory_bytes
from vllm.multimodal.video import (
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
)
raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes)
# Each API server process runs its own decoder surfaces and NVDEC/CUVID CUDA
# context on the GPU, outside the worker memory pool. Reserve that footprint
# per process so gpu_memory_utilization bounds total GPU usage across them.
num_api_servers = max(1, api_process_count)
per_server_decoder_bytes = (
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
)
decoder_reserved_bytes = (
num_api_servers * per_server_decoder_bytes
if mm_config.use_gpu_video_backend()
else 0
)
reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes
if reserved_bytes <= 0:
return available_kv_cache_memory_bytes
remaining = available_kv_cache_memory_bytes - reserved_bytes
if remaining <= 0:
raise ValueError(
f"frontend multimodal GPU decoding reserves "
f"{format_gib(reserved_bytes)} GiB "
f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, "
f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), "
f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is "
"available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a "
"different video backend, or increase gpu_memory_utilization."
)
logger.info_once(
"Reserving %s GiB of GPU memory for frontend multimodal decoding "
"(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context "
"across %d API server(s) @ %s GiB/server); "
"KV cache memory reduced to %s GiB.",
format_gib(reserved_bytes),
format_gib(raw_frame_reserved_bytes),
format_gib(decoder_reserved_bytes),
num_api_servers,
format_gib(per_server_decoder_bytes),
format_gib(remaining),
)
return remaining
+10 -81
View File
@@ -51,12 +51,7 @@ from vllm.distributed.weight_transfer import (
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
from vllm.multimodal.video import (
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
VIDEO_LOADER_REGISTRY,
)
from vllm.multimodal.gpu_ipc_memory import reserve_mm_ipc_gpu_memory
from vllm.platforms import current_platform
from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper
from vllm.sequence import IntermediateTensors
@@ -478,7 +473,11 @@ class Worker(WorkerBase):
"correspondingly."
)
logger.info(msg)
return self._reserve_mm_ipc_gpu_memory(kv_cache_memory_bytes)
return reserve_mm_ipc_gpu_memory(
kv_cache_memory_bytes,
self.model_config.multimodal_config,
getattr(self.parallel_config, "_api_process_count", 1),
)
# Execute a forward pass with dummy inputs to profile the memory usage
# of the model.
@@ -589,82 +588,12 @@ class Worker(WorkerBase):
suggested_util,
)
return self._reserve_mm_ipc_gpu_memory(
int(self.available_kv_cache_memory_bytes)
return reserve_mm_ipc_gpu_memory(
int(self.available_kv_cache_memory_bytes),
self.model_config.multimodal_config,
getattr(self.parallel_config, "_api_process_count", 1),
)
@staticmethod
def _uses_gpu_video_backend(mm_config) -> bool:
video_kwargs = mm_config.media_io_kwargs.get("video", {})
video_loader_backend = (
video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
)
codec_backend = video_kwargs.get("backend")
return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or (
codec_backend is not None
and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend)
)
def _reserve_mm_ipc_gpu_memory(self, available_kv_cache_memory_bytes: int) -> int:
"""Carve frontend multimodal GPU memory out of the KV cache.
The frontend (API-server) process allocates GPU memory for hardware
multimodal decoding. Raw decoded frames are bounded by
``mm_ipc_gpu_memory_gb`` and acquired by the frontend semaphore. Some
decoders also keep persistent surfaces around; reserve a fixed upper
bound for those when the corresponding backend is configured.
"""
mm_config = self.model_config.multimodal_config
if mm_config is None:
return available_kv_cache_memory_bytes
raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes)
# Each api_server_count process runs its OWN decoder surfaces + NVDEC/CUVID
# CUDA context on the GPU, outside this (worker) memory pool. Reserve that
# per-server footprint x api_server_count so gpu_memory_utilization bounds
# TOTAL GPU usage across all API-server processes. Without the multiply,
# HW decode overshoots the budget by ~(api_server_count-1) x per-server and
# OOMs at high gmu, while SW decode (no per-server GPU allocation) does not.
num_api_servers = max(1, getattr(self.parallel_config, "_api_process_count", 1))
per_server_decoder_bytes = (
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES
* PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
)
decoder_reserved_bytes = (
num_api_servers * per_server_decoder_bytes
if self._uses_gpu_video_backend(mm_config)
else 0
)
reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes
if reserved_bytes <= 0:
return available_kv_cache_memory_bytes
remaining = available_kv_cache_memory_bytes - reserved_bytes
if remaining <= 0:
raise ValueError(
f"frontend multimodal GPU decoding reserves "
f"{format_gib(reserved_bytes)} GiB "
f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, "
f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), "
f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is "
"available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a "
"different video backend, or increase gpu_memory_utilization."
)
logger.info_once(
"Reserving %s GiB of GPU memory for frontend multimodal decoding "
"(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context "
"across %d API server(s) @ %s GiB/server); "
"KV cache memory reduced to %s GiB.",
format_gib(reserved_bytes),
format_gib(raw_frame_reserved_bytes),
format_gib(decoder_reserved_bytes),
num_api_servers,
format_gib(per_server_decoder_bytes),
format_gib(remaining),
)
return remaining
def get_kv_connector_handshake_metadata(
self,
) -> dict[tuple[int, int], KVConnectorHandshakeMetadata] | None: