[Misc] Move PyNvVideoCodec stuff out of gpu worker (#49322)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -43,6 +43,15 @@ def test_language_model_only_affects_model_hash():
|
||||
assert base_hash != lm_only_hash
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend_arg", ["video_backend", "backend"])
|
||||
def test_use_gpu_video_backend_from_media_io_kwargs(backend_arg: str):
|
||||
config = MultiModalConfig(
|
||||
media_io_kwargs={"video": {backend_arg: "pynvvideocodec"}}
|
||||
)
|
||||
|
||||
assert config.use_gpu_video_backend()
|
||||
|
||||
|
||||
def test_mm_encoder_fp8_scale_path_requires_fp8():
|
||||
with pytest.raises(ValueError, match="mm_encoder_attn_dtype"):
|
||||
MultiModalConfig(mm_encoder_fp8_scale_path="/tmp/scales.json")
|
||||
|
||||
@@ -6,15 +6,43 @@ import time
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.config.multimodal as multimodal_config_module
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
from vllm.multimodal.gpu_ipc_memory import (
|
||||
MultiModalGPUMemoryPool,
|
||||
get_mm_gpu_ipc_pool,
|
||||
maybe_init_mm_gpu_ipc_pool,
|
||||
reserve_mm_ipc_gpu_memory,
|
||||
set_mm_gpu_ipc_pool,
|
||||
)
|
||||
from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
|
||||
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
|
||||
|
||||
def _mm_config(
|
||||
*,
|
||||
mm_ipc_gpu_memory_gb: float = 0,
|
||||
video_backend: str | None = None,
|
||||
) -> MultiModalConfig:
|
||||
video_kwargs = {} if video_backend is None else {"video_backend": video_backend}
|
||||
return MultiModalConfig(
|
||||
mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
|
||||
media_io_kwargs={"video": video_kwargs} if video_kwargs else {},
|
||||
)
|
||||
|
||||
|
||||
def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:
|
||||
return api_process_count * (
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
|
||||
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
|
||||
)
|
||||
|
||||
|
||||
def test_acquire_release_accounting():
|
||||
pool = MultiModalGPUMemoryPool(total_bytes=100)
|
||||
assert pool.available_bytes == 100
|
||||
@@ -143,3 +171,72 @@ def test_global_pool_splits_budget_across_api_processes():
|
||||
def test_global_pool_rejects_invalid_api_process_count():
|
||||
with pytest.raises(ValueError):
|
||||
maybe_init_mm_gpu_ipc_pool(2, api_process_count=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("video_backend", [None, "opencv"])
|
||||
def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
video_backend: str | None,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
multimodal_config_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
"opencv",
|
||||
)
|
||||
mm_config = _mm_config(
|
||||
mm_ipc_gpu_memory_gb=0.25,
|
||||
video_backend=video_backend,
|
||||
)
|
||||
|
||||
assert reserve_mm_ipc_gpu_memory(GiB_bytes, mm_config) == int(0.75 * GiB_bytes)
|
||||
|
||||
|
||||
def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
multimodal_config_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
"opencv",
|
||||
)
|
||||
mm_config = _mm_config(
|
||||
mm_ipc_gpu_memory_gb=0.25,
|
||||
video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
available_bytes = 4 * GiB_bytes
|
||||
|
||||
assert reserve_mm_ipc_gpu_memory(available_bytes, mm_config) == (
|
||||
available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget()
|
||||
)
|
||||
|
||||
|
||||
def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
multimodal_config_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
available_bytes = 4 * GiB_bytes
|
||||
|
||||
assert reserve_mm_ipc_gpu_memory(available_bytes, _mm_config()) == (
|
||||
available_bytes - _pynvvideocodec_decoder_budget()
|
||||
)
|
||||
|
||||
|
||||
def test_reserve_mm_ipc_gpu_memory_scales_decoder_budget_by_api_servers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
multimodal_config_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
available_bytes = 8 * GiB_bytes
|
||||
|
||||
assert reserve_mm_ipc_gpu_memory(
|
||||
available_bytes,
|
||||
_mm_config(),
|
||||
api_process_count=3,
|
||||
) == available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)
|
||||
|
||||
@@ -6,122 +6,13 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
import vllm.v1.worker.gpu_worker as gpu_worker_module
|
||||
from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
|
||||
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.v1.worker import startup_plan
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
from vllm.v1.worker.startup_plan import (
|
||||
maybe_apply_startup_plan,
|
||||
maybe_save_startup_plan,
|
||||
)
|
||||
|
||||
|
||||
def _worker_with_mm_config(
|
||||
mm_config: SimpleNamespace,
|
||||
*,
|
||||
api_process_count: int = 1,
|
||||
) -> Worker:
|
||||
worker = object.__new__(Worker)
|
||||
worker.model_config = SimpleNamespace(multimodal_config=mm_config)
|
||||
worker.parallel_config = SimpleNamespace(_api_process_count=api_process_count)
|
||||
return worker
|
||||
|
||||
|
||||
def _mm_config(
|
||||
*,
|
||||
mm_ipc_gpu_memory_gb: float = 0,
|
||||
video_backend: str | None = None,
|
||||
) -> SimpleNamespace:
|
||||
video_kwargs = {} if video_backend is None else {"video_backend": video_backend}
|
||||
return SimpleNamespace(
|
||||
mm_ipc_gpu_memory_gb=mm_ipc_gpu_memory_gb,
|
||||
media_io_kwargs={"video": video_kwargs} if video_kwargs else {},
|
||||
)
|
||||
|
||||
|
||||
def _pynvvideocodec_decoder_budget(api_process_count: int = 1) -> int:
|
||||
return api_process_count * (
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
|
||||
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("video_backend", [None, "opencv"])
|
||||
def test_reserve_mm_ipc_gpu_memory_raw_frame_budget_only(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
video_backend: str | None,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gpu_worker_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
"opencv",
|
||||
)
|
||||
worker = _worker_with_mm_config(
|
||||
_mm_config(mm_ipc_gpu_memory_gb=0.25, video_backend=video_backend)
|
||||
)
|
||||
|
||||
assert worker._reserve_mm_ipc_gpu_memory(GiB_bytes) == int(0.75 * GiB_bytes)
|
||||
|
||||
|
||||
def test_reserve_mm_ipc_gpu_memory_includes_pynvvideocodec_decoder_budget(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gpu_worker_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
"opencv",
|
||||
)
|
||||
worker = _worker_with_mm_config(
|
||||
_mm_config(
|
||||
mm_ipc_gpu_memory_gb=0.25,
|
||||
video_backend=PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
)
|
||||
available_bytes = 4 * GiB_bytes
|
||||
|
||||
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
|
||||
available_bytes - int(0.25 * GiB_bytes) - _pynvvideocodec_decoder_budget()
|
||||
)
|
||||
|
||||
|
||||
def test_reserve_mm_ipc_gpu_memory_uses_env_video_backend(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gpu_worker_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
worker = _worker_with_mm_config(_mm_config())
|
||||
available_bytes = 4 * GiB_bytes
|
||||
|
||||
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
|
||||
available_bytes - _pynvvideocodec_decoder_budget()
|
||||
)
|
||||
|
||||
|
||||
def test_reserve_mm_ipc_gpu_memory_scales_pynvvideocodec_budget_by_api_servers(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
gpu_worker_module.envs,
|
||||
"VLLM_VIDEO_LOADER_BACKEND",
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
)
|
||||
worker = _worker_with_mm_config(_mm_config(), api_process_count=3)
|
||||
available_bytes = 8 * GiB_bytes
|
||||
|
||||
assert worker._reserve_mm_ipc_gpu_memory(available_bytes) == (
|
||||
available_bytes - _pynvvideocodec_decoder_budget(api_process_count=3)
|
||||
)
|
||||
|
||||
|
||||
# Startup-plan persistence (vllm/v1/worker/startup_plan.py), applied and
|
||||
# saved by Worker.determine_available_memory / compile_or_warm_up_model.
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Any, Literal, TypeAlias, TypedDict, final
|
||||
from pydantic import ConfigDict, Field, field_validator, model_validator
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.utils import config
|
||||
from vllm.utils.hashing import safe_hash
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
@@ -344,5 +345,19 @@ class MultiModalConfig:
|
||||
kwargs = self.mm_processor_kwargs or {}
|
||||
return kwargs | dict(inference_kwargs)
|
||||
|
||||
def use_gpu_video_backend(self) -> bool:
|
||||
"""Return whether the configured video loader or codec uses the GPU."""
|
||||
from vllm.multimodal.video import VIDEO_LOADER_REGISTRY
|
||||
|
||||
video_kwargs = self.media_io_kwargs.get("video", {})
|
||||
video_loader_backend = (
|
||||
video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
|
||||
)
|
||||
codec_backend = video_kwargs.get("backend")
|
||||
return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or (
|
||||
codec_backend is not None
|
||||
and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend)
|
||||
)
|
||||
|
||||
def is_multimodal_pruning_enabled(self):
|
||||
return self.video_pruning_rate is not None and self.video_pruning_rate > 0
|
||||
|
||||
@@ -17,9 +17,14 @@ amount out of its KV-cache budget so the headroom physically exists.
|
||||
"""
|
||||
|
||||
import threading
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.mem_constants import GiB_bytes
|
||||
from vllm.utils.mem_utils import format_gib
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.multimodal import MultiModalConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -145,3 +150,67 @@ def maybe_init_mm_gpu_ipc_pool(
|
||||
api_process_count,
|
||||
)
|
||||
return pool
|
||||
|
||||
|
||||
def reserve_mm_ipc_gpu_memory(
|
||||
available_kv_cache_memory_bytes: int,
|
||||
mm_config: "MultiModalConfig | None",
|
||||
api_process_count: int = 1,
|
||||
) -> int:
|
||||
"""Carve frontend multimodal GPU memory out of the KV cache.
|
||||
|
||||
Raw decoded frames are bounded by ``mm_ipc_gpu_memory_gb`` and acquired by
|
||||
the frontend semaphore. Some decoders also keep persistent surfaces around;
|
||||
reserve a fixed upper bound for those when a GPU backend is configured.
|
||||
"""
|
||||
if mm_config is None:
|
||||
return available_kv_cache_memory_bytes
|
||||
|
||||
from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
|
||||
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
|
||||
)
|
||||
|
||||
raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes)
|
||||
# Each API server process runs its own decoder surfaces and NVDEC/CUVID CUDA
|
||||
# context on the GPU, outside the worker memory pool. Reserve that footprint
|
||||
# per process so gpu_memory_utilization bounds total GPU usage across them.
|
||||
num_api_servers = max(1, api_process_count)
|
||||
per_server_decoder_bytes = (
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES * PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
|
||||
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
|
||||
)
|
||||
decoder_reserved_bytes = (
|
||||
num_api_servers * per_server_decoder_bytes
|
||||
if mm_config.use_gpu_video_backend()
|
||||
else 0
|
||||
)
|
||||
reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes
|
||||
if reserved_bytes <= 0:
|
||||
return available_kv_cache_memory_bytes
|
||||
|
||||
remaining = available_kv_cache_memory_bytes - reserved_bytes
|
||||
if remaining <= 0:
|
||||
raise ValueError(
|
||||
f"frontend multimodal GPU decoding reserves "
|
||||
f"{format_gib(reserved_bytes)} GiB "
|
||||
f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, "
|
||||
f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), "
|
||||
f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is "
|
||||
"available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a "
|
||||
"different video backend, or increase gpu_memory_utilization."
|
||||
)
|
||||
logger.info_once(
|
||||
"Reserving %s GiB of GPU memory for frontend multimodal decoding "
|
||||
"(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context "
|
||||
"across %d API server(s) @ %s GiB/server); "
|
||||
"KV cache memory reduced to %s GiB.",
|
||||
format_gib(reserved_bytes),
|
||||
format_gib(raw_frame_reserved_bytes),
|
||||
format_gib(decoder_reserved_bytes),
|
||||
num_api_servers,
|
||||
format_gib(per_server_decoder_bytes),
|
||||
format_gib(remaining),
|
||||
)
|
||||
return remaining
|
||||
|
||||
@@ -51,12 +51,7 @@ from vllm.distributed.weight_transfer import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
|
||||
from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES,
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES,
|
||||
PYNVVIDEOCODEC_MAX_RETAINED_DECODERS,
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
)
|
||||
from vllm.multimodal.gpu_ipc_memory import reserve_mm_ipc_gpu_memory
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.profiler.wrapper import CudaProfilerWrapper, TorchProfilerWrapper
|
||||
from vllm.sequence import IntermediateTensors
|
||||
@@ -478,7 +473,11 @@ class Worker(WorkerBase):
|
||||
"correspondingly."
|
||||
)
|
||||
logger.info(msg)
|
||||
return self._reserve_mm_ipc_gpu_memory(kv_cache_memory_bytes)
|
||||
return reserve_mm_ipc_gpu_memory(
|
||||
kv_cache_memory_bytes,
|
||||
self.model_config.multimodal_config,
|
||||
getattr(self.parallel_config, "_api_process_count", 1),
|
||||
)
|
||||
|
||||
# Execute a forward pass with dummy inputs to profile the memory usage
|
||||
# of the model.
|
||||
@@ -589,82 +588,12 @@ class Worker(WorkerBase):
|
||||
suggested_util,
|
||||
)
|
||||
|
||||
return self._reserve_mm_ipc_gpu_memory(
|
||||
int(self.available_kv_cache_memory_bytes)
|
||||
return reserve_mm_ipc_gpu_memory(
|
||||
int(self.available_kv_cache_memory_bytes),
|
||||
self.model_config.multimodal_config,
|
||||
getattr(self.parallel_config, "_api_process_count", 1),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _uses_gpu_video_backend(mm_config) -> bool:
|
||||
video_kwargs = mm_config.media_io_kwargs.get("video", {})
|
||||
video_loader_backend = (
|
||||
video_kwargs.get("video_backend") or envs.VLLM_VIDEO_LOADER_BACKEND
|
||||
)
|
||||
codec_backend = video_kwargs.get("backend")
|
||||
return VIDEO_LOADER_REGISTRY.backend_requires_gpu(video_loader_backend) or (
|
||||
codec_backend is not None
|
||||
and VIDEO_LOADER_REGISTRY.backend_requires_gpu(codec_backend)
|
||||
)
|
||||
|
||||
def _reserve_mm_ipc_gpu_memory(self, available_kv_cache_memory_bytes: int) -> int:
|
||||
"""Carve frontend multimodal GPU memory out of the KV cache.
|
||||
|
||||
The frontend (API-server) process allocates GPU memory for hardware
|
||||
multimodal decoding. Raw decoded frames are bounded by
|
||||
``mm_ipc_gpu_memory_gb`` and acquired by the frontend semaphore. Some
|
||||
decoders also keep persistent surfaces around; reserve a fixed upper
|
||||
bound for those when the corresponding backend is configured.
|
||||
"""
|
||||
mm_config = self.model_config.multimodal_config
|
||||
if mm_config is None:
|
||||
return available_kv_cache_memory_bytes
|
||||
|
||||
raw_frame_reserved_bytes = int(mm_config.mm_ipc_gpu_memory_gb * GiB_bytes)
|
||||
# Each api_server_count process runs its OWN decoder surfaces + NVDEC/CUVID
|
||||
# CUDA context on the GPU, outside this (worker) memory pool. Reserve that
|
||||
# per-server footprint x api_server_count so gpu_memory_utilization bounds
|
||||
# TOTAL GPU usage across all API-server processes. Without the multiply,
|
||||
# HW decode overshoots the budget by ~(api_server_count-1) x per-server and
|
||||
# OOMs at high gmu, while SW decode (no per-server GPU allocation) does not.
|
||||
num_api_servers = max(1, getattr(self.parallel_config, "_api_process_count", 1))
|
||||
per_server_decoder_bytes = (
|
||||
PYNVVIDEOCODEC_DECODER_GPU_MEMORY_BYTES
|
||||
* PYNVVIDEOCODEC_MAX_RETAINED_DECODERS
|
||||
+ PYNVVIDEOCODEC_CUDA_CONTEXT_BYTES
|
||||
)
|
||||
decoder_reserved_bytes = (
|
||||
num_api_servers * per_server_decoder_bytes
|
||||
if self._uses_gpu_video_backend(mm_config)
|
||||
else 0
|
||||
)
|
||||
reserved_bytes = raw_frame_reserved_bytes + decoder_reserved_bytes
|
||||
if reserved_bytes <= 0:
|
||||
return available_kv_cache_memory_bytes
|
||||
|
||||
remaining = available_kv_cache_memory_bytes - reserved_bytes
|
||||
if remaining <= 0:
|
||||
raise ValueError(
|
||||
f"frontend multimodal GPU decoding reserves "
|
||||
f"{format_gib(reserved_bytes)} GiB "
|
||||
f"({format_gib(raw_frame_reserved_bytes)} GiB raw-frame budget, "
|
||||
f"{format_gib(decoder_reserved_bytes)} GiB decoder cache budget), "
|
||||
f"but only {format_gib(available_kv_cache_memory_bytes)} GiB is "
|
||||
"available for the KV cache. Reduce mm_ipc_gpu_memory_gb, use a "
|
||||
"different video backend, or increase gpu_memory_utilization."
|
||||
)
|
||||
logger.info_once(
|
||||
"Reserving %s GiB of GPU memory for frontend multimodal decoding "
|
||||
"(%s GiB raw-frame semaphore budget, %s GiB decoder+CUDA-context "
|
||||
"across %d API server(s) @ %s GiB/server); "
|
||||
"KV cache memory reduced to %s GiB.",
|
||||
format_gib(reserved_bytes),
|
||||
format_gib(raw_frame_reserved_bytes),
|
||||
format_gib(decoder_reserved_bytes),
|
||||
num_api_servers,
|
||||
format_gib(per_server_decoder_bytes),
|
||||
format_gib(remaining),
|
||||
)
|
||||
return remaining
|
||||
|
||||
def get_kv_connector_handshake_metadata(
|
||||
self,
|
||||
) -> dict[tuple[int, int], KVConnectorHandshakeMetadata] | None:
|
||||
|
||||
Reference in New Issue
Block a user