[VLM] Limit multimodal input cache by memory (#14805)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import re
|
||||
import sys
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from collections.abc import (Callable, Generator, ItemsView, Iterable, Mapping,
|
||||
@@ -11,14 +11,17 @@ from functools import lru_cache
|
||||
from typing import (TYPE_CHECKING, Generic, NamedTuple, Optional, Protocol,
|
||||
TypeVar, Union, cast)
|
||||
|
||||
import torch
|
||||
from cachetools import LRUCache
|
||||
from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from vllm.inputs import InputProcessingContext
|
||||
from vllm.jsontree import json_map_leaves, json_reduce_leaves
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
|
||||
encode_tokens)
|
||||
from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
|
||||
from vllm.utils import GiB_bytes, flatten_2d_lists, full_groupby
|
||||
|
||||
from .hasher import MultiModalHasher
|
||||
from .inputs import (MultiModalDataDict, MultiModalEncDecInputs,
|
||||
@@ -812,25 +815,50 @@ def find_mm_placeholders(
|
||||
return dict(full_groupby_modality(it))
|
||||
|
||||
|
||||
_V = TypeVar("_V", bound="Union[MultiModalKwargs, MultiModalKwargsItem]")
|
||||
|
||||
|
||||
class ProcessingCache:
|
||||
|
||||
def __init__(self, capacity: int) -> None:
|
||||
@staticmethod
|
||||
def get_lru_cache(
|
||||
capacity_gb: int,
|
||||
value_type: type[_V],
|
||||
) -> LRUCache[str, _V]:
|
||||
|
||||
def get_size(leaf: object) -> int:
|
||||
if isinstance(leaf, torch.Tensor):
|
||||
return leaf.nbytes # sys.getsizeof doesn't work for tensors
|
||||
|
||||
return sys.getsizeof(leaf)
|
||||
|
||||
return LRUCache[str, _V](
|
||||
GiB_bytes * capacity_gb,
|
||||
getsizeof=lambda x: json_reduce_leaves(
|
||||
lambda a, b: a + b,
|
||||
json_map_leaves(get_size, x),
|
||||
),
|
||||
)
|
||||
|
||||
def __init__(self, capacity_gb: int) -> None:
|
||||
super().__init__()
|
||||
|
||||
# DEBUG: Set to None to disable
|
||||
self.debug_cache_hit_ratio_steps: Optional[int] = None
|
||||
self.debug_cache_hits = 0
|
||||
self.debug_cache_total = 0
|
||||
|
||||
self._cache = LRUCache[str, MultiModalKwargsItem](capacity)
|
||||
self._cache = self.get_lru_cache(capacity_gb, MultiModalKwargsItem)
|
||||
|
||||
def _maybe_log_cache_stats(self) -> None:
|
||||
steps = self.debug_cache_hit_ratio_steps
|
||||
if not steps:
|
||||
return
|
||||
|
||||
cache_stats = self._cache.stat()
|
||||
if cache_stats.total % steps == 0:
|
||||
total = self.debug_cache_total
|
||||
if total > 0 and total % steps == 0:
|
||||
logger.debug("ProcessingCache: hit_ratio = %.2f",
|
||||
cache_stats.hit_ratio)
|
||||
self.debug_cache_hits / total)
|
||||
|
||||
def get(
|
||||
self,
|
||||
@@ -853,6 +881,13 @@ class ProcessingCache:
|
||||
cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
|
||||
**{modality: input_item},
|
||||
**input_kwargs)
|
||||
|
||||
if self.debug_cache_hit_ratio_steps:
|
||||
if cache_key in self._cache:
|
||||
self.debug_cache_hits += 1
|
||||
|
||||
self.debug_cache_total += 1
|
||||
|
||||
return self._cache.get(cache_key)
|
||||
|
||||
def put(
|
||||
@@ -870,7 +905,7 @@ class ProcessingCache:
|
||||
cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
|
||||
**{modality: input_item},
|
||||
**input_kwargs)
|
||||
self._cache.put(cache_key, output_kwargs)
|
||||
self._cache[cache_key] = output_kwargs
|
||||
|
||||
|
||||
class BaseProcessingInfo:
|
||||
|
||||
Reference in New Issue
Block a user