[Multimodal] Expose mm hash algothrim selection to cli args (#49686)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -17,6 +17,7 @@ from vllm.config.model_arch import (
|
||||
from vllm.config.multimodal import (
|
||||
MMCacheType,
|
||||
MMEncoderTPMode,
|
||||
MMHasherAlgorithm,
|
||||
MMTensorIPC,
|
||||
MultiModalConfig,
|
||||
)
|
||||
@@ -374,6 +375,7 @@ class ModelConfig:
|
||||
mm_processor_kwargs: InitVar[dict[str, Any] | None] = None
|
||||
mm_processor_cache_gb: InitVar[float | None] = None
|
||||
mm_processor_cache_type: InitVar[MMCacheType | None] = None
|
||||
mm_hasher_algorithm: InitVar[MMHasherAlgorithm | None] = None
|
||||
mm_shm_cache_max_object_size_mb: InitVar[int | None] = None
|
||||
mm_encoder_only: InitVar[bool | None] = None
|
||||
mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None
|
||||
@@ -503,6 +505,7 @@ class ModelConfig:
|
||||
mm_processor_kwargs: dict[str, Any] | None,
|
||||
mm_processor_cache_gb: float | None,
|
||||
mm_processor_cache_type: MMCacheType | None,
|
||||
mm_hasher_algorithm: MMHasherAlgorithm | None,
|
||||
mm_shm_cache_max_object_size_mb: int | None,
|
||||
mm_encoder_only: bool | None,
|
||||
mm_encoder_tp_mode: MMEncoderTPMode | None,
|
||||
@@ -737,6 +740,7 @@ class ModelConfig:
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
mm_processor_cache_gb=mm_processor_cache_gb,
|
||||
mm_processor_cache_type=mm_processor_cache_type,
|
||||
mm_hasher_algorithm=mm_hasher_algorithm,
|
||||
mm_shm_cache_max_object_size_mb=mm_shm_cache_max_object_size_mb,
|
||||
mm_encoder_only=mm_encoder_only,
|
||||
mm_encoder_tp_mode=mm_encoder_tp_mode,
|
||||
|
||||
@@ -3,13 +3,13 @@
|
||||
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypeAlias, TypedDict, final
|
||||
from typing import Any, Literal, TypeAlias, TypedDict, cast, final
|
||||
|
||||
from pydantic import ConfigDict, Field, field_validator, model_validator
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.utils import config
|
||||
from vllm.config.utils import config, get_from_deprecated_env_if_set
|
||||
from vllm.utils.hashing import safe_hash
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
@@ -63,6 +63,19 @@ MMEncoderTPMode = Literal["weights", "data"]
|
||||
MMCacheType = Literal["shm", "lru"]
|
||||
VideoPruningMethod = Literal["evs", "vidcom2"]
|
||||
MMTensorIPC = Literal["direct_rpc", "torch_shm"]
|
||||
MMHasherAlgorithm = Literal["blake3", "sha256", "sha512"]
|
||||
|
||||
|
||||
def _get_mm_hasher_algorithm() -> MMHasherAlgorithm:
|
||||
env_value = get_from_deprecated_env_if_set(
|
||||
"VLLM_MM_HASHER_ALGORITHM",
|
||||
"v0.27",
|
||||
"mm_hasher_algorithm",
|
||||
)
|
||||
env_value = "blake3" if env_value is None else env_value
|
||||
return cast(MMHasherAlgorithm, env_value.lower())
|
||||
|
||||
|
||||
MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions]
|
||||
"""
|
||||
A dictionary containing an entry for each modality type of dummy data.
|
||||
@@ -134,6 +147,11 @@ class MultiModalConfig:
|
||||
mm_processor_cache_type: MMCacheType = "lru"
|
||||
"""Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,
|
||||
use shared memory FIFO cache. If `lru`, use mirrored LRU cache."""
|
||||
mm_hasher_algorithm: MMHasherAlgorithm = Field(
|
||||
default_factory=_get_mm_hasher_algorithm
|
||||
)
|
||||
"""Hash algorithm to use for multi-modal input caching. Use `"sha256"` or
|
||||
`"sha512"` for FIPS-compliant deployments."""
|
||||
mm_shm_cache_max_object_size_mb: int = Field(default=128, ge=0)
|
||||
"""Size limit (in MiB) for each object stored in the multi-modal processor
|
||||
shared memory cache. Only effective when `mm_processor_cache_type` is
|
||||
|
||||
@@ -86,7 +86,12 @@ from vllm.config.model import (
|
||||
RunnerOption,
|
||||
TokenizerMode,
|
||||
)
|
||||
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MMTensorIPC
|
||||
from vllm.config.multimodal import (
|
||||
MMCacheType,
|
||||
MMEncoderTPMode,
|
||||
MMHasherAlgorithm,
|
||||
MMTensorIPC,
|
||||
)
|
||||
from vllm.config.observability import DetailedTraceModules
|
||||
from vllm.config.parallel import (
|
||||
All2AllBackend,
|
||||
@@ -566,6 +571,9 @@ class EngineArgs:
|
||||
mm_processor_cache_type: MMCacheType | None = (
|
||||
MultiModalConfig.mm_processor_cache_type
|
||||
)
|
||||
mm_hasher_algorithm: MMHasherAlgorithm = get_field(
|
||||
MultiModalConfig, "mm_hasher_algorithm"
|
||||
)
|
||||
mm_shm_cache_max_object_size_mb: int = (
|
||||
MultiModalConfig.mm_shm_cache_max_object_size_mb
|
||||
)
|
||||
@@ -1295,6 +1303,9 @@ class EngineArgs:
|
||||
multimodal_group.add_argument(
|
||||
"--mm-processor-cache-type", **multimodal_kwargs["mm_processor_cache_type"]
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-hasher-algorithm", **multimodal_kwargs["mm_hasher_algorithm"]
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-shm-cache-max-object-size-mb",
|
||||
**multimodal_kwargs["mm_shm_cache_max_object_size_mb"],
|
||||
@@ -1712,6 +1723,7 @@ class EngineArgs:
|
||||
mm_processor_cache_gb=self.mm_processor_cache_gb,
|
||||
mm_processor_cache_type=self.mm_processor_cache_type,
|
||||
mm_shm_cache_max_object_size_mb=self.mm_shm_cache_max_object_size_mb,
|
||||
mm_hasher_algorithm=self.mm_hasher_algorithm,
|
||||
mm_encoder_only=self.mm_encoder_only,
|
||||
mm_encoder_tp_mode=self.mm_encoder_tp_mode,
|
||||
mm_encoder_attn_backend=self.mm_encoder_attn_backend,
|
||||
|
||||
@@ -217,7 +217,10 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing
|
||||
)
|
||||
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
mm_hashes = inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
|
||||
mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}
|
||||
|
||||
|
||||
@@ -402,7 +402,10 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
mm_hashes = inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
|
||||
# For gemma3 we check `token_type_ids` as the key
|
||||
mm_token_type_ids = processed_data.pop("token_type_ids", None)
|
||||
|
||||
@@ -11,7 +11,7 @@ import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.multimodal import MMHasherAlgorithm
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .media import MediaWithBytes
|
||||
@@ -20,7 +20,9 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=3)
|
||||
def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]:
|
||||
def _get_hasher_factory(
|
||||
algorithm: MMHasherAlgorithm,
|
||||
) -> Callable[[], "hashlib._Hash"]:
|
||||
"""
|
||||
Get the hasher factory based on the configured algorithm.
|
||||
|
||||
@@ -32,7 +34,6 @@ def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]:
|
||||
|
||||
See: https://github.com/vllm-project/vllm/issues/18334
|
||||
"""
|
||||
algorithm = algorithm.lower()
|
||||
|
||||
if algorithm == "blake3":
|
||||
from blake3 import blake3
|
||||
@@ -43,7 +44,7 @@ def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]:
|
||||
elif algorithm == "sha512":
|
||||
return hashlib.sha512
|
||||
else:
|
||||
# This should never happen due to env_with_choices validation
|
||||
# This should never happen due to config validation
|
||||
raise ValueError(f"Unsupported hash algorithm: {algorithm}")
|
||||
|
||||
|
||||
@@ -162,8 +163,13 @@ class MultiModalHasher:
|
||||
yield from cls.serialize_item(obj)
|
||||
|
||||
@classmethod
|
||||
def hash_kwargs(cls, **kwargs: object) -> str:
|
||||
hasher_factory = _get_hasher_factory(envs.VLLM_MM_HASHER_ALGORITHM)
|
||||
def hash_kwargs(
|
||||
cls,
|
||||
algorithm: MMHasherAlgorithm,
|
||||
/,
|
||||
**kwargs: object,
|
||||
) -> str:
|
||||
hasher_factory = _get_hasher_factory(algorithm)
|
||||
hasher = hasher_factory()
|
||||
|
||||
for k, v in sorted(kwargs.items(), key=lambda kv: kv[0]):
|
||||
|
||||
@@ -229,40 +229,57 @@ Uses a list instead of a tensor if the dimensions of each element do not match.
|
||||
"""
|
||||
|
||||
|
||||
def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
|
||||
def nested_tensors_equal(
|
||||
a: NestedTensors,
|
||||
b: NestedTensors,
|
||||
check_dtype: bool = True,
|
||||
) -> bool:
|
||||
"""
|
||||
Equality check between
|
||||
[`NestedTensors`][vllm.multimodal.inputs.NestedTensors] objects.
|
||||
|
||||
If `check_dtype` is `True`, the tensors must have the same dtype.
|
||||
"""
|
||||
check_dtype_func = (
|
||||
lambda a, b, check_dtype: a.dtype == b.dtype if check_dtype else True
|
||||
)
|
||||
if isinstance(a, torch.Tensor):
|
||||
return isinstance(b, torch.Tensor) and torch.equal(a, b)
|
||||
return (
|
||||
isinstance(b, torch.Tensor)
|
||||
and torch.equal(a, b)
|
||||
and check_dtype_func(a, b, check_dtype)
|
||||
)
|
||||
elif isinstance(b, torch.Tensor):
|
||||
return isinstance(a, torch.Tensor) and torch.equal(b, a)
|
||||
return (
|
||||
isinstance(a, torch.Tensor)
|
||||
and torch.equal(b, a)
|
||||
and check_dtype_func(b, a, check_dtype)
|
||||
)
|
||||
|
||||
if isinstance(a, list):
|
||||
return (
|
||||
isinstance(b, list)
|
||||
and len(a) == len(b)
|
||||
and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b))
|
||||
and all(nested_tensors_equal(a_, b_, check_dtype) for a_, b_ in zip(a, b))
|
||||
)
|
||||
if isinstance(b, list):
|
||||
return (
|
||||
isinstance(a, list)
|
||||
and len(b) == len(a)
|
||||
and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a))
|
||||
and all(nested_tensors_equal(b_, a_, check_dtype) for b_, a_ in zip(b, a))
|
||||
)
|
||||
|
||||
if isinstance(a, tuple):
|
||||
return (
|
||||
isinstance(b, tuple)
|
||||
and len(a) == len(b)
|
||||
and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b))
|
||||
and all(nested_tensors_equal(a_, b_, check_dtype) for a_, b_ in zip(a, b))
|
||||
)
|
||||
if isinstance(b, tuple):
|
||||
return (
|
||||
isinstance(a, tuple)
|
||||
and len(b) == len(a)
|
||||
and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a))
|
||||
and all(nested_tensors_equal(b_, a_, check_dtype) for b_, a_ in zip(b, a))
|
||||
)
|
||||
|
||||
# Both a and b are scalars
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from vllm.config.multimodal import MMHasherAlgorithm
|
||||
from vllm.inputs import MultiModalHashes
|
||||
|
||||
from ..hasher import MultiModalHasher
|
||||
@@ -22,7 +23,11 @@ class ProcessorInputs:
|
||||
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def get_mm_hashes(self, model_id: str) -> MultiModalHashes:
|
||||
def get_mm_hashes(
|
||||
self,
|
||||
model_id: str,
|
||||
hash_algorithm: MMHasherAlgorithm,
|
||||
) -> MultiModalHashes:
|
||||
mm_data_items = self.mm_data_items
|
||||
mm_uuid_items = self.mm_uuid_items or {}
|
||||
hf_processor_mm_kwargs = self.hf_processor_mm_kwargs
|
||||
@@ -49,6 +54,7 @@ class ProcessorInputs:
|
||||
item = uuid_item if uuid_item is not None else item
|
||||
hashes.append(
|
||||
hasher.hash_kwargs(
|
||||
hash_algorithm,
|
||||
model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
@@ -61,6 +67,7 @@ class ProcessorInputs:
|
||||
else:
|
||||
mm_hashes[modality] = [
|
||||
hasher.hash_kwargs(
|
||||
hash_algorithm,
|
||||
model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
|
||||
@@ -1422,7 +1422,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
mm_hashes = inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
|
||||
mm_prompt_updates = self._get_mm_prompt_updates(
|
||||
inputs.mm_data_items,
|
||||
@@ -1454,7 +1457,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
mm_hashes = inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
|
||||
with timing_ctx.record("get_cache_missing_items"):
|
||||
mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
|
||||
|
||||
+31
-19
@@ -16,13 +16,13 @@ from typing_extensions import deprecated
|
||||
from vllm.inputs import MultiModalPlaceholders
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
|
||||
from .hasher import MultiModalHasher
|
||||
from .inputs import (
|
||||
BatchedTensorInputs,
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalSharedField,
|
||||
nested_tensors_equal,
|
||||
)
|
||||
from .media import (
|
||||
AudioMediaIO,
|
||||
@@ -165,11 +165,28 @@ def argsort_mm_positions(
|
||||
return [(modality, idx) for modality, idx, _ in sorted_flat_items]
|
||||
|
||||
|
||||
def _get_group_hash(elem: MultiModalFieldElem):
|
||||
if not isinstance(elem.field, MultiModalSharedField):
|
||||
return None
|
||||
def _can_batch_mm_items(
|
||||
left: MultiModalKwargsItem,
|
||||
right: MultiModalKwargsItem,
|
||||
) -> bool:
|
||||
if left.keys() != right.keys():
|
||||
return False
|
||||
|
||||
return MultiModalHasher.hash_kwargs(data=elem.data)
|
||||
for key, left_elem in left.items():
|
||||
right_elem = right[key]
|
||||
left_field, right_field = left_elem.field, right_elem.field
|
||||
is_shared_field = isinstance(left_field, MultiModalSharedField) and isinstance(
|
||||
right_field, MultiModalSharedField
|
||||
)
|
||||
if (type(left_field) is not type(right_field)) or (
|
||||
is_shared_field
|
||||
and not nested_tensors_equal(
|
||||
left_elem.data, right_elem.data, check_dtype=True
|
||||
)
|
||||
):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _batch_mm_items(
|
||||
@@ -217,26 +234,21 @@ def group_and_batch_mm_items(
|
||||
- `kwargs` is a dictionary of keyword arguments to pass to the model;
|
||||
- `num_items` is the corresponding number of items.
|
||||
"""
|
||||
group_ids = [
|
||||
tuple(
|
||||
(key, _get_group_hash(elem))
|
||||
for key, elem in sorted(item.items(), key=lambda kv: kv[0])
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
group_sizes = [sum(1 for _ in group) for _, group in groupby(group_ids)]
|
||||
|
||||
start_idx = 0
|
||||
for group_size in group_sizes:
|
||||
for end_idx in range(1, len(items) + 1):
|
||||
if end_idx < len(items) and _can_batch_mm_items(
|
||||
items[end_idx - 1], items[end_idx]
|
||||
):
|
||||
continue
|
||||
|
||||
group_data = _batch_mm_items(
|
||||
items[start_idx : start_idx + group_size],
|
||||
items[start_idx:end_idx],
|
||||
device=device,
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
|
||||
yield group_size, group_data
|
||||
|
||||
start_idx += group_size
|
||||
yield end_idx - start_idx, group_data
|
||||
start_idx = end_idx
|
||||
|
||||
assert start_idx == len(items)
|
||||
|
||||
|
||||
@@ -1276,8 +1276,8 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
embeds_prompt["prompt_embeds"] = full_embeds
|
||||
embeds_prompt["prompt_is_token_ids"] = is_token_ids_mask
|
||||
|
||||
@staticmethod
|
||||
def _apply_prompt_embeds_to_engine_input(
|
||||
self,
|
||||
engine_input: MultiModalInput,
|
||||
prompt_embeds_tensors: list[torch.Tensor],
|
||||
mm_updates: MultiModalPromptUpdates,
|
||||
@@ -1300,6 +1300,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
pe_kwargs_items: list[MultiModalKwargsItem] = []
|
||||
pe_hashes: list[str] = []
|
||||
pe_placeholders: list[PlaceholderRange] = []
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
for tensor, (start, length) in zip(
|
||||
prompt_embeds_tensors, positions, strict=True
|
||||
):
|
||||
@@ -1313,7 +1314,11 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
}
|
||||
)
|
||||
)
|
||||
pe_hashes.append(MultiModalHasher.hash_kwargs(prompt_embeds=tensor))
|
||||
pe_hashes.append(
|
||||
MultiModalHasher.hash_kwargs(
|
||||
mm_config.mm_hasher_algorithm, prompt_embeds=tensor
|
||||
)
|
||||
)
|
||||
# `is_embed=None` matches the existing image_embeds-style
|
||||
# "no encoder, just splice the tensor directly" semantics.
|
||||
pe_placeholders.append(
|
||||
|
||||
Reference in New Issue
Block a user