[Multimodal] Expose mm hash algothrim selection to cli args (#49686)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-07-31 05:25:19 +00:00
committed by GitHub
parent 0f17394564
commit 1180b604f8
17 changed files with 234 additions and 61 deletions
+8 -2
View File
@@ -396,7 +396,7 @@ FIPS compliance depends on many factors, so a vLLM deployment is not automatical
Operators running vLLM on FIPS-enabled hosts should select FIPS-approved algorithms via the following knobs:
- **Multimodal input hashing** — `VLLM_MM_HASHER_ALGORITHM` defaults to `blake3`, which is not FIPS-approved. Set it to `sha256` or `sha512` in FIPS-enabled environments.
- **Multimodal input hashing** — `--mm-hasher-algorithm` (config field `mm_hasher_algorithm`) defaults to `blake3`, which is not FIPS-approved. Set it to `sha256` or `sha512` in FIPS-enabled environments.
- **Prefix-cache hashing** — set `--prefix-caching-hash-algo` (config field `prefix_caching_hash_algo`) to `sha256` or `sha256_cbor`. The `xxhash` and `xxhash_cbor` options are not FIPS-approved.
- **TLS ciphers** — use `--ssl-ciphers` to restrict the API server's TLS handshake to FIPS-approved cipher suites that match your environment's policy.
@@ -408,7 +408,13 @@ vLLM uses MD5 in a few places to derive non-security cache keys (for example, co
Some dependencies expose hash implementations that are not FIPS-approved. vLLM only invokes them when the corresponding algorithm is selected, but operators with strict cryptographic controls may want to ensure the code paths are not exercised — and, where policy requires, that the packages themselves are absent:
- `blake3` — currently listed in `requirements/common.txt`, so a standard install pulls it in. It is imported lazily and only used when `VLLM_MM_HASHER_ALGORITHM=blake3` (the default). Setting `VLLM_MM_HASHER_ALGORITHM` to `sha256` or `sha512` is sufficient to keep the non-FIPS code path dormant. If your policy additionally forbids the package being present, uninstall it after `pip install` (`pip uninstall blake3`); vLLM will continue to function as long as `VLLM_MM_HASHER_ALGORITHM` is set to a non-blake3 value.
- `blake3` — currently listed in `requirements/common.txt`, so a standard
install pulls it in. It is imported lazily and only used when
`mm_hasher_algorithm=blake3` (the default). Setting
`--mm-hasher-algorithm sha256` or `--mm-hasher-algorithm sha512` is sufficient
to keep the non-FIPS code path dormant. If your policy additionally forbids
the package being present, uninstall it after installation; vLLM will
continue to function as long as a non-blake3 algorithm is selected.
- `xxhash` — a true optional dependency (not in `requirements/common.txt`). It is only imported when an `xxhash`-based prefix-cache algorithm is selected. Leave it uninstalled and select a `sha256`-based prefix-cache algorithm.
### Beyond hashing: other FIPS considerations
+5
View File
@@ -24,6 +24,11 @@ def test_mm_encoder_attn_backend_invalid():
MultiModalConfig(mm_encoder_attn_backend="not_a_backend") # type: ignore[arg-type]
def test_mm_hasher_algorithm_invalid():
with pytest.raises(ValueError, match="mm_hasher_algorithm"):
MultiModalConfig(mm_hasher_algorithm="md5") # type: ignore[arg-type]
def test_mm_encoder_attn_backend_hash_updates():
base_hash = MultiModalConfig().compute_hash()
overridden_hash = MultiModalConfig(
@@ -50,6 +50,7 @@ class _Tokenizer:
class _MMConfig:
enable_mm_embeds = False
mm_processor_cache_gb = 1
mm_hasher_algorithm = "blake3"
def merge_mm_processor_kwargs(self, kwargs):
return dict(kwargs)
+2 -1
View File
@@ -144,7 +144,8 @@ def _compare_caches(
for _ in range(int(item_capacity / hit_rate))
]
all_hashes = [
MultiModalHasher.hash_kwargs(item=item.get_data()) for item in all_items
MultiModalHasher.hash_kwargs("blake3", item=item.get_data())
for item in all_items
]
prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)
+58 -16
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import hashlib
import uuid
from io import BytesIO
from pathlib import Path
@@ -9,6 +10,7 @@ import pytest
import torch
from PIL import Image, ImageDraw
from vllm.config.multimodal import MMHasherAlgorithm
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.media.base import MediaWithBytes
from vllm.multimodal.media.image import ImageMediaIO
@@ -20,12 +22,26 @@ ASSETS_DIR = Path(__file__).parent / "assets"
assert ASSETS_DIR.exists()
@pytest.mark.parametrize("algorithm", ["sha256", "sha512"])
def test_hash_algorithm(algorithm: MMHasherAlgorithm):
hasher = getattr(hashlib, algorithm)()
for bytes_ in MultiModalHasher.iter_item_to_bytes("value", "test"):
hasher.update(bytes_)
assert MultiModalHasher.hash_kwargs(algorithm, value="test") == hasher.hexdigest()
def test_hash_algorithm_required():
with pytest.raises(TypeError, match="algorithm"):
MultiModalHasher.hash_kwargs(value="test") # type: ignore[call-arg]
def test_hash_single_item_different_shape():
x1 = torch.zeros(())
x2 = torch.zeros((1,))
hasher = MultiModalHasher
assert hasher.hash_kwargs(x=x1) != hasher.hash_kwargs(x=x2)
assert hasher.hash_kwargs("blake3", x=x1) != hasher.hash_kwargs("blake3", x=x2)
def test_hash_key_order_invariant():
@@ -33,7 +49,9 @@ def test_hash_key_order_invariant():
y = torch.ones((5, 10))
hasher = MultiModalHasher
assert hasher.hash_kwargs(x=x, y=y) == hasher.hash_kwargs(y=y, x=x)
assert hasher.hash_kwargs("blake3", x=x, y=y) == hasher.hash_kwargs(
"blake3", y=y, x=x
)
# NOTE: Images that are the same visually are allowed to have the same hash
@@ -44,7 +62,9 @@ def test_hash_collision_image_mode(mode_pair):
image2 = Image.new(mode2, size=(10, 10), color=1)
hasher = MultiModalHasher
assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)
assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(
"blake3", image=image2
)
def test_hash_collision_image_palette():
@@ -53,7 +73,9 @@ def test_hash_collision_image_palette():
image2 = Image.open(ASSETS_DIR / "image2.png")
hasher = MultiModalHasher
assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)
assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(
"blake3", image=image2
)
def test_hash_collision_image_transpose():
@@ -64,7 +86,9 @@ def test_hash_collision_image_transpose():
ImageDraw.Draw(image2).line([(0, 0), (0, 10)])
hasher = MultiModalHasher
assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)
assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(
"blake3", image=image2
)
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@@ -74,7 +98,9 @@ def test_hash_collision_tensor_shape(dtype):
arr2 = torch.zeros((10, 20, 5, 3), dtype=dtype)
hasher = MultiModalHasher
assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)
assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs(
"blake3", data=arr2
)
def test_hash_collision_array_shape():
@@ -83,7 +109,9 @@ def test_hash_collision_array_shape():
arr2 = np.zeros((10, 20, 5, 3))
hasher = MultiModalHasher
assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)
assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs(
"blake3", data=arr2
)
def test_hash_collision_video_num_frames():
@@ -104,8 +132,8 @@ def test_hash_collision_video_num_frames():
return items.get_all_items_for_hash()[0]
hasher = MultiModalHasher
assert hasher.hash_kwargs(video=item_for_hash(2)) != hasher.hash_kwargs(
video=item_for_hash(4)
assert hasher.hash_kwargs("blake3", video=item_for_hash(2)) != hasher.hash_kwargs(
"blake3", video=item_for_hash(4)
)
@@ -118,7 +146,9 @@ def test_hash_non_contiguous_array():
hasher = MultiModalHasher
# Both should be hashable and produce the same hashes
assert hasher.hash_kwargs(data=arr) == hasher.hash_kwargs(data=arr_c)
assert hasher.hash_kwargs("blake3", data=arr) == hasher.hash_kwargs(
"blake3", data=arr_c
)
def test_hash_image_exif_id():
@@ -133,9 +163,13 @@ def test_hash_image_exif_id():
hasher = MultiModalHasher
# first image has UUID in ImageID, so it should hash to that UUID
assert hasher.hash_kwargs(image=image1) == hasher.hash_kwargs(image=id.bytes)
assert hasher.hash_kwargs("blake3", image=image1) == hasher.hash_kwargs(
"blake3", image=id.bytes
)
# second image has non-UUID in ImageID, so it should hash to the image data
assert hasher.hash_kwargs(image=image2) == hasher.hash_kwargs(image=image2a)
assert hasher.hash_kwargs("blake3", image=image2) == hasher.hash_kwargs(
"blake3", image=image2a
)
def _rgba_png_bytes() -> bytes:
@@ -153,9 +187,15 @@ def test_hash_collision_media_io_config():
keep = ImageMediaIO(image_mode=None).load_bytes(data)
hasher = MultiModalHasher
assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=black)
assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=keep)
assert hasher.hash_kwargs(image=white) == hasher.hash_kwargs(image=white2)
assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs(
"blake3", image=black
)
assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs(
"blake3", image=keep
)
assert hasher.hash_kwargs("blake3", image=white) == hasher.hash_kwargs(
"blake3", image=white2
)
def test_hash_media_io_noop_config_preserves_hash():
@@ -169,4 +209,6 @@ def test_hash_media_io_noop_config_preserves_hash():
plain = MediaWithBytes(loaded.media, data)
hasher = MultiModalHasher
assert hasher.hash_kwargs(image=loaded) == hasher.hash_kwargs(image=plain)
assert hasher.hash_kwargs("blake3", image=loaded) == hasher.hash_kwargs(
"blake3", image=plain
)
+25
View File
@@ -199,6 +199,31 @@ def test_group_and_batch_mm_items_split_by_fieldset():
assert [num_items for num_items, _ in res] == [2, 1, 1, 1]
def test_group_and_batch_mm_items_splits_shared_data_by_dtype():
elem1 = MultiModalFieldElem(
data=torch.zeros(1, dtype=torch.int32),
field=MultiModalSharedField(batch_size=1),
)
elem2 = MultiModalFieldElem(
data=torch.zeros(1, dtype=torch.float32),
field=MultiModalSharedField(batch_size=1),
)
elem3 = MultiModalFieldElem(
data=[torch.zeros(1, dtype=torch.int32), torch.zeros(1, dtype=torch.float32)],
field=MultiModalSharedField(batch_size=1),
)
res = group_and_batch_mm_items(
[
MultiModalKwargsItem({"x": elem1}),
MultiModalKwargsItem({"x": elem2}),
MultiModalKwargsItem({"x": elem3}),
]
)
assert [num_items for num_items, _ in res] == [1, 1, 1]
def test_group_and_batch_mm_items_split_by_shared_data():
elem1 = MultiModalFieldElem(
data=torch.zeros(1, dtype=torch.uint8),
+4
View File
@@ -17,6 +17,7 @@ from vllm.config.model_arch import (
from vllm.config.multimodal import (
MMCacheType,
MMEncoderTPMode,
MMHasherAlgorithm,
MMTensorIPC,
MultiModalConfig,
)
@@ -374,6 +375,7 @@ class ModelConfig:
mm_processor_kwargs: InitVar[dict[str, Any] | None] = None
mm_processor_cache_gb: InitVar[float | None] = None
mm_processor_cache_type: InitVar[MMCacheType | None] = None
mm_hasher_algorithm: InitVar[MMHasherAlgorithm | None] = None
mm_shm_cache_max_object_size_mb: InitVar[int | None] = None
mm_encoder_only: InitVar[bool | None] = None
mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None
@@ -503,6 +505,7 @@ class ModelConfig:
mm_processor_kwargs: dict[str, Any] | None,
mm_processor_cache_gb: float | None,
mm_processor_cache_type: MMCacheType | None,
mm_hasher_algorithm: MMHasherAlgorithm | None,
mm_shm_cache_max_object_size_mb: int | None,
mm_encoder_only: bool | None,
mm_encoder_tp_mode: MMEncoderTPMode | None,
@@ -737,6 +740,7 @@ class ModelConfig:
mm_processor_kwargs=mm_processor_kwargs,
mm_processor_cache_gb=mm_processor_cache_gb,
mm_processor_cache_type=mm_processor_cache_type,
mm_hasher_algorithm=mm_hasher_algorithm,
mm_shm_cache_max_object_size_mb=mm_shm_cache_max_object_size_mb,
mm_encoder_only=mm_encoder_only,
mm_encoder_tp_mode=mm_encoder_tp_mode,
+20 -2
View File
@@ -3,13 +3,13 @@
from collections.abc import Mapping
from pathlib import Path
from typing import Any, Literal, TypeAlias, TypedDict, final
from typing import Any, Literal, TypeAlias, TypedDict, cast, final
from pydantic import ConfigDict, Field, field_validator, model_validator
from pydantic.dataclasses import dataclass
import vllm.envs as envs
from vllm.config.utils import config
from vllm.config.utils import config, get_from_deprecated_env_if_set
from vllm.utils.hashing import safe_hash
from vllm.v1.attention.backends.registry import AttentionBackendEnum
@@ -63,6 +63,19 @@ MMEncoderTPMode = Literal["weights", "data"]
MMCacheType = Literal["shm", "lru"]
VideoPruningMethod = Literal["evs", "vidcom2"]
MMTensorIPC = Literal["direct_rpc", "torch_shm"]
MMHasherAlgorithm = Literal["blake3", "sha256", "sha512"]
def _get_mm_hasher_algorithm() -> MMHasherAlgorithm:
env_value = get_from_deprecated_env_if_set(
"VLLM_MM_HASHER_ALGORITHM",
"v0.27",
"mm_hasher_algorithm",
)
env_value = "blake3" if env_value is None else env_value
return cast(MMHasherAlgorithm, env_value.lower())
MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions]
"""
A dictionary containing an entry for each modality type of dummy data.
@@ -134,6 +147,11 @@ class MultiModalConfig:
mm_processor_cache_type: MMCacheType = "lru"
"""Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,
use shared memory FIFO cache. If `lru`, use mirrored LRU cache."""
mm_hasher_algorithm: MMHasherAlgorithm = Field(
default_factory=_get_mm_hasher_algorithm
)
"""Hash algorithm to use for multi-modal input caching. Use `"sha256"` or
`"sha512"` for FIPS-compliant deployments."""
mm_shm_cache_max_object_size_mb: int = Field(default=128, ge=0)
"""Size limit (in MiB) for each object stored in the multi-modal processor
shared memory cache. Only effective when `mm_processor_cache_type` is
+13 -1
View File
@@ -86,7 +86,12 @@ from vllm.config.model import (
RunnerOption,
TokenizerMode,
)
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MMTensorIPC
from vllm.config.multimodal import (
MMCacheType,
MMEncoderTPMode,
MMHasherAlgorithm,
MMTensorIPC,
)
from vllm.config.observability import DetailedTraceModules
from vllm.config.parallel import (
All2AllBackend,
@@ -566,6 +571,9 @@ class EngineArgs:
mm_processor_cache_type: MMCacheType | None = (
MultiModalConfig.mm_processor_cache_type
)
mm_hasher_algorithm: MMHasherAlgorithm = get_field(
MultiModalConfig, "mm_hasher_algorithm"
)
mm_shm_cache_max_object_size_mb: int = (
MultiModalConfig.mm_shm_cache_max_object_size_mb
)
@@ -1295,6 +1303,9 @@ class EngineArgs:
multimodal_group.add_argument(
"--mm-processor-cache-type", **multimodal_kwargs["mm_processor_cache_type"]
)
multimodal_group.add_argument(
"--mm-hasher-algorithm", **multimodal_kwargs["mm_hasher_algorithm"]
)
multimodal_group.add_argument(
"--mm-shm-cache-max-object-size-mb",
**multimodal_kwargs["mm_shm_cache_max_object_size_mb"],
@@ -1712,6 +1723,7 @@ class EngineArgs:
mm_processor_cache_gb=self.mm_processor_cache_gb,
mm_processor_cache_type=self.mm_processor_cache_type,
mm_shm_cache_max_object_size_mb=self.mm_shm_cache_max_object_size_mb,
mm_hasher_algorithm=self.mm_hasher_algorithm,
mm_encoder_only=self.mm_encoder_only,
mm_encoder_tp_mode=self.mm_encoder_tp_mode,
mm_encoder_attn_backend=self.mm_encoder_attn_backend,
+4 -1
View File
@@ -217,7 +217,10 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing
)
with timing_ctx.record("get_mm_hashes"):
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
mm_hashes = inputs.get_mm_hashes(
self.info.model_id,
self.info.ctx.get_mm_config().mm_hasher_algorithm,
)
mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}
@@ -402,7 +402,10 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
# Use overrides if provided; fallback to data-dependent hashing.
with timing_ctx.record("get_mm_hashes"):
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
mm_hashes = inputs.get_mm_hashes(
self.info.model_id,
self.info.ctx.get_mm_config().mm_hasher_algorithm,
)
# For gemma3 we check `token_type_ids` as the key
mm_token_type_ids = processed_data.pop("token_type_ids", None)
+12 -6
View File
@@ -11,7 +11,7 @@ import numpy as np
import torch
from PIL import Image
import vllm.envs as envs
from vllm.config.multimodal import MMHasherAlgorithm
from vllm.logger import init_logger
from .media import MediaWithBytes
@@ -20,7 +20,9 @@ logger = init_logger(__name__)
@functools.lru_cache(maxsize=3)
def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]:
def _get_hasher_factory(
algorithm: MMHasherAlgorithm,
) -> Callable[[], "hashlib._Hash"]:
"""
Get the hasher factory based on the configured algorithm.
@@ -32,7 +34,6 @@ def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]:
See: https://github.com/vllm-project/vllm/issues/18334
"""
algorithm = algorithm.lower()
if algorithm == "blake3":
from blake3 import blake3
@@ -43,7 +44,7 @@ def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]:
elif algorithm == "sha512":
return hashlib.sha512
else:
# This should never happen due to env_with_choices validation
# This should never happen due to config validation
raise ValueError(f"Unsupported hash algorithm: {algorithm}")
@@ -162,8 +163,13 @@ class MultiModalHasher:
yield from cls.serialize_item(obj)
@classmethod
def hash_kwargs(cls, **kwargs: object) -> str:
hasher_factory = _get_hasher_factory(envs.VLLM_MM_HASHER_ALGORITHM)
def hash_kwargs(
cls,
algorithm: MMHasherAlgorithm,
/,
**kwargs: object,
) -> str:
hasher_factory = _get_hasher_factory(algorithm)
hasher = hasher_factory()
for k, v in sorted(kwargs.items(), key=lambda kv: kv[0]):
+24 -7
View File
@@ -229,40 +229,57 @@ Uses a list instead of a tensor if the dimensions of each element do not match.
"""
def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
def nested_tensors_equal(
a: NestedTensors,
b: NestedTensors,
check_dtype: bool = True,
) -> bool:
"""
Equality check between
[`NestedTensors`][vllm.multimodal.inputs.NestedTensors] objects.
If `check_dtype` is `True`, the tensors must have the same dtype.
"""
check_dtype_func = (
lambda a, b, check_dtype: a.dtype == b.dtype if check_dtype else True
)
if isinstance(a, torch.Tensor):
return isinstance(b, torch.Tensor) and torch.equal(a, b)
return (
isinstance(b, torch.Tensor)
and torch.equal(a, b)
and check_dtype_func(a, b, check_dtype)
)
elif isinstance(b, torch.Tensor):
return isinstance(a, torch.Tensor) and torch.equal(b, a)
return (
isinstance(a, torch.Tensor)
and torch.equal(b, a)
and check_dtype_func(b, a, check_dtype)
)
if isinstance(a, list):
return (
isinstance(b, list)
and len(a) == len(b)
and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b))
and all(nested_tensors_equal(a_, b_, check_dtype) for a_, b_ in zip(a, b))
)
if isinstance(b, list):
return (
isinstance(a, list)
and len(b) == len(a)
and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a))
and all(nested_tensors_equal(b_, a_, check_dtype) for b_, a_ in zip(b, a))
)
if isinstance(a, tuple):
return (
isinstance(b, tuple)
and len(a) == len(b)
and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b))
and all(nested_tensors_equal(a_, b_, check_dtype) for a_, b_ in zip(a, b))
)
if isinstance(b, tuple):
return (
isinstance(a, tuple)
and len(b) == len(a)
and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a))
and all(nested_tensors_equal(b_, a_, check_dtype) for b_, a_ in zip(b, a))
)
# Both a and b are scalars
+8 -1
View File
@@ -3,6 +3,7 @@
from collections.abc import Mapping
from dataclasses import dataclass, field
from vllm.config.multimodal import MMHasherAlgorithm
from vllm.inputs import MultiModalHashes
from ..hasher import MultiModalHasher
@@ -22,7 +23,11 @@ class ProcessorInputs:
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)
def get_mm_hashes(self, model_id: str) -> MultiModalHashes:
def get_mm_hashes(
self,
model_id: str,
hash_algorithm: MMHasherAlgorithm,
) -> MultiModalHashes:
mm_data_items = self.mm_data_items
mm_uuid_items = self.mm_uuid_items or {}
hf_processor_mm_kwargs = self.hf_processor_mm_kwargs
@@ -49,6 +54,7 @@ class ProcessorInputs:
item = uuid_item if uuid_item is not None else item
hashes.append(
hasher.hash_kwargs(
hash_algorithm,
model_id=model_id,
**{modality: item},
**hf_processor_mm_kwargs,
@@ -61,6 +67,7 @@ class ProcessorInputs:
else:
mm_hashes[modality] = [
hasher.hash_kwargs(
hash_algorithm,
model_id=model_id,
**{modality: item},
**hf_processor_mm_kwargs,
+8 -2
View File
@@ -1422,7 +1422,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
# Use overrides if provided; fallback to data-dependent hashing.
with timing_ctx.record("get_mm_hashes"):
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
mm_hashes = inputs.get_mm_hashes(
self.info.model_id,
self.info.ctx.get_mm_config().mm_hasher_algorithm,
)
mm_prompt_updates = self._get_mm_prompt_updates(
inputs.mm_data_items,
@@ -1454,7 +1457,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
return self._apply_hf_processor(inputs, timing_ctx)
with timing_ctx.record("get_mm_hashes"):
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
mm_hashes = inputs.get_mm_hashes(
self.info.model_id,
self.info.ctx.get_mm_config().mm_hasher_algorithm,
)
with timing_ctx.record("get_cache_missing_items"):
mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
+31 -19
View File
@@ -16,13 +16,13 @@ from typing_extensions import deprecated
from vllm.inputs import MultiModalPlaceholders
from vllm.utils.import_utils import LazyLoader
from .hasher import MultiModalHasher
from .inputs import (
BatchedTensorInputs,
MultiModalFeatureSpec,
MultiModalFieldElem,
MultiModalKwargsItem,
MultiModalSharedField,
nested_tensors_equal,
)
from .media import (
AudioMediaIO,
@@ -165,11 +165,28 @@ def argsort_mm_positions(
return [(modality, idx) for modality, idx, _ in sorted_flat_items]
def _get_group_hash(elem: MultiModalFieldElem):
if not isinstance(elem.field, MultiModalSharedField):
return None
def _can_batch_mm_items(
left: MultiModalKwargsItem,
right: MultiModalKwargsItem,
) -> bool:
if left.keys() != right.keys():
return False
return MultiModalHasher.hash_kwargs(data=elem.data)
for key, left_elem in left.items():
right_elem = right[key]
left_field, right_field = left_elem.field, right_elem.field
is_shared_field = isinstance(left_field, MultiModalSharedField) and isinstance(
right_field, MultiModalSharedField
)
if (type(left_field) is not type(right_field)) or (
is_shared_field
and not nested_tensors_equal(
left_elem.data, right_elem.data, check_dtype=True
)
):
return False
return True
def _batch_mm_items(
@@ -217,26 +234,21 @@ def group_and_batch_mm_items(
- `kwargs` is a dictionary of keyword arguments to pass to the model;
- `num_items` is the corresponding number of items.
"""
group_ids = [
tuple(
(key, _get_group_hash(elem))
for key, elem in sorted(item.items(), key=lambda kv: kv[0])
)
for item in items
]
group_sizes = [sum(1 for _ in group) for _, group in groupby(group_ids)]
start_idx = 0
for group_size in group_sizes:
for end_idx in range(1, len(items) + 1):
if end_idx < len(items) and _can_batch_mm_items(
items[end_idx - 1], items[end_idx]
):
continue
group_data = _batch_mm_items(
items[start_idx : start_idx + group_size],
items[start_idx:end_idx],
device=device,
pin_memory=pin_memory,
)
yield group_size, group_data
start_idx += group_size
yield end_idx - start_idx, group_data
start_idx = end_idx
assert start_idx == len(items)
+7 -2
View File
@@ -1276,8 +1276,8 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
embeds_prompt["prompt_embeds"] = full_embeds
embeds_prompt["prompt_is_token_ids"] = is_token_ids_mask
@staticmethod
def _apply_prompt_embeds_to_engine_input(
self,
engine_input: MultiModalInput,
prompt_embeds_tensors: list[torch.Tensor],
mm_updates: MultiModalPromptUpdates,
@@ -1300,6 +1300,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
pe_kwargs_items: list[MultiModalKwargsItem] = []
pe_hashes: list[str] = []
pe_placeholders: list[PlaceholderRange] = []
mm_config = self.model_config.get_multimodal_config()
for tensor, (start, length) in zip(
prompt_embeds_tensors, positions, strict=True
):
@@ -1313,7 +1314,11 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
}
)
)
pe_hashes.append(MultiModalHasher.hash_kwargs(prompt_embeds=tensor))
pe_hashes.append(
MultiModalHasher.hash_kwargs(
mm_config.mm_hasher_algorithm, prompt_embeds=tensor
)
)
# `is_embed=None` matches the existing image_embeds-style
# "no encoder, just splice the tensor directly" semantics.
pe_placeholders.append(