From 1180b604f8d5559a00d17b8f384ddcdbc9a41d16 Mon Sep 17 00:00:00 2001 From: Isotr0py Date: Fri, 31 Jul 2026 13:25:19 +0800 Subject: [PATCH] [Multimodal] Expose mm hash algothrim selection to cli args (#49686) Signed-off-by: Isotr0py --- docs/usage/security.md | 10 ++- tests/config/test_multimodal_config.py | 5 ++ .../multimodal/processing/test_moss_audio.py | 1 + tests/multimodal/test_cache.py | 3 +- tests/multimodal/test_hasher.py | 74 +++++++++++++++---- tests/multimodal/test_utils.py | 25 +++++++ vllm/config/model.py | 4 + vllm/config/multimodal.py | 22 +++++- vllm/engine/arg_utils.py | 14 +++- vllm/model_executor/models/terratorch.py | 5 +- .../models/transformers/multimodal.py | 5 +- vllm/multimodal/hasher.py | 18 +++-- vllm/multimodal/inputs.py | 31 ++++++-- vllm/multimodal/processing/inputs.py | 9 ++- vllm/multimodal/processing/processor.py | 10 ++- vllm/multimodal/utils.py | 50 ++++++++----- vllm/renderers/hf.py | 9 ++- 17 files changed, 234 insertions(+), 61 deletions(-) diff --git a/docs/usage/security.md b/docs/usage/security.md index 672d96e85e..7eb57c9aa2 100644 --- a/docs/usage/security.md +++ b/docs/usage/security.md @@ -396,7 +396,7 @@ FIPS compliance depends on many factors, so a vLLM deployment is not automatical Operators running vLLM on FIPS-enabled hosts should select FIPS-approved algorithms via the following knobs: -- **Multimodal input hashing** — `VLLM_MM_HASHER_ALGORITHM` defaults to `blake3`, which is not FIPS-approved. Set it to `sha256` or `sha512` in FIPS-enabled environments. +- **Multimodal input hashing** — `--mm-hasher-algorithm` (config field `mm_hasher_algorithm`) defaults to `blake3`, which is not FIPS-approved. Set it to `sha256` or `sha512` in FIPS-enabled environments. - **Prefix-cache hashing** — set `--prefix-caching-hash-algo` (config field `prefix_caching_hash_algo`) to `sha256` or `sha256_cbor`. The `xxhash` and `xxhash_cbor` options are not FIPS-approved. - **TLS ciphers** — use `--ssl-ciphers` to restrict the API server's TLS handshake to FIPS-approved cipher suites that match your environment's policy. @@ -408,7 +408,13 @@ vLLM uses MD5 in a few places to derive non-security cache keys (for example, co Some dependencies expose hash implementations that are not FIPS-approved. vLLM only invokes them when the corresponding algorithm is selected, but operators with strict cryptographic controls may want to ensure the code paths are not exercised — and, where policy requires, that the packages themselves are absent: -- `blake3` — currently listed in `requirements/common.txt`, so a standard install pulls it in. It is imported lazily and only used when `VLLM_MM_HASHER_ALGORITHM=blake3` (the default). Setting `VLLM_MM_HASHER_ALGORITHM` to `sha256` or `sha512` is sufficient to keep the non-FIPS code path dormant. If your policy additionally forbids the package being present, uninstall it after `pip install` (`pip uninstall blake3`); vLLM will continue to function as long as `VLLM_MM_HASHER_ALGORITHM` is set to a non-blake3 value. +- `blake3` — currently listed in `requirements/common.txt`, so a standard + install pulls it in. It is imported lazily and only used when + `mm_hasher_algorithm=blake3` (the default). Setting + `--mm-hasher-algorithm sha256` or `--mm-hasher-algorithm sha512` is sufficient + to keep the non-FIPS code path dormant. If your policy additionally forbids + the package being present, uninstall it after installation; vLLM will + continue to function as long as a non-blake3 algorithm is selected. - `xxhash` — a true optional dependency (not in `requirements/common.txt`). It is only imported when an `xxhash`-based prefix-cache algorithm is selected. Leave it uninstalled and select a `sha256`-based prefix-cache algorithm. ### Beyond hashing: other FIPS considerations diff --git a/tests/config/test_multimodal_config.py b/tests/config/test_multimodal_config.py index 7786cef857..5d0d755e51 100644 --- a/tests/config/test_multimodal_config.py +++ b/tests/config/test_multimodal_config.py @@ -24,6 +24,11 @@ def test_mm_encoder_attn_backend_invalid(): MultiModalConfig(mm_encoder_attn_backend="not_a_backend") # type: ignore[arg-type] +def test_mm_hasher_algorithm_invalid(): + with pytest.raises(ValueError, match="mm_hasher_algorithm"): + MultiModalConfig(mm_hasher_algorithm="md5") # type: ignore[arg-type] + + def test_mm_encoder_attn_backend_hash_updates(): base_hash = MultiModalConfig().compute_hash() overridden_hash = MultiModalConfig( diff --git a/tests/models/multimodal/processing/test_moss_audio.py b/tests/models/multimodal/processing/test_moss_audio.py index 6a18f63642..8a18d90f56 100644 --- a/tests/models/multimodal/processing/test_moss_audio.py +++ b/tests/models/multimodal/processing/test_moss_audio.py @@ -50,6 +50,7 @@ class _Tokenizer: class _MMConfig: enable_mm_embeds = False mm_processor_cache_gb = 1 + mm_hasher_algorithm = "blake3" def merge_mm_processor_kwargs(self, kwargs): return dict(kwargs) diff --git a/tests/multimodal/test_cache.py b/tests/multimodal/test_cache.py index bf297946f4..816d218dd6 100644 --- a/tests/multimodal/test_cache.py +++ b/tests/multimodal/test_cache.py @@ -144,7 +144,8 @@ def _compare_caches( for _ in range(int(item_capacity / hit_rate)) ] all_hashes = [ - MultiModalHasher.hash_kwargs(item=item.get_data()) for item in all_items + MultiModalHasher.hash_kwargs("blake3", item=item.get_data()) + for item in all_items ] prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0) diff --git a/tests/multimodal/test_hasher.py b/tests/multimodal/test_hasher.py index d177fb55e8..4810081ad4 100644 --- a/tests/multimodal/test_hasher.py +++ b/tests/multimodal/test_hasher.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import hashlib import uuid from io import BytesIO from pathlib import Path @@ -9,6 +10,7 @@ import pytest import torch from PIL import Image, ImageDraw +from vllm.config.multimodal import MMHasherAlgorithm from vllm.multimodal.hasher import MultiModalHasher from vllm.multimodal.media.base import MediaWithBytes from vllm.multimodal.media.image import ImageMediaIO @@ -20,12 +22,26 @@ ASSETS_DIR = Path(__file__).parent / "assets" assert ASSETS_DIR.exists() +@pytest.mark.parametrize("algorithm", ["sha256", "sha512"]) +def test_hash_algorithm(algorithm: MMHasherAlgorithm): + hasher = getattr(hashlib, algorithm)() + for bytes_ in MultiModalHasher.iter_item_to_bytes("value", "test"): + hasher.update(bytes_) + + assert MultiModalHasher.hash_kwargs(algorithm, value="test") == hasher.hexdigest() + + +def test_hash_algorithm_required(): + with pytest.raises(TypeError, match="algorithm"): + MultiModalHasher.hash_kwargs(value="test") # type: ignore[call-arg] + + def test_hash_single_item_different_shape(): x1 = torch.zeros(()) x2 = torch.zeros((1,)) hasher = MultiModalHasher - assert hasher.hash_kwargs(x=x1) != hasher.hash_kwargs(x=x2) + assert hasher.hash_kwargs("blake3", x=x1) != hasher.hash_kwargs("blake3", x=x2) def test_hash_key_order_invariant(): @@ -33,7 +49,9 @@ def test_hash_key_order_invariant(): y = torch.ones((5, 10)) hasher = MultiModalHasher - assert hasher.hash_kwargs(x=x, y=y) == hasher.hash_kwargs(y=y, x=x) + assert hasher.hash_kwargs("blake3", x=x, y=y) == hasher.hash_kwargs( + "blake3", y=y, x=x + ) # NOTE: Images that are the same visually are allowed to have the same hash @@ -44,7 +62,9 @@ def test_hash_collision_image_mode(mode_pair): image2 = Image.new(mode2, size=(10, 10), color=1) hasher = MultiModalHasher - assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2) + assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs( + "blake3", image=image2 + ) def test_hash_collision_image_palette(): @@ -53,7 +73,9 @@ def test_hash_collision_image_palette(): image2 = Image.open(ASSETS_DIR / "image2.png") hasher = MultiModalHasher - assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2) + assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs( + "blake3", image=image2 + ) def test_hash_collision_image_transpose(): @@ -64,7 +86,9 @@ def test_hash_collision_image_transpose(): ImageDraw.Draw(image2).line([(0, 0), (0, 10)]) hasher = MultiModalHasher - assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2) + assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs( + "blake3", image=image2 + ) @pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) @@ -74,7 +98,9 @@ def test_hash_collision_tensor_shape(dtype): arr2 = torch.zeros((10, 20, 5, 3), dtype=dtype) hasher = MultiModalHasher - assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2) + assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs( + "blake3", data=arr2 + ) def test_hash_collision_array_shape(): @@ -83,7 +109,9 @@ def test_hash_collision_array_shape(): arr2 = np.zeros((10, 20, 5, 3)) hasher = MultiModalHasher - assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2) + assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs( + "blake3", data=arr2 + ) def test_hash_collision_video_num_frames(): @@ -104,8 +132,8 @@ def test_hash_collision_video_num_frames(): return items.get_all_items_for_hash()[0] hasher = MultiModalHasher - assert hasher.hash_kwargs(video=item_for_hash(2)) != hasher.hash_kwargs( - video=item_for_hash(4) + assert hasher.hash_kwargs("blake3", video=item_for_hash(2)) != hasher.hash_kwargs( + "blake3", video=item_for_hash(4) ) @@ -118,7 +146,9 @@ def test_hash_non_contiguous_array(): hasher = MultiModalHasher # Both should be hashable and produce the same hashes - assert hasher.hash_kwargs(data=arr) == hasher.hash_kwargs(data=arr_c) + assert hasher.hash_kwargs("blake3", data=arr) == hasher.hash_kwargs( + "blake3", data=arr_c + ) def test_hash_image_exif_id(): @@ -133,9 +163,13 @@ def test_hash_image_exif_id(): hasher = MultiModalHasher # first image has UUID in ImageID, so it should hash to that UUID - assert hasher.hash_kwargs(image=image1) == hasher.hash_kwargs(image=id.bytes) + assert hasher.hash_kwargs("blake3", image=image1) == hasher.hash_kwargs( + "blake3", image=id.bytes + ) # second image has non-UUID in ImageID, so it should hash to the image data - assert hasher.hash_kwargs(image=image2) == hasher.hash_kwargs(image=image2a) + assert hasher.hash_kwargs("blake3", image=image2) == hasher.hash_kwargs( + "blake3", image=image2a + ) def _rgba_png_bytes() -> bytes: @@ -153,9 +187,15 @@ def test_hash_collision_media_io_config(): keep = ImageMediaIO(image_mode=None).load_bytes(data) hasher = MultiModalHasher - assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=black) - assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=keep) - assert hasher.hash_kwargs(image=white) == hasher.hash_kwargs(image=white2) + assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs( + "blake3", image=black + ) + assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs( + "blake3", image=keep + ) + assert hasher.hash_kwargs("blake3", image=white) == hasher.hash_kwargs( + "blake3", image=white2 + ) def test_hash_media_io_noop_config_preserves_hash(): @@ -169,4 +209,6 @@ def test_hash_media_io_noop_config_preserves_hash(): plain = MediaWithBytes(loaded.media, data) hasher = MultiModalHasher - assert hasher.hash_kwargs(image=loaded) == hasher.hash_kwargs(image=plain) + assert hasher.hash_kwargs("blake3", image=loaded) == hasher.hash_kwargs( + "blake3", image=plain + ) diff --git a/tests/multimodal/test_utils.py b/tests/multimodal/test_utils.py index 4e765ab1b8..ffc8ce504c 100644 --- a/tests/multimodal/test_utils.py +++ b/tests/multimodal/test_utils.py @@ -199,6 +199,31 @@ def test_group_and_batch_mm_items_split_by_fieldset(): assert [num_items for num_items, _ in res] == [2, 1, 1, 1] +def test_group_and_batch_mm_items_splits_shared_data_by_dtype(): + elem1 = MultiModalFieldElem( + data=torch.zeros(1, dtype=torch.int32), + field=MultiModalSharedField(batch_size=1), + ) + elem2 = MultiModalFieldElem( + data=torch.zeros(1, dtype=torch.float32), + field=MultiModalSharedField(batch_size=1), + ) + elem3 = MultiModalFieldElem( + data=[torch.zeros(1, dtype=torch.int32), torch.zeros(1, dtype=torch.float32)], + field=MultiModalSharedField(batch_size=1), + ) + + res = group_and_batch_mm_items( + [ + MultiModalKwargsItem({"x": elem1}), + MultiModalKwargsItem({"x": elem2}), + MultiModalKwargsItem({"x": elem3}), + ] + ) + + assert [num_items for num_items, _ in res] == [1, 1, 1] + + def test_group_and_batch_mm_items_split_by_shared_data(): elem1 = MultiModalFieldElem( data=torch.zeros(1, dtype=torch.uint8), diff --git a/vllm/config/model.py b/vllm/config/model.py index 341698a6d1..3b1afedfc4 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -17,6 +17,7 @@ from vllm.config.model_arch import ( from vllm.config.multimodal import ( MMCacheType, MMEncoderTPMode, + MMHasherAlgorithm, MMTensorIPC, MultiModalConfig, ) @@ -374,6 +375,7 @@ class ModelConfig: mm_processor_kwargs: InitVar[dict[str, Any] | None] = None mm_processor_cache_gb: InitVar[float | None] = None mm_processor_cache_type: InitVar[MMCacheType | None] = None + mm_hasher_algorithm: InitVar[MMHasherAlgorithm | None] = None mm_shm_cache_max_object_size_mb: InitVar[int | None] = None mm_encoder_only: InitVar[bool | None] = None mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None @@ -503,6 +505,7 @@ class ModelConfig: mm_processor_kwargs: dict[str, Any] | None, mm_processor_cache_gb: float | None, mm_processor_cache_type: MMCacheType | None, + mm_hasher_algorithm: MMHasherAlgorithm | None, mm_shm_cache_max_object_size_mb: int | None, mm_encoder_only: bool | None, mm_encoder_tp_mode: MMEncoderTPMode | None, @@ -737,6 +740,7 @@ class ModelConfig: mm_processor_kwargs=mm_processor_kwargs, mm_processor_cache_gb=mm_processor_cache_gb, mm_processor_cache_type=mm_processor_cache_type, + mm_hasher_algorithm=mm_hasher_algorithm, mm_shm_cache_max_object_size_mb=mm_shm_cache_max_object_size_mb, mm_encoder_only=mm_encoder_only, mm_encoder_tp_mode=mm_encoder_tp_mode, diff --git a/vllm/config/multimodal.py b/vllm/config/multimodal.py index ab5cf50fc3..6c85fc7c93 100644 --- a/vllm/config/multimodal.py +++ b/vllm/config/multimodal.py @@ -3,13 +3,13 @@ from collections.abc import Mapping from pathlib import Path -from typing import Any, Literal, TypeAlias, TypedDict, final +from typing import Any, Literal, TypeAlias, TypedDict, cast, final from pydantic import ConfigDict, Field, field_validator, model_validator from pydantic.dataclasses import dataclass import vllm.envs as envs -from vllm.config.utils import config +from vllm.config.utils import config, get_from_deprecated_env_if_set from vllm.utils.hashing import safe_hash from vllm.v1.attention.backends.registry import AttentionBackendEnum @@ -63,6 +63,19 @@ MMEncoderTPMode = Literal["weights", "data"] MMCacheType = Literal["shm", "lru"] VideoPruningMethod = Literal["evs", "vidcom2"] MMTensorIPC = Literal["direct_rpc", "torch_shm"] +MMHasherAlgorithm = Literal["blake3", "sha256", "sha512"] + + +def _get_mm_hasher_algorithm() -> MMHasherAlgorithm: + env_value = get_from_deprecated_env_if_set( + "VLLM_MM_HASHER_ALGORITHM", + "v0.27", + "mm_hasher_algorithm", + ) + env_value = "blake3" if env_value is None else env_value + return cast(MMHasherAlgorithm, env_value.lower()) + + MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions] """ A dictionary containing an entry for each modality type of dummy data. @@ -134,6 +147,11 @@ class MultiModalConfig: mm_processor_cache_type: MMCacheType = "lru" """Type of cache to use for the multi-modal preprocessor/mapper. If `shm`, use shared memory FIFO cache. If `lru`, use mirrored LRU cache.""" + mm_hasher_algorithm: MMHasherAlgorithm = Field( + default_factory=_get_mm_hasher_algorithm + ) + """Hash algorithm to use for multi-modal input caching. Use `"sha256"` or + `"sha512"` for FIPS-compliant deployments.""" mm_shm_cache_max_object_size_mb: int = Field(default=128, ge=0) """Size limit (in MiB) for each object stored in the multi-modal processor shared memory cache. Only effective when `mm_processor_cache_type` is diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index fd1fa77e71..3003d73859 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -86,7 +86,12 @@ from vllm.config.model import ( RunnerOption, TokenizerMode, ) -from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MMTensorIPC +from vllm.config.multimodal import ( + MMCacheType, + MMEncoderTPMode, + MMHasherAlgorithm, + MMTensorIPC, +) from vllm.config.observability import DetailedTraceModules from vllm.config.parallel import ( All2AllBackend, @@ -566,6 +571,9 @@ class EngineArgs: mm_processor_cache_type: MMCacheType | None = ( MultiModalConfig.mm_processor_cache_type ) + mm_hasher_algorithm: MMHasherAlgorithm = get_field( + MultiModalConfig, "mm_hasher_algorithm" + ) mm_shm_cache_max_object_size_mb: int = ( MultiModalConfig.mm_shm_cache_max_object_size_mb ) @@ -1295,6 +1303,9 @@ class EngineArgs: multimodal_group.add_argument( "--mm-processor-cache-type", **multimodal_kwargs["mm_processor_cache_type"] ) + multimodal_group.add_argument( + "--mm-hasher-algorithm", **multimodal_kwargs["mm_hasher_algorithm"] + ) multimodal_group.add_argument( "--mm-shm-cache-max-object-size-mb", **multimodal_kwargs["mm_shm_cache_max_object_size_mb"], @@ -1712,6 +1723,7 @@ class EngineArgs: mm_processor_cache_gb=self.mm_processor_cache_gb, mm_processor_cache_type=self.mm_processor_cache_type, mm_shm_cache_max_object_size_mb=self.mm_shm_cache_max_object_size_mb, + mm_hasher_algorithm=self.mm_hasher_algorithm, mm_encoder_only=self.mm_encoder_only, mm_encoder_tp_mode=self.mm_encoder_tp_mode, mm_encoder_attn_backend=self.mm_encoder_attn_backend, diff --git a/vllm/model_executor/models/terratorch.py b/vllm/model_executor/models/terratorch.py index e863b0bb51..b49f7827da 100644 --- a/vllm/model_executor/models/terratorch.py +++ b/vllm/model_executor/models/terratorch.py @@ -217,7 +217,10 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing ) with timing_ctx.record("get_mm_hashes"): - mm_hashes = inputs.get_mm_hashes(self.info.model_id) + mm_hashes = inputs.get_mm_hashes( + self.info.model_id, + self.info.ctx.get_mm_config().mm_hasher_algorithm, + ) mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]} diff --git a/vllm/model_executor/models/transformers/multimodal.py b/vllm/model_executor/models/transformers/multimodal.py index 5c6b0e286a..a87e28241f 100644 --- a/vllm/model_executor/models/transformers/multimodal.py +++ b/vllm/model_executor/models/transformers/multimodal.py @@ -402,7 +402,10 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]): # Use overrides if provided; fallback to data-dependent hashing. with timing_ctx.record("get_mm_hashes"): - mm_hashes = inputs.get_mm_hashes(self.info.model_id) + mm_hashes = inputs.get_mm_hashes( + self.info.model_id, + self.info.ctx.get_mm_config().mm_hasher_algorithm, + ) # For gemma3 we check `token_type_ids` as the key mm_token_type_ids = processed_data.pop("token_type_ids", None) diff --git a/vllm/multimodal/hasher.py b/vllm/multimodal/hasher.py index 2025f51133..3d86b33d51 100644 --- a/vllm/multimodal/hasher.py +++ b/vllm/multimodal/hasher.py @@ -11,7 +11,7 @@ import numpy as np import torch from PIL import Image -import vllm.envs as envs +from vllm.config.multimodal import MMHasherAlgorithm from vllm.logger import init_logger from .media import MediaWithBytes @@ -20,7 +20,9 @@ logger = init_logger(__name__) @functools.lru_cache(maxsize=3) -def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]: +def _get_hasher_factory( + algorithm: MMHasherAlgorithm, +) -> Callable[[], "hashlib._Hash"]: """ Get the hasher factory based on the configured algorithm. @@ -32,7 +34,6 @@ def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]: See: https://github.com/vllm-project/vllm/issues/18334 """ - algorithm = algorithm.lower() if algorithm == "blake3": from blake3 import blake3 @@ -43,7 +44,7 @@ def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]: elif algorithm == "sha512": return hashlib.sha512 else: - # This should never happen due to env_with_choices validation + # This should never happen due to config validation raise ValueError(f"Unsupported hash algorithm: {algorithm}") @@ -162,8 +163,13 @@ class MultiModalHasher: yield from cls.serialize_item(obj) @classmethod - def hash_kwargs(cls, **kwargs: object) -> str: - hasher_factory = _get_hasher_factory(envs.VLLM_MM_HASHER_ALGORITHM) + def hash_kwargs( + cls, + algorithm: MMHasherAlgorithm, + /, + **kwargs: object, + ) -> str: + hasher_factory = _get_hasher_factory(algorithm) hasher = hasher_factory() for k, v in sorted(kwargs.items(), key=lambda kv: kv[0]): diff --git a/vllm/multimodal/inputs.py b/vllm/multimodal/inputs.py index dc96c366ad..ab04dc71e8 100644 --- a/vllm/multimodal/inputs.py +++ b/vllm/multimodal/inputs.py @@ -229,40 +229,57 @@ Uses a list instead of a tensor if the dimensions of each element do not match. """ -def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool: +def nested_tensors_equal( + a: NestedTensors, + b: NestedTensors, + check_dtype: bool = True, +) -> bool: """ Equality check between [`NestedTensors`][vllm.multimodal.inputs.NestedTensors] objects. + + If `check_dtype` is `True`, the tensors must have the same dtype. """ + check_dtype_func = ( + lambda a, b, check_dtype: a.dtype == b.dtype if check_dtype else True + ) if isinstance(a, torch.Tensor): - return isinstance(b, torch.Tensor) and torch.equal(a, b) + return ( + isinstance(b, torch.Tensor) + and torch.equal(a, b) + and check_dtype_func(a, b, check_dtype) + ) elif isinstance(b, torch.Tensor): - return isinstance(a, torch.Tensor) and torch.equal(b, a) + return ( + isinstance(a, torch.Tensor) + and torch.equal(b, a) + and check_dtype_func(b, a, check_dtype) + ) if isinstance(a, list): return ( isinstance(b, list) and len(a) == len(b) - and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b)) + and all(nested_tensors_equal(a_, b_, check_dtype) for a_, b_ in zip(a, b)) ) if isinstance(b, list): return ( isinstance(a, list) and len(b) == len(a) - and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a)) + and all(nested_tensors_equal(b_, a_, check_dtype) for b_, a_ in zip(b, a)) ) if isinstance(a, tuple): return ( isinstance(b, tuple) and len(a) == len(b) - and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b)) + and all(nested_tensors_equal(a_, b_, check_dtype) for a_, b_ in zip(a, b)) ) if isinstance(b, tuple): return ( isinstance(a, tuple) and len(b) == len(a) - and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a)) + and all(nested_tensors_equal(b_, a_, check_dtype) for b_, a_ in zip(b, a)) ) # Both a and b are scalars diff --git a/vllm/multimodal/processing/inputs.py b/vllm/multimodal/processing/inputs.py index 35a37d2377..73a734d9c2 100644 --- a/vllm/multimodal/processing/inputs.py +++ b/vllm/multimodal/processing/inputs.py @@ -3,6 +3,7 @@ from collections.abc import Mapping from dataclasses import dataclass, field +from vllm.config.multimodal import MMHasherAlgorithm from vllm.inputs import MultiModalHashes from ..hasher import MultiModalHasher @@ -22,7 +23,11 @@ class ProcessorInputs: hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict) tokenization_kwargs: Mapping[str, object] = field(default_factory=dict) - def get_mm_hashes(self, model_id: str) -> MultiModalHashes: + def get_mm_hashes( + self, + model_id: str, + hash_algorithm: MMHasherAlgorithm, + ) -> MultiModalHashes: mm_data_items = self.mm_data_items mm_uuid_items = self.mm_uuid_items or {} hf_processor_mm_kwargs = self.hf_processor_mm_kwargs @@ -49,6 +54,7 @@ class ProcessorInputs: item = uuid_item if uuid_item is not None else item hashes.append( hasher.hash_kwargs( + hash_algorithm, model_id=model_id, **{modality: item}, **hf_processor_mm_kwargs, @@ -61,6 +67,7 @@ class ProcessorInputs: else: mm_hashes[modality] = [ hasher.hash_kwargs( + hash_algorithm, model_id=model_id, **{modality: item}, **hf_processor_mm_kwargs, diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index 7b24cd3fcb..f43b45ff7e 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -1422,7 +1422,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): # Use overrides if provided; fallback to data-dependent hashing. with timing_ctx.record("get_mm_hashes"): - mm_hashes = inputs.get_mm_hashes(self.info.model_id) + mm_hashes = inputs.get_mm_hashes( + self.info.model_id, + self.info.ctx.get_mm_config().mm_hasher_algorithm, + ) mm_prompt_updates = self._get_mm_prompt_updates( inputs.mm_data_items, @@ -1454,7 +1457,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]): return self._apply_hf_processor(inputs, timing_ctx) with timing_ctx.record("get_mm_hashes"): - mm_hashes = inputs.get_mm_hashes(self.info.model_id) + mm_hashes = inputs.get_mm_hashes( + self.info.model_id, + self.info.ctx.get_mm_config().mm_hasher_algorithm, + ) with timing_ctx.record("get_cache_missing_items"): mm_is_cached, mm_missing_data_items = self._get_cache_missing_items( diff --git a/vllm/multimodal/utils.py b/vllm/multimodal/utils.py index eae274a81e..25c811fd42 100644 --- a/vllm/multimodal/utils.py +++ b/vllm/multimodal/utils.py @@ -16,13 +16,13 @@ from typing_extensions import deprecated from vllm.inputs import MultiModalPlaceholders from vllm.utils.import_utils import LazyLoader -from .hasher import MultiModalHasher from .inputs import ( BatchedTensorInputs, MultiModalFeatureSpec, MultiModalFieldElem, MultiModalKwargsItem, MultiModalSharedField, + nested_tensors_equal, ) from .media import ( AudioMediaIO, @@ -165,11 +165,28 @@ def argsort_mm_positions( return [(modality, idx) for modality, idx, _ in sorted_flat_items] -def _get_group_hash(elem: MultiModalFieldElem): - if not isinstance(elem.field, MultiModalSharedField): - return None +def _can_batch_mm_items( + left: MultiModalKwargsItem, + right: MultiModalKwargsItem, +) -> bool: + if left.keys() != right.keys(): + return False - return MultiModalHasher.hash_kwargs(data=elem.data) + for key, left_elem in left.items(): + right_elem = right[key] + left_field, right_field = left_elem.field, right_elem.field + is_shared_field = isinstance(left_field, MultiModalSharedField) and isinstance( + right_field, MultiModalSharedField + ) + if (type(left_field) is not type(right_field)) or ( + is_shared_field + and not nested_tensors_equal( + left_elem.data, right_elem.data, check_dtype=True + ) + ): + return False + + return True def _batch_mm_items( @@ -217,26 +234,21 @@ def group_and_batch_mm_items( - `kwargs` is a dictionary of keyword arguments to pass to the model; - `num_items` is the corresponding number of items. """ - group_ids = [ - tuple( - (key, _get_group_hash(elem)) - for key, elem in sorted(item.items(), key=lambda kv: kv[0]) - ) - for item in items - ] - group_sizes = [sum(1 for _ in group) for _, group in groupby(group_ids)] - start_idx = 0 - for group_size in group_sizes: + for end_idx in range(1, len(items) + 1): + if end_idx < len(items) and _can_batch_mm_items( + items[end_idx - 1], items[end_idx] + ): + continue + group_data = _batch_mm_items( - items[start_idx : start_idx + group_size], + items[start_idx:end_idx], device=device, pin_memory=pin_memory, ) - yield group_size, group_data - - start_idx += group_size + yield end_idx - start_idx, group_data + start_idx = end_idx assert start_idx == len(items) diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 55e8de7bea..95210e4a47 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -1276,8 +1276,8 @@ class HfRenderer(BaseRenderer[HfTokenizer]): embeds_prompt["prompt_embeds"] = full_embeds embeds_prompt["prompt_is_token_ids"] = is_token_ids_mask - @staticmethod def _apply_prompt_embeds_to_engine_input( + self, engine_input: MultiModalInput, prompt_embeds_tensors: list[torch.Tensor], mm_updates: MultiModalPromptUpdates, @@ -1300,6 +1300,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]): pe_kwargs_items: list[MultiModalKwargsItem] = [] pe_hashes: list[str] = [] pe_placeholders: list[PlaceholderRange] = [] + mm_config = self.model_config.get_multimodal_config() for tensor, (start, length) in zip( prompt_embeds_tensors, positions, strict=True ): @@ -1313,7 +1314,11 @@ class HfRenderer(BaseRenderer[HfTokenizer]): } ) ) - pe_hashes.append(MultiModalHasher.hash_kwargs(prompt_embeds=tensor)) + pe_hashes.append( + MultiModalHasher.hash_kwargs( + mm_config.mm_hasher_algorithm, prompt_embeds=tensor + ) + ) # `is_embed=None` matches the existing image_embeds-style # "no encoder, just splice the tensor directly" semantics. pe_placeholders.append(