[Multimodal] Expose mm hash algothrim selection to cli args (#49686)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-07-31 05:25:19 +00:00
committed by GitHub
parent 0f17394564
commit 1180b604f8
17 changed files with 234 additions and 61 deletions
+5
View File
@@ -24,6 +24,11 @@ def test_mm_encoder_attn_backend_invalid():
MultiModalConfig(mm_encoder_attn_backend="not_a_backend") # type: ignore[arg-type]
def test_mm_hasher_algorithm_invalid():
with pytest.raises(ValueError, match="mm_hasher_algorithm"):
MultiModalConfig(mm_hasher_algorithm="md5") # type: ignore[arg-type]
def test_mm_encoder_attn_backend_hash_updates():
base_hash = MultiModalConfig().compute_hash()
overridden_hash = MultiModalConfig(
@@ -50,6 +50,7 @@ class _Tokenizer:
class _MMConfig:
enable_mm_embeds = False
mm_processor_cache_gb = 1
mm_hasher_algorithm = "blake3"
def merge_mm_processor_kwargs(self, kwargs):
return dict(kwargs)
+2 -1
View File
@@ -144,7 +144,8 @@ def _compare_caches(
for _ in range(int(item_capacity / hit_rate))
]
all_hashes = [
MultiModalHasher.hash_kwargs(item=item.get_data()) for item in all_items
MultiModalHasher.hash_kwargs("blake3", item=item.get_data())
for item in all_items
]
prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)
+58 -16
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import hashlib
import uuid
from io import BytesIO
from pathlib import Path
@@ -9,6 +10,7 @@ import pytest
import torch
from PIL import Image, ImageDraw
from vllm.config.multimodal import MMHasherAlgorithm
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.media.base import MediaWithBytes
from vllm.multimodal.media.image import ImageMediaIO
@@ -20,12 +22,26 @@ ASSETS_DIR = Path(__file__).parent / "assets"
assert ASSETS_DIR.exists()
@pytest.mark.parametrize("algorithm", ["sha256", "sha512"])
def test_hash_algorithm(algorithm: MMHasherAlgorithm):
hasher = getattr(hashlib, algorithm)()
for bytes_ in MultiModalHasher.iter_item_to_bytes("value", "test"):
hasher.update(bytes_)
assert MultiModalHasher.hash_kwargs(algorithm, value="test") == hasher.hexdigest()
def test_hash_algorithm_required():
with pytest.raises(TypeError, match="algorithm"):
MultiModalHasher.hash_kwargs(value="test") # type: ignore[call-arg]
def test_hash_single_item_different_shape():
x1 = torch.zeros(())
x2 = torch.zeros((1,))
hasher = MultiModalHasher
assert hasher.hash_kwargs(x=x1) != hasher.hash_kwargs(x=x2)
assert hasher.hash_kwargs("blake3", x=x1) != hasher.hash_kwargs("blake3", x=x2)
def test_hash_key_order_invariant():
@@ -33,7 +49,9 @@ def test_hash_key_order_invariant():
y = torch.ones((5, 10))
hasher = MultiModalHasher
assert hasher.hash_kwargs(x=x, y=y) == hasher.hash_kwargs(y=y, x=x)
assert hasher.hash_kwargs("blake3", x=x, y=y) == hasher.hash_kwargs(
"blake3", y=y, x=x
)
# NOTE: Images that are the same visually are allowed to have the same hash
@@ -44,7 +62,9 @@ def test_hash_collision_image_mode(mode_pair):
image2 = Image.new(mode2, size=(10, 10), color=1)
hasher = MultiModalHasher
assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)
assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(
"blake3", image=image2
)
def test_hash_collision_image_palette():
@@ -53,7 +73,9 @@ def test_hash_collision_image_palette():
image2 = Image.open(ASSETS_DIR / "image2.png")
hasher = MultiModalHasher
assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)
assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(
"blake3", image=image2
)
def test_hash_collision_image_transpose():
@@ -64,7 +86,9 @@ def test_hash_collision_image_transpose():
ImageDraw.Draw(image2).line([(0, 0), (0, 10)])
hasher = MultiModalHasher
assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)
assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(
"blake3", image=image2
)
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@@ -74,7 +98,9 @@ def test_hash_collision_tensor_shape(dtype):
arr2 = torch.zeros((10, 20, 5, 3), dtype=dtype)
hasher = MultiModalHasher
assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)
assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs(
"blake3", data=arr2
)
def test_hash_collision_array_shape():
@@ -83,7 +109,9 @@ def test_hash_collision_array_shape():
arr2 = np.zeros((10, 20, 5, 3))
hasher = MultiModalHasher
assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)
assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs(
"blake3", data=arr2
)
def test_hash_collision_video_num_frames():
@@ -104,8 +132,8 @@ def test_hash_collision_video_num_frames():
return items.get_all_items_for_hash()[0]
hasher = MultiModalHasher
assert hasher.hash_kwargs(video=item_for_hash(2)) != hasher.hash_kwargs(
video=item_for_hash(4)
assert hasher.hash_kwargs("blake3", video=item_for_hash(2)) != hasher.hash_kwargs(
"blake3", video=item_for_hash(4)
)
@@ -118,7 +146,9 @@ def test_hash_non_contiguous_array():
hasher = MultiModalHasher
# Both should be hashable and produce the same hashes
assert hasher.hash_kwargs(data=arr) == hasher.hash_kwargs(data=arr_c)
assert hasher.hash_kwargs("blake3", data=arr) == hasher.hash_kwargs(
"blake3", data=arr_c
)
def test_hash_image_exif_id():
@@ -133,9 +163,13 @@ def test_hash_image_exif_id():
hasher = MultiModalHasher
# first image has UUID in ImageID, so it should hash to that UUID
assert hasher.hash_kwargs(image=image1) == hasher.hash_kwargs(image=id.bytes)
assert hasher.hash_kwargs("blake3", image=image1) == hasher.hash_kwargs(
"blake3", image=id.bytes
)
# second image has non-UUID in ImageID, so it should hash to the image data
assert hasher.hash_kwargs(image=image2) == hasher.hash_kwargs(image=image2a)
assert hasher.hash_kwargs("blake3", image=image2) == hasher.hash_kwargs(
"blake3", image=image2a
)
def _rgba_png_bytes() -> bytes:
@@ -153,9 +187,15 @@ def test_hash_collision_media_io_config():
keep = ImageMediaIO(image_mode=None).load_bytes(data)
hasher = MultiModalHasher
assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=black)
assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=keep)
assert hasher.hash_kwargs(image=white) == hasher.hash_kwargs(image=white2)
assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs(
"blake3", image=black
)
assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs(
"blake3", image=keep
)
assert hasher.hash_kwargs("blake3", image=white) == hasher.hash_kwargs(
"blake3", image=white2
)
def test_hash_media_io_noop_config_preserves_hash():
@@ -169,4 +209,6 @@ def test_hash_media_io_noop_config_preserves_hash():
plain = MediaWithBytes(loaded.media, data)
hasher = MultiModalHasher
assert hasher.hash_kwargs(image=loaded) == hasher.hash_kwargs(image=plain)
assert hasher.hash_kwargs("blake3", image=loaded) == hasher.hash_kwargs(
"blake3", image=plain
)
+25
View File
@@ -199,6 +199,31 @@ def test_group_and_batch_mm_items_split_by_fieldset():
assert [num_items for num_items, _ in res] == [2, 1, 1, 1]
def test_group_and_batch_mm_items_splits_shared_data_by_dtype():
elem1 = MultiModalFieldElem(
data=torch.zeros(1, dtype=torch.int32),
field=MultiModalSharedField(batch_size=1),
)
elem2 = MultiModalFieldElem(
data=torch.zeros(1, dtype=torch.float32),
field=MultiModalSharedField(batch_size=1),
)
elem3 = MultiModalFieldElem(
data=[torch.zeros(1, dtype=torch.int32), torch.zeros(1, dtype=torch.float32)],
field=MultiModalSharedField(batch_size=1),
)
res = group_and_batch_mm_items(
[
MultiModalKwargsItem({"x": elem1}),
MultiModalKwargsItem({"x": elem2}),
MultiModalKwargsItem({"x": elem3}),
]
)
assert [num_items for num_items, _ in res] == [1, 1, 1]
def test_group_and_batch_mm_items_split_by_shared_data():
elem1 = MultiModalFieldElem(
data=torch.zeros(1, dtype=torch.uint8),