From a8d1aa9c99b8698a2a78b611b7a10c30e6b3995b Mon Sep 17 00:00:00 2001 From: Hotragn Pettugani <103170876+Hotragn@users.noreply.github.com> Date: Sat, 19 Sep 2026 00:06:48 -0400 Subject: [PATCH] [Bugfix][Multimodal] Frame the multi-modal hash digest input (#54283) Signed-off-by: hotragn Co-authored-by: Claude Opus 5 Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- tests/multimodal/test_hasher.py | 40 +++++++++++++++ tests/multimodal/test_processing.py | 37 ++++++++++++++ vllm/multimodal/hasher.py | 75 +++++++++++++++++++++++------ 3 files changed, 138 insertions(+), 14 deletions(-) diff --git a/tests/multimodal/test_hasher.py b/tests/multimodal/test_hasher.py index 1b738eeeb8..971d97a2ca 100644 --- a/tests/multimodal/test_hasher.py +++ b/tests/multimodal/test_hasher.py @@ -268,3 +268,43 @@ def test_hash_media_io_noop_config_preserves_hash(): assert hasher.hash_kwargs("blake3", image=loaded) == hasher.hash_kwargs( "blake3", image=plain ) + + +# The digest input is a concatenation of byte chunks, so it has to be uniquely +# decodable. Each case below is a pair of distinct processor kwargs that used to +# serialize to the same bytes, which made two requests share an mm hash -- and +# therefore share both the processor cache entry and the prefix-cache block key. +IMAGE = b"\x89PNG\r\n\x1a\n" + + +def _hash(**mm_processor_kwargs: object) -> str: + return MultiModalHasher.hash_kwargs( + "blake3", model_id="m", image=IMAGE, **mm_processor_kwargs + ) + + +def test_hash_collision_kwargs_key_value_boundary(): + # Both used to flatten to b"ab" + b"c". + assert _hash(**{"ab": "c"}) != _hash(**{"a": "bc"}) + + +def test_hash_collision_nested_vs_flattened_key(): + nested = _hash(size={"shortest_edge": 224}) + flattened = _hash(**{"size.shortest_edge": 224}) + assert nested != flattened + + +def test_hash_collision_sequence_vs_mapping(): + assert _hash(fps=[2, 4]) != _hash(fps={"0": 2, "1": 4}) + + +@pytest.mark.parametrize("empty", ["", b"", []]) +def test_hash_collision_none_vs_empty(empty): + assert _hash(video_pruning_rate=None) != _hash(video_pruning_rate=empty) + + +def test_hash_collision_empty_container_vs_omitted(): + omitted = _hash() + assert _hash(size={}) != omitted + assert _hash(size=[]) != omitted + assert _hash(size={}) != _hash(size=[]) diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 7d6a7fd7e1..de3e333886 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -1324,3 +1324,40 @@ def test_processor_inputs_hashes_ignore_unrelated_kwargs(): ) assert inputs.get_mm_hashes("test-model", "blake3") == {"image": ["image-uuid"]} + + +@pytest.mark.parametrize( + ("left", "right"), + [ + # Shifting the key/value boundary: both flatten to the dotted key + # "mm_processor_kwargs.abc" followed by no value bytes. + ({"ab": "c"}, {"a": "bc"}), + # A nested mapping and a caller-supplied dotted key flatten alike. + ({"size": {"shortest_edge": 224}}, {"size.shortest_edge": 224}), + # A sequence and a mapping keyed by stringified indices flatten alike. + ({"fps": [2, 4]}, {"fps": {"0": 2, "1": 4}}), + # None contributes the key alone, which a zero-byte value also does. + ({"video_pruning_rate": None}, {"video_pruning_rate": ""}), + # An empty container contributes nothing, as does omitting the key. + ({"size": {}}, {}), + ], +) +def test_processor_inputs_hashes_distinguish_kwargs_shapes(left, right): + """Distinct processor kwargs must not share a multi-modal hash. + + ``hf_processor_mm_kwargs`` is per-request input, so both the keys and the + values here are caller-controlled. The hash is the identity of the + processor cache entry and is mixed into the prefix-cache block key, so two + requests sharing one is a cross-request cache hit. + """ + image = random_image(np.random.RandomState(0), min_wh=8, max_wh=9) + mm_data_items = MultiModalDataParser().parse_mm_data({"image": [image]}) + + def hash_with(hf_processor_mm_kwargs): + return ProcessorInputs( + prompt=[], + mm_data_items=mm_data_items, + hf_processor_mm_kwargs=hf_processor_mm_kwargs, + ).get_mm_hashes("test-model", "blake3")["image"][0] + + assert hash_with(left) != hash_with(right) diff --git a/vllm/multimodal/hasher.py b/vllm/multimodal/hasher.py index eb4ab4d28a..ae52ceefc4 100644 --- a/vllm/multimodal/hasher.py +++ b/vllm/multimodal/hasher.py @@ -18,6 +18,28 @@ from .media import MediaWithBytes logger = init_logger(__name__) +# Framing for the digest input. The hash is built by feeding a stream of byte +# chunks to `hasher.update`, so the stream must be uniquely decodable: without +# an explicit length in front of every chunk and a tag in front of every +# container, distinct inputs serialize to identical bytes and share a cache +# entry. See `test_hash_collision_*` in tests/multimodal/test_hasher.py. +_LENGTH_BYTES = 8 +_TAG_NONE = b"\x00" +_TAG_SEQUENCE = b"\x01" +_TAG_MAPPING = b"\x02" +_TAG_LEAF = b"\x03" + + +def _encode_length(value: int) -> bytes: + return value.to_bytes(_LENGTH_BYTES, "little") + + +def _framed(chunk: bytes | memoryview) -> Iterable[bytes | memoryview]: + """Yield *chunk* preceded by its size in bytes.""" + size = chunk.nbytes if isinstance(chunk, memoryview) else len(chunk) + yield _encode_length(size) + yield chunk + @functools.lru_cache(maxsize=3) def _get_hasher_factory( @@ -61,20 +83,27 @@ def _get_image_id_bytes(image: Image.Image) -> bytes | None: class MultiModalHasher: + """Derives multi-modal cache keys. + + Every method here yields *framed* chunks: each chunk is preceded by its + length and each container by its kind, so that the concatenation fed to the + digest is uniquely decodable and distinct inputs cannot share a key. + """ + @classmethod def serialize_item(cls, obj: object) -> Iterable[bytes | memoryview]: # Simple cases if isinstance(obj, (bytes, memoryview)): - return (obj,) + return _framed(obj) if isinstance(obj, str): - return (obj.encode("utf-8"),) + return _framed(obj.encode("utf-8")) if isinstance(obj, (int, float)): - return (np.array(obj).tobytes(),) + return _framed(np.array(obj).tobytes()) if isinstance(obj, Image.Image): image_id = _get_image_id_bytes(obj) if image_id is not None: - return (image_id,) + return _framed(image_id) data = {"mode": obj.mode, "data": np.asarray(obj)} palette = obj.palette @@ -88,7 +117,7 @@ class MultiModalHasher: if isinstance(obj, MediaWithBytes) and isinstance(obj.media, Image.Image): image_id = _get_image_id_bytes(obj.media) if image_id is not None: - return (image_id,) + return _framed(image_id) if obj.io_config: return cls.iter_item_to_bytes( @@ -151,7 +180,7 @@ class MultiModalHasher: "No serialization method found for %s. Falling back to pickle.", type(obj) ) - return (pickle.dumps(obj),) + return _framed(pickle.dumps(obj)) @classmethod def iter_item_to_bytes( @@ -159,18 +188,36 @@ class MultiModalHasher: key: str, obj: object, ) -> Iterable[bytes | memoryview]: + """Yield the digest input for a single ``key``/``obj`` pair.""" + yield from _framed(key.encode("utf-8")) + yield from cls.iter_value_to_bytes(obj) + + @classmethod + def iter_value_to_bytes( + cls, + obj: object, + ) -> Iterable[bytes | memoryview]: + """Yield the digest input for a value, tagged by its container kind. + + Containers carry their kind and length so that a nested structure can + never serialize to the same bytes as a differently shaped one (a list + and a mapping keyed by stringified indices, for example). + """ if obj is None: - yield key.encode("utf-8") - return - # Recursive cases - if isinstance(obj, (list, tuple)): - for i, elem in enumerate(obj): - yield from cls.iter_item_to_bytes(f"{key}.{i}", elem) + yield _TAG_NONE + elif isinstance(obj, (list, tuple)): + yield _TAG_SEQUENCE + yield _encode_length(len(obj)) + for elem in obj: + yield from cls.iter_value_to_bytes(elem) elif isinstance(obj, dict): + yield _TAG_MAPPING + yield _encode_length(len(obj)) for k, v in obj.items(): - yield from cls.iter_item_to_bytes(f"{key}.{k}", v) + yield from _framed(str(k).encode("utf-8")) + yield from cls.iter_value_to_bytes(v) else: - yield key.encode("utf-8") + yield _TAG_LEAF yield from cls.serialize_item(obj) @classmethod