[Bugfix][Multimodal] Frame the multi-modal hash digest input (#54283)

Signed-off-by: hotragn <hotragn.pettugani_2024@woxsen.edu.in>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Hotragn Pettugani
2026-09-18 21:06:48 -07:00
committed by GitHub
co-authored by Claude Opus 5 mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 01c7bf8813
commit a8d1aa9c99
3 changed files with 138 additions and 14 deletions
+40
View File
@@ -268,3 +268,43 @@ def test_hash_media_io_noop_config_preserves_hash():
assert hasher.hash_kwargs("blake3", image=loaded) == hasher.hash_kwargs(
"blake3", image=plain
)
# The digest input is a concatenation of byte chunks, so it has to be uniquely
# decodable. Each case below is a pair of distinct processor kwargs that used to
# serialize to the same bytes, which made two requests share an mm hash -- and
# therefore share both the processor cache entry and the prefix-cache block key.
IMAGE = b"\x89PNG\r\n\x1a\n"
def _hash(**mm_processor_kwargs: object) -> str:
return MultiModalHasher.hash_kwargs(
"blake3", model_id="m", image=IMAGE, **mm_processor_kwargs
)
def test_hash_collision_kwargs_key_value_boundary():
# Both used to flatten to b"ab" + b"c".
assert _hash(**{"ab": "c"}) != _hash(**{"a": "bc"})
def test_hash_collision_nested_vs_flattened_key():
nested = _hash(size={"shortest_edge": 224})
flattened = _hash(**{"size.shortest_edge": 224})
assert nested != flattened
def test_hash_collision_sequence_vs_mapping():
assert _hash(fps=[2, 4]) != _hash(fps={"0": 2, "1": 4})
@pytest.mark.parametrize("empty", ["", b"", []])
def test_hash_collision_none_vs_empty(empty):
assert _hash(video_pruning_rate=None) != _hash(video_pruning_rate=empty)
def test_hash_collision_empty_container_vs_omitted():
omitted = _hash()
assert _hash(size={}) != omitted
assert _hash(size=[]) != omitted
assert _hash(size={}) != _hash(size=[])
+37
View File
@@ -1324,3 +1324,40 @@ def test_processor_inputs_hashes_ignore_unrelated_kwargs():
)
assert inputs.get_mm_hashes("test-model", "blake3") == {"image": ["image-uuid"]}
@pytest.mark.parametrize(
("left", "right"),
[
# Shifting the key/value boundary: both flatten to the dotted key
# "mm_processor_kwargs.abc" followed by no value bytes.
({"ab": "c"}, {"a": "bc"}),
# A nested mapping and a caller-supplied dotted key flatten alike.
({"size": {"shortest_edge": 224}}, {"size.shortest_edge": 224}),
# A sequence and a mapping keyed by stringified indices flatten alike.
({"fps": [2, 4]}, {"fps": {"0": 2, "1": 4}}),
# None contributes the key alone, which a zero-byte value also does.
({"video_pruning_rate": None}, {"video_pruning_rate": ""}),
# An empty container contributes nothing, as does omitting the key.
({"size": {}}, {}),
],
)
def test_processor_inputs_hashes_distinguish_kwargs_shapes(left, right):
"""Distinct processor kwargs must not share a multi-modal hash.
``hf_processor_mm_kwargs`` is per-request input, so both the keys and the
values here are caller-controlled. The hash is the identity of the
processor cache entry and is mixed into the prefix-cache block key, so two
requests sharing one is a cross-request cache hit.
"""
image = random_image(np.random.RandomState(0), min_wh=8, max_wh=9)
mm_data_items = MultiModalDataParser().parse_mm_data({"image": [image]})
def hash_with(hf_processor_mm_kwargs):
return ProcessorInputs(
prompt=[],
mm_data_items=mm_data_items,
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
).get_mm_hashes("test-model", "blake3")["image"][0]
assert hash_with(left) != hash_with(right)
+61 -14
View File
@@ -18,6 +18,28 @@ from .media import MediaWithBytes
logger = init_logger(__name__)
# Framing for the digest input. The hash is built by feeding a stream of byte
# chunks to `hasher.update`, so the stream must be uniquely decodable: without
# an explicit length in front of every chunk and a tag in front of every
# container, distinct inputs serialize to identical bytes and share a cache
# entry. See `test_hash_collision_*` in tests/multimodal/test_hasher.py.
_LENGTH_BYTES = 8
_TAG_NONE = b"\x00"
_TAG_SEQUENCE = b"\x01"
_TAG_MAPPING = b"\x02"
_TAG_LEAF = b"\x03"
def _encode_length(value: int) -> bytes:
return value.to_bytes(_LENGTH_BYTES, "little")
def _framed(chunk: bytes | memoryview) -> Iterable[bytes | memoryview]:
"""Yield *chunk* preceded by its size in bytes."""
size = chunk.nbytes if isinstance(chunk, memoryview) else len(chunk)
yield _encode_length(size)
yield chunk
@functools.lru_cache(maxsize=3)
def _get_hasher_factory(
@@ -61,20 +83,27 @@ def _get_image_id_bytes(image: Image.Image) -> bytes | None:
class MultiModalHasher:
"""Derives multi-modal cache keys.
Every method here yields *framed* chunks: each chunk is preceded by its
length and each container by its kind, so that the concatenation fed to the
digest is uniquely decodable and distinct inputs cannot share a key.
"""
@classmethod
def serialize_item(cls, obj: object) -> Iterable[bytes | memoryview]:
# Simple cases
if isinstance(obj, (bytes, memoryview)):
return (obj,)
return _framed(obj)
if isinstance(obj, str):
return (obj.encode("utf-8"),)
return _framed(obj.encode("utf-8"))
if isinstance(obj, (int, float)):
return (np.array(obj).tobytes(),)
return _framed(np.array(obj).tobytes())
if isinstance(obj, Image.Image):
image_id = _get_image_id_bytes(obj)
if image_id is not None:
return (image_id,)
return _framed(image_id)
data = {"mode": obj.mode, "data": np.asarray(obj)}
palette = obj.palette
@@ -88,7 +117,7 @@ class MultiModalHasher:
if isinstance(obj, MediaWithBytes) and isinstance(obj.media, Image.Image):
image_id = _get_image_id_bytes(obj.media)
if image_id is not None:
return (image_id,)
return _framed(image_id)
if obj.io_config:
return cls.iter_item_to_bytes(
@@ -151,7 +180,7 @@ class MultiModalHasher:
"No serialization method found for %s. Falling back to pickle.", type(obj)
)
return (pickle.dumps(obj),)
return _framed(pickle.dumps(obj))
@classmethod
def iter_item_to_bytes(
@@ -159,18 +188,36 @@ class MultiModalHasher:
key: str,
obj: object,
) -> Iterable[bytes | memoryview]:
"""Yield the digest input for a single ``key``/``obj`` pair."""
yield from _framed(key.encode("utf-8"))
yield from cls.iter_value_to_bytes(obj)
@classmethod
def iter_value_to_bytes(
cls,
obj: object,
) -> Iterable[bytes | memoryview]:
"""Yield the digest input for a value, tagged by its container kind.
Containers carry their kind and length so that a nested structure can
never serialize to the same bytes as a differently shaped one (a list
and a mapping keyed by stringified indices, for example).
"""
if obj is None:
yield key.encode("utf-8")
return
# Recursive cases
if isinstance(obj, (list, tuple)):
for i, elem in enumerate(obj):
yield from cls.iter_item_to_bytes(f"{key}.{i}", elem)
yield _TAG_NONE
elif isinstance(obj, (list, tuple)):
yield _TAG_SEQUENCE
yield _encode_length(len(obj))
for elem in obj:
yield from cls.iter_value_to_bytes(elem)
elif isinstance(obj, dict):
yield _TAG_MAPPING
yield _encode_length(len(obj))
for k, v in obj.items():
yield from cls.iter_item_to_bytes(f"{key}.{k}", v)
yield from _framed(str(k).encode("utf-8"))
yield from cls.iter_value_to_bytes(v)
else:
yield key.encode("utf-8")
yield _TAG_LEAF
yield from cls.serialize_item(obj)
@classmethod