[Bugfix][Multimodal] Frame the multi-modal hash digest input (#54283)
Signed-off-by: hotragn <hotragn.pettugani_2024@woxsen.edu.in> Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude Opus 5
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
01c7bf8813
commit
a8d1aa9c99
@@ -268,3 +268,43 @@ def test_hash_media_io_noop_config_preserves_hash():
|
||||
assert hasher.hash_kwargs("blake3", image=loaded) == hasher.hash_kwargs(
|
||||
"blake3", image=plain
|
||||
)
|
||||
|
||||
|
||||
# The digest input is a concatenation of byte chunks, so it has to be uniquely
|
||||
# decodable. Each case below is a pair of distinct processor kwargs that used to
|
||||
# serialize to the same bytes, which made two requests share an mm hash -- and
|
||||
# therefore share both the processor cache entry and the prefix-cache block key.
|
||||
IMAGE = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
|
||||
def _hash(**mm_processor_kwargs: object) -> str:
|
||||
return MultiModalHasher.hash_kwargs(
|
||||
"blake3", model_id="m", image=IMAGE, **mm_processor_kwargs
|
||||
)
|
||||
|
||||
|
||||
def test_hash_collision_kwargs_key_value_boundary():
|
||||
# Both used to flatten to b"ab" + b"c".
|
||||
assert _hash(**{"ab": "c"}) != _hash(**{"a": "bc"})
|
||||
|
||||
|
||||
def test_hash_collision_nested_vs_flattened_key():
|
||||
nested = _hash(size={"shortest_edge": 224})
|
||||
flattened = _hash(**{"size.shortest_edge": 224})
|
||||
assert nested != flattened
|
||||
|
||||
|
||||
def test_hash_collision_sequence_vs_mapping():
|
||||
assert _hash(fps=[2, 4]) != _hash(fps={"0": 2, "1": 4})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("empty", ["", b"", []])
|
||||
def test_hash_collision_none_vs_empty(empty):
|
||||
assert _hash(video_pruning_rate=None) != _hash(video_pruning_rate=empty)
|
||||
|
||||
|
||||
def test_hash_collision_empty_container_vs_omitted():
|
||||
omitted = _hash()
|
||||
assert _hash(size={}) != omitted
|
||||
assert _hash(size=[]) != omitted
|
||||
assert _hash(size={}) != _hash(size=[])
|
||||
|
||||
@@ -1324,3 +1324,40 @@ def test_processor_inputs_hashes_ignore_unrelated_kwargs():
|
||||
)
|
||||
|
||||
assert inputs.get_mm_hashes("test-model", "blake3") == {"image": ["image-uuid"]}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("left", "right"),
|
||||
[
|
||||
# Shifting the key/value boundary: both flatten to the dotted key
|
||||
# "mm_processor_kwargs.abc" followed by no value bytes.
|
||||
({"ab": "c"}, {"a": "bc"}),
|
||||
# A nested mapping and a caller-supplied dotted key flatten alike.
|
||||
({"size": {"shortest_edge": 224}}, {"size.shortest_edge": 224}),
|
||||
# A sequence and a mapping keyed by stringified indices flatten alike.
|
||||
({"fps": [2, 4]}, {"fps": {"0": 2, "1": 4}}),
|
||||
# None contributes the key alone, which a zero-byte value also does.
|
||||
({"video_pruning_rate": None}, {"video_pruning_rate": ""}),
|
||||
# An empty container contributes nothing, as does omitting the key.
|
||||
({"size": {}}, {}),
|
||||
],
|
||||
)
|
||||
def test_processor_inputs_hashes_distinguish_kwargs_shapes(left, right):
|
||||
"""Distinct processor kwargs must not share a multi-modal hash.
|
||||
|
||||
``hf_processor_mm_kwargs`` is per-request input, so both the keys and the
|
||||
values here are caller-controlled. The hash is the identity of the
|
||||
processor cache entry and is mixed into the prefix-cache block key, so two
|
||||
requests sharing one is a cross-request cache hit.
|
||||
"""
|
||||
image = random_image(np.random.RandomState(0), min_wh=8, max_wh=9)
|
||||
mm_data_items = MultiModalDataParser().parse_mm_data({"image": [image]})
|
||||
|
||||
def hash_with(hf_processor_mm_kwargs):
|
||||
return ProcessorInputs(
|
||||
prompt=[],
|
||||
mm_data_items=mm_data_items,
|
||||
hf_processor_mm_kwargs=hf_processor_mm_kwargs,
|
||||
).get_mm_hashes("test-model", "blake3")["image"][0]
|
||||
|
||||
assert hash_with(left) != hash_with(right)
|
||||
|
||||
+61
-14
@@ -18,6 +18,28 @@ from .media import MediaWithBytes
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Framing for the digest input. The hash is built by feeding a stream of byte
|
||||
# chunks to `hasher.update`, so the stream must be uniquely decodable: without
|
||||
# an explicit length in front of every chunk and a tag in front of every
|
||||
# container, distinct inputs serialize to identical bytes and share a cache
|
||||
# entry. See `test_hash_collision_*` in tests/multimodal/test_hasher.py.
|
||||
_LENGTH_BYTES = 8
|
||||
_TAG_NONE = b"\x00"
|
||||
_TAG_SEQUENCE = b"\x01"
|
||||
_TAG_MAPPING = b"\x02"
|
||||
_TAG_LEAF = b"\x03"
|
||||
|
||||
|
||||
def _encode_length(value: int) -> bytes:
|
||||
return value.to_bytes(_LENGTH_BYTES, "little")
|
||||
|
||||
|
||||
def _framed(chunk: bytes | memoryview) -> Iterable[bytes | memoryview]:
|
||||
"""Yield *chunk* preceded by its size in bytes."""
|
||||
size = chunk.nbytes if isinstance(chunk, memoryview) else len(chunk)
|
||||
yield _encode_length(size)
|
||||
yield chunk
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=3)
|
||||
def _get_hasher_factory(
|
||||
@@ -61,20 +83,27 @@ def _get_image_id_bytes(image: Image.Image) -> bytes | None:
|
||||
|
||||
|
||||
class MultiModalHasher:
|
||||
"""Derives multi-modal cache keys.
|
||||
|
||||
Every method here yields *framed* chunks: each chunk is preceded by its
|
||||
length and each container by its kind, so that the concatenation fed to the
|
||||
digest is uniquely decodable and distinct inputs cannot share a key.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def serialize_item(cls, obj: object) -> Iterable[bytes | memoryview]:
|
||||
# Simple cases
|
||||
if isinstance(obj, (bytes, memoryview)):
|
||||
return (obj,)
|
||||
return _framed(obj)
|
||||
if isinstance(obj, str):
|
||||
return (obj.encode("utf-8"),)
|
||||
return _framed(obj.encode("utf-8"))
|
||||
if isinstance(obj, (int, float)):
|
||||
return (np.array(obj).tobytes(),)
|
||||
return _framed(np.array(obj).tobytes())
|
||||
|
||||
if isinstance(obj, Image.Image):
|
||||
image_id = _get_image_id_bytes(obj)
|
||||
if image_id is not None:
|
||||
return (image_id,)
|
||||
return _framed(image_id)
|
||||
|
||||
data = {"mode": obj.mode, "data": np.asarray(obj)}
|
||||
palette = obj.palette
|
||||
@@ -88,7 +117,7 @@ class MultiModalHasher:
|
||||
if isinstance(obj, MediaWithBytes) and isinstance(obj.media, Image.Image):
|
||||
image_id = _get_image_id_bytes(obj.media)
|
||||
if image_id is not None:
|
||||
return (image_id,)
|
||||
return _framed(image_id)
|
||||
|
||||
if obj.io_config:
|
||||
return cls.iter_item_to_bytes(
|
||||
@@ -151,7 +180,7 @@ class MultiModalHasher:
|
||||
"No serialization method found for %s. Falling back to pickle.", type(obj)
|
||||
)
|
||||
|
||||
return (pickle.dumps(obj),)
|
||||
return _framed(pickle.dumps(obj))
|
||||
|
||||
@classmethod
|
||||
def iter_item_to_bytes(
|
||||
@@ -159,18 +188,36 @@ class MultiModalHasher:
|
||||
key: str,
|
||||
obj: object,
|
||||
) -> Iterable[bytes | memoryview]:
|
||||
"""Yield the digest input for a single ``key``/``obj`` pair."""
|
||||
yield from _framed(key.encode("utf-8"))
|
||||
yield from cls.iter_value_to_bytes(obj)
|
||||
|
||||
@classmethod
|
||||
def iter_value_to_bytes(
|
||||
cls,
|
||||
obj: object,
|
||||
) -> Iterable[bytes | memoryview]:
|
||||
"""Yield the digest input for a value, tagged by its container kind.
|
||||
|
||||
Containers carry their kind and length so that a nested structure can
|
||||
never serialize to the same bytes as a differently shaped one (a list
|
||||
and a mapping keyed by stringified indices, for example).
|
||||
"""
|
||||
if obj is None:
|
||||
yield key.encode("utf-8")
|
||||
return
|
||||
# Recursive cases
|
||||
if isinstance(obj, (list, tuple)):
|
||||
for i, elem in enumerate(obj):
|
||||
yield from cls.iter_item_to_bytes(f"{key}.{i}", elem)
|
||||
yield _TAG_NONE
|
||||
elif isinstance(obj, (list, tuple)):
|
||||
yield _TAG_SEQUENCE
|
||||
yield _encode_length(len(obj))
|
||||
for elem in obj:
|
||||
yield from cls.iter_value_to_bytes(elem)
|
||||
elif isinstance(obj, dict):
|
||||
yield _TAG_MAPPING
|
||||
yield _encode_length(len(obj))
|
||||
for k, v in obj.items():
|
||||
yield from cls.iter_item_to_bytes(f"{key}.{k}", v)
|
||||
yield from _framed(str(k).encode("utf-8"))
|
||||
yield from cls.iter_value_to_bytes(v)
|
||||
else:
|
||||
yield key.encode("utf-8")
|
||||
yield _TAG_LEAF
|
||||
yield from cls.serialize_item(obj)
|
||||
|
||||
@classmethod
|
||||
|
||||
Reference in New Issue
Block a user