[Multimodal] Expose mm hash algothrim selection to cli args (#49686)
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -396,7 +396,7 @@ FIPS compliance depends on many factors, so a vLLM deployment is not automatical
|
||||
|
||||
Operators running vLLM on FIPS-enabled hosts should select FIPS-approved algorithms via the following knobs:
|
||||
|
||||
- **Multimodal input hashing** — `VLLM_MM_HASHER_ALGORITHM` defaults to `blake3`, which is not FIPS-approved. Set it to `sha256` or `sha512` in FIPS-enabled environments.
|
||||
- **Multimodal input hashing** — `--mm-hasher-algorithm` (config field `mm_hasher_algorithm`) defaults to `blake3`, which is not FIPS-approved. Set it to `sha256` or `sha512` in FIPS-enabled environments.
|
||||
- **Prefix-cache hashing** — set `--prefix-caching-hash-algo` (config field `prefix_caching_hash_algo`) to `sha256` or `sha256_cbor`. The `xxhash` and `xxhash_cbor` options are not FIPS-approved.
|
||||
- **TLS ciphers** — use `--ssl-ciphers` to restrict the API server's TLS handshake to FIPS-approved cipher suites that match your environment's policy.
|
||||
|
||||
@@ -408,7 +408,13 @@ vLLM uses MD5 in a few places to derive non-security cache keys (for example, co
|
||||
|
||||
Some dependencies expose hash implementations that are not FIPS-approved. vLLM only invokes them when the corresponding algorithm is selected, but operators with strict cryptographic controls may want to ensure the code paths are not exercised — and, where policy requires, that the packages themselves are absent:
|
||||
|
||||
- `blake3` — currently listed in `requirements/common.txt`, so a standard install pulls it in. It is imported lazily and only used when `VLLM_MM_HASHER_ALGORITHM=blake3` (the default). Setting `VLLM_MM_HASHER_ALGORITHM` to `sha256` or `sha512` is sufficient to keep the non-FIPS code path dormant. If your policy additionally forbids the package being present, uninstall it after `pip install` (`pip uninstall blake3`); vLLM will continue to function as long as `VLLM_MM_HASHER_ALGORITHM` is set to a non-blake3 value.
|
||||
- `blake3` — currently listed in `requirements/common.txt`, so a standard
|
||||
install pulls it in. It is imported lazily and only used when
|
||||
`mm_hasher_algorithm=blake3` (the default). Setting
|
||||
`--mm-hasher-algorithm sha256` or `--mm-hasher-algorithm sha512` is sufficient
|
||||
to keep the non-FIPS code path dormant. If your policy additionally forbids
|
||||
the package being present, uninstall it after installation; vLLM will
|
||||
continue to function as long as a non-blake3 algorithm is selected.
|
||||
- `xxhash` — a true optional dependency (not in `requirements/common.txt`). It is only imported when an `xxhash`-based prefix-cache algorithm is selected. Leave it uninstalled and select a `sha256`-based prefix-cache algorithm.
|
||||
|
||||
### Beyond hashing: other FIPS considerations
|
||||
|
||||
@@ -24,6 +24,11 @@ def test_mm_encoder_attn_backend_invalid():
|
||||
MultiModalConfig(mm_encoder_attn_backend="not_a_backend") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_mm_hasher_algorithm_invalid():
|
||||
with pytest.raises(ValueError, match="mm_hasher_algorithm"):
|
||||
MultiModalConfig(mm_hasher_algorithm="md5") # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_mm_encoder_attn_backend_hash_updates():
|
||||
base_hash = MultiModalConfig().compute_hash()
|
||||
overridden_hash = MultiModalConfig(
|
||||
|
||||
@@ -50,6 +50,7 @@ class _Tokenizer:
|
||||
class _MMConfig:
|
||||
enable_mm_embeds = False
|
||||
mm_processor_cache_gb = 1
|
||||
mm_hasher_algorithm = "blake3"
|
||||
|
||||
def merge_mm_processor_kwargs(self, kwargs):
|
||||
return dict(kwargs)
|
||||
|
||||
@@ -144,7 +144,8 @@ def _compare_caches(
|
||||
for _ in range(int(item_capacity / hit_rate))
|
||||
]
|
||||
all_hashes = [
|
||||
MultiModalHasher.hash_kwargs(item=item.get_data()) for item in all_items
|
||||
MultiModalHasher.hash_kwargs("blake3", item=item.get_data())
|
||||
for item in all_items
|
||||
]
|
||||
|
||||
prompt_update = PromptInsertion("dummy", "target", "insertion").resolve(0)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import hashlib
|
||||
import uuid
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
@@ -9,6 +10,7 @@ import pytest
|
||||
import torch
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
from vllm.config.multimodal import MMHasherAlgorithm
|
||||
from vllm.multimodal.hasher import MultiModalHasher
|
||||
from vllm.multimodal.media.base import MediaWithBytes
|
||||
from vllm.multimodal.media.image import ImageMediaIO
|
||||
@@ -20,12 +22,26 @@ ASSETS_DIR = Path(__file__).parent / "assets"
|
||||
assert ASSETS_DIR.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("algorithm", ["sha256", "sha512"])
|
||||
def test_hash_algorithm(algorithm: MMHasherAlgorithm):
|
||||
hasher = getattr(hashlib, algorithm)()
|
||||
for bytes_ in MultiModalHasher.iter_item_to_bytes("value", "test"):
|
||||
hasher.update(bytes_)
|
||||
|
||||
assert MultiModalHasher.hash_kwargs(algorithm, value="test") == hasher.hexdigest()
|
||||
|
||||
|
||||
def test_hash_algorithm_required():
|
||||
with pytest.raises(TypeError, match="algorithm"):
|
||||
MultiModalHasher.hash_kwargs(value="test") # type: ignore[call-arg]
|
||||
|
||||
|
||||
def test_hash_single_item_different_shape():
|
||||
x1 = torch.zeros(())
|
||||
x2 = torch.zeros((1,))
|
||||
|
||||
hasher = MultiModalHasher
|
||||
assert hasher.hash_kwargs(x=x1) != hasher.hash_kwargs(x=x2)
|
||||
assert hasher.hash_kwargs("blake3", x=x1) != hasher.hash_kwargs("blake3", x=x2)
|
||||
|
||||
|
||||
def test_hash_key_order_invariant():
|
||||
@@ -33,7 +49,9 @@ def test_hash_key_order_invariant():
|
||||
y = torch.ones((5, 10))
|
||||
|
||||
hasher = MultiModalHasher
|
||||
assert hasher.hash_kwargs(x=x, y=y) == hasher.hash_kwargs(y=y, x=x)
|
||||
assert hasher.hash_kwargs("blake3", x=x, y=y) == hasher.hash_kwargs(
|
||||
"blake3", y=y, x=x
|
||||
)
|
||||
|
||||
|
||||
# NOTE: Images that are the same visually are allowed to have the same hash
|
||||
@@ -44,7 +62,9 @@ def test_hash_collision_image_mode(mode_pair):
|
||||
image2 = Image.new(mode2, size=(10, 10), color=1)
|
||||
|
||||
hasher = MultiModalHasher
|
||||
assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)
|
||||
assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(
|
||||
"blake3", image=image2
|
||||
)
|
||||
|
||||
|
||||
def test_hash_collision_image_palette():
|
||||
@@ -53,7 +73,9 @@ def test_hash_collision_image_palette():
|
||||
image2 = Image.open(ASSETS_DIR / "image2.png")
|
||||
|
||||
hasher = MultiModalHasher
|
||||
assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)
|
||||
assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(
|
||||
"blake3", image=image2
|
||||
)
|
||||
|
||||
|
||||
def test_hash_collision_image_transpose():
|
||||
@@ -64,7 +86,9 @@ def test_hash_collision_image_transpose():
|
||||
ImageDraw.Draw(image2).line([(0, 0), (0, 10)])
|
||||
|
||||
hasher = MultiModalHasher
|
||||
assert hasher.hash_kwargs(image=image1) != hasher.hash_kwargs(image=image2)
|
||||
assert hasher.hash_kwargs("blake3", image=image1) != hasher.hash_kwargs(
|
||||
"blake3", image=image2
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
@@ -74,7 +98,9 @@ def test_hash_collision_tensor_shape(dtype):
|
||||
arr2 = torch.zeros((10, 20, 5, 3), dtype=dtype)
|
||||
|
||||
hasher = MultiModalHasher
|
||||
assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)
|
||||
assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs(
|
||||
"blake3", data=arr2
|
||||
)
|
||||
|
||||
|
||||
def test_hash_collision_array_shape():
|
||||
@@ -83,7 +109,9 @@ def test_hash_collision_array_shape():
|
||||
arr2 = np.zeros((10, 20, 5, 3))
|
||||
|
||||
hasher = MultiModalHasher
|
||||
assert hasher.hash_kwargs(data=arr1) != hasher.hash_kwargs(data=arr2)
|
||||
assert hasher.hash_kwargs("blake3", data=arr1) != hasher.hash_kwargs(
|
||||
"blake3", data=arr2
|
||||
)
|
||||
|
||||
|
||||
def test_hash_collision_video_num_frames():
|
||||
@@ -104,8 +132,8 @@ def test_hash_collision_video_num_frames():
|
||||
return items.get_all_items_for_hash()[0]
|
||||
|
||||
hasher = MultiModalHasher
|
||||
assert hasher.hash_kwargs(video=item_for_hash(2)) != hasher.hash_kwargs(
|
||||
video=item_for_hash(4)
|
||||
assert hasher.hash_kwargs("blake3", video=item_for_hash(2)) != hasher.hash_kwargs(
|
||||
"blake3", video=item_for_hash(4)
|
||||
)
|
||||
|
||||
|
||||
@@ -118,7 +146,9 @@ def test_hash_non_contiguous_array():
|
||||
|
||||
hasher = MultiModalHasher
|
||||
# Both should be hashable and produce the same hashes
|
||||
assert hasher.hash_kwargs(data=arr) == hasher.hash_kwargs(data=arr_c)
|
||||
assert hasher.hash_kwargs("blake3", data=arr) == hasher.hash_kwargs(
|
||||
"blake3", data=arr_c
|
||||
)
|
||||
|
||||
|
||||
def test_hash_image_exif_id():
|
||||
@@ -133,9 +163,13 @@ def test_hash_image_exif_id():
|
||||
|
||||
hasher = MultiModalHasher
|
||||
# first image has UUID in ImageID, so it should hash to that UUID
|
||||
assert hasher.hash_kwargs(image=image1) == hasher.hash_kwargs(image=id.bytes)
|
||||
assert hasher.hash_kwargs("blake3", image=image1) == hasher.hash_kwargs(
|
||||
"blake3", image=id.bytes
|
||||
)
|
||||
# second image has non-UUID in ImageID, so it should hash to the image data
|
||||
assert hasher.hash_kwargs(image=image2) == hasher.hash_kwargs(image=image2a)
|
||||
assert hasher.hash_kwargs("blake3", image=image2) == hasher.hash_kwargs(
|
||||
"blake3", image=image2a
|
||||
)
|
||||
|
||||
|
||||
def _rgba_png_bytes() -> bytes:
|
||||
@@ -153,9 +187,15 @@ def test_hash_collision_media_io_config():
|
||||
keep = ImageMediaIO(image_mode=None).load_bytes(data)
|
||||
|
||||
hasher = MultiModalHasher
|
||||
assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=black)
|
||||
assert hasher.hash_kwargs(image=white) != hasher.hash_kwargs(image=keep)
|
||||
assert hasher.hash_kwargs(image=white) == hasher.hash_kwargs(image=white2)
|
||||
assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs(
|
||||
"blake3", image=black
|
||||
)
|
||||
assert hasher.hash_kwargs("blake3", image=white) != hasher.hash_kwargs(
|
||||
"blake3", image=keep
|
||||
)
|
||||
assert hasher.hash_kwargs("blake3", image=white) == hasher.hash_kwargs(
|
||||
"blake3", image=white2
|
||||
)
|
||||
|
||||
|
||||
def test_hash_media_io_noop_config_preserves_hash():
|
||||
@@ -169,4 +209,6 @@ def test_hash_media_io_noop_config_preserves_hash():
|
||||
|
||||
plain = MediaWithBytes(loaded.media, data)
|
||||
hasher = MultiModalHasher
|
||||
assert hasher.hash_kwargs(image=loaded) == hasher.hash_kwargs(image=plain)
|
||||
assert hasher.hash_kwargs("blake3", image=loaded) == hasher.hash_kwargs(
|
||||
"blake3", image=plain
|
||||
)
|
||||
|
||||
@@ -199,6 +199,31 @@ def test_group_and_batch_mm_items_split_by_fieldset():
|
||||
assert [num_items for num_items, _ in res] == [2, 1, 1, 1]
|
||||
|
||||
|
||||
def test_group_and_batch_mm_items_splits_shared_data_by_dtype():
|
||||
elem1 = MultiModalFieldElem(
|
||||
data=torch.zeros(1, dtype=torch.int32),
|
||||
field=MultiModalSharedField(batch_size=1),
|
||||
)
|
||||
elem2 = MultiModalFieldElem(
|
||||
data=torch.zeros(1, dtype=torch.float32),
|
||||
field=MultiModalSharedField(batch_size=1),
|
||||
)
|
||||
elem3 = MultiModalFieldElem(
|
||||
data=[torch.zeros(1, dtype=torch.int32), torch.zeros(1, dtype=torch.float32)],
|
||||
field=MultiModalSharedField(batch_size=1),
|
||||
)
|
||||
|
||||
res = group_and_batch_mm_items(
|
||||
[
|
||||
MultiModalKwargsItem({"x": elem1}),
|
||||
MultiModalKwargsItem({"x": elem2}),
|
||||
MultiModalKwargsItem({"x": elem3}),
|
||||
]
|
||||
)
|
||||
|
||||
assert [num_items for num_items, _ in res] == [1, 1, 1]
|
||||
|
||||
|
||||
def test_group_and_batch_mm_items_split_by_shared_data():
|
||||
elem1 = MultiModalFieldElem(
|
||||
data=torch.zeros(1, dtype=torch.uint8),
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.config.model_arch import (
|
||||
from vllm.config.multimodal import (
|
||||
MMCacheType,
|
||||
MMEncoderTPMode,
|
||||
MMHasherAlgorithm,
|
||||
MMTensorIPC,
|
||||
MultiModalConfig,
|
||||
)
|
||||
@@ -374,6 +375,7 @@ class ModelConfig:
|
||||
mm_processor_kwargs: InitVar[dict[str, Any] | None] = None
|
||||
mm_processor_cache_gb: InitVar[float | None] = None
|
||||
mm_processor_cache_type: InitVar[MMCacheType | None] = None
|
||||
mm_hasher_algorithm: InitVar[MMHasherAlgorithm | None] = None
|
||||
mm_shm_cache_max_object_size_mb: InitVar[int | None] = None
|
||||
mm_encoder_only: InitVar[bool | None] = None
|
||||
mm_encoder_tp_mode: InitVar[MMEncoderTPMode | None] = None
|
||||
@@ -503,6 +505,7 @@ class ModelConfig:
|
||||
mm_processor_kwargs: dict[str, Any] | None,
|
||||
mm_processor_cache_gb: float | None,
|
||||
mm_processor_cache_type: MMCacheType | None,
|
||||
mm_hasher_algorithm: MMHasherAlgorithm | None,
|
||||
mm_shm_cache_max_object_size_mb: int | None,
|
||||
mm_encoder_only: bool | None,
|
||||
mm_encoder_tp_mode: MMEncoderTPMode | None,
|
||||
@@ -737,6 +740,7 @@ class ModelConfig:
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
mm_processor_cache_gb=mm_processor_cache_gb,
|
||||
mm_processor_cache_type=mm_processor_cache_type,
|
||||
mm_hasher_algorithm=mm_hasher_algorithm,
|
||||
mm_shm_cache_max_object_size_mb=mm_shm_cache_max_object_size_mb,
|
||||
mm_encoder_only=mm_encoder_only,
|
||||
mm_encoder_tp_mode=mm_encoder_tp_mode,
|
||||
|
||||
@@ -3,13 +3,13 @@
|
||||
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, TypeAlias, TypedDict, final
|
||||
from typing import Any, Literal, TypeAlias, TypedDict, cast, final
|
||||
|
||||
from pydantic import ConfigDict, Field, field_validator, model_validator
|
||||
from pydantic.dataclasses import dataclass
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.utils import config
|
||||
from vllm.config.utils import config, get_from_deprecated_env_if_set
|
||||
from vllm.utils.hashing import safe_hash
|
||||
from vllm.v1.attention.backends.registry import AttentionBackendEnum
|
||||
|
||||
@@ -63,6 +63,19 @@ MMEncoderTPMode = Literal["weights", "data"]
|
||||
MMCacheType = Literal["shm", "lru"]
|
||||
VideoPruningMethod = Literal["evs", "vidcom2"]
|
||||
MMTensorIPC = Literal["direct_rpc", "torch_shm"]
|
||||
MMHasherAlgorithm = Literal["blake3", "sha256", "sha512"]
|
||||
|
||||
|
||||
def _get_mm_hasher_algorithm() -> MMHasherAlgorithm:
|
||||
env_value = get_from_deprecated_env_if_set(
|
||||
"VLLM_MM_HASHER_ALGORITHM",
|
||||
"v0.27",
|
||||
"mm_hasher_algorithm",
|
||||
)
|
||||
env_value = "blake3" if env_value is None else env_value
|
||||
return cast(MMHasherAlgorithm, env_value.lower())
|
||||
|
||||
|
||||
MMDummyOptions: TypeAlias = dict[str, BaseDummyOptions]
|
||||
"""
|
||||
A dictionary containing an entry for each modality type of dummy data.
|
||||
@@ -134,6 +147,11 @@ class MultiModalConfig:
|
||||
mm_processor_cache_type: MMCacheType = "lru"
|
||||
"""Type of cache to use for the multi-modal preprocessor/mapper. If `shm`,
|
||||
use shared memory FIFO cache. If `lru`, use mirrored LRU cache."""
|
||||
mm_hasher_algorithm: MMHasherAlgorithm = Field(
|
||||
default_factory=_get_mm_hasher_algorithm
|
||||
)
|
||||
"""Hash algorithm to use for multi-modal input caching. Use `"sha256"` or
|
||||
`"sha512"` for FIPS-compliant deployments."""
|
||||
mm_shm_cache_max_object_size_mb: int = Field(default=128, ge=0)
|
||||
"""Size limit (in MiB) for each object stored in the multi-modal processor
|
||||
shared memory cache. Only effective when `mm_processor_cache_type` is
|
||||
|
||||
@@ -86,7 +86,12 @@ from vllm.config.model import (
|
||||
RunnerOption,
|
||||
TokenizerMode,
|
||||
)
|
||||
from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MMTensorIPC
|
||||
from vllm.config.multimodal import (
|
||||
MMCacheType,
|
||||
MMEncoderTPMode,
|
||||
MMHasherAlgorithm,
|
||||
MMTensorIPC,
|
||||
)
|
||||
from vllm.config.observability import DetailedTraceModules
|
||||
from vllm.config.parallel import (
|
||||
All2AllBackend,
|
||||
@@ -566,6 +571,9 @@ class EngineArgs:
|
||||
mm_processor_cache_type: MMCacheType | None = (
|
||||
MultiModalConfig.mm_processor_cache_type
|
||||
)
|
||||
mm_hasher_algorithm: MMHasherAlgorithm = get_field(
|
||||
MultiModalConfig, "mm_hasher_algorithm"
|
||||
)
|
||||
mm_shm_cache_max_object_size_mb: int = (
|
||||
MultiModalConfig.mm_shm_cache_max_object_size_mb
|
||||
)
|
||||
@@ -1295,6 +1303,9 @@ class EngineArgs:
|
||||
multimodal_group.add_argument(
|
||||
"--mm-processor-cache-type", **multimodal_kwargs["mm_processor_cache_type"]
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-hasher-algorithm", **multimodal_kwargs["mm_hasher_algorithm"]
|
||||
)
|
||||
multimodal_group.add_argument(
|
||||
"--mm-shm-cache-max-object-size-mb",
|
||||
**multimodal_kwargs["mm_shm_cache_max_object_size_mb"],
|
||||
@@ -1712,6 +1723,7 @@ class EngineArgs:
|
||||
mm_processor_cache_gb=self.mm_processor_cache_gb,
|
||||
mm_processor_cache_type=self.mm_processor_cache_type,
|
||||
mm_shm_cache_max_object_size_mb=self.mm_shm_cache_max_object_size_mb,
|
||||
mm_hasher_algorithm=self.mm_hasher_algorithm,
|
||||
mm_encoder_only=self.mm_encoder_only,
|
||||
mm_encoder_tp_mode=self.mm_encoder_tp_mode,
|
||||
mm_encoder_attn_backend=self.mm_encoder_attn_backend,
|
||||
|
||||
@@ -217,7 +217,10 @@ class TerratorchMultiModalProcessor(BaseMultiModalProcessor[TerratorchProcessing
|
||||
)
|
||||
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
mm_hashes = inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
|
||||
mm_placeholders = {"image": [PlaceholderRange(offset=0, length=0)]}
|
||||
|
||||
|
||||
@@ -402,7 +402,10 @@ class MultiModalProcessor(BaseMultiModalProcessor[MultiModalProcessingInfo]):
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
mm_hashes = inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
|
||||
# For gemma3 we check `token_type_ids` as the key
|
||||
mm_token_type_ids = processed_data.pop("token_type_ids", None)
|
||||
|
||||
@@ -11,7 +11,7 @@ import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config.multimodal import MMHasherAlgorithm
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .media import MediaWithBytes
|
||||
@@ -20,7 +20,9 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=3)
|
||||
def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]:
|
||||
def _get_hasher_factory(
|
||||
algorithm: MMHasherAlgorithm,
|
||||
) -> Callable[[], "hashlib._Hash"]:
|
||||
"""
|
||||
Get the hasher factory based on the configured algorithm.
|
||||
|
||||
@@ -32,7 +34,6 @@ def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]:
|
||||
|
||||
See: https://github.com/vllm-project/vllm/issues/18334
|
||||
"""
|
||||
algorithm = algorithm.lower()
|
||||
|
||||
if algorithm == "blake3":
|
||||
from blake3 import blake3
|
||||
@@ -43,7 +44,7 @@ def _get_hasher_factory(algorithm: str) -> Callable[[], "hashlib._Hash"]:
|
||||
elif algorithm == "sha512":
|
||||
return hashlib.sha512
|
||||
else:
|
||||
# This should never happen due to env_with_choices validation
|
||||
# This should never happen due to config validation
|
||||
raise ValueError(f"Unsupported hash algorithm: {algorithm}")
|
||||
|
||||
|
||||
@@ -162,8 +163,13 @@ class MultiModalHasher:
|
||||
yield from cls.serialize_item(obj)
|
||||
|
||||
@classmethod
|
||||
def hash_kwargs(cls, **kwargs: object) -> str:
|
||||
hasher_factory = _get_hasher_factory(envs.VLLM_MM_HASHER_ALGORITHM)
|
||||
def hash_kwargs(
|
||||
cls,
|
||||
algorithm: MMHasherAlgorithm,
|
||||
/,
|
||||
**kwargs: object,
|
||||
) -> str:
|
||||
hasher_factory = _get_hasher_factory(algorithm)
|
||||
hasher = hasher_factory()
|
||||
|
||||
for k, v in sorted(kwargs.items(), key=lambda kv: kv[0]):
|
||||
|
||||
@@ -229,40 +229,57 @@ Uses a list instead of a tensor if the dimensions of each element do not match.
|
||||
"""
|
||||
|
||||
|
||||
def nested_tensors_equal(a: NestedTensors, b: NestedTensors) -> bool:
|
||||
def nested_tensors_equal(
|
||||
a: NestedTensors,
|
||||
b: NestedTensors,
|
||||
check_dtype: bool = True,
|
||||
) -> bool:
|
||||
"""
|
||||
Equality check between
|
||||
[`NestedTensors`][vllm.multimodal.inputs.NestedTensors] objects.
|
||||
|
||||
If `check_dtype` is `True`, the tensors must have the same dtype.
|
||||
"""
|
||||
check_dtype_func = (
|
||||
lambda a, b, check_dtype: a.dtype == b.dtype if check_dtype else True
|
||||
)
|
||||
if isinstance(a, torch.Tensor):
|
||||
return isinstance(b, torch.Tensor) and torch.equal(a, b)
|
||||
return (
|
||||
isinstance(b, torch.Tensor)
|
||||
and torch.equal(a, b)
|
||||
and check_dtype_func(a, b, check_dtype)
|
||||
)
|
||||
elif isinstance(b, torch.Tensor):
|
||||
return isinstance(a, torch.Tensor) and torch.equal(b, a)
|
||||
return (
|
||||
isinstance(a, torch.Tensor)
|
||||
and torch.equal(b, a)
|
||||
and check_dtype_func(b, a, check_dtype)
|
||||
)
|
||||
|
||||
if isinstance(a, list):
|
||||
return (
|
||||
isinstance(b, list)
|
||||
and len(a) == len(b)
|
||||
and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b))
|
||||
and all(nested_tensors_equal(a_, b_, check_dtype) for a_, b_ in zip(a, b))
|
||||
)
|
||||
if isinstance(b, list):
|
||||
return (
|
||||
isinstance(a, list)
|
||||
and len(b) == len(a)
|
||||
and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a))
|
||||
and all(nested_tensors_equal(b_, a_, check_dtype) for b_, a_ in zip(b, a))
|
||||
)
|
||||
|
||||
if isinstance(a, tuple):
|
||||
return (
|
||||
isinstance(b, tuple)
|
||||
and len(a) == len(b)
|
||||
and all(nested_tensors_equal(a_, b_) for a_, b_ in zip(a, b))
|
||||
and all(nested_tensors_equal(a_, b_, check_dtype) for a_, b_ in zip(a, b))
|
||||
)
|
||||
if isinstance(b, tuple):
|
||||
return (
|
||||
isinstance(a, tuple)
|
||||
and len(b) == len(a)
|
||||
and all(nested_tensors_equal(b_, a_) for b_, a_ in zip(b, a))
|
||||
and all(nested_tensors_equal(b_, a_, check_dtype) for b_, a_ in zip(b, a))
|
||||
)
|
||||
|
||||
# Both a and b are scalars
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from vllm.config.multimodal import MMHasherAlgorithm
|
||||
from vllm.inputs import MultiModalHashes
|
||||
|
||||
from ..hasher import MultiModalHasher
|
||||
@@ -22,7 +23,11 @@ class ProcessorInputs:
|
||||
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
tokenization_kwargs: Mapping[str, object] = field(default_factory=dict)
|
||||
|
||||
def get_mm_hashes(self, model_id: str) -> MultiModalHashes:
|
||||
def get_mm_hashes(
|
||||
self,
|
||||
model_id: str,
|
||||
hash_algorithm: MMHasherAlgorithm,
|
||||
) -> MultiModalHashes:
|
||||
mm_data_items = self.mm_data_items
|
||||
mm_uuid_items = self.mm_uuid_items or {}
|
||||
hf_processor_mm_kwargs = self.hf_processor_mm_kwargs
|
||||
@@ -49,6 +54,7 @@ class ProcessorInputs:
|
||||
item = uuid_item if uuid_item is not None else item
|
||||
hashes.append(
|
||||
hasher.hash_kwargs(
|
||||
hash_algorithm,
|
||||
model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
@@ -61,6 +67,7 @@ class ProcessorInputs:
|
||||
else:
|
||||
mm_hashes[modality] = [
|
||||
hasher.hash_kwargs(
|
||||
hash_algorithm,
|
||||
model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs,
|
||||
|
||||
@@ -1422,7 +1422,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
|
||||
# Use overrides if provided; fallback to data-dependent hashing.
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
mm_hashes = inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
|
||||
mm_prompt_updates = self._get_mm_prompt_updates(
|
||||
inputs.mm_data_items,
|
||||
@@ -1454,7 +1457,10 @@ class BaseMultiModalProcessor(ABC, Generic[_I]):
|
||||
return self._apply_hf_processor(inputs, timing_ctx)
|
||||
|
||||
with timing_ctx.record("get_mm_hashes"):
|
||||
mm_hashes = inputs.get_mm_hashes(self.info.model_id)
|
||||
mm_hashes = inputs.get_mm_hashes(
|
||||
self.info.model_id,
|
||||
self.info.ctx.get_mm_config().mm_hasher_algorithm,
|
||||
)
|
||||
|
||||
with timing_ctx.record("get_cache_missing_items"):
|
||||
mm_is_cached, mm_missing_data_items = self._get_cache_missing_items(
|
||||
|
||||
+31
-19
@@ -16,13 +16,13 @@ from typing_extensions import deprecated
|
||||
from vllm.inputs import MultiModalPlaceholders
|
||||
from vllm.utils.import_utils import LazyLoader
|
||||
|
||||
from .hasher import MultiModalHasher
|
||||
from .inputs import (
|
||||
BatchedTensorInputs,
|
||||
MultiModalFeatureSpec,
|
||||
MultiModalFieldElem,
|
||||
MultiModalKwargsItem,
|
||||
MultiModalSharedField,
|
||||
nested_tensors_equal,
|
||||
)
|
||||
from .media import (
|
||||
AudioMediaIO,
|
||||
@@ -165,11 +165,28 @@ def argsort_mm_positions(
|
||||
return [(modality, idx) for modality, idx, _ in sorted_flat_items]
|
||||
|
||||
|
||||
def _get_group_hash(elem: MultiModalFieldElem):
|
||||
if not isinstance(elem.field, MultiModalSharedField):
|
||||
return None
|
||||
def _can_batch_mm_items(
|
||||
left: MultiModalKwargsItem,
|
||||
right: MultiModalKwargsItem,
|
||||
) -> bool:
|
||||
if left.keys() != right.keys():
|
||||
return False
|
||||
|
||||
return MultiModalHasher.hash_kwargs(data=elem.data)
|
||||
for key, left_elem in left.items():
|
||||
right_elem = right[key]
|
||||
left_field, right_field = left_elem.field, right_elem.field
|
||||
is_shared_field = isinstance(left_field, MultiModalSharedField) and isinstance(
|
||||
right_field, MultiModalSharedField
|
||||
)
|
||||
if (type(left_field) is not type(right_field)) or (
|
||||
is_shared_field
|
||||
and not nested_tensors_equal(
|
||||
left_elem.data, right_elem.data, check_dtype=True
|
||||
)
|
||||
):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _batch_mm_items(
|
||||
@@ -217,26 +234,21 @@ def group_and_batch_mm_items(
|
||||
- `kwargs` is a dictionary of keyword arguments to pass to the model;
|
||||
- `num_items` is the corresponding number of items.
|
||||
"""
|
||||
group_ids = [
|
||||
tuple(
|
||||
(key, _get_group_hash(elem))
|
||||
for key, elem in sorted(item.items(), key=lambda kv: kv[0])
|
||||
)
|
||||
for item in items
|
||||
]
|
||||
group_sizes = [sum(1 for _ in group) for _, group in groupby(group_ids)]
|
||||
|
||||
start_idx = 0
|
||||
for group_size in group_sizes:
|
||||
for end_idx in range(1, len(items) + 1):
|
||||
if end_idx < len(items) and _can_batch_mm_items(
|
||||
items[end_idx - 1], items[end_idx]
|
||||
):
|
||||
continue
|
||||
|
||||
group_data = _batch_mm_items(
|
||||
items[start_idx : start_idx + group_size],
|
||||
items[start_idx:end_idx],
|
||||
device=device,
|
||||
pin_memory=pin_memory,
|
||||
)
|
||||
|
||||
yield group_size, group_data
|
||||
|
||||
start_idx += group_size
|
||||
yield end_idx - start_idx, group_data
|
||||
start_idx = end_idx
|
||||
|
||||
assert start_idx == len(items)
|
||||
|
||||
|
||||
@@ -1276,8 +1276,8 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
embeds_prompt["prompt_embeds"] = full_embeds
|
||||
embeds_prompt["prompt_is_token_ids"] = is_token_ids_mask
|
||||
|
||||
@staticmethod
|
||||
def _apply_prompt_embeds_to_engine_input(
|
||||
self,
|
||||
engine_input: MultiModalInput,
|
||||
prompt_embeds_tensors: list[torch.Tensor],
|
||||
mm_updates: MultiModalPromptUpdates,
|
||||
@@ -1300,6 +1300,7 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
pe_kwargs_items: list[MultiModalKwargsItem] = []
|
||||
pe_hashes: list[str] = []
|
||||
pe_placeholders: list[PlaceholderRange] = []
|
||||
mm_config = self.model_config.get_multimodal_config()
|
||||
for tensor, (start, length) in zip(
|
||||
prompt_embeds_tensors, positions, strict=True
|
||||
):
|
||||
@@ -1313,7 +1314,11 @@ class HfRenderer(BaseRenderer[HfTokenizer]):
|
||||
}
|
||||
)
|
||||
)
|
||||
pe_hashes.append(MultiModalHasher.hash_kwargs(prompt_embeds=tensor))
|
||||
pe_hashes.append(
|
||||
MultiModalHasher.hash_kwargs(
|
||||
mm_config.mm_hasher_algorithm, prompt_embeds=tensor
|
||||
)
|
||||
)
|
||||
# `is_embed=None` matches the existing image_embeds-style
|
||||
# "no encoder, just splice the tensor directly" semantics.
|
||||
pe_placeholders.append(
|
||||
|
||||
Reference in New Issue
Block a user