iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

[Feat][MM Hashing] include media_io_kwargs in multi-modal hashes (#54241)

Browse Source 
Signed-off-by: Jan Kwiecinski <jkwiecinski@nvidia.com>
This commit is contained in:
Jan Kwieciński
2026-09-01 13:17:01 +00:00
committed by GitHub
parent 55aa766dc8
commit 1f1f628859
6 changed files with 163 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/multimodal/test_processing.py
+22
View File
@@ -10,10 +10,13 @@ import pytest
from vllm.config import ModelConfig
from vllm.exceptions import VLLMValidationError
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.hasher import MultiModalHasher
from vllm.multimodal.parse import MultiModalDataParser
from vllm.multimodal.processing.context import (
InputProcessingContext,
overlay_modality_mm_kwargs,
)
from vllm.multimodal.processing.inputs import ProcessorInputs
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
PlaceholderFeaturesInfo,
@@ -1262,3 +1265,22 @@ def test_mm_processor_kwargs_merge_then_overlay_preserves_scoping():
assert overlay_modality_mm_kwargs(merged, "video")["size"] == size
assert "size" not in overlay_modality_mm_kwargs(merged, "image")
assert "size" not in overlay_modality_mm_kwargs(merged, None)
def test_processor_inputs_hashes_partial_uuids():
rng = np.random.RandomState(0)
images = [random_image(rng, min_wh=8, max_wh=9) for _ in range(2)]
inputs = ProcessorInputs(
prompt=[],
mm_data_items=MultiModalDataParser().parse_mm_data({"image": images}),
mm_uuid_items={"image": ["image-uuid", None]},
)
assert inputs.get_mm_hashes("test-model", "blake3") == {
"image": [
"image-uuid",
MultiModalHasher.hash_kwargs(
"blake3", model_id="test-model", image=images[1]
),
]
}
tests/renderers/test_multimodal_hashes.py
+92
View File
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.assets.video import VideoAsset
from vllm.config import ModelConfig, VllmConfig
from vllm.multimodal.utils import encode_video_url
from vllm.renderers.hf import HfRenderer
from vllm.renderers.params import ChatParams
from vllm.tokenizers.registry import cached_tokenizer_from_config
pytestmark = pytest.mark.cpu_test
video_data_url = encode_video_url(VideoAsset("baby_reading").np_ndarrays)
DEFAULT_MEDIA_IO_KWARGS = {"video": {"num_frames": 32}}
DEFAULT_MM_PROCESSOR_KWARGS = {"videos_kwargs": {"do_resize": True}}
def _build_video_renderer() -> HfRenderer:
model_config = ModelConfig(
model="llava-hf/llava-onevision-qwen2-0.5b-ov-hf",
max_model_len=4096,
mm_processor_cache_gb=4.0,
)
return HfRenderer(
VllmConfig(model_config=model_config),
cached_tokenizer_from_config(model_config),
)
def _get_message_hash(
renderer: HfRenderer,
messages: list[dict],
media_io_kwargs: dict | None = None,
mm_processor_kwargs: dict | None = None,
) -> str:
media_io_kwargs = media_io_kwargs or DEFAULT_MEDIA_IO_KWARGS
mm_processor_kwargs = mm_processor_kwargs or DEFAULT_MM_PROCESSOR_KWARGS
_, inputs = renderer.render_chat(
[messages],
ChatParams(
media_io_kwargs=media_io_kwargs,
mm_processor_kwargs=mm_processor_kwargs,
),
prompt_extras={"mm_processor_kwargs": mm_processor_kwargs},
)
return inputs[0]["mm_hashes"]["video"][0]
def _video_chat_messages(client_uuid: str):
return [
{
"role": "user",
"content": [
{
"type": "video_url",
"video_url": {"url": video_data_url},
"uuid": client_uuid,
},
{"type": "text", "text": "Describe the video."},
],
}
]
def test_mm_hash_includes_uuid_and_processing_kwargs():
renderer = _build_video_renderer()
messages = _video_chat_messages(video_data_url)
base_hash = _get_message_hash(renderer, messages)
media_io_hash = _get_message_hash(
renderer,
messages,
media_io_kwargs={"video": {"num_frames": 4}},
)
uuid_hash = _get_message_hash(
renderer,
_video_chat_messages("different-video-uuid"),
)
mm_processor_hash = _get_message_hash(
renderer,
messages,
mm_processor_kwargs={"videos_kwargs": {"do_resize": False}},
)
assert base_hash != media_io_hash
assert base_hash != uuid_hash
assert base_hash != mm_processor_hash
vllm/inputs/llm.py
+6
View File
@@ -73,6 +73,12 @@ class _PromptOptions(TypedDict):
if the model supports it.
"""
media_io_kwargs: NotRequired[dict[str, dict[str, Any]] | None]
"""
Optional per-modality media loading and decoding arguments.
Used only for hash derivation when using multi-modal UUIDs.
"""
mm_processor_kwargs: NotRequired[dict[str, Any] | None]
"""
Optional multi-modal processor kwargs to be forwarded to the
vllm/multimodal/processing/inputs.py
+34 -35
View File
@@ -21,6 +21,7 @@ class ProcessorInputs:
mm_data_items: MultiModalDataItems
mm_uuid_items: MultiModalUUIDItems | None = None
hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict)
media_io_kwargs: Mapping[str, Mapping[str, object]] = field(default_factory=dict)
def get_mm_hashes(
self,
@@ -29,49 +30,47 @@ class ProcessorInputs:
) -> MultiModalHashes:
mm_data_items = self.mm_data_items
mm_uuid_items = self.mm_uuid_items or {}
hf_processor_mm_kwargs = self.hf_processor_mm_kwargs
hash_factors = {
"media_io_kwargs": dict(self.media_io_kwargs),
"mm_processor_kwargs": dict(self.hf_processor_mm_kwargs),
}
has_hash_factors = any(hash_factors.values())
hash_factor_kwargs = hash_factors if has_hash_factors else {}
mm_hashes = dict[str, list[str]]()
hasher = MultiModalHasher
for modality, data_items in mm_data_items.items():
if modality in mm_uuid_items:
uuid_items = mm_uuid_items[modality]
uuid_items = (
mm_uuid_items[modality]
if modality in mm_uuid_items
else ([None] * len(data_items))
)
# For None entries, compute a hash; otherwise, use provided ID.
hashes: list[str] = []
for i, item in enumerate(data_items.get_all_items_for_hash()):
uuid_item = uuid_items[i]
# For None entries, compute a hash; otherwise, use provided ID.
hashes: list[str] = []
for i, item in enumerate(data_items.get_all_items_for_hash()):
uuid_item = uuid_items[i]
# NOTE: Even if a uuid_item is provided, we still compute a hash
# if `hf_processor_mm_kwargs` is provided.
# This is because the processed multimodal inputs can be different
# depending on the processor kwargs.
if uuid_item is None or hf_processor_mm_kwargs:
# NOTE: use provided hash string to hash with kwargs
# if available for better performance.
item = uuid_item if uuid_item is not None else item
hashes.append(
hasher.hash_kwargs(
hash_algorithm,
model_id=model_id,
**{modality: item},
**hf_processor_mm_kwargs,
)
# NOTE: Even if a uuid_item is provided, model output
# depends on hash_factor_kwargs, so they are taken into account.
if uuid_item is None or has_hash_factors:
# NOTE: use provided hash string to hash with kwargs
# if available for better performance.
item = uuid_item if uuid_item is not None else item
hashes.append(
hasher.hash_kwargs(
hash_algorithm,
model_id=model_id,
**{modality: item},
**hash_factor_kwargs,
)
else:
hashes.append(uuid_item)
mm_hashes[modality] = hashes
else:
mm_hashes[modality] = [
hasher.hash_kwargs(
hash_algorithm,
model_id=model_id,
**{modality: item},
**hf_processor_mm_kwargs,
)
for item in data_items.get_all_items_for_hash()
]
else:
# If there are no extra kwargs, use the client-provided UUID.
hashes.append(uuid_item)
mm_hashes[modality] = hashes
return mm_hashes
vllm/multimodal/processing/processor.py
+1
View File
@@ -1811,6 +1811,7 @@ class EncDecMultiModalProcessor(BaseMultiModalProcessor[_I]):
inputs.mm_data_items,
inputs.mm_uuid_items,
hf_processor_mm_kwargs=inputs.hf_processor_mm_kwargs,
media_io_kwargs=inputs.media_io_kwargs,
)
encoder_inputs = super().apply(encoder_processor_inputs, timing_ctx)
vllm/renderers/base.py
+8
View File
@@ -759,6 +759,7 @@ class BaseRenderer(ABC, Generic[_T]):
mm_data: MultiModalDataDict,
mm_uuids: MultiModalUUIDDict | None,
mm_processor_kwargs: Mapping[str, object] | None,
media_io_kwargs: Mapping[str, Mapping[str, object]] | None = None,
*,
skip_mm_cache: bool = False,
) -> "MultiModalInput":
@@ -781,6 +782,7 @@ class BaseRenderer(ABC, Generic[_T]):
mm_data_items,
mm_uuid_items,
hf_processor_mm_kwargs=mm_processor_kwargs or {},
media_io_kwargs=media_io_kwargs or {},
)
mm_timing_ctx = self._mm_timing_registry.get(mm_req_id)
@@ -809,6 +811,7 @@ class BaseRenderer(ABC, Generic[_T]):
multi_modal_data,
mm_processor_kwargs=prompt.get("mm_processor_kwargs"),
mm_uuids=prompt.get("multi_modal_uuids"),
media_io_kwargs=prompt.get("media_io_kwargs"),
skip_mm_cache=skip_mm_cache,
)
else:
@@ -871,6 +874,7 @@ class BaseRenderer(ABC, Generic[_T]):
multi_modal_data,
mm_processor_kwargs=prompt.get("mm_processor_kwargs"),
mm_uuids=prompt.get("multi_modal_uuids"),
media_io_kwargs=prompt.get("media_io_kwargs"),
skip_mm_cache=skip_mm_cache,
)
else:
@@ -1082,6 +1086,8 @@ class BaseRenderer(ABC, Generic[_T]):
tok_prompts = self.tokenize_prompts(dict_prompts, tok_params)
prompt_extras = dict(prompt_extras or {})
prompt_extras["media_io_kwargs"] = chat_params.media_io_kwargs or {}
self._apply_prompt_extras(tok_prompts, prompt_extras)
eng_prompts = [
@@ -1118,6 +1124,8 @@ class BaseRenderer(ABC, Generic[_T]):
tok_prompts = await self.tokenize_prompts_async(dict_prompts, tok_params)
prompt_extras = dict(prompt_extras or {})
prompt_extras["media_io_kwargs"] = chat_params.media_io_kwargs or {}
self._apply_prompt_extras(tok_prompts, prompt_extras)
eng_prompts = await asyncio.gather(