[V1] Extend beyond image modality and support mixed-modality inference with Llava-OneVision (#11685)

Signed-off-by: Roger Wang <ywang@roblox.com>
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Roger Wang
2025-01-06 19:58:16 +00:00
committed by GitHub
co-authored by DarkLight1337
parent e20c92bb61
commit 91b361ae89
17 changed files with 633 additions and 279 deletions
+30 -62
View File
@@ -1,4 +1,3 @@
import pickle
import re
from abc import ABC, abstractmethod
from collections import defaultdict
@@ -7,18 +6,16 @@ from dataclasses import dataclass, field
from functools import lru_cache
from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
import numpy as np
import torch
from blake3 import blake3
from PIL import Image
from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
from vllm import envs
from vllm.inputs import DummyData, InputProcessingContext
from vllm.logger import init_logger
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
encode_tokens)
from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
from .hasher import MultiModalHasher
from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
MultiModalInputsV2, MultiModalKwargs,
MultiModalKwargsItem, PlaceholderRange)
@@ -486,56 +483,6 @@ class ProcessingCache:
logger.debug("ProcessingCache: hit_ratio = %.2f",
cache_stats.hit_ratio)
def _serialize_item(self, obj: object) -> bytes:
# Simple cases
if isinstance(obj, str):
return obj.encode("utf-8")
if isinstance(obj, bytes):
return obj
if isinstance(obj, Image.Image):
return obj.tobytes()
# Convertible to NumPy arrays
if isinstance(obj, torch.Tensor):
obj = obj.numpy()
if isinstance(obj, (int, float)):
obj = np.array(obj)
if isinstance(obj, np.ndarray):
return obj.tobytes()
logger.warning(
"No serialization method found for %s. "
"Falling back to pickle.", type(obj))
return pickle.dumps(obj)
def _item_to_bytes(
self,
key: str,
obj: object,
) -> Iterable[tuple[bytes, bytes]]:
# Recursive cases
if isinstance(obj, (list, tuple)):
for i, elem in enumerate(obj):
yield from self._item_to_bytes(f"{key}.{i}", elem)
elif isinstance(obj, dict):
for k, v in obj.items():
yield from self._item_to_bytes(f"{key}.{k}", v)
else:
key_bytes = self._serialize_item(key)
value_bytes = self._serialize_item(obj)
yield key_bytes, value_bytes
def _hash_kwargs(self, **kwargs: object) -> str:
hasher = blake3()
for k, v in kwargs.items():
for k_bytes, v_bytes in self._item_to_bytes(k, v):
hasher.update(k_bytes)
hasher.update(v_bytes)
return hasher.hexdigest()
def get(
self,
model_id: str,
@@ -554,9 +501,9 @@ class ProcessingCache:
"""
self._maybe_log_cache_stats()
cache_key = self._hash_kwargs(model_id=model_id,
**{modality: input_item},
**input_kwargs)
cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
**{modality: input_item},
**input_kwargs)
return self._cache.get(cache_key)
def put(
@@ -571,9 +518,9 @@ class ProcessingCache:
Put a processed multi-modal item into the cache
according to its dependencies (see :meth:`get`).
"""
cache_key = self._hash_kwargs(model_id=model_id,
**{modality: input_item},
**input_kwargs)
cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
**{modality: input_item},
**input_kwargs)
self._cache.put(cache_key, output_kwargs)
@@ -1049,6 +996,24 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
"""
mm_items = self._to_mm_items(mm_data)
# Create MM hashes (only used in V1)
# TODO: Use these hash keys for caching operations in apply_hf_processor
# instead of rehashing.
if envs.VLLM_USE_V1:
model_id = self.ctx.model_config.model
mm_hashes = {
modality: [
MultiModalHasher.hash_kwargs(model_id=model_id,
**{modality: item},
**hf_processor_mm_kwargs)
for item in items
]
for modality, items in mm_items.items()
}
else:
mm_hashes = None
prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
prompt_text,
mm_items,
@@ -1122,6 +1087,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
prompt=prompt_text,
prompt_token_ids=prompt_ids,
mm_kwargs=mm_kwargs,
mm_hashes=mm_hashes,
mm_placeholders=mm_placeholder_ranges,
)
@@ -1174,7 +1140,9 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
"tokens.")
total_len = len(prompt_token_ids)
if total_len > seq_len:
# V0 does not support chunked prefill.
if total_len > seq_len and not envs.VLLM_USE_V1:
logger.warning(
"The context length (%d) of the model is too short "
"to hold the multi-modal embeddings in the worst case "