[V1] Extend beyond image modality and support mixed-modality inference with Llava-OneVision (#11685)
Signed-off-by: Roger Wang <ywang@roblox.com> Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
DarkLight1337
parent
e20c92bb61
commit
91b361ae89
@@ -1,4 +1,3 @@
|
||||
import pickle
|
||||
import re
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
@@ -7,18 +6,16 @@ from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from typing import Any, NamedTuple, Optional, Protocol, TypeVar, Union
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from blake3 import blake3
|
||||
from PIL import Image
|
||||
from transformers import BatchFeature, PretrainedConfig, ProcessorMixin
|
||||
|
||||
from vllm import envs
|
||||
from vllm.inputs import DummyData, InputProcessingContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils.tokenizer import (AnyTokenizer, decode_tokens,
|
||||
encode_tokens)
|
||||
from vllm.utils import LRUCache, flatten_2d_lists, full_groupby
|
||||
|
||||
from .hasher import MultiModalHasher
|
||||
from .inputs import (MultiModalDataDict, MultiModalFieldConfig,
|
||||
MultiModalInputsV2, MultiModalKwargs,
|
||||
MultiModalKwargsItem, PlaceholderRange)
|
||||
@@ -486,56 +483,6 @@ class ProcessingCache:
|
||||
logger.debug("ProcessingCache: hit_ratio = %.2f",
|
||||
cache_stats.hit_ratio)
|
||||
|
||||
def _serialize_item(self, obj: object) -> bytes:
|
||||
# Simple cases
|
||||
if isinstance(obj, str):
|
||||
return obj.encode("utf-8")
|
||||
if isinstance(obj, bytes):
|
||||
return obj
|
||||
if isinstance(obj, Image.Image):
|
||||
return obj.tobytes()
|
||||
|
||||
# Convertible to NumPy arrays
|
||||
if isinstance(obj, torch.Tensor):
|
||||
obj = obj.numpy()
|
||||
if isinstance(obj, (int, float)):
|
||||
obj = np.array(obj)
|
||||
if isinstance(obj, np.ndarray):
|
||||
return obj.tobytes()
|
||||
|
||||
logger.warning(
|
||||
"No serialization method found for %s. "
|
||||
"Falling back to pickle.", type(obj))
|
||||
|
||||
return pickle.dumps(obj)
|
||||
|
||||
def _item_to_bytes(
|
||||
self,
|
||||
key: str,
|
||||
obj: object,
|
||||
) -> Iterable[tuple[bytes, bytes]]:
|
||||
# Recursive cases
|
||||
if isinstance(obj, (list, tuple)):
|
||||
for i, elem in enumerate(obj):
|
||||
yield from self._item_to_bytes(f"{key}.{i}", elem)
|
||||
elif isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
yield from self._item_to_bytes(f"{key}.{k}", v)
|
||||
else:
|
||||
key_bytes = self._serialize_item(key)
|
||||
value_bytes = self._serialize_item(obj)
|
||||
yield key_bytes, value_bytes
|
||||
|
||||
def _hash_kwargs(self, **kwargs: object) -> str:
|
||||
hasher = blake3()
|
||||
|
||||
for k, v in kwargs.items():
|
||||
for k_bytes, v_bytes in self._item_to_bytes(k, v):
|
||||
hasher.update(k_bytes)
|
||||
hasher.update(v_bytes)
|
||||
|
||||
return hasher.hexdigest()
|
||||
|
||||
def get(
|
||||
self,
|
||||
model_id: str,
|
||||
@@ -554,9 +501,9 @@ class ProcessingCache:
|
||||
"""
|
||||
self._maybe_log_cache_stats()
|
||||
|
||||
cache_key = self._hash_kwargs(model_id=model_id,
|
||||
**{modality: input_item},
|
||||
**input_kwargs)
|
||||
cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
|
||||
**{modality: input_item},
|
||||
**input_kwargs)
|
||||
return self._cache.get(cache_key)
|
||||
|
||||
def put(
|
||||
@@ -571,9 +518,9 @@ class ProcessingCache:
|
||||
Put a processed multi-modal item into the cache
|
||||
according to its dependencies (see :meth:`get`).
|
||||
"""
|
||||
cache_key = self._hash_kwargs(model_id=model_id,
|
||||
**{modality: input_item},
|
||||
**input_kwargs)
|
||||
cache_key = MultiModalHasher.hash_kwargs(model_id=model_id,
|
||||
**{modality: input_item},
|
||||
**input_kwargs)
|
||||
self._cache.put(cache_key, output_kwargs)
|
||||
|
||||
|
||||
@@ -1049,6 +996,24 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
|
||||
"""
|
||||
mm_items = self._to_mm_items(mm_data)
|
||||
|
||||
# Create MM hashes (only used in V1)
|
||||
# TODO: Use these hash keys for caching operations in apply_hf_processor
|
||||
# instead of rehashing.
|
||||
|
||||
if envs.VLLM_USE_V1:
|
||||
model_id = self.ctx.model_config.model
|
||||
mm_hashes = {
|
||||
modality: [
|
||||
MultiModalHasher.hash_kwargs(model_id=model_id,
|
||||
**{modality: item},
|
||||
**hf_processor_mm_kwargs)
|
||||
for item in items
|
||||
]
|
||||
for modality, items in mm_items.items()
|
||||
}
|
||||
else:
|
||||
mm_hashes = None
|
||||
|
||||
prompt_ids, mm_kwargs = self._cached_apply_hf_processor(
|
||||
prompt_text,
|
||||
mm_items,
|
||||
@@ -1122,6 +1087,7 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
|
||||
prompt=prompt_text,
|
||||
prompt_token_ids=prompt_ids,
|
||||
mm_kwargs=mm_kwargs,
|
||||
mm_hashes=mm_hashes,
|
||||
mm_placeholders=mm_placeholder_ranges,
|
||||
)
|
||||
|
||||
@@ -1174,7 +1140,9 @@ class BaseMultiModalProcessor(ProcessingMixin, ABC):
|
||||
"tokens.")
|
||||
|
||||
total_len = len(prompt_token_ids)
|
||||
if total_len > seq_len:
|
||||
|
||||
# V0 does not support chunked prefill.
|
||||
if total_len > seq_len and not envs.VLLM_USE_V1:
|
||||
logger.warning(
|
||||
"The context length (%d) of the model is too short "
|
||||
"to hold the multi-modal embeddings in the worst case "
|
||||
|
||||
Reference in New Issue
Block a user