[MultiModal] add support for numpy array embeddings (#38119)

Signed-off-by: guillaume_guy <guillaume.guy@airbnb.com>
Signed-off-by: Guillaume Guy <guillaume.c.guy@gmail.com>
Co-authored-by: guillaume_guy <guillaume.guy@airbnb.com>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
Guillaume Guy
2026-03-25 20:13:04 +00:00
committed by GitHub
co-authored by guillaume_guy Cyrus Leung
parent 978fc18bf0
commit 70a2152830
2 changed files with 63 additions and 3 deletions
+17 -3
View File
@@ -4,6 +4,7 @@
from io import BytesIO
from pathlib import Path
import numpy as np
import pybase64
import torch
from PIL import Image
@@ -13,6 +14,8 @@ from vllm.utils.serial_utils import tensor2base64
from ..image import convert_image_mode, rgba_to_rgb
from .base import MediaIO, MediaWithBytes
MAGIC_NUMPY_PREFIX = b"\x93NUMPY" # https://numpy.org/devdocs/reference/generated/numpy.lib.format.html#format-version-1-0
class ImageMediaIO(MediaIO[Image.Image]):
"""Configuration values can be user-provided either by --media-io-kwargs or
@@ -104,7 +107,7 @@ class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
def __init__(self) -> None:
super().__init__()
def load_bytes(self, data: bytes) -> torch.Tensor:
def _load_pickled_torch(self, data: bytes) -> torch.Tensor:
buffer = BytesIO(data)
# Enable sparse tensor integrity checks to prevent out-of-bounds
# writes from maliciously crafted tensors
@@ -112,12 +115,23 @@ class ImageEmbeddingMediaIO(MediaIO[torch.Tensor]):
tensor = torch.load(buffer, weights_only=True)
return tensor.to_dense()
def _load_numpy(self, data: bytes) -> torch.Tensor:
with BytesIO(data) as buffer:
return torch.from_numpy(np.load(buffer))
def load_bytes(self, data: bytes) -> torch.Tensor:
if data[:6] == MAGIC_NUMPY_PREFIX:
return self._load_numpy(data)
return self._load_pickled_torch(data)
def load_base64(self, media_type: str, data: str) -> torch.Tensor:
return self.load_bytes(pybase64.b64decode(data, validate=True))
def load_file(self, filepath: Path) -> torch.Tensor:
# Enable sparse tensor integrity checks to prevent out-of-bounds
# writes from maliciously crafted tensors
if filepath.suffix == ".npy":
return torch.from_numpy(np.load(filepath))
with torch.sparse.check_sparse_tensor_invariants():
tensor = torch.load(filepath, weights_only=True)
return tensor.to_dense()