[Frontend] Add video embeds input support (#54242)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-08-31 03:20:47 +00:00
committed by GitHub
parent 5707355209
commit 555ea65e8c
8 changed files with 461 additions and 24 deletions
@@ -10,7 +10,7 @@ clusters:
• decode (language-model inference)
For MM input we:
1. Extract *every* image/audio item.
1. Extract *every* image/audio/video item.
2. Fire N concurrent requests to the encoder cluster
(one request per item, with **all text removed**).
3. Wait for all of them to succeed.
@@ -61,7 +61,16 @@ encoder_rr_lock = asyncio.Lock()
###############################################################################
MM_TYPES = {"image_url", "audio_url", "input_audio"}
MM_TYPES = {"image_url", "audio_url", "input_audio", "video_url"}
# The embeds content type each MM item is rewritten to once the encoder has
# published its embedding out of band.
EMBEDS_TYPES = {
"image_url": "image_embeds",
"audio_url": "audio_embeds",
"input_audio": "audio_embeds",
"video_url": "video_embeds",
}
def encoder_rr_assignment(
@@ -94,7 +103,9 @@ def content_uuid(item: dict) -> str:
the unmodified path, which hashes the content, would keep it. That asymmetry
silently biases any comparison between the two.
"""
url = (item.get("image_url") or item.get("audio_url") or {}).get("url") or ""
url = (
item.get("image_url") or item.get("audio_url") or item.get("video_url") or {}
).get("url") or ""
payload = url or json.dumps(item, sort_keys=True)
return hashlib.sha256(payload.encode()).hexdigest()
@@ -103,19 +114,22 @@ def _b64_tensor(values: list) -> str:
import torch
buf = io.BytesIO()
grid = torch.tensor(values, dtype=torch.long)
# Downstream stacks per item, so hand over a flat (t, h, w).
torch.save(grid.reshape(-1)[:3], buf)
flat = [v for item in values for v in (item if isinstance(item, list) else [item])]
# Floats stay float64 so timestamp strings format exactly as the
# encoder computed them.
dtype = torch.float64 if any(isinstance(v, float) for v in flat) else None
# Downstream stacks per item, so hand over a flat vector.
torch.save(torch.tensor(flat, dtype=dtype), buf)
return base64.b64encode(buf.getvalue()).decode()
def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
"""Replace each image item with a metadata-only reference for the decoder.
"""Replace each media item with a metadata-only reference for the decoder.
The decoder does not need the pixels: the encoder instance already produced
the embedding and published it through the EC connector under the same uuid.
Sending only the grid lets the decoder size the placeholder range without
re-running the image transform.
re-running the media transform.
`item_meta` holds what the encoder reported for each item (its cache key and
the grid its processor actually produced), so the grid is never re-derived
@@ -146,25 +160,22 @@ def rewrite_for_decode(req_data: dict, item_meta: dict[int, dict]) -> dict:
# processor cache); let the decoder process the media itself.
new_content.append(item)
continue
embeds_type = EMBEDS_TYPES[item["type"]]
new_content.append(
{
"type": "image_embeds",
"image_embeds": metadata,
"uuid": item_uuid,
}
{"type": embeds_type, embeds_type: metadata, "uuid": item_uuid}
)
rewritten += 1
new_messages.append({**msg, "content": new_content})
if not rewritten:
return req_data
logger.info("Rewrote %d image item(s) as metadata references", rewritten)
logger.info("Rewrote %d media item(s) as metadata references", rewritten)
return {**req_data, "messages": new_messages}
def extract_mm_items(request_data: dict) -> list[dict]:
"""
Return *all* image/audio items that appear anywhere in `messages`.
Return *all* image/audio/video items that appear anywhere in `messages`.
Each returned dict looks like:
{ "type": "image_url", "image_url": {...} }
@@ -98,6 +98,16 @@ def qwen25omni_model_config_image_embeds():
)
@pytest.fixture(scope="function")
def qwen25omni_model_config_video_embeds():
return ModelConfig(
QWEN25OMNI_MODEL_ID,
runner="generate",
limit_mm_per_prompt={"video": 2},
enable_mm_embeds=True,
)
@pytest.fixture(scope="function")
def qwen2_audio_model_config():
return ModelConfig(
@@ -969,6 +979,133 @@ async def test_parse_chat_messages_audio_embeds_async(
_assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None])
def test_parse_chat_messages_empty_video_embeds_with_uuid(
qwen25omni_model_config_video_embeds,
):
"""Test video_embeds with UUID (no actual embeds data)."""
uuid = "test-video-uuid-123"
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this video"},
{"type": "video_embeds", "video_embeds": None, "uuid": uuid},
],
}
],
qwen25omni_model_config_video_embeds,
content_format="string",
)
# Should have video in mm_data as None (UUID provided)
assert mm_data is not None
assert "video" in mm_data
assert isinstance(mm_data["video"], list)
assert len(mm_data["video"]) == 1
assert mm_data["video"][0] is None
# UUID should be recorded
_assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[uuid])
def test_parse_chat_messages_video_embeds_with_string(
qwen25omni_model_config_video_embeds,
):
"""Test video_embeds with base64 string embedding data."""
hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()
video_embedding = torch.randn(1, 128, hidden_size)
base64_video_embedding = tensor2base64(video_embedding)
conversation, mm_data, mm_uuids = parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this video"},
{
"type": "video_embeds",
"video_embeds": base64_video_embedding,
},
],
}
],
qwen25omni_model_config_video_embeds,
content_format="string",
)
# Should have video embedding in mm_data (single tensor, not a list)
assert mm_data is not None
assert "video" in mm_data
assert isinstance(mm_data["video"], torch.Tensor)
assert mm_data["video"].shape == video_embedding.shape
# No UUID provided
assert mm_uuids is not None
assert "video" in mm_uuids
_assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
@pytest.mark.asyncio
async def test_parse_chat_messages_video_embeds_async(
qwen25omni_model_config_video_embeds,
):
"""Test video_embeds with async futures."""
hidden_size = qwen25omni_model_config_video_embeds.get_inputs_embeds_size()
video_embedding = torch.randn(1, 128, hidden_size)
base64_video_embedding = tensor2base64(video_embedding)
conversation, mm_data, mm_uuids = await parse_chat_messages_async(
[
{
"role": "user",
"content": [
{"type": "text", "text": "Describe this video"},
{
"type": "video_embeds",
"video_embeds": base64_video_embedding,
},
],
}
],
qwen25omni_model_config_video_embeds,
content_format="string",
)
# Should have video embedding in mm_data (single tensor, not a list)
assert mm_data is not None
assert "video" in mm_data
assert isinstance(mm_data["video"], torch.Tensor)
assert mm_data["video"].shape == video_embedding.shape
# No UUID provided
assert mm_uuids is not None
assert "video" in mm_uuids
_assert_mm_uuids(mm_uuids, 1, modality="video", expected_uuids=[None])
def test_parse_chat_messages_video_and_video_embeds_mixed(
qwen25omni_model_config_video_embeds,
video_url,
):
"""Test that mixing raw video and video_embeds is rejected."""
with pytest.raises(VLLMValidationError, match="Mixing raw video"):
parse_chat_messages(
[
{
"role": "user",
"content": [
{"type": "video_url", "video_url": {"url": video_url}},
{"type": "video_embeds", "video_embeds": None, "uuid": "x"},
],
}
],
qwen25omni_model_config_video_embeds,
content_format="string",
)
def test_parse_chat_messages_multiple_image_embeds(
phi3v_model_config_image_embeds,
):
@@ -11,8 +11,10 @@ from typing import Any
import numpy as np
import pytest
from vllm.config import ModelConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
from ...registry import HF_EXAMPLE_MODELS
from ...utils import build_model_context
MODEL_ID = "Qwen/Qwen3-VL-4B-Instruct"
@@ -186,6 +188,89 @@ def test_processor_multi_video_list_kwargs(
)
def _build_video_embeds_mm_data(
hidden_size: int,
grid_thw: tuple[int, int, int] = (2, 4, 4),
) -> dict[str, Any]:
"""Create an embeds-only video item as an EPD consumer receives it:
pre-computed embeddings plus the metadata published by the encoder."""
import torch
t, h, w = grid_thw
num_tokens = t * h * w // 4 # spatial_merge_size ** 2
return {
"video": {
"video_embeds": torch.zeros(num_tokens, hidden_size),
"video_grid_thw": torch.tensor([grid_thw]),
"timestamps": torch.tensor([[float(i) for i in range(t)]]),
}
}
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_video_embeds_with_timestamps(model_id: str) -> None:
"""Embeds-only video input must size the placeholder range from the
grid and the real timestamps published by the encoder (EC consumer
path); synthesized or missing timestamps would change the token count
and break embedding merging downstream."""
# `build_model_context` forces enable_mm_embeds off, so build the
# config directly; keep the same online-availability skip behavior.
HF_EXAMPLE_MODELS.find_hf_info(model_id).check_available_online(on_fail="skip")
model_config = ModelConfig(
model_id,
runner="generate",
limit_mm_per_prompt={"image": 0, "video": 1},
enable_mm_embeds=True,
)
processor = MULTIMODAL_REGISTRY.create_processor(model_config)
tokenizer = processor.info.get_tokenizer()
grid_thw = (2, 4, 4)
mm_data = _build_video_embeds_mm_data(
model_config.get_inputs_embeds_size(), grid_thw
)
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(mm_data),
hf_processor_mm_kwargs={},
)
video_phs = processed["mm_placeholders"].get("video", [])
assert len(video_phs) == 1, (
f"Expected exactly 1 video placeholder, got {len(video_phs)}"
)
t, h, w = grid_thw
tokens_per_frame = h * w // 4
timestamp_tokens = sum(
len(tokenizer.encode(f"<{float(i):.1f} seconds>", add_special_tokens=False))
for i in range(t)
)
# per frame: timestamp tokens + vision_start + video tokens + vision_end
expected_len = timestamp_tokens + t * (tokens_per_frame + 2)
assert video_phs[0].length == expected_len
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_processor_video_embeds_missing_timestamps(model_id: str) -> None:
"""Timestamps are required metadata for video embeds: they size the
placeholder range, so omitting them must fail loudly at parse time
instead of silently producing a wrong prompt."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
mm_data = _build_video_embeds_mm_data(ctx.model_config.get_inputs_embeds_size())
del mm_data["video"]["timestamps"]
with pytest.raises(ValueError, match="timestamps"):
processor.info.parse_mm_data(mm_data)
@pytest.mark.parametrize("model_id", [MODEL_ID])
def test_dummy_video_spreads_budget_when_frame_cap_enabled(model_id: str) -> None:
"""Regression test for memory profiling with ``cap_pixels_per_frame``.
@@ -14,6 +14,7 @@ from vllm.distributed.ec_transfer.ec_connector.base import (
ECConnectorRole,
)
from vllm.logger import init_logger
from vllm.utils.collection_utils import is_list_of
from vllm.v1.core.sched.output import SchedulerOutput
if TYPE_CHECKING:
@@ -221,11 +222,15 @@ class ECExampleConnector(ECConnectorBase):
# back to processing the media itself.
if feature.data is not None:
wanted = self._placeholder_metadata_fields(feature.modality)
metadata = {
key: value.tolist()
for key, value in feature.data.get_data().items()
if key in wanted and isinstance(value, torch.Tensor)
}
for key, value in feature.data.get_data().items():
if key not in wanted:
continue
if isinstance(value, torch.Tensor):
metadata[key] = value.tolist()
elif is_list_of(value, (int, float)):
# Some metadata (e.g. Qwen3-VL video timestamps) is
# produced as a plain list rather than a tensor.
metadata[key] = value
items.append({"mm_hash": feature.identifier, **metadata})
if not items:
+115
View File
@@ -178,6 +178,22 @@ class ChatCompletionContentPartAudioEmbedsParam(TypedDict, total=False):
"""
class ChatCompletionContentPartVideoEmbedsParam(TypedDict, total=False):
video_embeds: str | dict[str, str] | None
"""
The video embeddings. It can be either:
- A single base64 string representing a serialized torch tensor.
- A dictionary where each value is a base64 string.
"""
type: Required[Literal["video_embeds"]]
"""The type of the content part."""
uuid: str | None
"""
User-provided UUID of a media. User must guarantee that it is properly
generated and unique for different medias.
"""
class ChatCompletionContentPartPromptEmbedsParam(TypedDict, total=False):
data: Required[str]
"""
@@ -324,6 +340,7 @@ ChatCompletionContentPartParam: TypeAlias = (
| CustomChatCompletionContentSimpleImageParam
| ChatCompletionContentPartImageEmbedsParam
| ChatCompletionContentPartAudioEmbedsParam
| ChatCompletionContentPartVideoEmbedsParam
| ChatCompletionContentPartPromptEmbedsParam
| CustomChatCompletionContentSimpleAudioParam
| CustomChatCompletionContentSimpleVideoParam
@@ -415,6 +432,7 @@ ModalityStr = Literal[
"video",
"image_embeds",
"audio_embeds",
"video_embeds",
"vision_chunk",
"prompt_embeds",
]
@@ -755,6 +773,11 @@ def _resolve_items(
"Mixing raw audio and embedding inputs is not allowed",
parameter="audio_embeds",
)
if "video" in items_by_modality and "video_embeds" in items_by_modality:
raise VLLMValidationError(
"Mixing raw video and embedding inputs is not allowed",
parameter="video_embeds",
)
# `prompt_embeds` bypasses HF MM processors. Every other modality requires one.
processor_modalities = items_by_modality.keys() - {"prompt_embeds"}
if processor_modalities and mm_processor is None:
@@ -789,6 +812,14 @@ def _resolve_items(
if "video" in items_by_modality:
mm_data["video"] = [data for data, uuid in items_by_modality["video"]]
mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video"]]
if "video_embeds" in items_by_modality:
assert mm_processor is not None
mm_data["video"] = _get_embeds_data(
"video",
[data for data, uuid in items_by_modality["video_embeds"]],
mm_processor,
)
mm_uuids["video"] = [uuid for data, uuid in items_by_modality["video_embeds"]]
if "vision_chunk" in items_by_modality:
assert mm_processor is not None
# Process vision_chunk items - extract from (data, modality) tuples
@@ -941,6 +972,14 @@ class BaseMultiModalContentParser(ABC):
def parse_video(self, video_url: str | None, uuid: str | None = None) -> None:
raise NotImplementedError
@abstractmethod
def parse_video_embeds(
self,
video_embeds: str | dict[str, str] | None,
uuid: str | None = None,
) -> None:
raise NotImplementedError
class MultiModalContentParser(BaseMultiModalContentParser):
def __init__(
@@ -1096,6 +1135,32 @@ class MultiModalContentParser(BaseMultiModalContentParser):
audio_placeholder = self._tracker.add("audio", (audio, uuid))
self._add_placeholder("audio", audio_placeholder)
def parse_video_embeds(
self,
video_embeds: str | dict[str, str] | None,
uuid: str | None = None,
) -> None:
mm_config = self.model_config.get_multimodal_config()
if not mm_config.enable_mm_embeds:
raise VLLMValidationError(
"You must set `--enable-mm-embeds` to input `video_embeds`",
parameter="video_embeds",
)
if isinstance(video_embeds, dict):
embeds = {
k: self._connector.fetch_video_embedding(v)
for k, v in video_embeds.items()
}
placeholder = self._tracker.add("video_embeds", (embeds, uuid))
elif isinstance(video_embeds, str):
embedding = self._connector.fetch_video_embedding(video_embeds)
placeholder = self._tracker.add("video_embeds", (embedding, uuid))
else:
placeholder = self._tracker.add("video_embeds", (None, uuid))
self._add_placeholder("video", placeholder)
class AsyncMultiModalContentParser(BaseMultiModalContentParser):
def __init__(
@@ -1304,6 +1369,43 @@ class AsyncMultiModalContentParser(BaseMultiModalContentParser):
)
self._add_placeholder("audio", audio_placeholder)
def parse_video_embeds(
self,
video_embeds: str | dict[str, str] | None,
uuid: str | None = None,
) -> None:
mm_config = self.model_config.get_multimodal_config()
if not mm_config.enable_mm_embeds:
raise VLLMValidationError(
"You must set `--enable-mm-embeds` to input `video_embeds`",
parameter="video_embeds",
)
placeholder = self._tracker.add(
"video_embeds",
partial(self._video_embeds_with_uuid_async, video_embeds, uuid),
)
self._add_placeholder("video", placeholder)
async def _video_embeds_with_uuid_async(
self,
video_embeds: str | dict[str, str] | None,
uuid: str | None,
):
if isinstance(video_embeds, dict):
tensors = await asyncio.gather(
*(
self._connector.fetch_video_embedding_async(v)
for v in video_embeds.values()
)
)
embeds = dict(zip(video_embeds, tensors))
elif isinstance(video_embeds, str):
embeds = await self._connector.fetch_video_embedding_async(video_embeds)
else:
embeds = None
return embeds, uuid
@dataclass
class ChatTemplateConfig:
@@ -1481,6 +1583,7 @@ def _get_full_multimodal_text_prompt(
_TextParser = partial(cast, ChatCompletionContentPartTextParam)
_ImageEmbedsParser = partial(cast, ChatCompletionContentPartImageEmbedsParam)
_AudioEmbedsParser = partial(cast, ChatCompletionContentPartAudioEmbedsParam)
_VideoEmbedsParser = partial(cast, ChatCompletionContentPartVideoEmbedsParam)
_PromptEmbedsParser = partial(cast, ChatCompletionContentPartPromptEmbedsParam)
_InputAudioParser = partial(cast, ChatCompletionContentPartInputAudioParam)
_RefusalParser = partial(cast, ChatCompletionContentPartRefusalParam)
@@ -1507,6 +1610,7 @@ MM_PARSER_MAP: dict[
"image_url": lambda part: _ImageParser(part).get("image_url", {}).get("url", None),
"image_embeds": lambda part: _ImageEmbedsParser(part).get("image_embeds", None),
"audio_embeds": lambda part: _AudioEmbedsParser(part).get("audio_embeds", None),
"video_embeds": lambda part: _VideoEmbedsParser(part).get("video_embeds", None),
"prompt_embeds": lambda part: _PromptEmbedsParser(part).get("data", None),
"image_pil": lambda part: _PILImageParser(part).get("image_pil", None),
"audio_url": lambda part: _AudioParser(part).get("audio_url", {}).get("url", None),
@@ -1605,6 +1709,13 @@ def _parse_chat_message_content_mm_part(
)
audio_embeds = audio_params.get("audio_embeds", None)
return "audio_embeds", audio_embeds
if "video_embeds" in part:
# "video_embeds" could be None if UUID is provided.
video_embeds_params = cast( # type: ignore[assignment]
ChatCompletionContentPartVideoEmbedsParam, part
)
video_embeds = video_embeds_params.get("video_embeds", None)
return "video_embeds", video_embeds
if "prompt_embeds" in part:
prompt_embeds_params = cast( # type: ignore[assignment]
ChatCompletionContentPartPromptEmbedsParam, part
@@ -1780,6 +1891,10 @@ def _parse_chat_message_content_part(
content = cast(str | dict[str, str], content) if content is not None else None
mm_parser.parse_audio_embeds(content, uuid)
modality = "audio"
elif part_type == "video_embeds":
content = cast(str | dict[str, str], content) if content is not None else None
mm_parser.parse_video_embeds(content, uuid)
modality = "video"
elif part_type == "prompt_embeds":
if not content:
raise VLLMValidationError(
+15 -2
View File
@@ -894,6 +894,19 @@ class Qwen3_VisionTransformer(nn.Module):
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
class Qwen3VLMultiModalDataParser(Qwen2VLMultiModalDataParser):
# Timestamps are part of the prompt replacement (each frame group's
# "<X.X seconds>" text), so they size the placeholder range and must
# travel with the grid when embeddings are delivered out of band.
embedding_fields = {
**Qwen2VLMultiModalDataParser.embedding_fields,
"video": {
**Qwen2VLMultiModalDataParser.embedding_fields["video"],
"timestamps": "metadata",
},
}
class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
def get_hf_config(self):
return self.ctx.get_hf_config(Qwen3VLConfig)
@@ -912,7 +925,7 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
return self.get_hf_processor(**kwargs).video_processor
def get_data_parser(self):
return Qwen2VLMultiModalDataParser(
return Qwen3VLMultiModalDataParser(
self.get_hf_config().vision_config.spatial_merge_size,
video_needs_metadata=True,
expected_hidden_size=self._get_expected_hidden_size(),
@@ -1609,7 +1622,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
def get_video_repl(
*,
tokens_per_frame: list[int],
timestamps: list[float | int],
timestamps: list[float | int] | torch.Tensor,
tokenizer: TokenizerLike,
vision_start_token_id: int,
vision_end_token_id: int,
+26 -1
View File
@@ -35,7 +35,7 @@ from vllm.utils.registry import ExtensionManager
from .audio import AudioEmbeddingMediaIO, AudioMediaIO
from .base import MediaIO, MediaWithBytes
from .image import ImageEmbeddingMediaIO, ImageMediaIO
from .video import VideoMediaIO
from .video import VideoEmbeddingMediaIO, VideoMediaIO
logger = init_logger(__name__)
@@ -653,3 +653,28 @@ class MediaConnector:
return await loop.run_in_executor(
global_thread_pool, audio_embedding_io.load_base64, "", data
)
def fetch_video_embedding(
self,
data: str,
) -> torch.Tensor:
"""
Load video embedding from a URL.
"""
video_embedding_io = VideoEmbeddingMediaIO()
return video_embedding_io.load_base64("", data)
async def fetch_video_embedding_async(
self,
data: str,
) -> torch.Tensor:
"""
Asynchronously load video embedding from a URL.
"""
video_embedding_io = VideoEmbeddingMediaIO()
loop = asyncio.get_running_loop()
return await loop.run_in_executor(
global_thread_pool, video_embedding_io.load_base64, "", data
)
+47 -1
View File
@@ -1,20 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from functools import partial
from io import BytesIO
from pathlib import Path
from typing import Any
import numpy as np
import numpy.typing as npt
import pybase64
import torch
from PIL import Image
from vllm import envs
from vllm.logger import init_logger
from vllm.utils.serial_utils import tensor2base64
from vllm.utils.sparse_utils import check_sparse_tensor_invariants_threadsafe
from ..video import VIDEO_LOADER_REGISTRY
from .base import MediaIO, MediaWithBytes
from .image import ImageMediaIO
from .image import MAGIC_NUMPY_PREFIX, ImageMediaIO
logger = init_logger(__name__)
@@ -194,3 +198,45 @@ class VideoMediaIO(MediaIO[MediaWithBytes[tuple[npt.NDArray, dict[str, Any]]]]):
msg = "Only JPEG format is supported for now."
raise NotImplementedError(msg)
class VideoEmbeddingMediaIO(MediaIO[torch.Tensor]):
"""Video embedding MediaIO implementation.
Configuration values can be user-provided either by --media-io-kwargs or
by the runtime API field "media_io_kwargs". Ensure proper validation and
error handling.
"""
def __init__(self) -> None:
super().__init__()
def _load_pickled_torch(self, data: bytes) -> torch.Tensor:
buffer = BytesIO(data)
with check_sparse_tensor_invariants_threadsafe():
tensor = torch.load(buffer, weights_only=True)
return tensor.to_dense()
def _load_numpy(self, data: bytes) -> torch.Tensor:
with BytesIO(data) as buffer:
return torch.from_numpy(np.load(buffer))
def load_bytes(self, data: bytes) -> torch.Tensor:
if data[:6] == MAGIC_NUMPY_PREFIX:
return self._load_numpy(data)
return self._load_pickled_torch(data)
def load_base64(self, media_type: str, data: str) -> torch.Tensor:
return self.load_bytes(pybase64.b64decode(data, validate=True))
def load_file(self, filepath: Path) -> torch.Tensor:
if filepath.suffix == ".npy":
return torch.from_numpy(np.load(filepath))
with check_sparse_tensor_invariants_threadsafe():
tensor = torch.load(filepath, weights_only=True)
return tensor.to_dense()
def encode_base64(self, media: torch.Tensor) -> str:
return tensor2base64(media)