Feature/video support in random mm dataset (#25963)

Signed-off-by: Eugene Khvedchenia <ekhvedchenia@nvidia.com>
Signed-off-by: Eugene Khvedchenya <ekhvedchenia@nvidia.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
Eugene Khvedchenya
2025-10-29 18:24:52 +08:00
committed by GitHub
co-authored by Roger Wang
parent 1a33aacf82
commit 5e72216d17
3 changed files with 601 additions and 25 deletions
+80 -25
View File
@@ -27,8 +27,10 @@ from copy import deepcopy
from dataclasses import dataclass
from functools import cache
from io import BytesIO
from tempfile import NamedTemporaryFile
from typing import Any, cast
import cv2
import numpy as np
from PIL import Image
from transformers import PreTrainedTokenizerBase
@@ -498,9 +500,13 @@ class RandomDataset(BenchmarkDataset):
num_requests, range_ratio, input_len, output_len, tokenizer
)
# Generate prefix once
prefix_token_ids = self.get_prefix(tokenizer, prefix_len)
vocab_size = tokenizer.vocab_size
prohibited_tokens = tokenizer.all_special_ids
all_tokens = np.arange(vocab_size)
allowed_tokens = np.array(list(set(all_tokens) - set(prohibited_tokens)))
# Generate prefix once
prefix_token_ids = self.get_prefix(allowed_tokens, prefix_len)
requests = []
token_mismatch_total = 0
@@ -513,6 +519,7 @@ class RandomDataset(BenchmarkDataset):
input_len=int(input_lens[i]),
offset=int(offsets[i]),
index=i,
allowed_tokens=allowed_tokens,
)
token_mismatch_total += token_mismatch
requests.append(
@@ -553,13 +560,17 @@ class RandomDataset(BenchmarkDataset):
return requests
def get_prefix(
self, tokenizer: PreTrainedTokenizerBase, prefix_len: int
self,
allowed_tokens: np.ndarray,
prefix_len: int,
) -> list[int]:
"""
Get the prefix for the dataset.
"""
return (
self._rng.integers(0, tokenizer.vocab_size, size=prefix_len).tolist()
allowed_tokens[
self._rng.integers(0, len(allowed_tokens), size=prefix_len)
].tolist()
if prefix_len > 0
else []
)
@@ -623,6 +634,7 @@ class RandomDataset(BenchmarkDataset):
input_len: int,
offset: int,
index: int,
allowed_tokens: np.ndarray,
) -> tuple[str, int, int]:
"""
Returns (prompt, total_input_len).
@@ -636,8 +648,11 @@ class RandomDataset(BenchmarkDataset):
To avoid uncontrolled change of the prompt length,
the encoded sequence is truncated before being decoded again.
"""
# Build the inner sequence by sampling sequentially from the vocab
inner_seq = ((offset + index + np.arange(input_len)) % vocab_size).tolist()
# Build the inner sequence by sampling
# sequentially from the allowed tokens
inner_seq = allowed_tokens[
(offset + index + np.arange(input_len)) % len(allowed_tokens)
].tolist()
token_sequence = prefix_token_ids + inner_seq
# Decode, then re-encode and truncate to preserve token count invariants
@@ -772,7 +787,7 @@ class RandomMultiModalDataset(RandomDataset):
Status:
- Images: supported via synthetic RGB data.
- Video: not yet supported (TODO: implement video generation method).
- Video: supported via synthetic RGB data.
- Audio: not yet supported.
Sampling overview:
@@ -782,7 +797,7 @@ class RandomMultiModalDataset(RandomDataset):
The maximum is further clamped to the sum of per-modality limits.
2) Each item’s modality and shape is sampled from `bucket_config`, a dict
mapping (height, width, num_frames) → probability. We treat
`num_frames`=1 as image and and `num_frames` > 1 as video.
`num_frames`=1 as image and `num_frames` > 1 as video.
Entries with zero probability are removed and the rest are renormalized
to sum to 1.
3) Per-modality hard caps are enforced via `limit_mm_per_prompt`.
@@ -797,8 +812,7 @@ class RandomMultiModalDataset(RandomDataset):
"""
IS_MULTIMODAL = True
# NOTE: video sampling is WIP. Setting it to 0.
DEFAULT_LIMIT_MM_PER_PROMPT = {"image": 255, "video": 0}
DEFAULT_LIMIT_MM_PER_PROMPT = {"image": 255, "video": 1}
DEFAULT_BASE_ITEMS_PER_REQUEST = 1
DEFAULT_NUM_MM_ITEMS_RANGE_RATIO = 0.0
@@ -828,12 +842,47 @@ class RandomMultiModalDataset(RandomDataset):
)
return Image.fromarray(random_pixels)
def generate_synthetic_video(self, width: int, height: int, num_frames: int) -> Any:
def generate_synthetic_video(
self, width: int, height: int, num_frames: int
) -> dict:
"""Generate synthetic video with random values.
TODO: Finish this method.
Creates a video with random pixel values, encodes it to MP4 format,
and returns the content as bytes.
"""
raise NotImplementedError("Video sampling is WIP.")
random_pixels = self._rng.integers(
0,
256,
(num_frames, height, width, 3),
dtype=np.uint8,
)
# Create a temporary video file in memory
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
fps = 30 # frames per second
with NamedTemporaryFile(suffix=".mp4", delete_on_close=False) as temp_file:
temp_path = temp_file.name
# Create video writer
video_writer = cv2.VideoWriter(
temp_path, fourcc=fourcc, fps=fps, frameSize=(width, height)
)
if not video_writer.isOpened():
raise RuntimeError("Failed to create video writer")
for frame in random_pixels:
video_writer.write(frame)
video_writer.release()
temp_file.close()
# Read the video file content
with open(temp_path, "rb") as f:
video_content = f.read()
return {"bytes": video_content}
def map_config_to_modality(self, config: tuple[int, int, int]) -> str:
"""Map the configuration to the modality."""
@@ -1044,16 +1093,6 @@ class RandomMultiModalDataset(RandomDataset):
enable_multimodal_chat: bool = DEFAULT_ENABLE_MULTIMODAL_CHAT,
**kwargs,
) -> list[SampleRequest]:
# NOTE: Video sampling is WIP. Raise error if video is in bucket config
# and probability is non-zero.
if any(
self.map_config_to_modality(cfg) == "video" and p > 0
for cfg, p in bucket_config.items()
):
raise NotImplementedError(
"Video sampling not implemented; set its probability to 0."
)
# Get the sampling parameters for the dataset
input_lens, output_lens, offsets = self.get_sampling_params(
num_requests, range_ratio, input_len, output_len, tokenizer
@@ -1071,9 +1110,24 @@ class RandomMultiModalDataset(RandomDataset):
bucket_config,
)
# Generate prefix once
prefix_token_ids = self.get_prefix(tokenizer, prefix_len)
vocab_size = tokenizer.vocab_size
# Can't use tokenizer.all_special_ids since
# it returns ONLY ids from special_tokens_map.json
# We want to exclude placeholder tokens and all
# tokens that indicate start/end of image as it
# may break prompt replacement logic.
prohibited_tokens = list(
tok_id
for tok_id, token in tokenizer.added_tokens_decoder.items()
if token.special
)
all_tokens = np.arange(vocab_size)
allowed_tokens = np.array(list(set(all_tokens) - set(prohibited_tokens)))
logger.debug(
"Sampling from %d out of %d (vocab size)", len(allowed_tokens), vocab_size
)
# Generate prefix once
prefix_token_ids = self.get_prefix(allowed_tokens, prefix_len)
# Add synthetic multimodal items to each request
mm_requests = []
token_mismatch_total = 0
@@ -1086,6 +1140,7 @@ class RandomMultiModalDataset(RandomDataset):
input_len=int(input_lens[i]),
offset=int(offsets[i]),
index=i,
allowed_tokens=allowed_tokens,
)
token_mismatch_total += token_mismatch
# Get multimodal item iterator for a given request