Feature/video support in random mm dataset (#25963)
Signed-off-by: Eugene Khvedchenia <ekhvedchenia@nvidia.com> Signed-off-by: Eugene Khvedchenya <ekhvedchenia@nvidia.com> Co-authored-by: Roger Wang <hey@rogerw.io>
This commit is contained in:
co-authored by
Roger Wang
parent
1a33aacf82
commit
5e72216d17
+80
-25
@@ -27,8 +27,10 @@ from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from io import BytesIO
|
||||
from tempfile import NamedTemporaryFile
|
||||
from typing import Any, cast
|
||||
|
||||
import cv2
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
@@ -498,9 +500,13 @@ class RandomDataset(BenchmarkDataset):
|
||||
num_requests, range_ratio, input_len, output_len, tokenizer
|
||||
)
|
||||
|
||||
# Generate prefix once
|
||||
prefix_token_ids = self.get_prefix(tokenizer, prefix_len)
|
||||
vocab_size = tokenizer.vocab_size
|
||||
prohibited_tokens = tokenizer.all_special_ids
|
||||
all_tokens = np.arange(vocab_size)
|
||||
allowed_tokens = np.array(list(set(all_tokens) - set(prohibited_tokens)))
|
||||
|
||||
# Generate prefix once
|
||||
prefix_token_ids = self.get_prefix(allowed_tokens, prefix_len)
|
||||
|
||||
requests = []
|
||||
token_mismatch_total = 0
|
||||
@@ -513,6 +519,7 @@ class RandomDataset(BenchmarkDataset):
|
||||
input_len=int(input_lens[i]),
|
||||
offset=int(offsets[i]),
|
||||
index=i,
|
||||
allowed_tokens=allowed_tokens,
|
||||
)
|
||||
token_mismatch_total += token_mismatch
|
||||
requests.append(
|
||||
@@ -553,13 +560,17 @@ class RandomDataset(BenchmarkDataset):
|
||||
return requests
|
||||
|
||||
def get_prefix(
|
||||
self, tokenizer: PreTrainedTokenizerBase, prefix_len: int
|
||||
self,
|
||||
allowed_tokens: np.ndarray,
|
||||
prefix_len: int,
|
||||
) -> list[int]:
|
||||
"""
|
||||
Get the prefix for the dataset.
|
||||
"""
|
||||
return (
|
||||
self._rng.integers(0, tokenizer.vocab_size, size=prefix_len).tolist()
|
||||
allowed_tokens[
|
||||
self._rng.integers(0, len(allowed_tokens), size=prefix_len)
|
||||
].tolist()
|
||||
if prefix_len > 0
|
||||
else []
|
||||
)
|
||||
@@ -623,6 +634,7 @@ class RandomDataset(BenchmarkDataset):
|
||||
input_len: int,
|
||||
offset: int,
|
||||
index: int,
|
||||
allowed_tokens: np.ndarray,
|
||||
) -> tuple[str, int, int]:
|
||||
"""
|
||||
Returns (prompt, total_input_len).
|
||||
@@ -636,8 +648,11 @@ class RandomDataset(BenchmarkDataset):
|
||||
To avoid uncontrolled change of the prompt length,
|
||||
the encoded sequence is truncated before being decoded again.
|
||||
"""
|
||||
# Build the inner sequence by sampling sequentially from the vocab
|
||||
inner_seq = ((offset + index + np.arange(input_len)) % vocab_size).tolist()
|
||||
# Build the inner sequence by sampling
|
||||
# sequentially from the allowed tokens
|
||||
inner_seq = allowed_tokens[
|
||||
(offset + index + np.arange(input_len)) % len(allowed_tokens)
|
||||
].tolist()
|
||||
token_sequence = prefix_token_ids + inner_seq
|
||||
|
||||
# Decode, then re-encode and truncate to preserve token count invariants
|
||||
@@ -772,7 +787,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
|
||||
Status:
|
||||
- Images: supported via synthetic RGB data.
|
||||
- Video: not yet supported (TODO: implement video generation method).
|
||||
- Video: supported via synthetic RGB data.
|
||||
- Audio: not yet supported.
|
||||
|
||||
Sampling overview:
|
||||
@@ -782,7 +797,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
The maximum is further clamped to the sum of per-modality limits.
|
||||
2) Each item’s modality and shape is sampled from `bucket_config`, a dict
|
||||
mapping (height, width, num_frames) → probability. We treat
|
||||
`num_frames`=1 as image and and `num_frames` > 1 as video.
|
||||
`num_frames`=1 as image and `num_frames` > 1 as video.
|
||||
Entries with zero probability are removed and the rest are renormalized
|
||||
to sum to 1.
|
||||
3) Per-modality hard caps are enforced via `limit_mm_per_prompt`.
|
||||
@@ -797,8 +812,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
"""
|
||||
|
||||
IS_MULTIMODAL = True
|
||||
# NOTE: video sampling is WIP. Setting it to 0.
|
||||
DEFAULT_LIMIT_MM_PER_PROMPT = {"image": 255, "video": 0}
|
||||
DEFAULT_LIMIT_MM_PER_PROMPT = {"image": 255, "video": 1}
|
||||
|
||||
DEFAULT_BASE_ITEMS_PER_REQUEST = 1
|
||||
DEFAULT_NUM_MM_ITEMS_RANGE_RATIO = 0.0
|
||||
@@ -828,12 +842,47 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
)
|
||||
return Image.fromarray(random_pixels)
|
||||
|
||||
def generate_synthetic_video(self, width: int, height: int, num_frames: int) -> Any:
|
||||
def generate_synthetic_video(
|
||||
self, width: int, height: int, num_frames: int
|
||||
) -> dict:
|
||||
"""Generate synthetic video with random values.
|
||||
|
||||
TODO: Finish this method.
|
||||
Creates a video with random pixel values, encodes it to MP4 format,
|
||||
and returns the content as bytes.
|
||||
"""
|
||||
raise NotImplementedError("Video sampling is WIP.")
|
||||
random_pixels = self._rng.integers(
|
||||
0,
|
||||
256,
|
||||
(num_frames, height, width, 3),
|
||||
dtype=np.uint8,
|
||||
)
|
||||
|
||||
# Create a temporary video file in memory
|
||||
fourcc = cv2.VideoWriter_fourcc(*"mp4v")
|
||||
fps = 30 # frames per second
|
||||
|
||||
with NamedTemporaryFile(suffix=".mp4", delete_on_close=False) as temp_file:
|
||||
temp_path = temp_file.name
|
||||
|
||||
# Create video writer
|
||||
video_writer = cv2.VideoWriter(
|
||||
temp_path, fourcc=fourcc, fps=fps, frameSize=(width, height)
|
||||
)
|
||||
|
||||
if not video_writer.isOpened():
|
||||
raise RuntimeError("Failed to create video writer")
|
||||
|
||||
for frame in random_pixels:
|
||||
video_writer.write(frame)
|
||||
|
||||
video_writer.release()
|
||||
temp_file.close()
|
||||
|
||||
# Read the video file content
|
||||
with open(temp_path, "rb") as f:
|
||||
video_content = f.read()
|
||||
|
||||
return {"bytes": video_content}
|
||||
|
||||
def map_config_to_modality(self, config: tuple[int, int, int]) -> str:
|
||||
"""Map the configuration to the modality."""
|
||||
@@ -1044,16 +1093,6 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
enable_multimodal_chat: bool = DEFAULT_ENABLE_MULTIMODAL_CHAT,
|
||||
**kwargs,
|
||||
) -> list[SampleRequest]:
|
||||
# NOTE: Video sampling is WIP. Raise error if video is in bucket config
|
||||
# and probability is non-zero.
|
||||
if any(
|
||||
self.map_config_to_modality(cfg) == "video" and p > 0
|
||||
for cfg, p in bucket_config.items()
|
||||
):
|
||||
raise NotImplementedError(
|
||||
"Video sampling not implemented; set its probability to 0."
|
||||
)
|
||||
|
||||
# Get the sampling parameters for the dataset
|
||||
input_lens, output_lens, offsets = self.get_sampling_params(
|
||||
num_requests, range_ratio, input_len, output_len, tokenizer
|
||||
@@ -1071,9 +1110,24 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
bucket_config,
|
||||
)
|
||||
|
||||
# Generate prefix once
|
||||
prefix_token_ids = self.get_prefix(tokenizer, prefix_len)
|
||||
vocab_size = tokenizer.vocab_size
|
||||
# Can't use tokenizer.all_special_ids since
|
||||
# it returns ONLY ids from special_tokens_map.json
|
||||
# We want to exclude placeholder tokens and all
|
||||
# tokens that indicate start/end of image as it
|
||||
# may break prompt replacement logic.
|
||||
prohibited_tokens = list(
|
||||
tok_id
|
||||
for tok_id, token in tokenizer.added_tokens_decoder.items()
|
||||
if token.special
|
||||
)
|
||||
all_tokens = np.arange(vocab_size)
|
||||
allowed_tokens = np.array(list(set(all_tokens) - set(prohibited_tokens)))
|
||||
logger.debug(
|
||||
"Sampling from %d out of %d (vocab size)", len(allowed_tokens), vocab_size
|
||||
)
|
||||
# Generate prefix once
|
||||
prefix_token_ids = self.get_prefix(allowed_tokens, prefix_len)
|
||||
# Add synthetic multimodal items to each request
|
||||
mm_requests = []
|
||||
token_mismatch_total = 0
|
||||
@@ -1086,6 +1140,7 @@ class RandomMultiModalDataset(RandomDataset):
|
||||
input_len=int(input_lens[i]),
|
||||
offset=int(offsets[i]),
|
||||
index=i,
|
||||
allowed_tokens=allowed_tokens,
|
||||
)
|
||||
token_mismatch_total += token_mismatch
|
||||
# Get multimodal item iterator for a given request
|
||||
|
||||
Reference in New Issue
Block a user