[V1] Override mm_counts for dummy data creation (#15703)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -3,7 +3,7 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Generic, NamedTuple, TypeVar, cast
|
||||
from typing import Generic, NamedTuple, Optional, TypeVar, cast
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
@@ -160,17 +160,19 @@ class MultiModalProfiler(Generic[_I]):
|
||||
def get_and_validate_mm_inputs(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Optional[Mapping[str, int]] = None,
|
||||
) -> tuple[MultiModalInputs, Mapping[str, int]]:
|
||||
mm_counts = self.get_mm_limits()
|
||||
if mm_counts is None:
|
||||
mm_counts = self.get_mm_limits()
|
||||
|
||||
info = self.processing_info
|
||||
mm_max_tokens_per_item = info.get_mm_max_tokens_per_item(
|
||||
seq_len, mm_counts)
|
||||
|
||||
if mm_counts.keys() != mm_max_tokens_per_item.keys():
|
||||
if mm_counts.keys() - mm_max_tokens_per_item.keys():
|
||||
raise AssertionError(
|
||||
"The keys returned by `get_supported_mm_limits` "
|
||||
f"({set(mm_counts.keys())}) should be the same as those "
|
||||
f"({set(mm_counts.keys())}) should be a subset of those "
|
||||
"returned by `get_mm_max_tokens_per_item` "
|
||||
f"({set(mm_max_tokens_per_item.keys())})")
|
||||
|
||||
@@ -193,8 +195,12 @@ class MultiModalProfiler(Generic[_I]):
|
||||
"tokens.")
|
||||
return mm_inputs, total_placeholders_by_modality
|
||||
|
||||
def get_encoder_dummy_data(self, seq_len: int) -> DummyEncoderData:
|
||||
mm_inputs, _ = self.get_and_validate_mm_inputs(seq_len)
|
||||
def get_encoder_dummy_data(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Optional[Mapping[str, int]] = None,
|
||||
) -> DummyEncoderData:
|
||||
mm_inputs, _ = self.get_and_validate_mm_inputs(seq_len, mm_counts)
|
||||
mm_inputs = cast(MultiModalEncDecInputs, mm_inputs)
|
||||
|
||||
# For encoder-decoder models, use encoder prompt token ids instead of
|
||||
@@ -207,9 +213,15 @@ class MultiModalProfiler(Generic[_I]):
|
||||
|
||||
return DummyEncoderData(encoder_prompt_token_ids)
|
||||
|
||||
def get_decoder_dummy_data(self, seq_len: int) -> DummyDecoderData:
|
||||
(mm_inputs, total_placeholders_by_modality
|
||||
) = self.get_and_validate_mm_inputs(seq_len)
|
||||
def get_decoder_dummy_data(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Optional[Mapping[str, int]] = None,
|
||||
) -> DummyDecoderData:
|
||||
(
|
||||
mm_inputs,
|
||||
total_placeholders_by_modality,
|
||||
) = self.get_and_validate_mm_inputs(seq_len, mm_counts)
|
||||
|
||||
prompt_token_ids = mm_inputs["prompt_token_ids"]
|
||||
total_len = len(prompt_token_ids)
|
||||
|
||||
Reference in New Issue
Block a user