[Model] Add support for the multi-modal Llama 3.2 model (#8811)
Co-authored-by: simon-mo <xmo@berkeley.edu> Co-authored-by: Chang Su <chang.s.su@oracle.com> Co-authored-by: Simon Mo <simon.mo@hey.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
co-authored by
simon-mo
Chang Su
Simon Mo
Roger Wang
Roger Wang
parent
4f1ba0844b
commit
770ec6024f
@@ -139,6 +139,12 @@ class EncoderDecoderLLMInputs(LLMInputs):
|
||||
available.
|
||||
"""
|
||||
|
||||
encoder_multi_modal_data: NotRequired[Optional["MultiModalDataDict"]]
|
||||
"""
|
||||
Optional multi-modal data to pass to the encoder model,
|
||||
if the model supports it.
|
||||
"""
|
||||
|
||||
|
||||
_T1 = TypeVar("_T1",
|
||||
bound=SingletonPromptInputs,
|
||||
|
||||
@@ -128,6 +128,7 @@ class InputPreprocessor:
|
||||
def _prepare_decoder_input_ids_for_generation(
|
||||
self,
|
||||
decoder_input_ids: Optional[List[int]],
|
||||
force_bos: bool = True,
|
||||
) -> List[int]:
|
||||
"""
|
||||
Prepares `decoder_input_ids` for generation with encoder-decoder models.
|
||||
@@ -157,8 +158,8 @@ class InputPreprocessor:
|
||||
# use decoder_start_token_id as decoder_input_ids
|
||||
decoder_input_ids = self._get_default_enc_dec_decoder_prompt()
|
||||
|
||||
if (len(decoder_input_ids) == 0
|
||||
or decoder_input_ids[0] != decoder_start_token_id):
|
||||
if force_bos and (len(decoder_input_ids) == 0
|
||||
or decoder_input_ids[0] != decoder_start_token_id):
|
||||
decoder_input_ids = [decoder_start_token_id] + decoder_input_ids
|
||||
|
||||
return decoder_input_ids
|
||||
@@ -295,18 +296,25 @@ class InputPreprocessor:
|
||||
encoder_prompt, encoder_prompt_ids, encoder_mm_data = encoder_comps
|
||||
decoder_prompt, decoder_prompt_ids, decoder_mm_data = decoder_comps
|
||||
|
||||
if encoder_mm_data is not None or decoder_mm_data is not None:
|
||||
raise ValueError("Multi-modal encoder-decoder models are "
|
||||
"not supported yet")
|
||||
if decoder_mm_data is not None:
|
||||
raise ValueError(
|
||||
"Multi-modality decoder inputs of encoder-decoder models are "
|
||||
"not supported yet")
|
||||
|
||||
decoder_prompt_ids = (
|
||||
self._prepare_decoder_input_ids_for_generation(decoder_prompt_ids))
|
||||
# For Multi-Modal models (e.g., mllama), the text input can be
|
||||
# <|image|><|begin_of_text|>hello world. And we should not add
|
||||
# another <|begin_of_text|> to the beginning.
|
||||
decoder_prompt_ids = (self._prepare_decoder_input_ids_for_generation(
|
||||
decoder_prompt_ids,
|
||||
force_bos=(encoder_mm_data is None and decoder_mm_data is None)))
|
||||
|
||||
return EncoderDecoderLLMInputs(
|
||||
prompt_token_ids=decoder_prompt_ids,
|
||||
prompt=decoder_prompt,
|
||||
multi_modal_data=decoder_mm_data,
|
||||
encoder_prompt_token_ids=encoder_prompt_ids,
|
||||
encoder_prompt=encoder_prompt,
|
||||
encoder_multi_modal_data=encoder_mm_data,
|
||||
)
|
||||
|
||||
def _process_encoder_decoder_prompt(
|
||||
|
||||
+48
-6
@@ -112,6 +112,8 @@ class InputRegistry:
|
||||
def __init__(self) -> None:
|
||||
self._dummy_factories_by_model_type: Dict[Type[nn.Module],
|
||||
DummyDataFactory] = {}
|
||||
self._dummy_encoder_factories_by_model_type: Dict[
|
||||
Type[nn.Module], DummyDataFactory] = {}
|
||||
self._input_processors_by_model_type: Dict[Type[nn.Module],
|
||||
InputProcessor] = {}
|
||||
|
||||
@@ -162,11 +164,44 @@ class InputRegistry:
|
||||
return self._dummy_factories_by_model_type \
|
||||
.get(model_cls, self._default_dummy_data_factory)
|
||||
|
||||
def register_dummy_encoder_data(self, factory: DummyDataFactory):
|
||||
"""
|
||||
Register a dummy encoder data factory to a model class
|
||||
|
||||
This is similar to :meth:`~register_dummy_data`, but for encoder input.
|
||||
"""
|
||||
|
||||
def wrapper(model_cls: N) -> N:
|
||||
if model_cls in self._dummy_encoder_factories_by_model_type:
|
||||
logger.warning(
|
||||
"Model class %s already has dummy encoder data "
|
||||
"registered to %s. It is overwritten by the new one.",
|
||||
model_cls, self)
|
||||
|
||||
self._dummy_encoder_factories_by_model_type[model_cls] = factory
|
||||
|
||||
return model_cls
|
||||
|
||||
return wrapper
|
||||
|
||||
def _get_dummy_encoder_data_factory(self, model_cls: Type[nn.Module]):
|
||||
if model_cls in self._dummy_encoder_factories_by_model_type:
|
||||
dummy_factory = self._dummy_encoder_factories_by_model_type[
|
||||
model_cls]
|
||||
else:
|
||||
logger.warning(
|
||||
"No dummy encoder data factory registered to %s. "
|
||||
"Using the dummy data factory for the model instead.",
|
||||
model_cls)
|
||||
dummy_factory = self._get_dummy_data_factory(model_cls)
|
||||
return dummy_factory
|
||||
|
||||
def dummy_data_for_profiling(
|
||||
self,
|
||||
model_config: "ModelConfig",
|
||||
seq_len: int,
|
||||
mm_registry: "MultiModalRegistry",
|
||||
is_encoder_data: bool = False,
|
||||
) -> Tuple["SequenceData", Optional["MultiModalDataDict"]]:
|
||||
"""
|
||||
Create dummy data for profiling the memory usage of a model.
|
||||
@@ -184,8 +219,10 @@ class InputRegistry:
|
||||
from vllm.model_executor.model_loader import get_model_architecture
|
||||
|
||||
model_cls, _ = get_model_architecture(model_config)
|
||||
dummy_factory = self._get_dummy_data_factory(model_cls)
|
||||
|
||||
if is_encoder_data:
|
||||
dummy_factory = self._get_dummy_encoder_data_factory(model_cls)
|
||||
else:
|
||||
dummy_factory = self._get_dummy_data_factory(model_cls)
|
||||
mm_counts = mm_registry.get_mm_limits_per_prompt(model_config)
|
||||
mm_processor_kwargs = get_allowed_kwarg_only_overrides(
|
||||
dummy_factory, overrides=model_config.mm_processor_kwargs)
|
||||
@@ -196,10 +233,15 @@ class InputRegistry:
|
||||
|
||||
# Having more tokens is over-conservative but otherwise fine
|
||||
num_tokens = seq_data.prompt_token_ids
|
||||
assert len(num_tokens) >= seq_len, (
|
||||
f"Expected at least {seq_len} dummy tokens for profiling, "
|
||||
f"but found {len(num_tokens)} tokens instead.")
|
||||
|
||||
if len(num_tokens) < seq_len:
|
||||
if is_encoder_data:
|
||||
logger.warning(
|
||||
"Expected at least %d dummy encoder tokens for profiling, "
|
||||
"but found %d tokens instead.", seq_len, len(num_tokens))
|
||||
else:
|
||||
raise AssertionError(
|
||||
f"Expected at least {seq_len} dummy tokens for profiling, "
|
||||
f"but found {len(num_tokens)} tokens instead.")
|
||||
if mm_data is not None:
|
||||
for k, v in mm_data.items():
|
||||
num_items = len(v) if isinstance(v, list) else 1
|
||||
|
||||
Reference in New Issue
Block a user