[Neuron] Add Multi-Modal model support for Neuron (#18921)
Signed-off-by: Satyajith Chilappagari <satchill@amazon.com> Co-authored-by: Ashraf Mahgoub <ashymahg@amazon.com> Co-authored-by: Rohith Nallamaddi <nalrohit@amazon.com> Co-authored-by: FeliciaLuo <luof@amazon.com> Co-authored-by: Elaine Zhao <elaineyz@amazon.com>
This commit is contained in:
co-authored by
Ashraf Mahgoub
Rohith Nallamaddi
FeliciaLuo
Elaine Zhao
parent
b8b904795d
commit
2a50ef5760
@@ -3,6 +3,8 @@
|
||||
from typing import List, Optional, Set
|
||||
|
||||
import torch
|
||||
from neuronx_distributed_inference.models.mllama.aspect_ratio_utils import (
|
||||
get_all_supported_aspect_ratios)
|
||||
from neuronx_distributed_inference.modules.generation.sampling import (
|
||||
prepare_sampling_params)
|
||||
from neuronx_distributed_inference.modules.lora_serving import (
|
||||
@@ -17,7 +19,7 @@ from vllm.model_executor import SamplingMetadata
|
||||
from vllm.model_executor.layers.sampler import SamplerOutput
|
||||
from vllm.model_executor.model_loader.neuronx_distributed import (
|
||||
_get_model_architecture, get_neuron_model)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.multimodal import MultiModalKwargs
|
||||
from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
|
||||
from vllm.worker.neuron_model_runner import (ModelInputForNeuron,
|
||||
NeuronModelRunner)
|
||||
@@ -121,42 +123,28 @@ class NeuronxDistributedModelRunner(NeuronModelRunner):
|
||||
sampling_params = self.get_nxd_sampling_params(
|
||||
model_input.sampling_metadata)
|
||||
|
||||
if model_input.multi_modal_kwargs.get('image') is not None:
|
||||
pixel_values = []
|
||||
aspect_ratios = []
|
||||
num_chunks = []
|
||||
has_image = []
|
||||
for multi_modal_input in model_input.multi_modal_kwargs.get(
|
||||
'image'):
|
||||
image_tensors = self.get_multi_modal_data_neuron(
|
||||
multi_modal_input.squeeze(0))
|
||||
pixel_values.append(image_tensors[0])
|
||||
aspect_ratios.append(image_tensors[1])
|
||||
num_chunks.append(image_tensors[2])
|
||||
has_image.append(image_tensors[3])
|
||||
|
||||
pixel_values = torch.cat(pixel_values, dim=0)
|
||||
aspect_ratios = torch.cat(aspect_ratios, dim=0)
|
||||
num_chunks = torch.cat(num_chunks, dim=0)
|
||||
has_image = torch.cat(has_image, dim=0)
|
||||
|
||||
if model_input.multi_modal_kwargs.get('pixel_values') is not None:
|
||||
hidden_states = self.model(
|
||||
input_ids=model_input.input_tokens,
|
||||
positions=model_input.input_positions,
|
||||
seq_ids=model_input.input_block_ids,
|
||||
pixel_values=pixel_values,
|
||||
aspect_ratios=aspect_ratios,
|
||||
pixel_values=model_input.multi_modal_kwargs.get(
|
||||
'pixel_values'),
|
||||
aspect_ratios=model_input.multi_modal_kwargs.get(
|
||||
'aspect_ratios'),
|
||||
sampling_params=sampling_params,
|
||||
num_chunks=num_chunks,
|
||||
has_image=has_image,
|
||||
num_chunks=model_input.multi_modal_kwargs.get('num_chunks'),
|
||||
has_image=model_input.multi_modal_kwargs.get(
|
||||
'has_image').squeeze(1),
|
||||
)
|
||||
else:
|
||||
empty_pixel_values = torch.zeros([1, 1, 4, 3, 560, 560],
|
||||
bs = model_input.input_tokens.shape[0] if (model_input.input_tokens
|
||||
is not None) else 1
|
||||
empty_pixel_values = torch.zeros([bs, 1, 4, 3, 560, 560],
|
||||
dtype=torch.bfloat16)
|
||||
empty_aspect_ratios = torch.ones([1, 1, 2], dtype=torch.int64)
|
||||
num_chunks = torch.tensor([[1]
|
||||
]) # dummy num_chunks, will not be used
|
||||
has_image = torch.tensor([0])
|
||||
empty_aspect_ratios = torch.ones([bs, 1, 2], dtype=torch.int64)
|
||||
num_chunks = torch.zeros((bs, 1), dtype=torch.int32)
|
||||
has_image = torch.zeros([bs], dtype=torch.int32)
|
||||
hidden_states = self.model(
|
||||
input_ids=model_input.input_tokens,
|
||||
positions=model_input.input_positions,
|
||||
@@ -175,6 +163,27 @@ class NeuronxDistributedModelRunner(NeuronModelRunner):
|
||||
|
||||
return [output]
|
||||
|
||||
def process_multi_modal_data_neuron(self, mm_data):
|
||||
# Neuron uses aspect_ratios instead of aspect_ratio_ids
|
||||
all_supported_aspect_ratios = get_all_supported_aspect_ratios(
|
||||
self.model.config.vision_config.max_num_tiles)
|
||||
aspect_ratio_ids = mm_data.get("aspect_ratio_ids")
|
||||
mm_data["aspect_ratios"] = torch.tensor(
|
||||
all_supported_aspect_ratios[aspect_ratio_ids]).unsqueeze(0)
|
||||
|
||||
# Neuron's num_chunks is HF's num_tiles
|
||||
mm_data["num_chunks"] = mm_data.get("num_tiles")
|
||||
|
||||
# Input has an image if it has pixel_values
|
||||
bs = mm_data["num_chunks"].shape[0]
|
||||
pixel_values = mm_data.get("pixel_values")
|
||||
if pixel_values is not None and not torch.all(pixel_values == 0):
|
||||
mm_data["has_image"] = torch.ones(bs)
|
||||
|
||||
else:
|
||||
mm_data["has_image"] = torch.zeros(bs)
|
||||
return mm_data
|
||||
|
||||
def _get_lora_adapter_ids(self, seq_group_metadata_list):
|
||||
# set LoRA adapter IDs for multi-lora serving
|
||||
batch_size = len(seq_group_metadata_list)
|
||||
@@ -200,7 +209,6 @@ class NeuronxDistributedModelRunner(NeuronModelRunner):
|
||||
virtual_engine: int = 0,
|
||||
finished_requests_ids: Optional[List[str]] = None
|
||||
) -> ModelInputForNeuron:
|
||||
multi_modal_kwargs = None
|
||||
# NOTE: We assume that all sequences in the group are all prompts or
|
||||
# all decodes.
|
||||
is_prompt = seq_group_metadata_list[0].is_prompt
|
||||
@@ -223,6 +231,14 @@ class NeuronxDistributedModelRunner(NeuronModelRunner):
|
||||
sampling_params.top_p = top_p
|
||||
sampling_params.temperature = temperature
|
||||
|
||||
# we need multi_modal_data for later tokens as well
|
||||
multi_modal_kwargs_list: List[MultiModalKwargs] = []
|
||||
for seq_group_metadata in seq_group_metadata_list:
|
||||
mm_data = seq_group_metadata.multi_modal_data
|
||||
if mm_data:
|
||||
multi_modal_kwargs_list.append(mm_data)
|
||||
multi_modal_kwargs = MultiModalKwargs.batch(multi_modal_kwargs_list)
|
||||
|
||||
lora_adapter_ids = self._get_lora_adapter_ids(seq_group_metadata_list)
|
||||
|
||||
sampling_metadata = SamplingMetadata.prepare(
|
||||
@@ -236,18 +252,6 @@ class NeuronxDistributedModelRunner(NeuronModelRunner):
|
||||
self.pin_memory,
|
||||
generators=self.get_generators(finished_requests_ids))
|
||||
|
||||
if current_platform.use_transformers_neuronx(
|
||||
) and not self._on_device_sampling_disabled:
|
||||
# Once the request IDs are changed in current iteration, we will
|
||||
# update the on-device sampling parameters.
|
||||
current_batch_request_ids = [
|
||||
seq_group_meta_data.request_id
|
||||
for seq_group_meta_data in seq_group_metadata_list
|
||||
]
|
||||
if current_batch_request_ids != self._previous_batch_request_ids:
|
||||
self._update_neuron_sampling_params(seq_group_metadata_list)
|
||||
self._previous_batch_request_ids = current_batch_request_ids
|
||||
|
||||
return ModelInputForNeuron(input_tokens=input_tokens,
|
||||
input_positions=input_positions,
|
||||
input_block_ids=input_block_ids,
|
||||
|
||||
Reference in New Issue
Block a user