[Perf][Multimodal] Sort multimodal encoder inputs by modality (#60196)
Signed-off-by: Jaebok Lee <jaebok9541@naver.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
295c062db8
commit
702ce313f4
@@ -150,9 +150,12 @@ class EncoderRunner:
|
||||
def execute_mm_encoder(
|
||||
self, mm_kwargs: list[tuple[str, MultiModalKwargsItem]]
|
||||
) -> list[torch.Tensor]:
|
||||
# Stable-sort items by modality so each modality is encoded in as few
|
||||
# batches as possible. The returned outputs are restored to input order.
|
||||
order = sorted(range(len(mm_kwargs)), key=lambda i: mm_kwargs[i][0])
|
||||
encoder_outputs: list[torch.Tensor] = []
|
||||
for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
|
||||
mm_kwargs, device=self.device, pin_memory=PIN_MEMORY
|
||||
[mm_kwargs[i] for i in order], device=self.device, pin_memory=PIN_MEMORY
|
||||
):
|
||||
cg_manager = self.cudagraph_manager
|
||||
cudagraph_output = (
|
||||
@@ -169,7 +172,7 @@ class EncoderRunner:
|
||||
)
|
||||
sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items)
|
||||
encoder_outputs.extend(batch_outputs)
|
||||
return encoder_outputs
|
||||
return [encoder_outputs[i] for i in np.argsort(order)]
|
||||
|
||||
@contextmanager
|
||||
def timed_encoder_operation(self, request_ids: Collection[str]):
|
||||
|
||||
@@ -6,6 +6,7 @@ import numpy as np
|
||||
|
||||
from vllm.lora.layers import LoRAMapping, LoRAMappingType
|
||||
from vllm.lora.worker_manager import WorkerLoRAManager
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.v1.worker.gpu.lora_utils import LoraState
|
||||
from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache
|
||||
|
||||
@@ -31,7 +32,7 @@ def set_active_mm_loras(
|
||||
encoder_token_counts: list[int] = []
|
||||
connector_token_counts: list[int] = []
|
||||
|
||||
# iterate through images
|
||||
mm_items: list[tuple[int, MultiModalFeatureSpec]] = []
|
||||
for req_id, encoder_input_ids in scheduled_encoder_inputs.items():
|
||||
req_idx = req_id_to_index.get(req_id)
|
||||
if req_idx is None:
|
||||
@@ -39,27 +40,29 @@ def set_active_mm_loras(
|
||||
|
||||
lora_id = int(lora_state.lora_ids[req_idx])
|
||||
mm_features = encoder_cache.mm_features[req_id]
|
||||
|
||||
# iterate through visual tokens
|
||||
for mm_input_id in encoder_input_ids:
|
||||
pos_info = mm_features[mm_input_id].mm_position
|
||||
|
||||
tower_tokens, connector_tokens = model.get_mm_lora_token_counts(
|
||||
modality=mm_features[mm_input_id].modality,
|
||||
mm_kwargs=mm_features[mm_input_id].data,
|
||||
num_mm_embeds=pos_info.get_num_embeds(),
|
||||
)
|
||||
|
||||
prompt_lora_mapping.append(lora_id)
|
||||
token_lora_mapping.extend([lora_id] * tower_tokens)
|
||||
encoder_token_counts.append(tower_tokens)
|
||||
connector_token_counts.append(connector_tokens)
|
||||
mm_items.extend((lora_id, mm_features[i]) for i in encoder_input_ids)
|
||||
|
||||
if lora_id > 0:
|
||||
lora_request = lora_state.lora_requests.get(req_id)
|
||||
if lora_request is not None:
|
||||
lora_requests.add(lora_request)
|
||||
|
||||
# Runs items sorted by modality.
|
||||
mm_items.sort(key=lambda x: x[1].modality)
|
||||
for lora_id, mm_feature in mm_items:
|
||||
pos_info = mm_feature.mm_position
|
||||
|
||||
tower_tokens, connector_tokens = model.get_mm_lora_token_counts(
|
||||
modality=mm_feature.modality,
|
||||
mm_kwargs=mm_feature.data,
|
||||
num_mm_embeds=pos_info.get_num_embeds(),
|
||||
)
|
||||
|
||||
prompt_lora_mapping.append(lora_id)
|
||||
token_lora_mapping.extend([lora_id] * tower_tokens)
|
||||
encoder_token_counts.append(tower_tokens)
|
||||
connector_token_counts.append(connector_tokens)
|
||||
|
||||
if not prompt_lora_mapping:
|
||||
return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user