[Perf][Multimodal] Sort multimodal encoder inputs by modality (#60196)

Signed-off-by: Jaebok Lee <jaebok9541@naver.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
zzaebok
2026-10-09 10:55:36 +08:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 295c062db8
commit 702ce313f4
5 changed files with 112 additions and 27 deletions
+5 -2
View File
@@ -150,9 +150,12 @@ class EncoderRunner:
def execute_mm_encoder(
self, mm_kwargs: list[tuple[str, MultiModalKwargsItem]]
) -> list[torch.Tensor]:
# Stable-sort items by modality so each modality is encoded in as few
# batches as possible. The returned outputs are restored to input order.
order = sorted(range(len(mm_kwargs)), key=lambda i: mm_kwargs[i][0])
encoder_outputs: list[torch.Tensor] = []
for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
mm_kwargs, device=self.device, pin_memory=PIN_MEMORY
[mm_kwargs[i] for i in order], device=self.device, pin_memory=PIN_MEMORY
):
cg_manager = self.cudagraph_manager
cudagraph_output = (
@@ -169,7 +172,7 @@ class EncoderRunner:
)
sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items)
encoder_outputs.extend(batch_outputs)
return encoder_outputs
return [encoder_outputs[i] for i in np.argsort(order)]
@contextmanager
def timed_encoder_operation(self, request_ids: Collection[str]):