[Perf][Multimodal] Sort multimodal encoder inputs by modality (#60196)
Signed-off-by: Jaebok Lee <jaebok9541@naver.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
295c062db8
commit
702ce313f4
@@ -150,9 +150,12 @@ class EncoderRunner:
|
||||
def execute_mm_encoder(
|
||||
self, mm_kwargs: list[tuple[str, MultiModalKwargsItem]]
|
||||
) -> list[torch.Tensor]:
|
||||
# Stable-sort items by modality so each modality is encoded in as few
|
||||
# batches as possible. The returned outputs are restored to input order.
|
||||
order = sorted(range(len(mm_kwargs)), key=lambda i: mm_kwargs[i][0])
|
||||
encoder_outputs: list[torch.Tensor] = []
|
||||
for modality, num_items, mm_kwargs_batch in group_and_batch_mm_kwargs(
|
||||
mm_kwargs, device=self.device, pin_memory=PIN_MEMORY
|
||||
[mm_kwargs[i] for i in order], device=self.device, pin_memory=PIN_MEMORY
|
||||
):
|
||||
cg_manager = self.cudagraph_manager
|
||||
cudagraph_output = (
|
||||
@@ -169,7 +172,7 @@ class EncoderRunner:
|
||||
)
|
||||
sanity_check_mm_encoder_outputs(batch_outputs, expected_num_items=num_items)
|
||||
encoder_outputs.extend(batch_outputs)
|
||||
return encoder_outputs
|
||||
return [encoder_outputs[i] for i in np.argsort(order)]
|
||||
|
||||
@contextmanager
|
||||
def timed_encoder_operation(self, request_ids: Collection[str]):
|
||||
|
||||
Reference in New Issue
Block a user