[Perf][Multimodal] Sort multimodal encoder inputs by modality (#60196)
Signed-off-by: Jaebok Lee <jaebok9541@naver.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
295c062db8
commit
702ce313f4
@@ -78,7 +78,7 @@ def test_encoder_graph_unsupported_modality_uses_eager_output():
|
||||
"vllm.v1.worker.gpu.mm.encoder_runner.group_and_batch_mm_kwargs",
|
||||
return_value=[("audio", 1, {"audio_values": torch.zeros(2)})],
|
||||
):
|
||||
result = runner.execute_mm_encoder([])
|
||||
result = runner.execute_mm_encoder([("audio", MagicMock())])
|
||||
assert len(result) == 1
|
||||
assert result[0] is output
|
||||
manager.execute.assert_not_called()
|
||||
@@ -290,6 +290,21 @@ def test_execute_mm_encoder_caches_outputs_without_gathering():
|
||||
state.encoder_runner.gather_mm_embeddings.assert_not_called()
|
||||
|
||||
|
||||
def test_execute_mm_encoder_sorts_by_modality():
|
||||
"""Items are encoded sorted by modality; outputs keep the input order."""
|
||||
mm_kwargs = [("video", 0), ("image", 1), ("video", 2), ("image", 3)]
|
||||
runner = _make_runner([], [])
|
||||
runner.model = MagicMock()
|
||||
runner.model.embed_multimodal.side_effect = lambda x: [torch.full((1, 1), x)]
|
||||
with patch(
|
||||
"vllm.v1.worker.gpu.mm.encoder_runner.group_and_batch_mm_kwargs",
|
||||
side_effect=lambda items, **_: [(m, 1, {"x": x}) for m, x in items],
|
||||
) as group:
|
||||
outputs = runner.execute_mm_encoder(mm_kwargs)
|
||||
assert group.call_args.args[0] == [mm_kwargs[i] for i in (1, 3, 0, 2)]
|
||||
assert [int(o) for o in outputs] == [0, 1, 2, 3]
|
||||
|
||||
|
||||
def test_execute_mm_encoder_is_a_noop_without_scheduled_items():
|
||||
"""A step that schedules no encoder input must not touch the encoder."""
|
||||
cache = EncoderCache()
|
||||
|
||||
@@ -624,6 +624,70 @@ def test_set_active_mm_loras_builds_tower_and_connector_mappings():
|
||||
assert connector_mapping.index_mapping == ((7,) * 14 + (7,) * 13 + (0,) * 12)
|
||||
|
||||
|
||||
def test_set_active_mm_loras_follows_modality_sorted_encoder_order():
|
||||
model = Mock()
|
||||
model.get_mm_lora_token_counts.side_effect = (
|
||||
lambda *, modality, mm_kwargs, num_mm_embeds: (num_mm_embeds, None)
|
||||
)
|
||||
lora_manager = Mock()
|
||||
lora_manager.supports_tower_connector_lora.return_value = True
|
||||
|
||||
encoder_cache = EncoderCache()
|
||||
for req_id, modality, length in [("req-a", "video", 1), ("req-b", "image", 2)]:
|
||||
encoder_cache.mm_features[req_id] = [
|
||||
MultiModalFeatureSpec(
|
||||
data=None,
|
||||
modality=modality,
|
||||
identifier=f"{req_id}-0",
|
||||
mm_position=PlaceholderRange(offset=0, length=length),
|
||||
)
|
||||
]
|
||||
lora_state = LoraState(max_num_reqs=2)
|
||||
lora_request = LoRARequest("vision-lora", 7, "/tmp/vision-lora")
|
||||
lora_state.add_request("req-a", 0, lora_request)
|
||||
lora_state.add_request("req-b", 1, None)
|
||||
|
||||
set_active_mm_loras(
|
||||
model=model,
|
||||
lora_manager=lora_manager,
|
||||
encoder_cache=encoder_cache,
|
||||
req_id_to_index={"req-a": 0, "req-b": 1},
|
||||
lora_state=lora_state,
|
||||
scheduled_encoder_inputs={"req-a": [0], "req-b": [0]},
|
||||
)
|
||||
|
||||
# The image is encoded before the video, so its tokens come first.
|
||||
_, tower_mapping = lora_manager.set_active_adapters.call_args_list[0].args
|
||||
assert tower_mapping.index_mapping == (0, 0, 7)
|
||||
|
||||
|
||||
def test_batch_mm_inputs_from_scheduler_sorts_by_modality():
|
||||
features = {
|
||||
f"req{i}": MultiModalFeatureSpec(
|
||||
data=Mock(),
|
||||
modality=m,
|
||||
identifier=f"hash{i}",
|
||||
mm_position=PlaceholderRange(offset=i, length=1),
|
||||
)
|
||||
for i, m in enumerate(["video", "image", "video"])
|
||||
}
|
||||
runner = SimpleNamespace(
|
||||
requests={r: SimpleNamespace(mm_features=[f]) for r, f in features.items()}
|
||||
)
|
||||
scheduler_output = SimpleNamespace(
|
||||
scheduled_encoder_inputs={r: [0] for r in features}
|
||||
)
|
||||
|
||||
mm_hashes, mm_kwargs, mm_lora_refs = GPUModelRunner._batch_mm_inputs_from_scheduler(
|
||||
runner, scheduler_output
|
||||
)
|
||||
|
||||
order = ["req1", "req0", "req2"]
|
||||
assert mm_hashes == [features[r].identifier for r in order]
|
||||
assert mm_kwargs == [(features[r].modality, features[r].data) for r in order]
|
||||
assert mm_lora_refs == [(r, features[r].mm_position) for r in order]
|
||||
|
||||
|
||||
def test_update_states_new_request(model_runner, dist_init):
|
||||
req_id = "req_0"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user