[MM] Move get_dummy_processor_inputs into MM processor (#57967)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-21 09:39:20 -07:00
committed by GitHub
parent 4e61ad8ee1
commit 7853700d4c
10 changed files with 131 additions and 135 deletions
@@ -159,7 +159,6 @@ def get_token_prompt(
processor: BaseMultiModalProcessor,
mm_data: MultiModalDataDict,
) -> list[int]:
dummy_inputs = processor.dummy_inputs
tokenizer: TokenizerLike = processor.info.get_tokenizer()
model_config = processor.info.ctx.model_config
@@ -174,7 +173,7 @@ def get_token_prompt(
mm_counts = {k: len(vs) for k, vs in parsed_data.items()}
if is_mistral_tokenizer(tokenizer):
inputs = dummy_inputs.get_dummy_processor_inputs(
inputs = processor.get_dummy_inputs(
model_config.max_model_len,
mm_counts,
mm_options=MultiModalDummyOptions(),
@@ -182,7 +181,7 @@ def get_token_prompt(
mm_data=mm_data, # type: ignore[call-arg]
)
else:
inputs = dummy_inputs.get_dummy_processor_inputs(
inputs = processor.get_dummy_inputs(
model_config.max_model_len,
mm_counts,
mm_options=MultiModalDummyOptions(),
@@ -90,13 +90,12 @@ def create_batched_mm_kwargs(
size_factors: tuple[float, ...] = (1.0, 0.5, 0.25),
) -> Iterable[tuple[str, int, BatchedTensorInputs]]:
processing_info = processor.info
dummy_inputs = processor.dummy_inputs
supported_mm_limits = processing_info.get_supported_mm_limits()
mm_counts = {
modality: 3 if limit is None else limit
for modality, limit in supported_mm_limits.items()
}
processor_inputs = dummy_inputs.get_dummy_processor_inputs(
processor_inputs = processor.get_dummy_inputs(
seq_len=model_config.max_model_len,
mm_counts=mm_counts,
mm_options=MultiModalDummyOptions(),
+6 -8
View File
@@ -4,7 +4,7 @@
These tests exercise:
- Zero-limit modalities are filtered from mm_counts passed to
get_dummy_processor_inputs (e.g. --limit-mm-per-prompt image=0 ...)
get_dummy_inputs (e.g. --limit-mm-per-prompt image=0 ...)
- The warmup runs through the processor-only cache and clears it afterwards,
so its dummy inputs never land in the sender cache used by the serving path
- MM warmup is skipped entirely when mm_processor is None
@@ -92,7 +92,7 @@ class TestMmWarmupZeroLimitFiltering:
BaseRenderer.warmup(renderer, ChatParams())
get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
get_inputs = renderer.mm_processor.get_dummy_inputs
get_inputs.assert_called_once()
_, kwargs = get_inputs.call_args
assert "video" not in kwargs["mm_counts"]
@@ -104,7 +104,7 @@ class TestMmWarmupZeroLimitFiltering:
BaseRenderer.warmup(renderer, ChatParams())
get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
get_inputs = renderer.mm_processor.get_dummy_inputs
get_inputs.assert_called_once()
_, kwargs = get_inputs.call_args
assert kwargs["mm_counts"] == {}
@@ -115,7 +115,7 @@ class TestMmWarmupZeroLimitFiltering:
BaseRenderer.warmup(renderer, ChatParams())
get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
get_inputs = renderer.mm_processor.get_dummy_inputs
get_inputs.assert_called_once()
_, kwargs = get_inputs.call_args
assert kwargs["mm_counts"] == {"image": 1, "video": 1}
@@ -134,7 +134,7 @@ class TestMmWarmupRunsNormally:
BaseRenderer.warmup(renderer, ChatParams())
get_inputs = renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs
get_inputs = renderer.mm_processor.get_dummy_inputs
assert get_inputs.call_args.kwargs["seq_len"] == expected_seq_len
def test_processor_apply_called(self):
@@ -207,9 +207,7 @@ class TestWarmupFaultIsolation:
def test_mm_failure_is_swallowed_and_cache_cleared(self):
renderer = _make_renderer_mock({"image": 1})
# the warmup blows up before apply()
renderer.mm_processor.dummy_inputs.get_dummy_processor_inputs.side_effect = (
RuntimeError("mm boom")
)
renderer.mm_processor.get_dummy_inputs.side_effect = RuntimeError("mm boom")
BaseRenderer.warmup(renderer, ChatParams()) # must not raise