[Refactor] Use dynamic MM cache in processor (#57674)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2026-09-19 15:38:00 +00:00
committed by GitHub
parent 41c4a3ed4e
commit 751f6807d9
23 changed files with 343 additions and 401 deletions
+4 -10
View File
@@ -5837,21 +5837,15 @@ class GPUModelRunner(
assert self.mm_budget is not None
# Don't use `max_items_per_batch` here to avoid redundant computation
dummy_mm_inputs = self.mm_registry.get_dummy_mm_inputs(
self.model_config,
mm_counts={modality: 1},
cache=self.mm_budget.cache,
dummy_mm_inputs = self.mm_budget.get_dummy_encoder_profile_inputs(
modality,
max_items_per_batch,
)
dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
# We use the cache so that the item is saved to the cache,
# but not read from the cache
assert dummy_mm_item is not None, "Item should not already be cached"
return next(
mm_kwargs_batch
for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs(
[(modality, dummy_mm_item)] * max_items_per_batch,
dummy_mm_inputs,
device=self.device,
pin_memory=PIN_MEMORY,
)