[Refactor] Use dynamic MM cache in processor (#57674)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -5837,21 +5837,15 @@ class GPUModelRunner(
|
||||
assert self.mm_budget is not None
|
||||
|
||||
# Don't use `max_items_per_batch` here to avoid redundant computation
|
||||
dummy_mm_inputs = self.mm_registry.get_dummy_mm_inputs(
|
||||
self.model_config,
|
||||
mm_counts={modality: 1},
|
||||
cache=self.mm_budget.cache,
|
||||
dummy_mm_inputs = self.mm_budget.get_dummy_encoder_profile_inputs(
|
||||
modality,
|
||||
max_items_per_batch,
|
||||
)
|
||||
dummy_mm_item = dummy_mm_inputs["mm_kwargs"][modality][0]
|
||||
|
||||
# We use the cache so that the item is saved to the cache,
|
||||
# but not read from the cache
|
||||
assert dummy_mm_item is not None, "Item should not already be cached"
|
||||
|
||||
return next(
|
||||
mm_kwargs_batch
|
||||
for _, _, mm_kwargs_batch in group_and_batch_mm_kwargs(
|
||||
[(modality, dummy_mm_item)] * max_items_per_batch,
|
||||
dummy_mm_inputs,
|
||||
device=self.device,
|
||||
pin_memory=PIN_MEMORY,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user