[MRV2] Add encoder cache profiling implementation (#47985)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Isotr0py
2026-07-20 20:18:14 -07:00
committed by GitHub
parent 94ed0bf4e0
commit ea0e9c8f2e
4 changed files with 93 additions and 2 deletions
+21
View File
@@ -43,6 +43,10 @@ from vllm.model_executor.layers.mamba.ops.ssu_dispatch import (
)
from vllm.model_executor.model_loader import get_model_loader
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.encoder_budget import (
MultiModalBudget,
get_dummy_encoder_profile_inputs,
)
from vllm.sequence import IntermediateTensors
from vllm.tasks import SupportedTask
from vllm.utils.math_utils import cdiv
@@ -671,6 +675,22 @@ class GPUModelRunner(LoRAModelRunnerMixin):
@torch.inference_mode()
def profile_run(self) -> None:
if self.supports_mm_inputs and self.is_first_pp_rank:
mm_config = self.model_config.multimodal_config
if mm_config is not None and not mm_config.skip_mm_profiling:
mm_budget = MultiModalBudget(
self.vllm_config,
self.mm_registry,
enable_cache=False,
)
dummy_mm_inputs = get_dummy_encoder_profile_inputs(
self.mm_registry,
mm_budget,
)
self.model_state.encoder_runner.profile_encoder_cache(
dummy_mm_inputs, mm_budget
)
hidden_states, sample_hidden_states = self._dummy_run(
self.max_num_tokens, skip_attn=True, is_profile=True
)
@@ -685,6 +705,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
torch.accelerator.synchronize()
del hidden_states, sample_hidden_states
self.reset_encoder_cache()
gc.collect()
def post_kv_cache_wake_up(self) -> None: