[Metrics] Log multi-modal cache stats and fix reset (#26285)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-10-10 01:45:55 -07:00
committed by GitHub
parent 6f0f570c43
commit ad430a67ca
25 changed files with 586 additions and 235 deletions
+3 -1
View File
@@ -463,6 +463,7 @@ class AsyncLLM(EngineClient):
output_processor = self.output_processor
log_stats = self.log_stats
logger_manager = self.logger_manager
processor = self.processor
async def output_handler():
try:
@@ -511,6 +512,7 @@ class AsyncLLM(EngineClient):
engine_idx=outputs.engine_index,
scheduler_stats=outputs.scheduler_stats,
iteration_stats=iteration_stats,
mm_cache_stats=processor.stat_mm_cache(),
)
except Exception as e:
logger.exception("AsyncLLM output_handler failed.")
@@ -660,7 +662,7 @@ class AsyncLLM(EngineClient):
await asyncio.gather(*coros)
async def reset_mm_cache(self) -> None:
self.processor.clear_cache()
self.processor.clear_mm_cache()
await self.engine_core.reset_mm_cache_async()
async def reset_prefix_cache(self, device: Optional[Device] = None) -> None:
+5 -2
View File
@@ -319,7 +319,7 @@ class EngineCore:
)
engine_core_outputs = self.scheduler.update_from_output(
scheduler_output, model_output
) # type: ignore
)
return (engine_core_outputs, scheduler_output.total_num_scheduled_tokens > 0)
@@ -400,16 +400,19 @@ class EngineCore:
def reset_mm_cache(self):
# NOTE: Since this is mainly for debugging, we don't attempt to
# re-sync the internal caches (P0 processor, P0 mirror, P1 mirror)
# re-sync the internal caches (P0 sender, P1 receiver)
if self.scheduler.has_unfinished_requests():
logger.warning(
"Resetting the multi-modal cache when requests are "
"in progress may lead to desynced internal caches."
)
# The cache either exists in EngineCore or WorkerWrapperBase
if self.mm_receiver_cache is not None:
self.mm_receiver_cache.clear_cache()
self.model_executor.reset_mm_cache()
def reset_prefix_cache(self):
self.scheduler.reset_prefix_cache()
+3 -1
View File
@@ -306,9 +306,11 @@ class LLMEngine:
# 4) Record stats
if self.logger_manager is not None:
assert outputs.scheduler_stats is not None
self.logger_manager.record(
scheduler_stats=outputs.scheduler_stats,
iteration_stats=iteration_stats,
mm_cache_stats=self.processor.stat_mm_cache(),
)
self.do_log_stats_with_interval()
@@ -321,7 +323,7 @@ class LLMEngine:
self.engine_core.profile(False)
def reset_mm_cache(self):
self.processor.clear_cache()
self.processor.clear_mm_cache()
self.engine_core.reset_mm_cache()
def reset_prefix_cache(self, device: Optional[Device] = None):
+6 -2
View File
@@ -21,6 +21,7 @@ from vllm.sampling_params import SamplingParams
from vllm.transformers_utils.tokenizer import AnyTokenizer
from vllm.utils import length_from_prompt_token_ids_or_embeds
from vllm.v1.engine import EngineCoreRequest
from vllm.v1.metrics.stats import MultiModalCacheStats
from vllm.v1.structured_output.backend_guidance import validate_guidance_grammar
from vllm.v1.structured_output.backend_lm_format_enforcer import (
validate_structured_output_request_lm_format_enforcer,
@@ -573,5 +574,8 @@ class Processor:
# check that chunked prefill does not truncate them
# max_batch_len = self.scheduler_config.max_num_batched_tokens
def clear_cache(self) -> None:
self.input_preprocessor.clear_cache()
def stat_mm_cache(self) -> Optional[MultiModalCacheStats]:
return self.input_preprocessor.stat_mm_cache()
def clear_mm_cache(self) -> None:
self.input_preprocessor.clear_mm_cache()