[Metrics] Log multi-modal cache stats and fix reset (#26285)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -463,6 +463,7 @@ class AsyncLLM(EngineClient):
|
||||
output_processor = self.output_processor
|
||||
log_stats = self.log_stats
|
||||
logger_manager = self.logger_manager
|
||||
processor = self.processor
|
||||
|
||||
async def output_handler():
|
||||
try:
|
||||
@@ -511,6 +512,7 @@ class AsyncLLM(EngineClient):
|
||||
engine_idx=outputs.engine_index,
|
||||
scheduler_stats=outputs.scheduler_stats,
|
||||
iteration_stats=iteration_stats,
|
||||
mm_cache_stats=processor.stat_mm_cache(),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception("AsyncLLM output_handler failed.")
|
||||
@@ -660,7 +662,7 @@ class AsyncLLM(EngineClient):
|
||||
await asyncio.gather(*coros)
|
||||
|
||||
async def reset_mm_cache(self) -> None:
|
||||
self.processor.clear_cache()
|
||||
self.processor.clear_mm_cache()
|
||||
await self.engine_core.reset_mm_cache_async()
|
||||
|
||||
async def reset_prefix_cache(self, device: Optional[Device] = None) -> None:
|
||||
|
||||
@@ -319,7 +319,7 @@ class EngineCore:
|
||||
)
|
||||
engine_core_outputs = self.scheduler.update_from_output(
|
||||
scheduler_output, model_output
|
||||
) # type: ignore
|
||||
)
|
||||
|
||||
return (engine_core_outputs, scheduler_output.total_num_scheduled_tokens > 0)
|
||||
|
||||
@@ -400,16 +400,19 @@ class EngineCore:
|
||||
|
||||
def reset_mm_cache(self):
|
||||
# NOTE: Since this is mainly for debugging, we don't attempt to
|
||||
# re-sync the internal caches (P0 processor, P0 mirror, P1 mirror)
|
||||
# re-sync the internal caches (P0 sender, P1 receiver)
|
||||
if self.scheduler.has_unfinished_requests():
|
||||
logger.warning(
|
||||
"Resetting the multi-modal cache when requests are "
|
||||
"in progress may lead to desynced internal caches."
|
||||
)
|
||||
|
||||
# The cache either exists in EngineCore or WorkerWrapperBase
|
||||
if self.mm_receiver_cache is not None:
|
||||
self.mm_receiver_cache.clear_cache()
|
||||
|
||||
self.model_executor.reset_mm_cache()
|
||||
|
||||
def reset_prefix_cache(self):
|
||||
self.scheduler.reset_prefix_cache()
|
||||
|
||||
|
||||
@@ -306,9 +306,11 @@ class LLMEngine:
|
||||
# 4) Record stats
|
||||
if self.logger_manager is not None:
|
||||
assert outputs.scheduler_stats is not None
|
||||
|
||||
self.logger_manager.record(
|
||||
scheduler_stats=outputs.scheduler_stats,
|
||||
iteration_stats=iteration_stats,
|
||||
mm_cache_stats=self.processor.stat_mm_cache(),
|
||||
)
|
||||
self.do_log_stats_with_interval()
|
||||
|
||||
@@ -321,7 +323,7 @@ class LLMEngine:
|
||||
self.engine_core.profile(False)
|
||||
|
||||
def reset_mm_cache(self):
|
||||
self.processor.clear_cache()
|
||||
self.processor.clear_mm_cache()
|
||||
self.engine_core.reset_mm_cache()
|
||||
|
||||
def reset_prefix_cache(self, device: Optional[Device] = None):
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.sampling_params import SamplingParams
|
||||
from vllm.transformers_utils.tokenizer import AnyTokenizer
|
||||
from vllm.utils import length_from_prompt_token_ids_or_embeds
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.metrics.stats import MultiModalCacheStats
|
||||
from vllm.v1.structured_output.backend_guidance import validate_guidance_grammar
|
||||
from vllm.v1.structured_output.backend_lm_format_enforcer import (
|
||||
validate_structured_output_request_lm_format_enforcer,
|
||||
@@ -573,5 +574,8 @@ class Processor:
|
||||
# check that chunked prefill does not truncate them
|
||||
# max_batch_len = self.scheduler_config.max_num_batched_tokens
|
||||
|
||||
def clear_cache(self) -> None:
|
||||
self.input_preprocessor.clear_cache()
|
||||
def stat_mm_cache(self) -> Optional[MultiModalCacheStats]:
|
||||
return self.input_preprocessor.stat_mm_cache()
|
||||
|
||||
def clear_mm_cache(self) -> None:
|
||||
self.input_preprocessor.clear_mm_cache()
|
||||
|
||||
Reference in New Issue
Block a user