From 833483f3578a2c4766977d0fe431bd32a01ce250 Mon Sep 17 00:00:00 2001 From: hotTea <45391346+hotTea123@users.noreply.github.com> Date: Fri, 24 Jul 2026 17:52:00 +0800 Subject: [PATCH] Encoder cache extension hooks (#48218) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: hotTea <958436561@qq.com> Signed-off-by: hanxi-java <634498162@qq.com> Co-authored-by: hanxi-java <634498162@qq.com> Co-authored-by: 韩熙 <63780107+hanxi-java@users.noreply.github.com> --- .../worker/test_gpu_model_runner_mm_gather.py | 1 + vllm/config/__init__.py | 3 + vllm/config/ec_manager_config.py | 29 ++++++++++ vllm/config/vllm.py | 5 ++ vllm/v1/core/encoder_cache_manager.py | 4 ++ vllm/v1/core/sched/output.py | 5 +- vllm/v1/core/sched/scheduler.py | 16 +++-- vllm/v1/worker/gpu_model_runner.py | 58 ++++++++++++++++--- 8 files changed, 106 insertions(+), 15 deletions(-) create mode 100644 vllm/config/ec_manager_config.py diff --git a/tests/v1/worker/test_gpu_model_runner_mm_gather.py b/tests/v1/worker/test_gpu_model_runner_mm_gather.py index 586acd5046..a86e77c8f5 100644 --- a/tests/v1/worker/test_gpu_model_runner_mm_gather.py +++ b/tests/v1/worker/test_gpu_model_runner_mm_gather.py @@ -46,6 +46,7 @@ def _gather(features, cached, *, num_scheduled, shift, num_computed=0): input_batch=SimpleNamespace(req_ids=["req0"]), requests={"req0": req_state}, encoder_cache=encoder_cache, + _get_encoder_output_from_cache=lambda mm_hash: encoder_cache.get(mm_hash), is_multimodal_pruning_enabled=False, uses_mrope=False, ) diff --git a/vllm/config/__init__.py b/vllm/config/__init__.py index 82ab1842fe..6070a3f823 100644 --- a/vllm/config/__init__.py +++ b/vllm/config/__init__.py @@ -11,6 +11,7 @@ from vllm.config.compilation import ( ) from vllm.config.device import DeviceConfig from vllm.config.diffusion import DiffusionConfig +from vllm.config.ec_manager_config import EncoderCacheManagerConfig from vllm.config.ec_transfer import ECTransferConfig from vllm.config.kernel import KernelConfig from vllm.config.kv_events import KVEventsConfig @@ -75,6 +76,8 @@ __all__ = [ "DeviceConfig", # From vllm.config.diffusion "DiffusionConfig", + # From vllm.config.ec_manager_config + "EncoderCacheManagerConfig", # From vllm.config.ec_transfer "ECTransferConfig", # From vllm.config.kernel diff --git a/vllm/config/ec_manager_config.py b/vllm/config/ec_manager_config.py new file mode 100644 index 0000000000..9e7cda64e6 --- /dev/null +++ b/vllm/config/ec_manager_config.py @@ -0,0 +1,29 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from abc import ABC + +from vllm.config.utils import config +from vllm.utils.import_utils import resolve_obj_by_qualname + + +@config +class EncoderCacheManagerConfig: + encoder_cache_manager_cls: str | None = None + """The qualified class name of the custom encoder cache manager. + """ + + def get_encoder_cache_manager_obj(self): + cls_path = self.encoder_cache_manager_cls + if cls_path is None: + return None + return resolve_obj_by_qualname(cls_path) + + +class EncoderCacheManagerMetadata(ABC): # noqa: B024 + """ + Abstract Metadata used to communicate between the + Scheduler EncoderCacheManager and Worker EncoderCacheManager. + """ + + pass diff --git a/vllm/config/vllm.py b/vllm/config/vllm.py index bda5013a5f..c05e35cbbc 100644 --- a/vllm/config/vllm.py +++ b/vllm/config/vllm.py @@ -32,6 +32,7 @@ from .cache import CacheConfig from .compilation import CompilationConfig, CompilationMode, CUDAGraphMode from .device import DeviceConfig from .diffusion import DiffusionConfig +from .ec_manager_config import EncoderCacheManagerConfig from .ec_transfer import ECTransferConfig from .kernel import KernelConfig from .kv_events import KVEventsConfig @@ -367,6 +368,10 @@ class VllmConfig: """The configurations for event publishing.""" ec_transfer_config: ECTransferConfig | None = None """The configurations for distributed EC cache transfer.""" + ec_manager_config: EncoderCacheManagerConfig = Field( + default_factory=EncoderCacheManagerConfig + ) + """The configurations for custom encoder cache manager.""" reasoning_config: ReasoningConfig | None = None """The configurations for reasoning model.""" # some opaque config, only used to provide additional information diff --git a/vllm/v1/core/encoder_cache_manager.py b/vllm/v1/core/encoder_cache_manager.py index d479239e3b..e2eacb2d02 100644 --- a/vllm/v1/core/encoder_cache_manager.py +++ b/vllm/v1/core/encoder_cache_manager.py @@ -5,6 +5,7 @@ from collections import OrderedDict from collections.abc import Mapping from typing import TYPE_CHECKING +from vllm.config.ec_manager_config import EncoderCacheManagerMetadata from vllm.logger import init_logger from vllm.v1.request import Request @@ -265,6 +266,9 @@ class EncoderCacheManager: self.freed = [] return freed + def get_manager_metadata(self) -> EncoderCacheManagerMetadata | None: + return None + def compute_mm_encoder_budget( scheduler_config: "SchedulerConfig", diff --git a/vllm/v1/core/sched/output.py b/vllm/v1/core/sched/output.py index 5667482e94..dab86c015e 100644 --- a/vllm/v1/core/sched/output.py +++ b/vllm/v1/core/sched/output.py @@ -5,6 +5,8 @@ from dataclasses import dataclass from functools import cached_property from typing import TYPE_CHECKING +from vllm.config.ec_manager_config import EncoderCacheManagerMetadata + if TYPE_CHECKING: import numpy as np import numpy.typing as npt @@ -246,7 +248,8 @@ class SchedulerOutput: # EC Cache Connector metadata ec_connector_metadata: ECConnectorMetadata | None = None - + # EC Cache Manager metadata + ec_manager_metadata: EncoderCacheManagerMetadata | None = None # Block IDs freshly allocated from the pool during this scheduling step. # The worker zeros the corresponding GPU memory before the blocks are used, # preventing stale NaN/data from corrupting attention or SSM computation. diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 1190b93cc8..7536f6b060 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -227,12 +227,15 @@ class Scheduler(SchedulerInterface): mm_budget.encoder_compute_budget if mm_budget else 0 ) encoder_cache_size = mm_budget.encoder_cache_size if mm_budget else 0 - self.encoder_cache_manager = ( - EncoderDecoderCacheManager(cache_size=encoder_cache_size) - if self.is_encoder_decoder - else EncoderCacheManager(cache_size=encoder_cache_size) - ) - + manager_cls_obj = vllm_config.ec_manager_config.get_encoder_cache_manager_obj() + if manager_cls_obj is not None: + self.encoder_cache_manager = manager_cls_obj(cache_size=encoder_cache_size) + else: + self.encoder_cache_manager = ( + EncoderDecoderCacheManager(cache_size=encoder_cache_size) + if self.is_encoder_decoder + else EncoderCacheManager(cache_size=encoder_cache_size) + ) speculative_config = vllm_config.speculative_config self.use_eagle = False self.num_spec_tokens = vllm_config.num_speculative_tokens @@ -1151,6 +1154,7 @@ class Scheduler(SchedulerInterface): new_block_ids_to_zero=self._get_new_block_ids_to_zero(), kv_cache_block_copies=pending_kv_cache_block_copies, num_spec_tokens_to_schedule=num_spec_tokens_to_schedule, + ec_manager_metadata=self.encoder_cache_manager.get_manager_metadata(), ) # NOTE(Kuntai): this function is designed for multiple purposes: diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index f7646682e7..40d2ec0e4d 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -37,6 +37,7 @@ from vllm.config import ( update_config, ) from vllm.config.cache import CacheConfig +from vllm.config.ec_manager_config import EncoderCacheManagerMetadata from vllm.config.model import PROCESSED_LOGPROBS_MODES from vllm.distributed.ec_transfer import get_ec_transfer, has_ec_transfer from vllm.distributed.eplb.eplb_state import EplbState @@ -1164,6 +1165,22 @@ class GPUModelRunner( self.async_output_copy_stream = stream return stream + def _on_request_state_removed( + self, + req_id: str, + req_state: CachedRequestState | None, + ) -> None: + """Hook for platform runners to clean request-scoped side caches.""" + del req_id, req_state + + def _process_encoder_cache_scheduler_output( + self, + scheduler_output: "SchedulerOutput", + ) -> None: + """Apply scheduler-side encoder cache lifecycle updates.""" + for mm_hash in scheduler_output.free_encoder_mm_hashes: + self.encoder_cache.pop(mm_hash, None) + def _update_states(self, scheduler_output: "SchedulerOutput") -> Callable | None: """Update the cached states and the persistent batch with the scheduler output. @@ -1176,7 +1193,8 @@ class GPUModelRunner( """ # Remove finished requests from the cached states. for req_id in scheduler_output.finished_req_ids: - self.requests.pop(req_id, None) + req_state = self.requests.pop(req_id, None) + self._on_request_state_removed(req_id, req_state) self.num_prompt_logprobs.pop(req_id, None) self.late_interaction_runner.on_requests_finished( scheduler_output.finished_req_ids @@ -1202,8 +1220,7 @@ class GPUModelRunner( ) # Free the cached encoder outputs. - for mm_hash in scheduler_output.free_encoder_mm_hashes: - self.encoder_cache.pop(mm_hash, None) + self._process_encoder_cache_scheduler_output(scheduler_output) # Remove the unscheduled requests from the persistent batch. # NOTE(woosuk): The unscheduled requests are either preempted requests @@ -2958,6 +2975,18 @@ class GPUModelRunner( return mm_hashes, mm_kwargs, mm_lora_refs + def _cache_encoder_output( + self, + mm_hash: str, + output: torch.Tensor, + ec_manager_metadata: "EncoderCacheManagerMetadata | None", + free_encoder_mm_hashes: list[str], + ) -> None: + """Store an encoder output for later multimodal embedding gather.""" + del ec_manager_metadata, free_encoder_mm_hashes + self.encoder_cache[mm_hash] = output + self.maybe_save_ec_to_connector(self.encoder_cache, mm_hash) + def _execute_mm_encoder( self, scheduler_output: "SchedulerOutput" ) -> list[torch.Tensor]: @@ -2983,8 +3012,12 @@ class GPUModelRunner( pe_tensor = mm_kwargs[i][1]["embedding"].data assert isinstance(pe_tensor, torch.Tensor) - self.encoder_cache[mm_hashes[i]] = pe_tensor.to(self.device) - self.maybe_save_ec_to_connector(self.encoder_cache, mm_hashes[i]) + self._cache_encoder_output( + mm_hashes[i], + pe_tensor.to(self.device), + scheduler_output.ec_manager_metadata, + scheduler_output.free_encoder_mm_hashes, + ) # Filter out `prompt_embeds` items from mm_kwargs/mm_hashes/mm_lora_refs # since they don't require further encoder processing. mm_hashes = [h for i, h in enumerate(mm_hashes) if i not in pe_indices] @@ -3161,12 +3194,21 @@ class GPUModelRunner( # Cache the encoder outputs by mm_hash for mm_hash, output in zip(mm_hashes, encoder_outputs): - self.encoder_cache[mm_hash] = output + self._cache_encoder_output( + mm_hash, + output, + scheduler_output.ec_manager_metadata, + scheduler_output.free_encoder_mm_hashes, + ) logger.debug("Finish execute for mm hash %s", mm_hash) - self.maybe_save_ec_to_connector(self.encoder_cache, mm_hash) return encoder_outputs + def _get_encoder_output_from_cache(self, mm_hash: str) -> torch.Tensor | None: + """Return a cached encoder output for multimodal + embedding gather.""" + return self.encoder_cache.get(mm_hash, None) + def _gather_mm_embeddings( self, scheduler_output: "SchedulerOutput", @@ -3220,7 +3262,7 @@ class GPUModelRunner( continue mm_hash = mm_feature.identifier - encoder_output = self.encoder_cache.get(mm_hash, None) + encoder_output = self._get_encoder_output_from_cache(mm_hash) if encoder_output is None: # A feature starting at/after the processed boundary is only # reached via the drafter's +1 look-ahead and might not be