From 7bbbf7c8e5040f7ebd374e8cbc657e01af1136dd Mon Sep 17 00:00:00 2001 From: hotTea <45391346+hotTea123@users.noreply.github.com> Date: Thu, 13 Aug 2026 16:08:01 +0800 Subject: [PATCH] [Core] Configure custom encoder cache managers from VllmConfig (#51251) Signed-off-by: hotTea <958436561@qq.com> Signed-off-by: HMCCMH Co-authored-by: HMCCMH --- vllm/config/ec_manager_config.py | 14 ++++++++++++-- vllm/engine/arg_utils.py | 10 ++++++++++ vllm/v1/core/encoder_cache_manager.py | 7 +++++++ vllm/v1/core/sched/scheduler.py | 13 +++++++------ 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/vllm/config/ec_manager_config.py b/vllm/config/ec_manager_config.py index 9e7cda64e6..a59d2cdc62 100644 --- a/vllm/config/ec_manager_config.py +++ b/vllm/config/ec_manager_config.py @@ -2,6 +2,8 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from abc import ABC +from dataclasses import field +from typing import Any from vllm.config.utils import config from vllm.utils.import_utils import resolve_obj_by_qualname @@ -10,8 +12,16 @@ from vllm.utils.import_utils import resolve_obj_by_qualname @config class EncoderCacheManagerConfig: encoder_cache_manager_cls: str | None = None - """The qualified class name of the custom encoder cache manager. - """ + """Fully qualified class name of the custom encoder cache manager.""" + + manager_config: dict[str, Any] = field(default_factory=dict) + """Opaque configuration interpreted by the custom cache manager.""" + + def __post_init__(self) -> None: + if self.encoder_cache_manager_cls is None and self.manager_config: + raise ValueError( + "manager_config requires encoder_cache_manager_cls to be set." + ) def get_encoder_cache_manager_obj(self): cls_path = self.encoder_cache_manager_cls diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 3b9ead7f7e..bd8b42dc27 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -41,6 +41,7 @@ from vllm.config import ( DeviceConfig, DiffusionConfig, ECTransferConfig, + EncoderCacheManagerConfig, EPLBConfig, FaultToleranceConfig, KernelConfig, @@ -686,6 +687,9 @@ class EngineArgs: kv_events_config: KVEventsConfig | None = None ec_transfer_config: ECTransferConfig | None = None + ec_manager_config: EncoderCacheManagerConfig = get_field( + VllmConfig, "ec_manager_config" + ) reasoning_config: ReasoningConfig = get_field(VllmConfig, "reasoning_config") generation_config: str = ModelConfig.generation_config @@ -766,6 +770,8 @@ class EngineArgs: self.mamba_config = MambaConfig(**self.mamba_config) if isinstance(self.kernel_config, dict): self.kernel_config = KernelConfig(**self.kernel_config) + if isinstance(self.ec_manager_config, dict): + self.ec_manager_config = EncoderCacheManagerConfig(**self.ec_manager_config) if isinstance(self.eplb_config, dict): self.eplb_config = EPLBConfig(**self.eplb_config) if isinstance(self.weight_transfer_config, dict): @@ -1625,6 +1631,9 @@ class EngineArgs: vllm_group.add_argument( "--ec-transfer-config", **vllm_kwargs["ec_transfer_config"] ) + vllm_group.add_argument( + "--ec-manager-config", **vllm_kwargs["ec_manager_config"] + ) vllm_group.add_argument( "--compilation-config", "-cc", **vllm_kwargs["compilation_config"] ) @@ -2506,6 +2515,7 @@ class EngineArgs: kv_transfer_config=self.kv_transfer_config, kv_events_config=self.kv_events_config, ec_transfer_config=self.ec_transfer_config, + ec_manager_config=self.ec_manager_config, reasoning_config=self.reasoning_config, profiler_config=self.profiler_config, additional_config=self.additional_config, diff --git a/vllm/v1/core/encoder_cache_manager.py b/vllm/v1/core/encoder_cache_manager.py index e2eacb2d02..02133ff5b8 100644 --- a/vllm/v1/core/encoder_cache_manager.py +++ b/vllm/v1/core/encoder_cache_manager.py @@ -5,6 +5,7 @@ from collections import OrderedDict from collections.abc import Mapping from typing import TYPE_CHECKING +from vllm.config import VllmConfig from vllm.config.ec_manager_config import EncoderCacheManagerMetadata from vllm.logger import init_logger from vllm.v1.request import Request @@ -65,6 +66,12 @@ class EncoderCacheManager: last call to get_freed_mm_hashes(). This list is cleared on return. """ + @classmethod + def create_manager( + cls, *, cache_size: int, vllm_config: "VllmConfig" + ) -> "EncoderCacheManager": + return cls(cache_size=cache_size) + def __init__(self, cache_size: int): self.cache_size = cache_size self.num_free_slots = cache_size diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 4ebc248b84..455289a479 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -234,14 +234,15 @@ class Scheduler(SchedulerInterface): ) encoder_cache_size = mm_budget.encoder_cache_size if mm_budget else 0 manager_cls_obj = vllm_config.ec_manager_config.get_encoder_cache_manager_obj() - if manager_cls_obj is not None: - self.encoder_cache_manager = manager_cls_obj(cache_size=encoder_cache_size) - else: - self.encoder_cache_manager = ( - EncoderDecoderCacheManager(cache_size=encoder_cache_size) + if manager_cls_obj is None: + manager_cls_obj = ( + EncoderDecoderCacheManager if self.is_encoder_decoder - else EncoderCacheManager(cache_size=encoder_cache_size) + else EncoderCacheManager ) + self.encoder_cache_manager = manager_cls_obj.create_manager( + cache_size=encoder_cache_size, vllm_config=vllm_config + ) speculative_config = vllm_config.speculative_config self.use_eagle = False self.num_spec_tokens = vllm_config.num_speculative_tokens