[Core] Configure custom encoder cache managers from VllmConfig (#51251)

Signed-off-by: hotTea <958436561@qq.com>
Signed-off-by: HMCCMH <chenminghaoscu@163.com>
Co-authored-by: HMCCMH <chenminghaoscu@163.com>
This commit is contained in:
hotTea
2026-08-13 16:08:01 +08:00
committed by GitHub
co-authored by HMCCMH
parent 399f97424e
commit 7bbbf7c8e5
4 changed files with 36 additions and 8 deletions
+12 -2
View File
@@ -2,6 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from abc import ABC from abc import ABC
from dataclasses import field
from typing import Any
from vllm.config.utils import config from vllm.config.utils import config
from vllm.utils.import_utils import resolve_obj_by_qualname from vllm.utils.import_utils import resolve_obj_by_qualname
@@ -10,8 +12,16 @@ from vllm.utils.import_utils import resolve_obj_by_qualname
@config @config
class EncoderCacheManagerConfig: class EncoderCacheManagerConfig:
encoder_cache_manager_cls: str | None = None encoder_cache_manager_cls: str | None = None
"""The qualified class name of the custom encoder cache manager. """Fully qualified class name of the custom encoder cache manager."""
"""
manager_config: dict[str, Any] = field(default_factory=dict)
"""Opaque configuration interpreted by the custom cache manager."""
def __post_init__(self) -> None:
if self.encoder_cache_manager_cls is None and self.manager_config:
raise ValueError(
"manager_config requires encoder_cache_manager_cls to be set."
)
def get_encoder_cache_manager_obj(self): def get_encoder_cache_manager_obj(self):
cls_path = self.encoder_cache_manager_cls cls_path = self.encoder_cache_manager_cls
+10
View File
@@ -41,6 +41,7 @@ from vllm.config import (
DeviceConfig, DeviceConfig,
DiffusionConfig, DiffusionConfig,
ECTransferConfig, ECTransferConfig,
EncoderCacheManagerConfig,
EPLBConfig, EPLBConfig,
FaultToleranceConfig, FaultToleranceConfig,
KernelConfig, KernelConfig,
@@ -686,6 +687,9 @@ class EngineArgs:
kv_events_config: KVEventsConfig | None = None kv_events_config: KVEventsConfig | None = None
ec_transfer_config: ECTransferConfig | None = None ec_transfer_config: ECTransferConfig | None = None
ec_manager_config: EncoderCacheManagerConfig = get_field(
VllmConfig, "ec_manager_config"
)
reasoning_config: ReasoningConfig = get_field(VllmConfig, "reasoning_config") reasoning_config: ReasoningConfig = get_field(VllmConfig, "reasoning_config")
generation_config: str = ModelConfig.generation_config generation_config: str = ModelConfig.generation_config
@@ -766,6 +770,8 @@ class EngineArgs:
self.mamba_config = MambaConfig(**self.mamba_config) self.mamba_config = MambaConfig(**self.mamba_config)
if isinstance(self.kernel_config, dict): if isinstance(self.kernel_config, dict):
self.kernel_config = KernelConfig(**self.kernel_config) self.kernel_config = KernelConfig(**self.kernel_config)
if isinstance(self.ec_manager_config, dict):
self.ec_manager_config = EncoderCacheManagerConfig(**self.ec_manager_config)
if isinstance(self.eplb_config, dict): if isinstance(self.eplb_config, dict):
self.eplb_config = EPLBConfig(**self.eplb_config) self.eplb_config = EPLBConfig(**self.eplb_config)
if isinstance(self.weight_transfer_config, dict): if isinstance(self.weight_transfer_config, dict):
@@ -1625,6 +1631,9 @@ class EngineArgs:
vllm_group.add_argument( vllm_group.add_argument(
"--ec-transfer-config", **vllm_kwargs["ec_transfer_config"] "--ec-transfer-config", **vllm_kwargs["ec_transfer_config"]
) )
vllm_group.add_argument(
"--ec-manager-config", **vllm_kwargs["ec_manager_config"]
)
vllm_group.add_argument( vllm_group.add_argument(
"--compilation-config", "-cc", **vllm_kwargs["compilation_config"] "--compilation-config", "-cc", **vllm_kwargs["compilation_config"]
) )
@@ -2506,6 +2515,7 @@ class EngineArgs:
kv_transfer_config=self.kv_transfer_config, kv_transfer_config=self.kv_transfer_config,
kv_events_config=self.kv_events_config, kv_events_config=self.kv_events_config,
ec_transfer_config=self.ec_transfer_config, ec_transfer_config=self.ec_transfer_config,
ec_manager_config=self.ec_manager_config,
reasoning_config=self.reasoning_config, reasoning_config=self.reasoning_config,
profiler_config=self.profiler_config, profiler_config=self.profiler_config,
additional_config=self.additional_config, additional_config=self.additional_config,
+7
View File
@@ -5,6 +5,7 @@ from collections import OrderedDict
from collections.abc import Mapping from collections.abc import Mapping
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
from vllm.config import VllmConfig
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.v1.request import Request from vllm.v1.request import Request
@@ -65,6 +66,12 @@ class EncoderCacheManager:
last call to get_freed_mm_hashes(). This list is cleared on return. last call to get_freed_mm_hashes(). This list is cleared on return.
""" """
@classmethod
def create_manager(
cls, *, cache_size: int, vllm_config: "VllmConfig"
) -> "EncoderCacheManager":
return cls(cache_size=cache_size)
def __init__(self, cache_size: int): def __init__(self, cache_size: int):
self.cache_size = cache_size self.cache_size = cache_size
self.num_free_slots = cache_size self.num_free_slots = cache_size
+7 -6
View File
@@ -234,14 +234,15 @@ class Scheduler(SchedulerInterface):
) )
encoder_cache_size = mm_budget.encoder_cache_size if mm_budget else 0 encoder_cache_size = mm_budget.encoder_cache_size if mm_budget else 0
manager_cls_obj = vllm_config.ec_manager_config.get_encoder_cache_manager_obj() manager_cls_obj = vllm_config.ec_manager_config.get_encoder_cache_manager_obj()
if manager_cls_obj is not None: if manager_cls_obj is None:
self.encoder_cache_manager = manager_cls_obj(cache_size=encoder_cache_size) manager_cls_obj = (
else: EncoderDecoderCacheManager
self.encoder_cache_manager = (
EncoderDecoderCacheManager(cache_size=encoder_cache_size)
if self.is_encoder_decoder if self.is_encoder_decoder
else EncoderCacheManager(cache_size=encoder_cache_size) else EncoderCacheManager
) )
self.encoder_cache_manager = manager_cls_obj.create_manager(
cache_size=encoder_cache_size, vllm_config=vllm_config
)
speculative_config = vllm_config.speculative_config speculative_config = vllm_config.speculative_config
self.use_eagle = False self.use_eagle = False
self.num_spec_tokens = vllm_config.num_speculative_tokens self.num_spec_tokens = vllm_config.num_speculative_tokens