[Core] Configure custom encoder cache managers from VllmConfig (#51251)

Signed-off-by: hotTea <958436561@qq.com>
Signed-off-by: HMCCMH <chenminghaoscu@163.com>
Co-authored-by: HMCCMH <chenminghaoscu@163.com>
This commit is contained in:
hotTea
2026-08-13 16:08:01 +08:00
committed by GitHub
co-authored by HMCCMH
parent 399f97424e
commit 7bbbf7c8e5
4 changed files with 36 additions and 8 deletions
+7
View File
@@ -5,6 +5,7 @@ from collections import OrderedDict
from collections.abc import Mapping
from typing import TYPE_CHECKING
from vllm.config import VllmConfig
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
from vllm.logger import init_logger
from vllm.v1.request import Request
@@ -65,6 +66,12 @@ class EncoderCacheManager:
last call to get_freed_mm_hashes(). This list is cleared on return.
"""
@classmethod
def create_manager(
cls, *, cache_size: int, vllm_config: "VllmConfig"
) -> "EncoderCacheManager":
return cls(cache_size=cache_size)
def __init__(self, cache_size: int):
self.cache_size = cache_size
self.num_free_slots = cache_size
+7 -6
View File
@@ -234,14 +234,15 @@ class Scheduler(SchedulerInterface):
)
encoder_cache_size = mm_budget.encoder_cache_size if mm_budget else 0
manager_cls_obj = vllm_config.ec_manager_config.get_encoder_cache_manager_obj()
if manager_cls_obj is not None:
self.encoder_cache_manager = manager_cls_obj(cache_size=encoder_cache_size)
else:
self.encoder_cache_manager = (
EncoderDecoderCacheManager(cache_size=encoder_cache_size)
if manager_cls_obj is None:
manager_cls_obj = (
EncoderDecoderCacheManager
if self.is_encoder_decoder
else EncoderCacheManager(cache_size=encoder_cache_size)
else EncoderCacheManager
)
self.encoder_cache_manager = manager_cls_obj.create_manager(
cache_size=encoder_cache_size, vllm_config=vllm_config
)
speculative_config = vllm_config.speculative_config
self.use_eagle = False
self.num_spec_tokens = vllm_config.num_speculative_tokens