[Core] Configure custom encoder cache managers from VllmConfig (#51251)
Signed-off-by: hotTea <958436561@qq.com> Signed-off-by: HMCCMH <chenminghaoscu@163.com> Co-authored-by: HMCCMH <chenminghaoscu@163.com>
This commit is contained in:
@@ -5,6 +5,7 @@ from collections import OrderedDict
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.request import Request
|
||||
@@ -65,6 +66,12 @@ class EncoderCacheManager:
|
||||
last call to get_freed_mm_hashes(). This list is cleared on return.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def create_manager(
|
||||
cls, *, cache_size: int, vllm_config: "VllmConfig"
|
||||
) -> "EncoderCacheManager":
|
||||
return cls(cache_size=cache_size)
|
||||
|
||||
def __init__(self, cache_size: int):
|
||||
self.cache_size = cache_size
|
||||
self.num_free_slots = cache_size
|
||||
|
||||
@@ -234,14 +234,15 @@ class Scheduler(SchedulerInterface):
|
||||
)
|
||||
encoder_cache_size = mm_budget.encoder_cache_size if mm_budget else 0
|
||||
manager_cls_obj = vllm_config.ec_manager_config.get_encoder_cache_manager_obj()
|
||||
if manager_cls_obj is not None:
|
||||
self.encoder_cache_manager = manager_cls_obj(cache_size=encoder_cache_size)
|
||||
else:
|
||||
self.encoder_cache_manager = (
|
||||
EncoderDecoderCacheManager(cache_size=encoder_cache_size)
|
||||
if manager_cls_obj is None:
|
||||
manager_cls_obj = (
|
||||
EncoderDecoderCacheManager
|
||||
if self.is_encoder_decoder
|
||||
else EncoderCacheManager(cache_size=encoder_cache_size)
|
||||
else EncoderCacheManager
|
||||
)
|
||||
self.encoder_cache_manager = manager_cls_obj.create_manager(
|
||||
cache_size=encoder_cache_size, vllm_config=vllm_config
|
||||
)
|
||||
speculative_config = vllm_config.speculative_config
|
||||
self.use_eagle = False
|
||||
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
||||
|
||||
Reference in New Issue
Block a user