[Core] Configure custom encoder cache managers from VllmConfig (#51251)
Signed-off-by: hotTea <958436561@qq.com> Signed-off-by: HMCCMH <chenminghaoscu@163.com> Co-authored-by: HMCCMH <chenminghaoscu@163.com>
This commit is contained in:
@@ -2,6 +2,8 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
from abc import ABC
|
from abc import ABC
|
||||||
|
from dataclasses import field
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from vllm.config.utils import config
|
from vllm.config.utils import config
|
||||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||||
@@ -10,8 +12,16 @@ from vllm.utils.import_utils import resolve_obj_by_qualname
|
|||||||
@config
|
@config
|
||||||
class EncoderCacheManagerConfig:
|
class EncoderCacheManagerConfig:
|
||||||
encoder_cache_manager_cls: str | None = None
|
encoder_cache_manager_cls: str | None = None
|
||||||
"""The qualified class name of the custom encoder cache manager.
|
"""Fully qualified class name of the custom encoder cache manager."""
|
||||||
"""
|
|
||||||
|
manager_config: dict[str, Any] = field(default_factory=dict)
|
||||||
|
"""Opaque configuration interpreted by the custom cache manager."""
|
||||||
|
|
||||||
|
def __post_init__(self) -> None:
|
||||||
|
if self.encoder_cache_manager_cls is None and self.manager_config:
|
||||||
|
raise ValueError(
|
||||||
|
"manager_config requires encoder_cache_manager_cls to be set."
|
||||||
|
)
|
||||||
|
|
||||||
def get_encoder_cache_manager_obj(self):
|
def get_encoder_cache_manager_obj(self):
|
||||||
cls_path = self.encoder_cache_manager_cls
|
cls_path = self.encoder_cache_manager_cls
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ from vllm.config import (
|
|||||||
DeviceConfig,
|
DeviceConfig,
|
||||||
DiffusionConfig,
|
DiffusionConfig,
|
||||||
ECTransferConfig,
|
ECTransferConfig,
|
||||||
|
EncoderCacheManagerConfig,
|
||||||
EPLBConfig,
|
EPLBConfig,
|
||||||
FaultToleranceConfig,
|
FaultToleranceConfig,
|
||||||
KernelConfig,
|
KernelConfig,
|
||||||
@@ -686,6 +687,9 @@ class EngineArgs:
|
|||||||
kv_events_config: KVEventsConfig | None = None
|
kv_events_config: KVEventsConfig | None = None
|
||||||
|
|
||||||
ec_transfer_config: ECTransferConfig | None = None
|
ec_transfer_config: ECTransferConfig | None = None
|
||||||
|
ec_manager_config: EncoderCacheManagerConfig = get_field(
|
||||||
|
VllmConfig, "ec_manager_config"
|
||||||
|
)
|
||||||
reasoning_config: ReasoningConfig = get_field(VllmConfig, "reasoning_config")
|
reasoning_config: ReasoningConfig = get_field(VllmConfig, "reasoning_config")
|
||||||
|
|
||||||
generation_config: str = ModelConfig.generation_config
|
generation_config: str = ModelConfig.generation_config
|
||||||
@@ -766,6 +770,8 @@ class EngineArgs:
|
|||||||
self.mamba_config = MambaConfig(**self.mamba_config)
|
self.mamba_config = MambaConfig(**self.mamba_config)
|
||||||
if isinstance(self.kernel_config, dict):
|
if isinstance(self.kernel_config, dict):
|
||||||
self.kernel_config = KernelConfig(**self.kernel_config)
|
self.kernel_config = KernelConfig(**self.kernel_config)
|
||||||
|
if isinstance(self.ec_manager_config, dict):
|
||||||
|
self.ec_manager_config = EncoderCacheManagerConfig(**self.ec_manager_config)
|
||||||
if isinstance(self.eplb_config, dict):
|
if isinstance(self.eplb_config, dict):
|
||||||
self.eplb_config = EPLBConfig(**self.eplb_config)
|
self.eplb_config = EPLBConfig(**self.eplb_config)
|
||||||
if isinstance(self.weight_transfer_config, dict):
|
if isinstance(self.weight_transfer_config, dict):
|
||||||
@@ -1625,6 +1631,9 @@ class EngineArgs:
|
|||||||
vllm_group.add_argument(
|
vllm_group.add_argument(
|
||||||
"--ec-transfer-config", **vllm_kwargs["ec_transfer_config"]
|
"--ec-transfer-config", **vllm_kwargs["ec_transfer_config"]
|
||||||
)
|
)
|
||||||
|
vllm_group.add_argument(
|
||||||
|
"--ec-manager-config", **vllm_kwargs["ec_manager_config"]
|
||||||
|
)
|
||||||
vllm_group.add_argument(
|
vllm_group.add_argument(
|
||||||
"--compilation-config", "-cc", **vllm_kwargs["compilation_config"]
|
"--compilation-config", "-cc", **vllm_kwargs["compilation_config"]
|
||||||
)
|
)
|
||||||
@@ -2506,6 +2515,7 @@ class EngineArgs:
|
|||||||
kv_transfer_config=self.kv_transfer_config,
|
kv_transfer_config=self.kv_transfer_config,
|
||||||
kv_events_config=self.kv_events_config,
|
kv_events_config=self.kv_events_config,
|
||||||
ec_transfer_config=self.ec_transfer_config,
|
ec_transfer_config=self.ec_transfer_config,
|
||||||
|
ec_manager_config=self.ec_manager_config,
|
||||||
reasoning_config=self.reasoning_config,
|
reasoning_config=self.reasoning_config,
|
||||||
profiler_config=self.profiler_config,
|
profiler_config=self.profiler_config,
|
||||||
additional_config=self.additional_config,
|
additional_config=self.additional_config,
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ from collections import OrderedDict
|
|||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
|
from vllm.config import VllmConfig
|
||||||
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
|
from vllm.config.ec_manager_config import EncoderCacheManagerMetadata
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.v1.request import Request
|
from vllm.v1.request import Request
|
||||||
@@ -65,6 +66,12 @@ class EncoderCacheManager:
|
|||||||
last call to get_freed_mm_hashes(). This list is cleared on return.
|
last call to get_freed_mm_hashes(). This list is cleared on return.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def create_manager(
|
||||||
|
cls, *, cache_size: int, vllm_config: "VllmConfig"
|
||||||
|
) -> "EncoderCacheManager":
|
||||||
|
return cls(cache_size=cache_size)
|
||||||
|
|
||||||
def __init__(self, cache_size: int):
|
def __init__(self, cache_size: int):
|
||||||
self.cache_size = cache_size
|
self.cache_size = cache_size
|
||||||
self.num_free_slots = cache_size
|
self.num_free_slots = cache_size
|
||||||
|
|||||||
@@ -234,14 +234,15 @@ class Scheduler(SchedulerInterface):
|
|||||||
)
|
)
|
||||||
encoder_cache_size = mm_budget.encoder_cache_size if mm_budget else 0
|
encoder_cache_size = mm_budget.encoder_cache_size if mm_budget else 0
|
||||||
manager_cls_obj = vllm_config.ec_manager_config.get_encoder_cache_manager_obj()
|
manager_cls_obj = vllm_config.ec_manager_config.get_encoder_cache_manager_obj()
|
||||||
if manager_cls_obj is not None:
|
if manager_cls_obj is None:
|
||||||
self.encoder_cache_manager = manager_cls_obj(cache_size=encoder_cache_size)
|
manager_cls_obj = (
|
||||||
else:
|
EncoderDecoderCacheManager
|
||||||
self.encoder_cache_manager = (
|
|
||||||
EncoderDecoderCacheManager(cache_size=encoder_cache_size)
|
|
||||||
if self.is_encoder_decoder
|
if self.is_encoder_decoder
|
||||||
else EncoderCacheManager(cache_size=encoder_cache_size)
|
else EncoderCacheManager
|
||||||
)
|
)
|
||||||
|
self.encoder_cache_manager = manager_cls_obj.create_manager(
|
||||||
|
cache_size=encoder_cache_size, vllm_config=vllm_config
|
||||||
|
)
|
||||||
speculative_config = vllm_config.speculative_config
|
speculative_config = vllm_config.speculative_config
|
||||||
self.use_eagle = False
|
self.use_eagle = False
|
||||||
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
self.num_spec_tokens = vllm_config.num_speculative_tokens
|
||||||
|
|||||||
Reference in New Issue
Block a user