[Core] Encoder separation for Encode-Prefill-Decode Disaggregation (#25233)
Signed-off-by: n00909098 <nguyen.kha.long@huawei.com> Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com> Signed-off-by: herotai214 <herotai214@gmail.com> Signed-off-by: Khuong Le <khuong.le.manh@huawei.com> Signed-off-by: Khuong Le <lemanhkhuong2611@gmail.com> Co-authored-by: n00909098 <nguyen.kha.long@huawei.com> Co-authored-by: knlnguyen1802 <knlnguyen1802@gmail.com> Co-authored-by: herotai214 <herotai214@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Khuong Le <khuong.le.manh@huawei.com> Co-authored-by: Khuong Le <lemanhkhuong2611@gmail.com>
This commit is contained in:
co-authored by
n00909098
knlnguyen1802
herotai214
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Khuong Le
Khuong Le
parent
cbb799e314
commit
4ccffe561f
@@ -0,0 +1,46 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm import envs
|
||||
from vllm.distributed.ec_transfer.ec_connector.base import (
|
||||
ECConnectorBase,
|
||||
ECConnectorRole,
|
||||
)
|
||||
from vllm.distributed.ec_transfer.ec_connector.factory import ECConnectorFactory
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
_EC_CONNECTOR_AGENT: ECConnectorBase | None = None
|
||||
|
||||
|
||||
def get_ec_transfer() -> ECConnectorBase:
|
||||
assert _EC_CONNECTOR_AGENT is not None, "disaggregated EC cache is not initialized"
|
||||
return _EC_CONNECTOR_AGENT
|
||||
|
||||
|
||||
def has_ec_transfer() -> bool:
|
||||
return _EC_CONNECTOR_AGENT is not None
|
||||
|
||||
|
||||
def ensure_ec_transfer_initialized(vllm_config: "VllmConfig") -> None:
|
||||
"""
|
||||
Initialize EC cache connector.
|
||||
"""
|
||||
|
||||
global _EC_CONNECTOR_AGENT
|
||||
|
||||
if vllm_config.ec_transfer_config is None:
|
||||
return
|
||||
|
||||
if (
|
||||
vllm_config.ec_transfer_config.is_ec_transfer_instance
|
||||
and _EC_CONNECTOR_AGENT is None
|
||||
):
|
||||
if envs.VLLM_USE_V1:
|
||||
_EC_CONNECTOR_AGENT = ECConnectorFactory.create_connector(
|
||||
config=vllm_config, role=ECConnectorRole.WORKER
|
||||
)
|
||||
else:
|
||||
raise ValueError("V0 is no longer supported")
|
||||
Reference in New Issue
Block a user