[Feature] Add fault tolerance framework (simplified) for DP+EP external LB deployments (#44428)
Signed-off-by: fangyuchu <fangyuchu@qq.com> Signed-off-by: a798347923 <2645302020@qq.com> Signed-off-by: TianZhuo <2770730562@qq.com> Signed-off-by: a798347923 <39047817+a798347923@users.noreply.github.com> Signed-off-by: 205150940 <112750056+205150940@users.noreply.github.com> Signed-off-by: w00689259 <wangzhuo66@huawei.com> Signed-off-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com> Signed-off-by: zWaNg3 <389750525@qq.com> Signed-off-by: yzchang-plus <1078477584@qq.com> Signed-off-by: Jade Zheng <zheng.shoujian@outlook.com> Co-authored-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com> Co-authored-by: a798347923 <2645302020@qq.com> Co-authored-by: TianZhuo <2770730562@qq.com> Co-authored-by: 205150940 <112750056+205150940@users.noreply.github.com> Co-authored-by: a798347923 <39047817+a798347923@users.noreply.github.com> Co-authored-by: w00689259 <wangzhuo66@huawei.com> Co-authored-by: zWaNg3 <389750525@qq.com> Co-authored-by: yzchang-plus <1078477584@qq.com> Co-authored-by: Jade Zheng <zheng.shoujian@outlook.com>
This commit is contained in:
co-authored by
zWaNg3
a798347923
TianZhuo
205150940
a798347923
w00689259
zWaNg3
yzchang-plus
Jade Zheng
parent
33ef67e9fb
commit
0b0bd2b5f6
@@ -20,6 +20,7 @@ from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
from vllm.v1.fault_tolerance.utils import FaultToleranceRequest, FaultToleranceResult
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.engine import PauseMode
|
||||
@@ -234,6 +235,16 @@ class EngineClient(ABC):
|
||||
"""Perform a collective RPC call to the given path."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def handle_fault(
|
||||
self, fault_tolerance_request: FaultToleranceRequest
|
||||
) -> FaultToleranceResult:
|
||||
"""send fault tolerance instruction to the engine"""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_status(self):
|
||||
"""Get fault tolerance status of all engines."""
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_supported_tasks(self) -> tuple[SupportedTask, ...]:
|
||||
"""Get supported tasks"""
|
||||
raise NotImplementedError
|
||||
|
||||
Reference in New Issue
Block a user