[Feature] Add fault tolerance framework (simplified) for DP+EP external LB deployments (#44428)
Signed-off-by: fangyuchu <fangyuchu@qq.com> Signed-off-by: a798347923 <2645302020@qq.com> Signed-off-by: TianZhuo <2770730562@qq.com> Signed-off-by: a798347923 <39047817+a798347923@users.noreply.github.com> Signed-off-by: 205150940 <112750056+205150940@users.noreply.github.com> Signed-off-by: w00689259 <wangzhuo66@huawei.com> Signed-off-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com> Signed-off-by: zWaNg3 <389750525@qq.com> Signed-off-by: yzchang-plus <1078477584@qq.com> Signed-off-by: Jade Zheng <zheng.shoujian@outlook.com> Co-authored-by: zWaNg3 <37772915+zWaNg3@users.noreply.github.com> Co-authored-by: a798347923 <2645302020@qq.com> Co-authored-by: TianZhuo <2770730562@qq.com> Co-authored-by: 205150940 <112750056+205150940@users.noreply.github.com> Co-authored-by: a798347923 <39047817+a798347923@users.noreply.github.com> Co-authored-by: w00689259 <wangzhuo66@huawei.com> Co-authored-by: zWaNg3 <389750525@qq.com> Co-authored-by: yzchang-plus <1078477584@qq.com> Co-authored-by: Jade Zheng <zheng.shoujian@outlook.com>
This commit is contained in:
co-authored by
zWaNg3
a798347923
TianZhuo
205150940
a798347923
w00689259
zWaNg3
yzchang-plus
Jade Zheng
parent
33ef67e9fb
commit
0b0bd2b5f6
@@ -269,6 +269,13 @@ def build_app(
|
||||
|
||||
register_pooling_api_routers(app, supported_tasks, model_config)
|
||||
|
||||
if args.enable_fault_tolerance:
|
||||
from vllm.entrypoints.serve.fault_tolerance.api_router import (
|
||||
register_fault_tolerance_api_router,
|
||||
)
|
||||
|
||||
register_fault_tolerance_api_router(app)
|
||||
|
||||
# Endpoint plugins are attached last so their routes are registered after all core
|
||||
# routers. This runs even for the CPU only render server. A plugin eligible for
|
||||
# the `render` task still gets its routes registered. It receives
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import uuid
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.fault_tolerance.utils import FaultToleranceRequest
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
_ALLOWED_INSTRUCTIONS = {"retry"}
|
||||
|
||||
|
||||
def _validate_payload(body: dict) -> tuple[str, dict]:
|
||||
if not isinstance(body, dict):
|
||||
raise HTTPException(400, "Request body must be a JSON object.")
|
||||
instruction = body.get("instruction")
|
||||
if not instruction:
|
||||
raise HTTPException(400, "'instruction' is required.")
|
||||
if instruction not in _ALLOWED_INSTRUCTIONS:
|
||||
raise HTTPException(400, f"Invalid instruction: '{instruction}'.")
|
||||
params = body.get("params", {})
|
||||
if not isinstance(params, dict):
|
||||
raise HTTPException(400, "'params' must be an object.")
|
||||
return instruction, params
|
||||
|
||||
|
||||
@router.post(
|
||||
"/fault_tolerance/apply",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.ACCEPTED.value: {"model": dict},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def process_fault_tolerance_instruction(
|
||||
raw_request: Request, background_tasks: BackgroundTasks
|
||||
):
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(400, "Invalid JSON format") from e
|
||||
|
||||
instruction, params = _validate_payload(body)
|
||||
ft_request = FaultToleranceRequest(
|
||||
instruction=instruction,
|
||||
params=params,
|
||||
request_id=str(uuid.uuid4()),
|
||||
)
|
||||
|
||||
client: EngineClient = raw_request.app.state.engine_client
|
||||
# Recovery runs cross-rank collective ops that only complete once every rank
|
||||
# has been dispatched. Run it in the background and return immediately so the
|
||||
# orchestrator can dispatch to all ranks without blocking; completion is
|
||||
# observed by polling GET /fault_tolerance/status.
|
||||
background_tasks.add_task(_run_fault_recovery, client, ft_request)
|
||||
return JSONResponse(
|
||||
status_code=HTTPStatus.ACCEPTED.value,
|
||||
content={
|
||||
"message": "Request accepted; poll /fault_tolerance/status for updates.",
|
||||
"request_id": ft_request.request_id,
|
||||
},
|
||||
background=background_tasks,
|
||||
)
|
||||
|
||||
|
||||
async def _run_fault_recovery(
|
||||
client: EngineClient, ft_request: FaultToleranceRequest
|
||||
) -> None:
|
||||
"""Drive recovery to completion after the 202 response is sent."""
|
||||
try:
|
||||
result = await client.handle_fault(ft_request)
|
||||
except Exception:
|
||||
logger.exception("[FT] Recovery dispatch failed.")
|
||||
return
|
||||
if not result.success:
|
||||
logger.error(
|
||||
"[FT] Recovery failed for request %s: %s",
|
||||
ft_request.request_id,
|
||||
result.reason,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/fault_tolerance/status")
|
||||
async def get_status(raw_request: Request):
|
||||
client: EngineClient = raw_request.app.state.engine_client
|
||||
return JSONResponse(content=await client.get_status())
|
||||
|
||||
|
||||
def register_fault_tolerance_api_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
Reference in New Issue
Block a user