Signed-off-by: lqh-hh <liuqihang6@huawei.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: lqh-hh <liuqihang6@huawei.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: 2017403603 <705328646@qq.com> Co-authored-by: Julien Mancuso <jmancuso@nvidia.com> Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
119 lines
4.1 KiB
Python
119 lines
4.1 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
|
|
import json
|
|
from http import HTTPStatus
|
|
|
|
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
|
from fastapi.responses import JSONResponse
|
|
|
|
from vllm.distributed.elastic_ep.external_elastic_ep import ExternalElasticEPScalePhase
|
|
from vllm.engine.protocol import EngineClient
|
|
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
|
|
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
|
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
|
from vllm.logger import init_logger
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
def engine_client(request: Request) -> EngineClient:
|
|
return request.app.state.engine_client
|
|
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
@router.post(
|
|
"/scale_elastic_ep",
|
|
dependencies=[Depends(validate_json_request)],
|
|
responses={
|
|
HTTPStatus.OK.value: {"model": dict},
|
|
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
|
HTTPStatus.REQUEST_TIMEOUT.value: {"model": ErrorResponse},
|
|
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
|
},
|
|
)
|
|
async def scale_elastic_ep(raw_request: Request):
|
|
try:
|
|
body = await raw_request.json()
|
|
except json.JSONDecodeError as e:
|
|
raise HTTPException(status_code=400, detail="Invalid JSON format") from e
|
|
|
|
new_data_parallel_size = body.get("new_data_parallel_size")
|
|
drain_timeout = body.get("drain_timeout", 120) # Default 2 minutes
|
|
|
|
if new_data_parallel_size is None:
|
|
raise HTTPException(
|
|
status_code=400, detail="new_data_parallel_size is required"
|
|
)
|
|
|
|
if not isinstance(new_data_parallel_size, int) or new_data_parallel_size <= 0:
|
|
raise HTTPException(
|
|
status_code=400,
|
|
detail="new_data_parallel_size must be a positive integer",
|
|
)
|
|
|
|
if not isinstance(drain_timeout, int) or drain_timeout <= 0:
|
|
raise HTTPException(
|
|
status_code=400, detail="drain_timeout must be a positive integer"
|
|
)
|
|
|
|
client = engine_client(raw_request)
|
|
try:
|
|
await client.scale_elastic_ep(new_data_parallel_size, drain_timeout)
|
|
return JSONResponse(
|
|
{
|
|
"message": f"Scaled to {new_data_parallel_size} data parallel engines",
|
|
}
|
|
)
|
|
except TimeoutError as e:
|
|
raise HTTPException(
|
|
status_code=408,
|
|
detail="Scale failed due to request drain timeout "
|
|
f"after {drain_timeout} seconds",
|
|
) from e
|
|
except ValueError as e:
|
|
raise HTTPException(status_code=400, detail=str(e)) from e
|
|
except Exception as e:
|
|
logger.error("Scale failed: %s", e)
|
|
raise HTTPException(status_code=500, detail="Scale failed") from e
|
|
|
|
|
|
@router.post("/is_scaling_elastic_ep")
|
|
async def is_scaling_elastic_ep(raw_request: Request):
|
|
# External operation status comes from the shared store. Middleware gating
|
|
# remains process-local, so scaling requests must reach every old-rank API.
|
|
try:
|
|
status = await engine_client(raw_request).get_external_elastic_ep_status()
|
|
except Exception as e:
|
|
logger.warning("Failed to query external Elastic EP status: %s", e)
|
|
raise HTTPException(
|
|
status_code=503,
|
|
detail="External Elastic EP status is temporarily unavailable",
|
|
) from e
|
|
if status is None:
|
|
# Non-external EEP modes retain the process-local middleware state.
|
|
is_scaling = get_scaling_elastic_ep()
|
|
status = {
|
|
"phase": (
|
|
ExternalElasticEPScalePhase.COMMITTING.value
|
|
if is_scaling
|
|
else ExternalElasticEPScalePhase.IDLE.value
|
|
),
|
|
"epoch": None,
|
|
"error": None,
|
|
"requested_data_parallel_size": None,
|
|
}
|
|
else:
|
|
is_scaling = status["phase"] in (
|
|
ExternalElasticEPScalePhase.PREPARING.value,
|
|
ExternalElasticEPScalePhase.COMMITTING.value,
|
|
)
|
|
return JSONResponse({"is_scaling_elastic_ep": is_scaling, **status})
|
|
|
|
|
|
def attach_router(app: FastAPI):
|
|
app.include_router(router)
|