[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)
Signed-off-by: lqh-hh <liuqihang6@huawei.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: lqh-hh <liuqihang6@huawei.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: 2017403603 <705328646@qq.com> Co-authored-by: Julien Mancuso <jmancuso@nvidia.com> Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
lqh-hh
Tyler Michael Smith
OpenAI Codex
2017403603
Julien Mancuso
Itay Alroy
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
fb3fec970e
commit
dd9e07a25f
@@ -328,7 +328,10 @@ def terminate_if_errored(server: uvicorn.Server, engine: EngineClient):
|
||||
for this request.
|
||||
"""
|
||||
engine_errored = engine.errored and not engine.is_running
|
||||
if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine_errored:
|
||||
keep_alive = (
|
||||
envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH or engine.should_keep_api_server_alive
|
||||
)
|
||||
if not keep_alive and engine_errored:
|
||||
server.should_exit = True
|
||||
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from http import HTTPStatus
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.distributed.elastic_ep.external_elastic_ep import ExternalElasticEPScalePhase
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
@@ -82,7 +83,35 @@ async def scale_elastic_ep(raw_request: Request):
|
||||
|
||||
@router.post("/is_scaling_elastic_ep")
|
||||
async def is_scaling_elastic_ep(raw_request: Request):
|
||||
return JSONResponse({"is_scaling_elastic_ep": get_scaling_elastic_ep()})
|
||||
# External operation status comes from the shared store. Middleware gating
|
||||
# remains process-local, so scaling requests must reach every old-rank API.
|
||||
try:
|
||||
status = await engine_client(raw_request).get_external_elastic_ep_status()
|
||||
except Exception as e:
|
||||
logger.warning("Failed to query external Elastic EP status: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="External Elastic EP status is temporarily unavailable",
|
||||
) from e
|
||||
if status is None:
|
||||
# Non-external EEP modes retain the process-local middleware state.
|
||||
is_scaling = get_scaling_elastic_ep()
|
||||
status = {
|
||||
"phase": (
|
||||
ExternalElasticEPScalePhase.COMMITTING.value
|
||||
if is_scaling
|
||||
else ExternalElasticEPScalePhase.IDLE.value
|
||||
),
|
||||
"epoch": None,
|
||||
"error": None,
|
||||
"requested_data_parallel_size": None,
|
||||
}
|
||||
else:
|
||||
is_scaling = status["phase"] in (
|
||||
ExternalElasticEPScalePhase.PREPARING.value,
|
||||
ExternalElasticEPScalePhase.COMMITTING.value,
|
||||
)
|
||||
return JSONResponse({"is_scaling_elastic_ep": is_scaling, **status})
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
|
||||
@@ -6,8 +6,12 @@ from collections.abc import Awaitable
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.types import ASGIApp, Receive, Scope, Send
|
||||
|
||||
# Global variable to track scaling state
|
||||
# Process-local states used to pause serving during scaling and after this rank
|
||||
# has been removed from the external-LB topology. A retired API process remains
|
||||
# available for health and scaling-status queries until its orchestrator exits it.
|
||||
_scaling_elastic_ep = False
|
||||
_elastic_ep_rank_retired = False
|
||||
_SCALING_OBSERVABILITY_PATHS = frozenset({"/health", "/is_scaling_elastic_ep"})
|
||||
|
||||
|
||||
def get_scaling_elastic_ep():
|
||||
@@ -19,12 +23,20 @@ def set_scaling_elastic_ep(value):
|
||||
_scaling_elastic_ep = value
|
||||
|
||||
|
||||
class ScalingMiddleware:
|
||||
"""Middleware that checks if the model is currently scaling and
|
||||
returns a 503 Service Unavailable response if it is.
|
||||
def get_elastic_ep_rank_retired():
|
||||
return _elastic_ep_rank_retired
|
||||
|
||||
This middleware applies to all HTTP requests and prevents
|
||||
processing when the model is in a scaling state.
|
||||
|
||||
def set_elastic_ep_rank_retired(value):
|
||||
global _elastic_ep_rank_retired
|
||||
_elastic_ep_rank_retired = value
|
||||
|
||||
|
||||
class ScalingMiddleware:
|
||||
"""Pause serving while scaling or after the local rank is retired.
|
||||
|
||||
Health and scaling-status requests remain available so an external
|
||||
orchestrator can observe the operation and remove a retired process.
|
||||
"""
|
||||
|
||||
def __init__(self, app: ASGIApp) -> None:
|
||||
@@ -34,13 +46,18 @@ class ScalingMiddleware:
|
||||
if scope["type"] != "http":
|
||||
return self.app(scope, receive, send)
|
||||
|
||||
# Check global scaling state
|
||||
if get_scaling_elastic_ep():
|
||||
# Return 503 Service Unavailable response
|
||||
# Keep observability available while serving is paused or this rank is
|
||||
# waiting for the external orchestrator to remove its process.
|
||||
rank_retired = get_elastic_ep_rank_retired()
|
||||
serving_paused = get_scaling_elastic_ep() or rank_retired
|
||||
if serving_paused and scope["path"] not in _SCALING_OBSERVABILITY_PATHS:
|
||||
error = (
|
||||
"This data-parallel rank has been retired and is awaiting shutdown."
|
||||
if rank_retired
|
||||
else "The model is currently scaling. Please try again later."
|
||||
)
|
||||
response = JSONResponse(
|
||||
content={
|
||||
"error": "The model is currently scaling. Please try again later."
|
||||
},
|
||||
content={"error": error},
|
||||
status_code=503,
|
||||
)
|
||||
return response(scope, receive, send)
|
||||
|
||||
Reference in New Issue
Block a user