[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)

Signed-off-by: lqh-hh <liuqihang6@huawei.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: lqh-hh <liuqihang6@huawei.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: 2017403603 <705328646@qq.com>
Co-authored-by: Julien Mancuso <jmancuso@nvidia.com>
Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
fangyuchu
2026-10-15 21:29:02 -04:00
committed by GitHub
co-authored by lqh-hh Tyler Michael Smith OpenAI Codex 2017403603 Julien Mancuso Itay Alroy Codex mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent fb3fec970e
commit dd9e07a25f
18 changed files with 1638 additions and 167 deletions
+4 -1
View File
@@ -328,7 +328,10 @@ def terminate_if_errored(server: uvicorn.Server, engine: EngineClient):
for this request.
"""
engine_errored = engine.errored and not engine.is_running
if not envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH and engine_errored:
keep_alive = (
envs.VLLM_KEEP_ALIVE_ON_ENGINE_DEATH or engine.should_keep_api_server_alive
)
if not keep_alive and engine_errored:
server.should_exit = True
@@ -8,6 +8,7 @@ from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
from fastapi.responses import JSONResponse
from vllm.distributed.elastic_ep.external_elastic_ep import ExternalElasticEPScalePhase
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
@@ -82,7 +83,35 @@ async def scale_elastic_ep(raw_request: Request):
@router.post("/is_scaling_elastic_ep")
async def is_scaling_elastic_ep(raw_request: Request):
return JSONResponse({"is_scaling_elastic_ep": get_scaling_elastic_ep()})
# External operation status comes from the shared store. Middleware gating
# remains process-local, so scaling requests must reach every old-rank API.
try:
status = await engine_client(raw_request).get_external_elastic_ep_status()
except Exception as e:
logger.warning("Failed to query external Elastic EP status: %s", e)
raise HTTPException(
status_code=503,
detail="External Elastic EP status is temporarily unavailable",
) from e
if status is None:
# Non-external EEP modes retain the process-local middleware state.
is_scaling = get_scaling_elastic_ep()
status = {
"phase": (
ExternalElasticEPScalePhase.COMMITTING.value
if is_scaling
else ExternalElasticEPScalePhase.IDLE.value
),
"epoch": None,
"error": None,
"requested_data_parallel_size": None,
}
else:
is_scaling = status["phase"] in (
ExternalElasticEPScalePhase.PREPARING.value,
ExternalElasticEPScalePhase.COMMITTING.value,
)
return JSONResponse({"is_scaling_elastic_ep": is_scaling, **status})
def attach_router(app: FastAPI):
+29 -12
View File
@@ -6,8 +6,12 @@ from collections.abc import Awaitable
from fastapi.responses import JSONResponse
from starlette.types import ASGIApp, Receive, Scope, Send
# Global variable to track scaling state
# Process-local states used to pause serving during scaling and after this rank
# has been removed from the external-LB topology. A retired API process remains
# available for health and scaling-status queries until its orchestrator exits it.
_scaling_elastic_ep = False
_elastic_ep_rank_retired = False
_SCALING_OBSERVABILITY_PATHS = frozenset({"/health", "/is_scaling_elastic_ep"})
def get_scaling_elastic_ep():
@@ -19,12 +23,20 @@ def set_scaling_elastic_ep(value):
_scaling_elastic_ep = value
class ScalingMiddleware:
"""Middleware that checks if the model is currently scaling and
returns a 503 Service Unavailable response if it is.
def get_elastic_ep_rank_retired():
return _elastic_ep_rank_retired
This middleware applies to all HTTP requests and prevents
processing when the model is in a scaling state.
def set_elastic_ep_rank_retired(value):
global _elastic_ep_rank_retired
_elastic_ep_rank_retired = value
class ScalingMiddleware:
"""Pause serving while scaling or after the local rank is retired.
Health and scaling-status requests remain available so an external
orchestrator can observe the operation and remove a retired process.
"""
def __init__(self, app: ASGIApp) -> None:
@@ -34,13 +46,18 @@ class ScalingMiddleware:
if scope["type"] != "http":
return self.app(scope, receive, send)
# Check global scaling state
if get_scaling_elastic_ep():
# Return 503 Service Unavailable response
# Keep observability available while serving is paused or this rank is
# waiting for the external orchestrator to remove its process.
rank_retired = get_elastic_ep_rank_retired()
serving_paused = get_scaling_elastic_ep() or rank_retired
if serving_paused and scope["path"] not in _SCALING_OBSERVABILITY_PATHS:
error = (
"This data-parallel rank has been retired and is awaiting shutdown."
if rank_retired
else "The model is currently scaling. Please try again later."
)
response = JSONResponse(
content={
"error": "The model is currently scaling. Please try again later."
},
content={"error": error},
status_code=503,
)
return response(scope, receive, send)