Files
fangyuchulqh-hhTyler Michael SmithOpenAI Codex2017403603Julien MancusoItay AlroyCodexmergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
dd9e07a25f [Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)
Signed-off-by: lqh-hh <liuqihang6@huawei.com>
Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: lqh-hh <liuqihang6@huawei.com>
Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: 2017403603 <705328646@qq.com>
Co-authored-by: Julien Mancuso <jmancuso@nvidia.com>
Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
2026-10-15 21:29:02 -04:00

66 lines
2.2 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Awaitable
from fastapi.responses import JSONResponse
from starlette.types import ASGIApp, Receive, Scope, Send
# Process-local states used to pause serving during scaling and after this rank
# has been removed from the external-LB topology. A retired API process remains
# available for health and scaling-status queries until its orchestrator exits it.
_scaling_elastic_ep = False
_elastic_ep_rank_retired = False
_SCALING_OBSERVABILITY_PATHS = frozenset({"/health", "/is_scaling_elastic_ep"})
def get_scaling_elastic_ep():
return _scaling_elastic_ep
def set_scaling_elastic_ep(value):
global _scaling_elastic_ep
_scaling_elastic_ep = value
def get_elastic_ep_rank_retired():
return _elastic_ep_rank_retired
def set_elastic_ep_rank_retired(value):
global _elastic_ep_rank_retired
_elastic_ep_rank_retired = value
class ScalingMiddleware:
"""Pause serving while scaling or after the local rank is retired.
Health and scaling-status requests remain available so an external
orchestrator can observe the operation and remove a retired process.
"""
def __init__(self, app: ASGIApp) -> None:
self.app = app
def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
if scope["type"] != "http":
return self.app(scope, receive, send)
# Keep observability available while serving is paused or this rank is
# waiting for the external orchestrator to remove its process.
rank_retired = get_elastic_ep_rank_retired()
serving_paused = get_scaling_elastic_ep() or rank_retired
if serving_paused and scope["path"] not in _SCALING_OBSERVABILITY_PATHS:
error = (
"This data-parallel rank has been retired and is awaiting shutdown."
if rank_retired
else "The model is currently scaling. Please try again later."
)
response = JSONResponse(
content={"error": error},
status_code=503,
)
return response(scope, receive, send)
return self.app(scope, receive, send)