Signed-off-by: lqh-hh <liuqihang6@huawei.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: lqh-hh <liuqihang6@huawei.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: 2017403603 <705328646@qq.com> Co-authored-by: Julien Mancuso <jmancuso@nvidia.com> Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
66 lines
2.2 KiB
Python
66 lines
2.2 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from collections.abc import Awaitable
|
|
|
|
from fastapi.responses import JSONResponse
|
|
from starlette.types import ASGIApp, Receive, Scope, Send
|
|
|
|
# Process-local states used to pause serving during scaling and after this rank
|
|
# has been removed from the external-LB topology. A retired API process remains
|
|
# available for health and scaling-status queries until its orchestrator exits it.
|
|
_scaling_elastic_ep = False
|
|
_elastic_ep_rank_retired = False
|
|
_SCALING_OBSERVABILITY_PATHS = frozenset({"/health", "/is_scaling_elastic_ep"})
|
|
|
|
|
|
def get_scaling_elastic_ep():
|
|
return _scaling_elastic_ep
|
|
|
|
|
|
def set_scaling_elastic_ep(value):
|
|
global _scaling_elastic_ep
|
|
_scaling_elastic_ep = value
|
|
|
|
|
|
def get_elastic_ep_rank_retired():
|
|
return _elastic_ep_rank_retired
|
|
|
|
|
|
def set_elastic_ep_rank_retired(value):
|
|
global _elastic_ep_rank_retired
|
|
_elastic_ep_rank_retired = value
|
|
|
|
|
|
class ScalingMiddleware:
|
|
"""Pause serving while scaling or after the local rank is retired.
|
|
|
|
Health and scaling-status requests remain available so an external
|
|
orchestrator can observe the operation and remove a retired process.
|
|
"""
|
|
|
|
def __init__(self, app: ASGIApp) -> None:
|
|
self.app = app
|
|
|
|
def __call__(self, scope: Scope, receive: Receive, send: Send) -> Awaitable[None]:
|
|
if scope["type"] != "http":
|
|
return self.app(scope, receive, send)
|
|
|
|
# Keep observability available while serving is paused or this rank is
|
|
# waiting for the external orchestrator to remove its process.
|
|
rank_retired = get_elastic_ep_rank_retired()
|
|
serving_paused = get_scaling_elastic_ep() or rank_retired
|
|
if serving_paused and scope["path"] not in _SCALING_OBSERVABILITY_PATHS:
|
|
error = (
|
|
"This data-parallel rank has been retired and is awaiting shutdown."
|
|
if rank_retired
|
|
else "The model is currently scaling. Please try again later."
|
|
)
|
|
response = JSONResponse(
|
|
content={"error": error},
|
|
status_code=503,
|
|
)
|
|
return response(scope, receive, send)
|
|
|
|
return self.app(scope, receive, send)
|