[Elastic EP] Async preparation (#47288)

Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
Itay Alroy
2026-07-28 05:19:13 -07:00
committed by GitHub
parent d2bfc6fe20
commit 35efdf6b34
21 changed files with 701 additions and 677 deletions
@@ -12,10 +12,7 @@ from vllm.engine.protocol import EngineClient
from vllm.entrypoints.openai.engine.protocol import (
ErrorResponse,
)
from vllm.entrypoints.serve.elastic_ep.middleware import (
get_scaling_elastic_ep,
set_scaling_elastic_ep,
)
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
from vllm.logger import init_logger
@@ -64,8 +61,6 @@ async def scale_elastic_ep(raw_request: Request):
status_code=400, detail="drain_timeout must be a positive integer"
)
# Set scaling flag to prevent new requests
set_scaling_elastic_ep(True)
client = engine_client(raw_request)
try:
await client.scale_elastic_ep(new_data_parallel_size, drain_timeout)
@@ -83,8 +78,6 @@ async def scale_elastic_ep(raw_request: Request):
except Exception as e:
logger.error("Scale failed: %s", e)
raise HTTPException(status_code=500, detail="Scale failed") from e
finally:
set_scaling_elastic_ep(False)
@router.post("/is_scaling_elastic_ep")