[Elastic EP] Async preparation (#47288)
Signed-off-by: Itay Alroy <ialroy@nvidia.com>
This commit is contained in:
@@ -12,10 +12,7 @@ from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.engine.protocol import (
|
||||
ErrorResponse,
|
||||
)
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import (
|
||||
get_scaling_elastic_ep,
|
||||
set_scaling_elastic_ep,
|
||||
)
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
@@ -64,8 +61,6 @@ async def scale_elastic_ep(raw_request: Request):
|
||||
status_code=400, detail="drain_timeout must be a positive integer"
|
||||
)
|
||||
|
||||
# Set scaling flag to prevent new requests
|
||||
set_scaling_elastic_ep(True)
|
||||
client = engine_client(raw_request)
|
||||
try:
|
||||
await client.scale_elastic_ep(new_data_parallel_size, drain_timeout)
|
||||
@@ -83,8 +78,6 @@ async def scale_elastic_ep(raw_request: Request):
|
||||
except Exception as e:
|
||||
logger.error("Scale failed: %s", e)
|
||||
raise HTTPException(status_code=500, detail="Scale failed") from e
|
||||
finally:
|
||||
set_scaling_elastic_ep(False)
|
||||
|
||||
|
||||
@router.post("/is_scaling_elastic_ep")
|
||||
|
||||
Reference in New Issue
Block a user