[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)
Signed-off-by: lqh-hh <liuqihang6@huawei.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: lqh-hh <liuqihang6@huawei.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: 2017403603 <705328646@qq.com> Co-authored-by: Julien Mancuso <jmancuso@nvidia.com> Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
lqh-hh
Tyler Michael Smith
OpenAI Codex
2017403603
Julien Mancuso
Itay Alroy
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
fb3fec970e
commit
dd9e07a25f
@@ -22,7 +22,11 @@ from vllm.distributed.weight_transfer.base import (
|
||||
)
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.engine.protocol import EngineClient, StreamingInput
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import set_scaling_elastic_ep
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import (
|
||||
get_elastic_ep_rank_retired,
|
||||
set_elastic_ep_rank_retired,
|
||||
set_scaling_elastic_ep,
|
||||
)
|
||||
from vllm.exceptions import (
|
||||
GracefulHTTPError,
|
||||
MaxQueuedTokensError,
|
||||
@@ -918,6 +922,12 @@ class AsyncLLM(EngineClient):
|
||||
mm_cache_stats=renderer.stat_mm_cache(),
|
||||
)
|
||||
except Exception as e:
|
||||
if get_elastic_ep_rank_retired() and isinstance(e, EngineDeadError):
|
||||
logger.info(
|
||||
"[Elastic EP] Retired rank stopped receiving EngineCore outputs"
|
||||
)
|
||||
output_processor.propagate_error(e)
|
||||
return
|
||||
logger.exception("AsyncLLM output_handler failed.")
|
||||
output_processor.propagate_error(e)
|
||||
|
||||
@@ -1103,6 +1113,8 @@ class AsyncLLM(EngineClient):
|
||||
|
||||
async def check_health(self) -> None:
|
||||
logger.debug("Called check_health.")
|
||||
if get_elastic_ep_rank_retired():
|
||||
return
|
||||
if self.errored:
|
||||
raise self.dead_error
|
||||
|
||||
@@ -1252,8 +1264,13 @@ class AsyncLLM(EngineClient):
|
||||
"waiting for requests to drain before scaling"
|
||||
)
|
||||
await self.wait_for_requests_to_drain(drain_timeout)
|
||||
except BaseException:
|
||||
set_scaling_elastic_ep(False)
|
||||
except BaseException as e:
|
||||
if self.vllm_config.parallel_config.data_parallel_external_lb:
|
||||
await self.engine_core.abort_external_elastic_ep(
|
||||
str(e) or type(e).__name__
|
||||
)
|
||||
else:
|
||||
set_scaling_elastic_ep(False)
|
||||
raise
|
||||
|
||||
async def scale_elastic_ep(
|
||||
@@ -1266,7 +1283,8 @@ class AsyncLLM(EngineClient):
|
||||
async def _scale_elastic_ep(
|
||||
self, new_data_parallel_size: int, drain_timeout: int
|
||||
) -> None:
|
||||
old_data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
|
||||
parallel_config = self.vllm_config.parallel_config
|
||||
old_data_parallel_size = parallel_config.data_parallel_size
|
||||
if old_data_parallel_size == new_data_parallel_size:
|
||||
logger.info(
|
||||
"Data parallel size is already %s, skipping scale",
|
||||
@@ -1297,9 +1315,36 @@ class AsyncLLM(EngineClient):
|
||||
if envs.VLLM_ELASTIC_EP_DRAIN_REQUESTS:
|
||||
await self._drain_requests_for_elastic_ep(drain_timeout)
|
||||
|
||||
await self.engine_core.commit_elastic_ep()
|
||||
self.vllm_config.parallel_config.data_parallel_size = new_data_parallel_size
|
||||
set_scaling_elastic_ep(False)
|
||||
rank_will_retire = (
|
||||
parallel_config.data_parallel_external_lb
|
||||
and new_data_parallel_size < old_data_parallel_size
|
||||
and parallel_config.data_parallel_rank >= new_data_parallel_size
|
||||
)
|
||||
if rank_will_retire:
|
||||
set_elastic_ep_rank_retired(True)
|
||||
|
||||
commit_succeeded = False
|
||||
try:
|
||||
await self.engine_core.commit_elastic_ep()
|
||||
parallel_config.data_parallel_size = new_data_parallel_size
|
||||
commit_succeeded = True
|
||||
if rank_will_retire:
|
||||
logger.info(
|
||||
"[Elastic EP] Data-parallel rank %s is retired and awaiting "
|
||||
"external process shutdown",
|
||||
parallel_config.data_parallel_rank,
|
||||
)
|
||||
finally:
|
||||
if rank_will_retire and not commit_succeeded:
|
||||
set_elastic_ep_rank_retired(False)
|
||||
# A failed commit may leave engines partially reconfigured or paused.
|
||||
if commit_succeeded:
|
||||
set_scaling_elastic_ep(False)
|
||||
|
||||
async def get_external_elastic_ep_status(
|
||||
self,
|
||||
) -> dict[str, str | int | None] | None:
|
||||
return await self.engine_core.get_external_elastic_ep_status()
|
||||
|
||||
async def handle_fault(
|
||||
self, fault_tolerance_request: FaultToleranceRequest
|
||||
@@ -1323,6 +1368,10 @@ class AsyncLLM(EngineClient):
|
||||
def errored(self) -> bool:
|
||||
return self.engine_core.resources.engine_dead or not self.is_running
|
||||
|
||||
@property
|
||||
def should_keep_api_server_alive(self) -> bool:
|
||||
return get_elastic_ep_rank_retired()
|
||||
|
||||
@property
|
||||
def dead_error(self) -> BaseException:
|
||||
return EngineDeadError()
|
||||
|
||||
Reference in New Issue
Block a user