[Feature] Add External LB Mode Support for Elastic EP Scaling (#43202)
Signed-off-by: lqh-hh <liuqihang6@huawei.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: lqh-hh <liuqihang6@huawei.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: 2017403603 <705328646@qq.com> Co-authored-by: Julien Mancuso <jmancuso@nvidia.com> Co-authored-by: Itay Alroy <75032521+itayalroy@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
lqh-hh
Tyler Michael Smith
OpenAI Codex
2017403603
Julien Mancuso
Itay Alroy
Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
fb3fec970e
commit
dd9e07a25f
@@ -51,7 +51,7 @@ def _traffic_loop(
|
||||
stop: threading.Event,
|
||||
finished: threading.Event,
|
||||
is_probe: bool = False,
|
||||
) -> list[tuple[float, float, int | None]]:
|
||||
) -> list[tuple[float, float, int | None, str | None]]:
|
||||
url = server.url_for("is_scaling_elastic_ep" if is_probe else "v1/completions")
|
||||
payload = {"model": MODEL_NAME, "prompt": "Hello", "max_tokens": 4}
|
||||
headers = None if dp_rank is None else {"X-data-parallel-rank": str(dp_rank)}
|
||||
@@ -60,15 +60,18 @@ def _traffic_loop(
|
||||
is_ready = False
|
||||
while not stop.is_set():
|
||||
request_start = time.perf_counter()
|
||||
phase = None
|
||||
try:
|
||||
response = requests.post(
|
||||
url, json=request_payload, headers=headers, timeout=120
|
||||
)
|
||||
status_code = response.status_code
|
||||
if is_probe and status_code == 200:
|
||||
phase = response.json()["phase"]
|
||||
except requests.exceptions.RequestException:
|
||||
status_code = None
|
||||
responses.append((request_start, time.perf_counter(), status_code))
|
||||
if status_code == 200:
|
||||
responses.append((request_start, time.perf_counter(), status_code, phase))
|
||||
if status_code == 200 and (not is_probe or phase in ("idle", "completed")):
|
||||
if not is_ready:
|
||||
ready.wait(timeout=120)
|
||||
is_ready = True
|
||||
@@ -78,14 +81,16 @@ def _traffic_loop(
|
||||
return responses
|
||||
|
||||
|
||||
def _downtime(responses: list[tuple[float, float, int | None]]) -> float:
|
||||
rejected = [end for _, end, status in responses if status == 503]
|
||||
if not rejected:
|
||||
def _downtime(responses: list[tuple[float, float, int | None, str | None]]) -> float:
|
||||
committing = [end for _, end, _, phase in responses if phase == "committing"]
|
||||
if not committing:
|
||||
return 0
|
||||
recovered = next(
|
||||
end for _, end, status in responses if status == 200 and end > rejected[-1]
|
||||
end
|
||||
for _, end, status, phase in responses
|
||||
if status == 200 and phase in ("idle", "completed") and end > committing[-1]
|
||||
)
|
||||
return recovered - rejected[0]
|
||||
return recovered - committing[0]
|
||||
|
||||
|
||||
def _scale_with_traffic(
|
||||
@@ -124,16 +129,20 @@ def _scale_with_traffic(
|
||||
bad_statuses = {
|
||||
status
|
||||
for responses in [probe_result, *results]
|
||||
for _, _, status in responses
|
||||
for _, _, status, _ in responses
|
||||
if status not in (200, 503)
|
||||
}
|
||||
assert not bad_statuses, f"traffic got unexpected statuses {bad_statuses}"
|
||||
probe_503 = [start for start, _, status in probe_result if status == 503]
|
||||
assert probe_503, "Scaling probe did not observe commit"
|
||||
probe_committing = [
|
||||
start for start, _, _, phase in probe_result if phase == "committing"
|
||||
]
|
||||
assert probe_committing, "Scaling probe did not observe commit"
|
||||
assert not results or any(
|
||||
status == 200 and start_time <= request_start and request_end < probe_503[0]
|
||||
status == 200
|
||||
and start_time <= request_start
|
||||
and request_end < probe_committing[0]
|
||||
for responses in results
|
||||
for request_start, request_end, status in responses
|
||||
for request_start, request_end, status, _ in responses
|
||||
), "No request completed successfully during preparation"
|
||||
|
||||
print(
|
||||
|
||||
Reference in New Issue
Block a user