[Frontend] Add reusable TP1 initialized-engine snapshots (#51360)
Signed-off-by: Nils Matteson <nilsmatteson@icloud.com> Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: elehayym <52448798+Yuzu23@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Taneem Ibrahim
elehayym
Andreas Karatzas
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
8bf051cd51
commit
13058faa49
@@ -0,0 +1,252 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.snapshot.manifest import ReleaseMarker, _validation_path, _write_json_atomic
|
||||
from vllm.snapshot.types import Oracle, oracles_match
|
||||
|
||||
_CANARY_PROMPT = "The capital of France is"
|
||||
|
||||
|
||||
class SnapshotCanaryError(RuntimeError):
|
||||
"""The initialized engine did not produce a valid snapshot oracle."""
|
||||
|
||||
|
||||
class SnapshotBarrierError(RuntimeError):
|
||||
"""The controller supplied an invalid snapshot release marker."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ListenerConfig:
|
||||
host: str | None
|
||||
port: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ControlArgs:
|
||||
ready_file: Path
|
||||
release_file: Path
|
||||
release_timeout_s: float
|
||||
|
||||
|
||||
def write_ready_atomic(path: Path, oracle: Oracle) -> None:
|
||||
_write_json_atomic(
|
||||
path,
|
||||
{
|
||||
"sampled_token_logprob": oracle.sampled_token_logprob,
|
||||
"token_ids": oracle.token_ids,
|
||||
"text": oracle.text,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def read_release_marker(path: Path) -> ListenerConfig:
|
||||
path = Path(path)
|
||||
path_stat = path.lstat()
|
||||
if stat.S_ISLNK(path_stat.st_mode):
|
||||
raise SnapshotBarrierError("release marker must not be a symlink")
|
||||
try:
|
||||
payload = json.loads(path.read_text())
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
raise SnapshotBarrierError("release marker is not valid JSON") from error
|
||||
try:
|
||||
marker = ReleaseMarker.model_validate(payload)
|
||||
except ValidationError as error:
|
||||
raise SnapshotBarrierError(
|
||||
f"release marker is invalid: {_validation_path(error)}"
|
||||
) from error
|
||||
return ListenerConfig(host=marker.host, port=marker.port)
|
||||
|
||||
|
||||
def parse_control_args(argv: list[str]) -> tuple[ControlArgs, list[str]]:
|
||||
parser = argparse.ArgumentParser(add_help=False)
|
||||
parser.add_argument("--ready-file", type=Path, required=True)
|
||||
parser.add_argument("--release-file", type=Path, required=True)
|
||||
parser.add_argument("--release-timeout-s", type=float, default=900.0)
|
||||
control_args, remaining = parser.parse_known_args(argv)
|
||||
if remaining and remaining[0] == "--":
|
||||
remaining = remaining[1:]
|
||||
if control_args.release_timeout_s <= 0:
|
||||
raise ValueError("release timeout must be positive")
|
||||
return (
|
||||
ControlArgs(
|
||||
ready_file=control_args.ready_file,
|
||||
release_file=control_args.release_file,
|
||||
release_timeout_s=control_args.release_timeout_s,
|
||||
),
|
||||
remaining,
|
||||
)
|
||||
|
||||
|
||||
async def _release_reloadable_state(engine: Any) -> None:
|
||||
"""Discard model and KV state before the process image is captured."""
|
||||
await engine.sleep(level=2)
|
||||
|
||||
|
||||
async def _restore_reloadable_state(engine: Any) -> None:
|
||||
"""Rebuild state discarded by ``_release_reloadable_state``."""
|
||||
await engine.wake_up(tags=["weights"])
|
||||
await engine.collective_rpc("reload_weights")
|
||||
await engine.wake_up(tags=["kv_cache"])
|
||||
|
||||
|
||||
def oracle_from_request_output(request_output: Any) -> Oracle:
|
||||
try:
|
||||
candidate = request_output.outputs[0]
|
||||
token_ids = tuple(candidate.token_ids)
|
||||
(sampled_token_id,) = token_ids
|
||||
return Oracle(
|
||||
token_ids=token_ids,
|
||||
text=candidate.text,
|
||||
sampled_token_logprob=candidate.logprobs[0][sampled_token_id].logprob,
|
||||
)
|
||||
except ValueError as error:
|
||||
raise SnapshotCanaryError(
|
||||
"snapshot canary must produce exactly one finite token logprob"
|
||||
) from error
|
||||
except (AttributeError, IndexError, KeyError, TypeError) as error:
|
||||
raise SnapshotCanaryError(
|
||||
"snapshot canary did not return sampled token logprob"
|
||||
) from error
|
||||
|
||||
|
||||
async def run_engine_canary(engine: Any) -> Oracle:
|
||||
from vllm import SamplingParams
|
||||
|
||||
final_output = None
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0,
|
||||
min_tokens=1,
|
||||
max_tokens=1,
|
||||
seed=0,
|
||||
logprobs=0,
|
||||
)
|
||||
async for output in engine.generate(
|
||||
_CANARY_PROMPT,
|
||||
sampling_params,
|
||||
request_id="vllm-snapshot-canary",
|
||||
):
|
||||
final_output = output
|
||||
if final_output is None:
|
||||
raise SnapshotCanaryError("canary generation returned no output")
|
||||
return oracle_from_request_output(final_output)
|
||||
|
||||
|
||||
def detach_snapshot_streams() -> None:
|
||||
"""Remove launch-log descriptors from the reusable process image."""
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
sink = os.open(os.devnull, os.O_WRONLY)
|
||||
try:
|
||||
os.dup2(sink, sys.stdout.fileno())
|
||||
os.dup2(sink, sys.stderr.fileno())
|
||||
finally:
|
||||
os.close(sink)
|
||||
|
||||
|
||||
async def wait_for_release_marker(
|
||||
path: Path,
|
||||
*,
|
||||
timeout_s: float,
|
||||
poll_interval_s: float = 0.05,
|
||||
) -> ListenerConfig:
|
||||
if timeout_s <= 0:
|
||||
raise ValueError("release timeout must be positive")
|
||||
if poll_interval_s <= 0:
|
||||
raise ValueError("release poll interval must be positive")
|
||||
remaining_s = timeout_s
|
||||
while not path.exists():
|
||||
if remaining_s <= 0:
|
||||
raise TimeoutError(f"release marker not found before timeout: {path}")
|
||||
delay_s = min(poll_interval_s, remaining_s)
|
||||
await asyncio.sleep(delay_s)
|
||||
remaining_s -= delay_s
|
||||
return read_release_marker(path)
|
||||
|
||||
|
||||
def parse_vllm_args(argv: list[str]) -> Any:
|
||||
from vllm.entrypoints.launchers.cli_args import (
|
||||
make_arg_parser,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
parser = FlexibleArgumentParser(prog="python -m vllm.snapshot.server")
|
||||
args = make_arg_parser(parser).parse_args(argv)
|
||||
if args.model_tag is not None:
|
||||
args.model = args.model_tag
|
||||
if args.grpc or args.headless:
|
||||
raise ValueError("snapshot server requires the HTTP frontend")
|
||||
if args.api_server_count not in (None, 1):
|
||||
raise ValueError("snapshot server supports one API server")
|
||||
args.api_server_count = None
|
||||
validate_parsed_serve_args(args)
|
||||
return args
|
||||
|
||||
|
||||
async def run_vllm_snapshot_child(control: ControlArgs, args: Any) -> None:
|
||||
from vllm.entrypoints.launchers.api_server.entry import (
|
||||
build_and_serve,
|
||||
build_async_engine_client,
|
||||
)
|
||||
from vllm.entrypoints.launchers.launcher import setup_server
|
||||
|
||||
args.enable_sleep_mode = True
|
||||
|
||||
async with build_async_engine_client(args) as engine:
|
||||
oracle = await run_engine_canary(engine)
|
||||
try:
|
||||
await _release_reloadable_state(engine)
|
||||
await _restore_reloadable_state(engine)
|
||||
rehearsal_oracle = await run_engine_canary(engine)
|
||||
if not oracles_match(oracle, rehearsal_oracle):
|
||||
raise SnapshotCanaryError("snapshot rehearsal changed canary output")
|
||||
await _release_reloadable_state(engine)
|
||||
except SnapshotCanaryError:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise SnapshotCanaryError("snapshot rehearsal failed") from error
|
||||
|
||||
detach_snapshot_streams()
|
||||
write_ready_atomic(control.ready_file, oracle)
|
||||
listener = await wait_for_release_marker(
|
||||
control.release_file,
|
||||
timeout_s=control.release_timeout_s,
|
||||
)
|
||||
args.host = listener.host
|
||||
args.port = listener.port
|
||||
await _restore_reloadable_state(engine)
|
||||
listen_address, sock = setup_server(args, reuse_port=False)
|
||||
try:
|
||||
shutdown_task = await build_and_serve(
|
||||
engine,
|
||||
listen_address,
|
||||
sock,
|
||||
args,
|
||||
)
|
||||
await shutdown_task
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
control, remaining = parse_control_args(sys.argv[1:] if argv is None else argv)
|
||||
args = parse_vllm_args(remaining)
|
||||
import uvloop
|
||||
|
||||
uvloop.run(run_vllm_snapshot_child(control, args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user