[Frontend] Add reusable TP1 initialized-engine snapshots (#51360)

Signed-off-by: Nils Matteson <nilsmatteson@icloud.com>
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com>
Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Co-authored-by: elehayym <52448798+Yuzu23@users.noreply.github.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Nils Matteson
2026-09-21 23:18:49 +00:00
committed by GitHub
co-authored by Taneem Ibrahim elehayym Andreas Karatzas mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 8bf051cd51
commit 13058faa49
16 changed files with 3773 additions and 10 deletions
+4
View File
@@ -0,0 +1,4 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Create and restore explicitly compatible vLLM engine snapshots."""
+132
View File
@@ -0,0 +1,132 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Create and restore transaction orchestration for local snapshots."""
import argparse
import shutil
import sys
from pathlib import Path
from vllm.snapshot.manifest import (
read_manifest,
validate_artifact_root,
validate_identity,
)
from vllm.snapshot.runtime import (
LocalSnapshotTools,
SnapshotCreateError,
SnapshotRestoreError,
_error_detail,
)
from vllm.snapshot.types import Oracle, oracles_match
def _child_engine_argv(
args: argparse.Namespace, engine_argv: tuple[str, ...] | None
) -> tuple[str, ...]:
"""Build the child argv from the first argv source that is present.
In order: an explicit ``engine_argv`` from the caller, the process argv
tail when invoked as ``vllm snapshot create ...``, then the bare model from
the parsed args. Whichever source wins, snapshot options are stripped and
sleep mode is appended.
"""
if not engine_argv:
process_argv = tuple(sys.argv[1:])
if process_argv[:2] == ("snapshot", "create"):
engine_argv = process_argv[2:]
else:
model = getattr(args, "model_tag", None) or getattr(args, "model", None)
if not model:
raise SnapshotCreateError("snapshot create requires a model argument")
engine_argv = (str(model),)
child_argv: list[str] = []
iterator = iter(engine_argv)
for item in iterator:
option, separator, _ = item.partition("=")
option = option.replace("_", "-")
if len(option) > 2 and "--snapshot-dir".startswith(option):
if not separator:
next(iterator, None)
else:
child_argv.append(item)
if "--enable-sleep-mode" not in child_argv:
child_argv.append("--enable-sleep-mode")
return tuple(child_argv)
def create_snapshot(
args: argparse.Namespace,
*,
engine_argv: tuple[str, ...] | None = None,
tools: LocalSnapshotTools | None = None,
) -> None:
target = Path(args.snapshot_dir).absolute()
toolset = tools or LocalSnapshotTools()
toolset.preflight("create", target)
if target.exists() or target.is_symlink():
raise SnapshotCreateError(f"snapshot target already exists: {target}")
validate_artifact_root(target, creating=True)
target.parent.mkdir(mode=0o700, parents=False, exist_ok=True)
validate_artifact_root(target.parent, creating=False)
target.mkdir(mode=0o700)
validate_artifact_root(target, creating=False)
published = False
root_pid: int | None = None
try:
child_argv = _child_engine_argv(args, engine_argv)
root_pid = toolset.launch_child(target, child_argv)
oracle = toolset.wait_ready(target, root_pid)
inventory = toolset.inventory(root_pid, target)
toolset.dump(target, inventory)
toolset.verify_dead(inventory)
manifest = toolset.make_manifest(args, child_argv, inventory, oracle, target)
toolset.publish(target, manifest)
published = True
except BaseException:
if root_pid is not None:
toolset.abort_create(root_pid)
raise
finally:
# Runs even when abort_create raised, so no partial artifact is left.
if not published and target.exists():
shutil.rmtree(target)
def restore_snapshot(
args: argparse.Namespace, *, tools: LocalSnapshotTools | None = None
) -> None:
artifact = Path(args.snapshot_dir).absolute()
toolset = tools or LocalSnapshotTools()
toolset.preflight("restore", artifact)
manifest = read_manifest(artifact)
validate_identity(manifest, toolset.current_identity(manifest.gpu_uuid))
root_pid: int | None = None
try:
root_pid = toolset.restore(artifact, manifest)
toolset.release(artifact, args.host, args.port)
toolset.wait_listener(root_pid, args.host, args.port)
actual = toolset.request_oracle(args.host, args.port, manifest)
expected = Oracle(
token_ids=manifest.oracle_token_ids,
text=manifest.oracle_text,
sampled_token_logprob=manifest.oracle_sampled_token_logprob,
)
if not oracles_match(expected, actual):
raise SnapshotRestoreError(
f"snapshot oracle mismatch: expected={expected!r}, actual={actual!r}"
)
toolset.complete_restore(root_pid)
except BaseException as error:
if root_pid is not None:
try:
toolset.cleanup(root_pid)
except BaseException as cleanup_error:
raise SnapshotRestoreError(
f"snapshot restore failed: {_error_detail(error)}; "
f"cleanup failed: {_error_detail(cleanup_error)}"
) from error
if isinstance(error, SnapshotRestoreError):
raise
raise SnapshotRestoreError(str(error)) from error
+277
View File
@@ -0,0 +1,277 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import math
import os
import stat
from pathlib import Path
from typing import Annotated, Any, Literal
from pydantic import (
BaseModel,
ConfigDict,
Field,
StrictFloat,
StrictInt,
StrictStr,
ValidationError,
field_validator,
model_validator,
)
from typing_extensions import Self
class SnapshotCompatibilityError(RuntimeError):
"""The snapshot identity does not match the requested runtime."""
class SnapshotSecurityError(RuntimeError):
"""The snapshot artifact does not meet the private-path contract."""
_MAX_PID = 2**31 - 1
_PositivePid = Annotated[int, Field(gt=0, le=_MAX_PID)]
_OracleTokenId = Annotated[int, Field(ge=0)]
class SnapshotRuntimeIdentity(BaseModel):
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
vllm_version: str
python_version: str
torch_version: str
cuda_runtime: str
driver_version: str
criu_version: str
cuda_checkpoint_sha256: str
kernel_release: str
host_id: str
gpu_name: str
gpu_uuid: str
environment: tuple[tuple[str, str], ...]
class SnapshotManifest(SnapshotRuntimeIdentity):
schema_version: Literal[1]
boundary: Literal["post-engine-init-reloadable-state-released"]
created_at: str
artifact_bytes: Annotated[int, Field(ge=0)]
model: str
served_model_name: str
model_revision: str
tokenizer_revision: str
engine_argv: tuple[str, ...]
process_tree: Annotated[tuple[_PositivePid, ...], Field(min_length=1)]
cuda_holders: Annotated[tuple[_PositivePid, ...], Field(min_length=1)]
external_cache_files: tuple[tuple[str, str], ...] = ()
oracle_token_ids: Annotated[
tuple[_OracleTokenId, ...], Field(min_length=1, max_length=1)
]
oracle_text: str
oracle_sampled_token_logprob: float
@field_validator("schema_version", mode="before")
@classmethod
def _require_schema_version_one(cls, value: Any) -> Any:
if type(value) is not int or value != 1:
raise ValueError("schema version must be integer 1")
return value
@field_validator("oracle_sampled_token_logprob", mode="before")
@classmethod
def _require_finite_float(cls, value: Any) -> Any:
if type(value) is not float or not math.isfinite(value):
raise ValueError("sampled token log probability must be a finite float")
return value
@model_validator(mode="after")
def _validate_process_relationships(self) -> Self:
if len(set(self.process_tree)) != len(self.process_tree):
raise ValueError("process_tree contains duplicate PIDs")
if len(set(self.cuda_holders)) != len(self.cuda_holders):
raise ValueError("cuda_holders contains duplicate PIDs")
if not set(self.cuda_holders).issubset(self.process_tree):
raise ValueError("cuda_holders must be a subset of process_tree")
return self
class ReleaseMarker(BaseModel):
"""Release marker the controller writes; unknown keys are ignored."""
release: Literal[True]
host: StrictStr | None = None
port: Annotated[StrictInt, Field(ge=1, le=65535)]
@field_validator("release", mode="before")
@classmethod
def _require_json_true(cls, value: Any) -> Any:
# A literal accepts 1 as True; the barrier requires JSON true itself.
if value is not True:
raise ValueError("release must be JSON true")
return value
class ReadyMarker(BaseModel):
"""Ready marker the child writes; unknown keys are ignored."""
token_ids: tuple[StrictInt, ...]
text: StrictStr
sampled_token_logprob: StrictFloat | StrictInt
def validate_identity(
expected: SnapshotRuntimeIdentity, actual: SnapshotRuntimeIdentity
) -> None:
"""Require an exact match for every field that can affect compatibility."""
# ``expected`` may be a manifest, so compare the identity fields only.
fields = set(SnapshotRuntimeIdentity.model_fields)
expected_values = expected.model_dump(include=fields)
actual_values = actual.model_dump(include=fields)
differing = [
name
for name in SnapshotRuntimeIdentity.model_fields
if expected_values[name] != actual_values[name]
]
if differing:
raise SnapshotCompatibilityError(f"snapshot mismatch: {', '.join(differing)}")
def _existing_path_chain(path: Path):
current = path
while True:
if current.exists() or current.is_symlink():
yield current
if current.parent == current:
break
current = current.parent
def _is_trusted_sticky_directory(path_stat: os.stat_result) -> bool:
return (
path_stat.st_uid == 0
and stat.S_ISDIR(path_stat.st_mode)
and bool(path_stat.st_mode & stat.S_ISVTX)
)
def validate_artifact_root(path: Path, *, creating: bool) -> None:
"""Validate that an artifact path cannot be replaced by another user."""
path = Path(os.path.abspath(path))
if path.is_symlink():
raise SnapshotSecurityError(f"snapshot path is a symlink: {path}")
if not creating and not path.exists():
raise SnapshotSecurityError(f"snapshot path does not exist: {path}")
effective_uid = os.geteuid()
for component in _existing_path_chain(path):
component_stat = component.lstat()
if stat.S_ISLNK(component_stat.st_mode):
raise SnapshotSecurityError(
f"snapshot path contains a symlink: {component}"
)
if component_stat.st_uid not in (0, effective_uid):
raise SnapshotSecurityError(
f"snapshot path has an untrusted owner: {component}"
)
if component_stat.st_mode & (
stat.S_IWGRP | stat.S_IWOTH
) and not _is_trusted_sticky_directory(component_stat):
raise SnapshotSecurityError(
f"snapshot path has a group- or world-writable ancestor: {component}"
)
if path.exists():
path_stat = path.lstat()
if not stat.S_ISDIR(path_stat.st_mode):
raise SnapshotSecurityError(f"snapshot path is not a directory: {path}")
if path_stat.st_uid != effective_uid:
raise SnapshotSecurityError(
f"snapshot directory is not owned by the current user: {path}"
)
if stat.S_IMODE(path_stat.st_mode) & 0o077:
raise SnapshotSecurityError(
f"snapshot directory must have mode 0700: {path}"
)
def _validation_path(error: ValidationError) -> str:
detail = error.errors(include_url=False)[0]
location = detail["loc"]
if not location:
return "JSON" if detail["type"] == "json_invalid" else "root"
return ".".join(map(str, location))
def _fsync_directory(path: Path) -> None:
directory_fd = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
try:
os.fsync(directory_fd)
finally:
os.close(directory_fd)
def _write_json_atomic(path: Path, payload: object, *, overwrite: bool = False) -> None:
temporary = path.with_name(f"{path.name}.tmp")
if overwrite:
temporary.unlink(missing_ok=True)
elif path.exists() or path.is_symlink():
raise FileExistsError(path)
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
descriptor = os.open(temporary, flags, 0o600)
try:
with os.fdopen(descriptor, "wb") as output:
output.write(
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
)
output.flush()
os.fsync(output.fileno())
os.replace(temporary, path)
_fsync_directory(path.parent)
except BaseException:
temporary.unlink(missing_ok=True)
raise
def write_manifest_atomic(path: Path, manifest: SnapshotManifest) -> None:
"""Write a new private manifest and make its directory entry durable."""
path = Path(path)
validate_artifact_root(path, creating=False)
destination = path / "manifest.json"
try:
_write_json_atomic(destination, manifest.model_dump(mode="json"))
except FileExistsError as error:
raise SnapshotSecurityError(
f"manifest already exists: {destination}"
) from error
def read_manifest(path: Path) -> SnapshotManifest:
"""Read a snapshot published by atomically installing its manifest."""
path = Path(path)
validate_artifact_root(path, creating=False)
manifest_path = path / "manifest.json"
if manifest_path.is_symlink():
raise SnapshotSecurityError(f"manifest is a symlink: {manifest_path}")
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
try:
file_descriptor = os.open(manifest_path, flags)
except FileNotFoundError as error:
raise SnapshotSecurityError(
f"manifest does not exist: {manifest_path}"
) from error
try:
with os.fdopen(file_descriptor, encoding="utf-8") as manifest_file:
payload = manifest_file.read()
except UnicodeDecodeError as error:
raise SnapshotCompatibilityError("invalid snapshot manifest: JSON") from error
try:
return SnapshotManifest.model_validate_json(payload, strict=True)
except ValidationError as error:
raise SnapshotCompatibilityError(
f"invalid snapshot manifest: {_validation_path(error)}"
) from error
def inspect_snapshot(path: Path) -> dict[str, object]:
return read_manifest(path).model_dump(mode="json")
File diff suppressed because it is too large Load Diff
+252
View File
@@ -0,0 +1,252 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import argparse
import asyncio
import json
import os
import stat
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Any
from pydantic import ValidationError
from vllm.snapshot.manifest import ReleaseMarker, _validation_path, _write_json_atomic
from vllm.snapshot.types import Oracle, oracles_match
_CANARY_PROMPT = "The capital of France is"
class SnapshotCanaryError(RuntimeError):
"""The initialized engine did not produce a valid snapshot oracle."""
class SnapshotBarrierError(RuntimeError):
"""The controller supplied an invalid snapshot release marker."""
@dataclass(frozen=True)
class ListenerConfig:
host: str | None
port: int
@dataclass(frozen=True)
class ControlArgs:
ready_file: Path
release_file: Path
release_timeout_s: float
def write_ready_atomic(path: Path, oracle: Oracle) -> None:
_write_json_atomic(
path,
{
"sampled_token_logprob": oracle.sampled_token_logprob,
"token_ids": oracle.token_ids,
"text": oracle.text,
},
)
def read_release_marker(path: Path) -> ListenerConfig:
path = Path(path)
path_stat = path.lstat()
if stat.S_ISLNK(path_stat.st_mode):
raise SnapshotBarrierError("release marker must not be a symlink")
try:
payload = json.loads(path.read_text())
except (OSError, json.JSONDecodeError) as error:
raise SnapshotBarrierError("release marker is not valid JSON") from error
try:
marker = ReleaseMarker.model_validate(payload)
except ValidationError as error:
raise SnapshotBarrierError(
f"release marker is invalid: {_validation_path(error)}"
) from error
return ListenerConfig(host=marker.host, port=marker.port)
def parse_control_args(argv: list[str]) -> tuple[ControlArgs, list[str]]:
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("--ready-file", type=Path, required=True)
parser.add_argument("--release-file", type=Path, required=True)
parser.add_argument("--release-timeout-s", type=float, default=900.0)
control_args, remaining = parser.parse_known_args(argv)
if remaining and remaining[0] == "--":
remaining = remaining[1:]
if control_args.release_timeout_s <= 0:
raise ValueError("release timeout must be positive")
return (
ControlArgs(
ready_file=control_args.ready_file,
release_file=control_args.release_file,
release_timeout_s=control_args.release_timeout_s,
),
remaining,
)
async def _release_reloadable_state(engine: Any) -> None:
"""Discard model and KV state before the process image is captured."""
await engine.sleep(level=2)
async def _restore_reloadable_state(engine: Any) -> None:
"""Rebuild state discarded by ``_release_reloadable_state``."""
await engine.wake_up(tags=["weights"])
await engine.collective_rpc("reload_weights")
await engine.wake_up(tags=["kv_cache"])
def oracle_from_request_output(request_output: Any) -> Oracle:
try:
candidate = request_output.outputs[0]
token_ids = tuple(candidate.token_ids)
(sampled_token_id,) = token_ids
return Oracle(
token_ids=token_ids,
text=candidate.text,
sampled_token_logprob=candidate.logprobs[0][sampled_token_id].logprob,
)
except ValueError as error:
raise SnapshotCanaryError(
"snapshot canary must produce exactly one finite token logprob"
) from error
except (AttributeError, IndexError, KeyError, TypeError) as error:
raise SnapshotCanaryError(
"snapshot canary did not return sampled token logprob"
) from error
async def run_engine_canary(engine: Any) -> Oracle:
from vllm import SamplingParams
final_output = None
sampling_params = SamplingParams(
temperature=0,
min_tokens=1,
max_tokens=1,
seed=0,
logprobs=0,
)
async for output in engine.generate(
_CANARY_PROMPT,
sampling_params,
request_id="vllm-snapshot-canary",
):
final_output = output
if final_output is None:
raise SnapshotCanaryError("canary generation returned no output")
return oracle_from_request_output(final_output)
def detach_snapshot_streams() -> None:
"""Remove launch-log descriptors from the reusable process image."""
sys.stdout.flush()
sys.stderr.flush()
sink = os.open(os.devnull, os.O_WRONLY)
try:
os.dup2(sink, sys.stdout.fileno())
os.dup2(sink, sys.stderr.fileno())
finally:
os.close(sink)
async def wait_for_release_marker(
path: Path,
*,
timeout_s: float,
poll_interval_s: float = 0.05,
) -> ListenerConfig:
if timeout_s <= 0:
raise ValueError("release timeout must be positive")
if poll_interval_s <= 0:
raise ValueError("release poll interval must be positive")
remaining_s = timeout_s
while not path.exists():
if remaining_s <= 0:
raise TimeoutError(f"release marker not found before timeout: {path}")
delay_s = min(poll_interval_s, remaining_s)
await asyncio.sleep(delay_s)
remaining_s -= delay_s
return read_release_marker(path)
def parse_vllm_args(argv: list[str]) -> Any:
from vllm.entrypoints.launchers.cli_args import (
make_arg_parser,
validate_parsed_serve_args,
)
from vllm.utils.argparse_utils import FlexibleArgumentParser
parser = FlexibleArgumentParser(prog="python -m vllm.snapshot.server")
args = make_arg_parser(parser).parse_args(argv)
if args.model_tag is not None:
args.model = args.model_tag
if args.grpc or args.headless:
raise ValueError("snapshot server requires the HTTP frontend")
if args.api_server_count not in (None, 1):
raise ValueError("snapshot server supports one API server")
args.api_server_count = None
validate_parsed_serve_args(args)
return args
async def run_vllm_snapshot_child(control: ControlArgs, args: Any) -> None:
from vllm.entrypoints.launchers.api_server.entry import (
build_and_serve,
build_async_engine_client,
)
from vllm.entrypoints.launchers.launcher import setup_server
args.enable_sleep_mode = True
async with build_async_engine_client(args) as engine:
oracle = await run_engine_canary(engine)
try:
await _release_reloadable_state(engine)
await _restore_reloadable_state(engine)
rehearsal_oracle = await run_engine_canary(engine)
if not oracles_match(oracle, rehearsal_oracle):
raise SnapshotCanaryError("snapshot rehearsal changed canary output")
await _release_reloadable_state(engine)
except SnapshotCanaryError:
raise
except Exception as error:
raise SnapshotCanaryError("snapshot rehearsal failed") from error
detach_snapshot_streams()
write_ready_atomic(control.ready_file, oracle)
listener = await wait_for_release_marker(
control.release_file,
timeout_s=control.release_timeout_s,
)
args.host = listener.host
args.port = listener.port
await _restore_reloadable_state(engine)
listen_address, sock = setup_server(args, reuse_port=False)
try:
shutdown_task = await build_and_serve(
engine,
listen_address,
sock,
args,
)
await shutdown_task
finally:
sock.close()
def main(argv: list[str] | None = None) -> None:
control, remaining = parse_control_args(sys.argv[1:] if argv is None else argv)
args = parse_vllm_args(remaining)
import uvloop
uvloop.run(run_vllm_snapshot_child(control, args))
if __name__ == "__main__":
main()
+37
View File
@@ -0,0 +1,37 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from dataclasses import dataclass
@dataclass(frozen=True)
class Oracle:
token_ids: tuple[int, ...]
text: str
sampled_token_logprob: float
def __post_init__(self) -> None:
if len(self.token_ids) != 1:
raise ValueError("snapshot oracle must contain exactly one token")
value = self.sampled_token_logprob
if (
isinstance(value, bool)
or not isinstance(value, (int, float))
or not math.isfinite(value)
):
raise ValueError("sampled token logprob must be finite")
object.__setattr__(self, "sampled_token_logprob", float(value))
def oracles_match(expected: Oracle, actual: Oracle) -> bool:
return (
expected.token_ids == actual.token_ids
and expected.text == actual.text
and math.isclose(
expected.sampled_token_logprob,
actual.sampled_token_logprob,
rel_tol=0.0,
abs_tol=1e-3,
)
)