[Frontend] Add reusable TP1 initialized-engine snapshots (#51360)
Signed-off-by: Nils Matteson <nilsmatteson@icloud.com> Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Co-authored-by: elehayym <52448798+Yuzu23@users.noreply.github.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Taneem Ibrahim
elehayym
Andreas Karatzas
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
8bf051cd51
commit
13058faa49
@@ -0,0 +1,4 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Create and restore explicitly compatible vLLM engine snapshots."""
|
||||
@@ -0,0 +1,132 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Create and restore transaction orchestration for local snapshots."""
|
||||
|
||||
import argparse
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from vllm.snapshot.manifest import (
|
||||
read_manifest,
|
||||
validate_artifact_root,
|
||||
validate_identity,
|
||||
)
|
||||
from vllm.snapshot.runtime import (
|
||||
LocalSnapshotTools,
|
||||
SnapshotCreateError,
|
||||
SnapshotRestoreError,
|
||||
_error_detail,
|
||||
)
|
||||
from vllm.snapshot.types import Oracle, oracles_match
|
||||
|
||||
|
||||
def _child_engine_argv(
|
||||
args: argparse.Namespace, engine_argv: tuple[str, ...] | None
|
||||
) -> tuple[str, ...]:
|
||||
"""Build the child argv from the first argv source that is present.
|
||||
|
||||
In order: an explicit ``engine_argv`` from the caller, the process argv
|
||||
tail when invoked as ``vllm snapshot create ...``, then the bare model from
|
||||
the parsed args. Whichever source wins, snapshot options are stripped and
|
||||
sleep mode is appended.
|
||||
"""
|
||||
if not engine_argv:
|
||||
process_argv = tuple(sys.argv[1:])
|
||||
if process_argv[:2] == ("snapshot", "create"):
|
||||
engine_argv = process_argv[2:]
|
||||
else:
|
||||
model = getattr(args, "model_tag", None) or getattr(args, "model", None)
|
||||
if not model:
|
||||
raise SnapshotCreateError("snapshot create requires a model argument")
|
||||
engine_argv = (str(model),)
|
||||
child_argv: list[str] = []
|
||||
iterator = iter(engine_argv)
|
||||
for item in iterator:
|
||||
option, separator, _ = item.partition("=")
|
||||
option = option.replace("_", "-")
|
||||
if len(option) > 2 and "--snapshot-dir".startswith(option):
|
||||
if not separator:
|
||||
next(iterator, None)
|
||||
else:
|
||||
child_argv.append(item)
|
||||
if "--enable-sleep-mode" not in child_argv:
|
||||
child_argv.append("--enable-sleep-mode")
|
||||
return tuple(child_argv)
|
||||
|
||||
|
||||
def create_snapshot(
|
||||
args: argparse.Namespace,
|
||||
*,
|
||||
engine_argv: tuple[str, ...] | None = None,
|
||||
tools: LocalSnapshotTools | None = None,
|
||||
) -> None:
|
||||
target = Path(args.snapshot_dir).absolute()
|
||||
toolset = tools or LocalSnapshotTools()
|
||||
toolset.preflight("create", target)
|
||||
if target.exists() or target.is_symlink():
|
||||
raise SnapshotCreateError(f"snapshot target already exists: {target}")
|
||||
validate_artifact_root(target, creating=True)
|
||||
target.parent.mkdir(mode=0o700, parents=False, exist_ok=True)
|
||||
validate_artifact_root(target.parent, creating=False)
|
||||
target.mkdir(mode=0o700)
|
||||
validate_artifact_root(target, creating=False)
|
||||
published = False
|
||||
root_pid: int | None = None
|
||||
try:
|
||||
child_argv = _child_engine_argv(args, engine_argv)
|
||||
root_pid = toolset.launch_child(target, child_argv)
|
||||
oracle = toolset.wait_ready(target, root_pid)
|
||||
inventory = toolset.inventory(root_pid, target)
|
||||
toolset.dump(target, inventory)
|
||||
toolset.verify_dead(inventory)
|
||||
manifest = toolset.make_manifest(args, child_argv, inventory, oracle, target)
|
||||
toolset.publish(target, manifest)
|
||||
published = True
|
||||
except BaseException:
|
||||
if root_pid is not None:
|
||||
toolset.abort_create(root_pid)
|
||||
raise
|
||||
finally:
|
||||
# Runs even when abort_create raised, so no partial artifact is left.
|
||||
if not published and target.exists():
|
||||
shutil.rmtree(target)
|
||||
|
||||
|
||||
def restore_snapshot(
|
||||
args: argparse.Namespace, *, tools: LocalSnapshotTools | None = None
|
||||
) -> None:
|
||||
artifact = Path(args.snapshot_dir).absolute()
|
||||
toolset = tools or LocalSnapshotTools()
|
||||
toolset.preflight("restore", artifact)
|
||||
manifest = read_manifest(artifact)
|
||||
validate_identity(manifest, toolset.current_identity(manifest.gpu_uuid))
|
||||
|
||||
root_pid: int | None = None
|
||||
try:
|
||||
root_pid = toolset.restore(artifact, manifest)
|
||||
toolset.release(artifact, args.host, args.port)
|
||||
toolset.wait_listener(root_pid, args.host, args.port)
|
||||
actual = toolset.request_oracle(args.host, args.port, manifest)
|
||||
expected = Oracle(
|
||||
token_ids=manifest.oracle_token_ids,
|
||||
text=manifest.oracle_text,
|
||||
sampled_token_logprob=manifest.oracle_sampled_token_logprob,
|
||||
)
|
||||
if not oracles_match(expected, actual):
|
||||
raise SnapshotRestoreError(
|
||||
f"snapshot oracle mismatch: expected={expected!r}, actual={actual!r}"
|
||||
)
|
||||
toolset.complete_restore(root_pid)
|
||||
except BaseException as error:
|
||||
if root_pid is not None:
|
||||
try:
|
||||
toolset.cleanup(root_pid)
|
||||
except BaseException as cleanup_error:
|
||||
raise SnapshotRestoreError(
|
||||
f"snapshot restore failed: {_error_detail(error)}; "
|
||||
f"cleanup failed: {_error_detail(cleanup_error)}"
|
||||
) from error
|
||||
if isinstance(error, SnapshotRestoreError):
|
||||
raise
|
||||
raise SnapshotRestoreError(str(error)) from error
|
||||
@@ -0,0 +1,277 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import (
|
||||
BaseModel,
|
||||
ConfigDict,
|
||||
Field,
|
||||
StrictFloat,
|
||||
StrictInt,
|
||||
StrictStr,
|
||||
ValidationError,
|
||||
field_validator,
|
||||
model_validator,
|
||||
)
|
||||
from typing_extensions import Self
|
||||
|
||||
|
||||
class SnapshotCompatibilityError(RuntimeError):
|
||||
"""The snapshot identity does not match the requested runtime."""
|
||||
|
||||
|
||||
class SnapshotSecurityError(RuntimeError):
|
||||
"""The snapshot artifact does not meet the private-path contract."""
|
||||
|
||||
|
||||
_MAX_PID = 2**31 - 1
|
||||
_PositivePid = Annotated[int, Field(gt=0, le=_MAX_PID)]
|
||||
_OracleTokenId = Annotated[int, Field(ge=0)]
|
||||
|
||||
|
||||
class SnapshotRuntimeIdentity(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid", frozen=True, strict=True)
|
||||
|
||||
vllm_version: str
|
||||
python_version: str
|
||||
torch_version: str
|
||||
cuda_runtime: str
|
||||
driver_version: str
|
||||
criu_version: str
|
||||
cuda_checkpoint_sha256: str
|
||||
kernel_release: str
|
||||
host_id: str
|
||||
gpu_name: str
|
||||
gpu_uuid: str
|
||||
environment: tuple[tuple[str, str], ...]
|
||||
|
||||
|
||||
class SnapshotManifest(SnapshotRuntimeIdentity):
|
||||
schema_version: Literal[1]
|
||||
boundary: Literal["post-engine-init-reloadable-state-released"]
|
||||
created_at: str
|
||||
artifact_bytes: Annotated[int, Field(ge=0)]
|
||||
model: str
|
||||
served_model_name: str
|
||||
model_revision: str
|
||||
tokenizer_revision: str
|
||||
engine_argv: tuple[str, ...]
|
||||
process_tree: Annotated[tuple[_PositivePid, ...], Field(min_length=1)]
|
||||
cuda_holders: Annotated[tuple[_PositivePid, ...], Field(min_length=1)]
|
||||
external_cache_files: tuple[tuple[str, str], ...] = ()
|
||||
oracle_token_ids: Annotated[
|
||||
tuple[_OracleTokenId, ...], Field(min_length=1, max_length=1)
|
||||
]
|
||||
oracle_text: str
|
||||
oracle_sampled_token_logprob: float
|
||||
|
||||
@field_validator("schema_version", mode="before")
|
||||
@classmethod
|
||||
def _require_schema_version_one(cls, value: Any) -> Any:
|
||||
if type(value) is not int or value != 1:
|
||||
raise ValueError("schema version must be integer 1")
|
||||
return value
|
||||
|
||||
@field_validator("oracle_sampled_token_logprob", mode="before")
|
||||
@classmethod
|
||||
def _require_finite_float(cls, value: Any) -> Any:
|
||||
if type(value) is not float or not math.isfinite(value):
|
||||
raise ValueError("sampled token log probability must be a finite float")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_process_relationships(self) -> Self:
|
||||
if len(set(self.process_tree)) != len(self.process_tree):
|
||||
raise ValueError("process_tree contains duplicate PIDs")
|
||||
if len(set(self.cuda_holders)) != len(self.cuda_holders):
|
||||
raise ValueError("cuda_holders contains duplicate PIDs")
|
||||
if not set(self.cuda_holders).issubset(self.process_tree):
|
||||
raise ValueError("cuda_holders must be a subset of process_tree")
|
||||
return self
|
||||
|
||||
|
||||
class ReleaseMarker(BaseModel):
|
||||
"""Release marker the controller writes; unknown keys are ignored."""
|
||||
|
||||
release: Literal[True]
|
||||
host: StrictStr | None = None
|
||||
port: Annotated[StrictInt, Field(ge=1, le=65535)]
|
||||
|
||||
@field_validator("release", mode="before")
|
||||
@classmethod
|
||||
def _require_json_true(cls, value: Any) -> Any:
|
||||
# A literal accepts 1 as True; the barrier requires JSON true itself.
|
||||
if value is not True:
|
||||
raise ValueError("release must be JSON true")
|
||||
return value
|
||||
|
||||
|
||||
class ReadyMarker(BaseModel):
|
||||
"""Ready marker the child writes; unknown keys are ignored."""
|
||||
|
||||
token_ids: tuple[StrictInt, ...]
|
||||
text: StrictStr
|
||||
sampled_token_logprob: StrictFloat | StrictInt
|
||||
|
||||
|
||||
def validate_identity(
|
||||
expected: SnapshotRuntimeIdentity, actual: SnapshotRuntimeIdentity
|
||||
) -> None:
|
||||
"""Require an exact match for every field that can affect compatibility."""
|
||||
# ``expected`` may be a manifest, so compare the identity fields only.
|
||||
fields = set(SnapshotRuntimeIdentity.model_fields)
|
||||
expected_values = expected.model_dump(include=fields)
|
||||
actual_values = actual.model_dump(include=fields)
|
||||
differing = [
|
||||
name
|
||||
for name in SnapshotRuntimeIdentity.model_fields
|
||||
if expected_values[name] != actual_values[name]
|
||||
]
|
||||
if differing:
|
||||
raise SnapshotCompatibilityError(f"snapshot mismatch: {', '.join(differing)}")
|
||||
|
||||
|
||||
def _existing_path_chain(path: Path):
|
||||
current = path
|
||||
while True:
|
||||
if current.exists() or current.is_symlink():
|
||||
yield current
|
||||
if current.parent == current:
|
||||
break
|
||||
current = current.parent
|
||||
|
||||
|
||||
def _is_trusted_sticky_directory(path_stat: os.stat_result) -> bool:
|
||||
return (
|
||||
path_stat.st_uid == 0
|
||||
and stat.S_ISDIR(path_stat.st_mode)
|
||||
and bool(path_stat.st_mode & stat.S_ISVTX)
|
||||
)
|
||||
|
||||
|
||||
def validate_artifact_root(path: Path, *, creating: bool) -> None:
|
||||
"""Validate that an artifact path cannot be replaced by another user."""
|
||||
path = Path(os.path.abspath(path))
|
||||
if path.is_symlink():
|
||||
raise SnapshotSecurityError(f"snapshot path is a symlink: {path}")
|
||||
if not creating and not path.exists():
|
||||
raise SnapshotSecurityError(f"snapshot path does not exist: {path}")
|
||||
|
||||
effective_uid = os.geteuid()
|
||||
for component in _existing_path_chain(path):
|
||||
component_stat = component.lstat()
|
||||
if stat.S_ISLNK(component_stat.st_mode):
|
||||
raise SnapshotSecurityError(
|
||||
f"snapshot path contains a symlink: {component}"
|
||||
)
|
||||
if component_stat.st_uid not in (0, effective_uid):
|
||||
raise SnapshotSecurityError(
|
||||
f"snapshot path has an untrusted owner: {component}"
|
||||
)
|
||||
if component_stat.st_mode & (
|
||||
stat.S_IWGRP | stat.S_IWOTH
|
||||
) and not _is_trusted_sticky_directory(component_stat):
|
||||
raise SnapshotSecurityError(
|
||||
f"snapshot path has a group- or world-writable ancestor: {component}"
|
||||
)
|
||||
|
||||
if path.exists():
|
||||
path_stat = path.lstat()
|
||||
if not stat.S_ISDIR(path_stat.st_mode):
|
||||
raise SnapshotSecurityError(f"snapshot path is not a directory: {path}")
|
||||
if path_stat.st_uid != effective_uid:
|
||||
raise SnapshotSecurityError(
|
||||
f"snapshot directory is not owned by the current user: {path}"
|
||||
)
|
||||
if stat.S_IMODE(path_stat.st_mode) & 0o077:
|
||||
raise SnapshotSecurityError(
|
||||
f"snapshot directory must have mode 0700: {path}"
|
||||
)
|
||||
|
||||
|
||||
def _validation_path(error: ValidationError) -> str:
|
||||
detail = error.errors(include_url=False)[0]
|
||||
location = detail["loc"]
|
||||
if not location:
|
||||
return "JSON" if detail["type"] == "json_invalid" else "root"
|
||||
return ".".join(map(str, location))
|
||||
|
||||
|
||||
def _fsync_directory(path: Path) -> None:
|
||||
directory_fd = os.open(path, os.O_RDONLY | getattr(os, "O_DIRECTORY", 0))
|
||||
try:
|
||||
os.fsync(directory_fd)
|
||||
finally:
|
||||
os.close(directory_fd)
|
||||
|
||||
|
||||
def _write_json_atomic(path: Path, payload: object, *, overwrite: bool = False) -> None:
|
||||
temporary = path.with_name(f"{path.name}.tmp")
|
||||
if overwrite:
|
||||
temporary.unlink(missing_ok=True)
|
||||
elif path.exists() or path.is_symlink():
|
||||
raise FileExistsError(path)
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
|
||||
descriptor = os.open(temporary, flags, 0o600)
|
||||
try:
|
||||
with os.fdopen(descriptor, "wb") as output:
|
||||
output.write(
|
||||
json.dumps(payload, sort_keys=True, separators=(",", ":")).encode()
|
||||
)
|
||||
output.flush()
|
||||
os.fsync(output.fileno())
|
||||
os.replace(temporary, path)
|
||||
_fsync_directory(path.parent)
|
||||
except BaseException:
|
||||
temporary.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def write_manifest_atomic(path: Path, manifest: SnapshotManifest) -> None:
|
||||
"""Write a new private manifest and make its directory entry durable."""
|
||||
path = Path(path)
|
||||
validate_artifact_root(path, creating=False)
|
||||
destination = path / "manifest.json"
|
||||
try:
|
||||
_write_json_atomic(destination, manifest.model_dump(mode="json"))
|
||||
except FileExistsError as error:
|
||||
raise SnapshotSecurityError(
|
||||
f"manifest already exists: {destination}"
|
||||
) from error
|
||||
|
||||
|
||||
def read_manifest(path: Path) -> SnapshotManifest:
|
||||
"""Read a snapshot published by atomically installing its manifest."""
|
||||
path = Path(path)
|
||||
validate_artifact_root(path, creating=False)
|
||||
manifest_path = path / "manifest.json"
|
||||
if manifest_path.is_symlink():
|
||||
raise SnapshotSecurityError(f"manifest is a symlink: {manifest_path}")
|
||||
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
||||
try:
|
||||
file_descriptor = os.open(manifest_path, flags)
|
||||
except FileNotFoundError as error:
|
||||
raise SnapshotSecurityError(
|
||||
f"manifest does not exist: {manifest_path}"
|
||||
) from error
|
||||
try:
|
||||
with os.fdopen(file_descriptor, encoding="utf-8") as manifest_file:
|
||||
payload = manifest_file.read()
|
||||
except UnicodeDecodeError as error:
|
||||
raise SnapshotCompatibilityError("invalid snapshot manifest: JSON") from error
|
||||
try:
|
||||
return SnapshotManifest.model_validate_json(payload, strict=True)
|
||||
except ValidationError as error:
|
||||
raise SnapshotCompatibilityError(
|
||||
f"invalid snapshot manifest: {_validation_path(error)}"
|
||||
) from error
|
||||
|
||||
|
||||
def inspect_snapshot(path: Path) -> dict[str, object]:
|
||||
return read_manifest(path).model_dump(mode="json")
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,252 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import stat
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.snapshot.manifest import ReleaseMarker, _validation_path, _write_json_atomic
|
||||
from vllm.snapshot.types import Oracle, oracles_match
|
||||
|
||||
_CANARY_PROMPT = "The capital of France is"
|
||||
|
||||
|
||||
class SnapshotCanaryError(RuntimeError):
|
||||
"""The initialized engine did not produce a valid snapshot oracle."""
|
||||
|
||||
|
||||
class SnapshotBarrierError(RuntimeError):
|
||||
"""The controller supplied an invalid snapshot release marker."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ListenerConfig:
|
||||
host: str | None
|
||||
port: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ControlArgs:
|
||||
ready_file: Path
|
||||
release_file: Path
|
||||
release_timeout_s: float
|
||||
|
||||
|
||||
def write_ready_atomic(path: Path, oracle: Oracle) -> None:
|
||||
_write_json_atomic(
|
||||
path,
|
||||
{
|
||||
"sampled_token_logprob": oracle.sampled_token_logprob,
|
||||
"token_ids": oracle.token_ids,
|
||||
"text": oracle.text,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def read_release_marker(path: Path) -> ListenerConfig:
|
||||
path = Path(path)
|
||||
path_stat = path.lstat()
|
||||
if stat.S_ISLNK(path_stat.st_mode):
|
||||
raise SnapshotBarrierError("release marker must not be a symlink")
|
||||
try:
|
||||
payload = json.loads(path.read_text())
|
||||
except (OSError, json.JSONDecodeError) as error:
|
||||
raise SnapshotBarrierError("release marker is not valid JSON") from error
|
||||
try:
|
||||
marker = ReleaseMarker.model_validate(payload)
|
||||
except ValidationError as error:
|
||||
raise SnapshotBarrierError(
|
||||
f"release marker is invalid: {_validation_path(error)}"
|
||||
) from error
|
||||
return ListenerConfig(host=marker.host, port=marker.port)
|
||||
|
||||
|
||||
def parse_control_args(argv: list[str]) -> tuple[ControlArgs, list[str]]:
|
||||
parser = argparse.ArgumentParser(add_help=False)
|
||||
parser.add_argument("--ready-file", type=Path, required=True)
|
||||
parser.add_argument("--release-file", type=Path, required=True)
|
||||
parser.add_argument("--release-timeout-s", type=float, default=900.0)
|
||||
control_args, remaining = parser.parse_known_args(argv)
|
||||
if remaining and remaining[0] == "--":
|
||||
remaining = remaining[1:]
|
||||
if control_args.release_timeout_s <= 0:
|
||||
raise ValueError("release timeout must be positive")
|
||||
return (
|
||||
ControlArgs(
|
||||
ready_file=control_args.ready_file,
|
||||
release_file=control_args.release_file,
|
||||
release_timeout_s=control_args.release_timeout_s,
|
||||
),
|
||||
remaining,
|
||||
)
|
||||
|
||||
|
||||
async def _release_reloadable_state(engine: Any) -> None:
|
||||
"""Discard model and KV state before the process image is captured."""
|
||||
await engine.sleep(level=2)
|
||||
|
||||
|
||||
async def _restore_reloadable_state(engine: Any) -> None:
|
||||
"""Rebuild state discarded by ``_release_reloadable_state``."""
|
||||
await engine.wake_up(tags=["weights"])
|
||||
await engine.collective_rpc("reload_weights")
|
||||
await engine.wake_up(tags=["kv_cache"])
|
||||
|
||||
|
||||
def oracle_from_request_output(request_output: Any) -> Oracle:
|
||||
try:
|
||||
candidate = request_output.outputs[0]
|
||||
token_ids = tuple(candidate.token_ids)
|
||||
(sampled_token_id,) = token_ids
|
||||
return Oracle(
|
||||
token_ids=token_ids,
|
||||
text=candidate.text,
|
||||
sampled_token_logprob=candidate.logprobs[0][sampled_token_id].logprob,
|
||||
)
|
||||
except ValueError as error:
|
||||
raise SnapshotCanaryError(
|
||||
"snapshot canary must produce exactly one finite token logprob"
|
||||
) from error
|
||||
except (AttributeError, IndexError, KeyError, TypeError) as error:
|
||||
raise SnapshotCanaryError(
|
||||
"snapshot canary did not return sampled token logprob"
|
||||
) from error
|
||||
|
||||
|
||||
async def run_engine_canary(engine: Any) -> Oracle:
|
||||
from vllm import SamplingParams
|
||||
|
||||
final_output = None
|
||||
sampling_params = SamplingParams(
|
||||
temperature=0,
|
||||
min_tokens=1,
|
||||
max_tokens=1,
|
||||
seed=0,
|
||||
logprobs=0,
|
||||
)
|
||||
async for output in engine.generate(
|
||||
_CANARY_PROMPT,
|
||||
sampling_params,
|
||||
request_id="vllm-snapshot-canary",
|
||||
):
|
||||
final_output = output
|
||||
if final_output is None:
|
||||
raise SnapshotCanaryError("canary generation returned no output")
|
||||
return oracle_from_request_output(final_output)
|
||||
|
||||
|
||||
def detach_snapshot_streams() -> None:
|
||||
"""Remove launch-log descriptors from the reusable process image."""
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
sink = os.open(os.devnull, os.O_WRONLY)
|
||||
try:
|
||||
os.dup2(sink, sys.stdout.fileno())
|
||||
os.dup2(sink, sys.stderr.fileno())
|
||||
finally:
|
||||
os.close(sink)
|
||||
|
||||
|
||||
async def wait_for_release_marker(
|
||||
path: Path,
|
||||
*,
|
||||
timeout_s: float,
|
||||
poll_interval_s: float = 0.05,
|
||||
) -> ListenerConfig:
|
||||
if timeout_s <= 0:
|
||||
raise ValueError("release timeout must be positive")
|
||||
if poll_interval_s <= 0:
|
||||
raise ValueError("release poll interval must be positive")
|
||||
remaining_s = timeout_s
|
||||
while not path.exists():
|
||||
if remaining_s <= 0:
|
||||
raise TimeoutError(f"release marker not found before timeout: {path}")
|
||||
delay_s = min(poll_interval_s, remaining_s)
|
||||
await asyncio.sleep(delay_s)
|
||||
remaining_s -= delay_s
|
||||
return read_release_marker(path)
|
||||
|
||||
|
||||
def parse_vllm_args(argv: list[str]) -> Any:
|
||||
from vllm.entrypoints.launchers.cli_args import (
|
||||
make_arg_parser,
|
||||
validate_parsed_serve_args,
|
||||
)
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
parser = FlexibleArgumentParser(prog="python -m vllm.snapshot.server")
|
||||
args = make_arg_parser(parser).parse_args(argv)
|
||||
if args.model_tag is not None:
|
||||
args.model = args.model_tag
|
||||
if args.grpc or args.headless:
|
||||
raise ValueError("snapshot server requires the HTTP frontend")
|
||||
if args.api_server_count not in (None, 1):
|
||||
raise ValueError("snapshot server supports one API server")
|
||||
args.api_server_count = None
|
||||
validate_parsed_serve_args(args)
|
||||
return args
|
||||
|
||||
|
||||
async def run_vllm_snapshot_child(control: ControlArgs, args: Any) -> None:
|
||||
from vllm.entrypoints.launchers.api_server.entry import (
|
||||
build_and_serve,
|
||||
build_async_engine_client,
|
||||
)
|
||||
from vllm.entrypoints.launchers.launcher import setup_server
|
||||
|
||||
args.enable_sleep_mode = True
|
||||
|
||||
async with build_async_engine_client(args) as engine:
|
||||
oracle = await run_engine_canary(engine)
|
||||
try:
|
||||
await _release_reloadable_state(engine)
|
||||
await _restore_reloadable_state(engine)
|
||||
rehearsal_oracle = await run_engine_canary(engine)
|
||||
if not oracles_match(oracle, rehearsal_oracle):
|
||||
raise SnapshotCanaryError("snapshot rehearsal changed canary output")
|
||||
await _release_reloadable_state(engine)
|
||||
except SnapshotCanaryError:
|
||||
raise
|
||||
except Exception as error:
|
||||
raise SnapshotCanaryError("snapshot rehearsal failed") from error
|
||||
|
||||
detach_snapshot_streams()
|
||||
write_ready_atomic(control.ready_file, oracle)
|
||||
listener = await wait_for_release_marker(
|
||||
control.release_file,
|
||||
timeout_s=control.release_timeout_s,
|
||||
)
|
||||
args.host = listener.host
|
||||
args.port = listener.port
|
||||
await _restore_reloadable_state(engine)
|
||||
listen_address, sock = setup_server(args, reuse_port=False)
|
||||
try:
|
||||
shutdown_task = await build_and_serve(
|
||||
engine,
|
||||
listen_address,
|
||||
sock,
|
||||
args,
|
||||
)
|
||||
await shutdown_task
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> None:
|
||||
control, remaining = parse_control_args(sys.argv[1:] if argv is None else argv)
|
||||
args = parse_vllm_args(remaining)
|
||||
import uvloop
|
||||
|
||||
uvloop.run(run_vllm_snapshot_child(control, args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Oracle:
|
||||
token_ids: tuple[int, ...]
|
||||
text: str
|
||||
sampled_token_logprob: float
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if len(self.token_ids) != 1:
|
||||
raise ValueError("snapshot oracle must contain exactly one token")
|
||||
value = self.sampled_token_logprob
|
||||
if (
|
||||
isinstance(value, bool)
|
||||
or not isinstance(value, (int, float))
|
||||
or not math.isfinite(value)
|
||||
):
|
||||
raise ValueError("sampled token logprob must be finite")
|
||||
object.__setattr__(self, "sampled_token_logprob", float(value))
|
||||
|
||||
|
||||
def oracles_match(expected: Oracle, actual: Oracle) -> bool:
|
||||
return (
|
||||
expected.token_ids == actual.token_ids
|
||||
and expected.text == actual.text
|
||||
and math.isclose(
|
||||
expected.sampled_token_logprob,
|
||||
actual.sampled_token_logprob,
|
||||
rel_tol=0.0,
|
||||
abs_tol=1e-3,
|
||||
)
|
||||
)
|
||||
Reference in New Issue
Block a user