[RL] P2P RDT weight sync (#43375)
This commit is contained in:
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Driver-side helpers to run the inference fleet as a standard ``vllm serve``
|
||||
process for the sharded-RDT examples.
|
||||
|
||||
The trainer reaches the server's weight-sync control plane over the RLHF HTTP
|
||||
routes (``HTTPVLLMWeightSyncClient``); generation uses ``/v1/completions``. RDT
|
||||
still needs the workers to be RayExecutorV2 tensor-transport actors sharing the
|
||||
trainer's Ray cluster, so the server is launched with the Ray v2 executor and
|
||||
inherits this process's env (``address=auto``). These helpers run only on the
|
||||
driver — they are not shipped to or imported by the Ray actors.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import ray
|
||||
import requests
|
||||
|
||||
|
||||
def launch_vllm_serve(
|
||||
model: str,
|
||||
*,
|
||||
tensor_parallel_size: int = 1,
|
||||
data_parallel_size: int = 1,
|
||||
enable_expert_parallel: bool = False,
|
||||
port: int = 8000,
|
||||
gpu_memory_utilization: float = 0.7,
|
||||
extra_args: list[str] | None = None,
|
||||
) -> subprocess.Popen:
|
||||
"""Start ``vllm serve`` for the RDT inference fleet.
|
||||
|
||||
``VLLM_SERVER_DEV_MODE`` exposes the RLHF weight-sync routes; the Ray v2
|
||||
executor makes the workers tensor-transport actors (RDT's data plane). The
|
||||
child inherits our env, so it joins the same Ray cluster (RAY_ADDRESS below)
|
||||
and picks up the editable vLLM install via the venv interpreter.
|
||||
"""
|
||||
vllm_bin = os.path.join(os.path.dirname(sys.executable), "vllm")
|
||||
if not os.path.exists(vllm_bin):
|
||||
vllm_bin = shutil.which("vllm") or "vllm"
|
||||
cmd = [
|
||||
vllm_bin,
|
||||
"serve",
|
||||
model,
|
||||
"--port",
|
||||
str(port),
|
||||
"--enforce-eager",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--gpu-memory-utilization",
|
||||
str(gpu_memory_utilization),
|
||||
"--distributed-executor-backend",
|
||||
"ray",
|
||||
"--weight-transfer-config",
|
||||
json.dumps({"backend": "sharded_rdt"}),
|
||||
]
|
||||
if tensor_parallel_size > 1:
|
||||
cmd += ["--tensor-parallel-size", str(tensor_parallel_size)]
|
||||
# Only engage the DP-ray backend when actually data-parallel: DP=1 with
|
||||
# data_parallel_backend=ray forces vLLM's DP-placement path, which fails.
|
||||
if data_parallel_size > 1:
|
||||
cmd += [
|
||||
"--data-parallel-size",
|
||||
str(data_parallel_size),
|
||||
"--data-parallel-backend",
|
||||
"ray",
|
||||
]
|
||||
if enable_expert_parallel:
|
||||
cmd.append("--enable-expert-parallel")
|
||||
if extra_args:
|
||||
cmd += extra_args
|
||||
env = dict(
|
||||
os.environ,
|
||||
VLLM_SERVER_DEV_MODE="1",
|
||||
VLLM_USE_RAY_V2_EXECUTOR_BACKEND="1",
|
||||
)
|
||||
# Pin the child to THIS Ray cluster. vLLM's ray DP backend reaches Ray
|
||||
# through a bare ``ray.init()``, which STARTS a cluster rather than
|
||||
# attaching, so wherever we started Ray ourselves (a bare CI runner) the
|
||||
# workers would come up in a second cluster and never see the producer
|
||||
# actors -- the failure is an actor lookup that cannot succeed. A bare
|
||||
# ``ray.init()`` does honour RAY_ADDRESS, so hand it over explicitly.
|
||||
if ray.is_initialized():
|
||||
env["RAY_ADDRESS"] = ray.get_runtime_context().gcs_address
|
||||
# Let the server's Ray workers use the ambient interpreter/install rather
|
||||
# than a workspace snapshot working_dir. On an Anyscale dev workspace the
|
||||
# RAY_RUNTIME_ENV_HOOK injects a git snapshot as the workers' working_dir,
|
||||
# which shadows an editable vLLM install (its compiled .so is gitignored, so
|
||||
# the snapshot lacks it). No-op off Anyscale (the var is unset there).
|
||||
env.pop("RAY_RUNTIME_ENV_HOOK", None)
|
||||
print(f"[serve] launching: {' '.join(cmd)}", flush=True)
|
||||
return subprocess.Popen(cmd, env=env)
|
||||
|
||||
|
||||
def wait_for_server(
|
||||
endpoint: str, proc: subprocess.Popen, timeout: float = 1800
|
||||
) -> None:
|
||||
"""Block until ``/health`` returns 200, failing fast if the server exits."""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
if proc.poll() is not None:
|
||||
raise RuntimeError(f"vllm serve exited early (code {proc.returncode})")
|
||||
try:
|
||||
if requests.get(f"{endpoint}/health", timeout=5).status_code == 200:
|
||||
print("[serve] server is healthy", flush=True)
|
||||
return
|
||||
except requests.RequestException:
|
||||
pass
|
||||
time.sleep(3)
|
||||
raise RuntimeError("vllm serve did not become healthy in time")
|
||||
|
||||
|
||||
def http_generate(
|
||||
endpoint: str, model: str, prompts: list[str], max_tokens: int = 16
|
||||
) -> list[str]:
|
||||
"""Greedy ``/v1/completions`` for each prompt; returns the generated texts."""
|
||||
outs = []
|
||||
for p in prompts:
|
||||
r = requests.post(
|
||||
f"{endpoint}/v1/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"prompt": p,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0,
|
||||
},
|
||||
timeout=120,
|
||||
)
|
||||
r.raise_for_status()
|
||||
outs.append(r.json()["choices"][0]["text"])
|
||||
return outs
|
||||
|
||||
|
||||
def pause_generation(endpoint: str, mode: str = "abort") -> None:
|
||||
requests.post(f"{endpoint}/pause", params={"mode": mode}, timeout=60)
|
||||
|
||||
|
||||
def resume_generation(endpoint: str) -> None:
|
||||
requests.post(f"{endpoint}/resume", timeout=60)
|
||||
|
||||
|
||||
def shutdown_server(proc: subprocess.Popen) -> None:
|
||||
print("[serve] shutting down vllm serve", flush=True)
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ``WeightSource`` for ``rlhf_sharded_rdt_small_ep.py``.
|
||||
|
||||
A trainer's producer job is to present its weights under the names the inference
|
||||
side expects, which for vLLM means HF CHECKPOINT names. Transformers does not
|
||||
store MoE experts that way -- it fuses each layer's into ``[E, ...]`` tensors --
|
||||
so that is the one conversion this file does. Kept beside the example rather than
|
||||
inside it so the example stays about the sync itself, alongside
|
||||
``rdt_vllm_serve.py``.
|
||||
"""
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer import ParamMeta, WeightSource
|
||||
from vllm.distributed.weight_transfer.base import materialize_full_tensor
|
||||
|
||||
# Transformers fuses a layer's experts into ``[E, ...]`` tensors; the checkpoint
|
||||
# stores them per expert. This matches the fused parameter names.
|
||||
_FUSED_EXPERT_RE = re.compile(r"^(.*\.experts)\.(gate_up_proj|down_proj)$")
|
||||
|
||||
|
||||
class CheckpointNameSource(WeightSource):
|
||||
"""`WeightSource` over an FSDP2 model that publishes HF CHECKPOINT names.
|
||||
|
||||
Transformers keeps each layer's experts fused as ``[E, ...]`` tensors, but
|
||||
vLLM's MoE loaders are written against the per-expert checkpoint entries --
|
||||
only ``qwen{2,3}_moe`` also accept the fused names, so publishing them
|
||||
straight from ``named_parameters()`` limits the example to one family and
|
||||
fails the consumer's bake on ``experts.gate_up_proj`` everywhere else.
|
||||
Splitting them back is also what a real trainer does, since converting an
|
||||
internal layout to checkpoint names is the normal producer job.
|
||||
|
||||
Every rank publishes the whole model: fused params are all-gathered and then
|
||||
sliced per expert. Ownership stays uniform (no ``held_names``) -- serving
|
||||
only the experts a rank holds is a real trainer's concern, not an example's.
|
||||
"""
|
||||
|
||||
def __init__(self, module: torch.nn.Module) -> None:
|
||||
self._module = module
|
||||
|
||||
@staticmethod
|
||||
def _expand(name: str, param: torch.Tensor) -> list[tuple[str, tuple]]:
|
||||
"""The checkpoint entries ``name`` contributes, as (name, shape).
|
||||
|
||||
``gate_up_proj`` is ``[E, 2I, H]`` with gate in rows ``:I`` and up in
|
||||
``I:``; ``down_proj`` is ``[E, H, I]``. Anything else is already a
|
||||
checkpoint name and passes through.
|
||||
"""
|
||||
m = _FUSED_EXPERT_RE.match(name)
|
||||
if m is None:
|
||||
return [(name, tuple(param.shape))]
|
||||
prefix, kind = m.group(1), m.group(2)
|
||||
experts, rows, cols = param.shape
|
||||
if kind == "down_proj":
|
||||
return [
|
||||
(f"{prefix}.{e}.down_proj.weight", (rows, cols)) for e in range(experts)
|
||||
]
|
||||
half = rows // 2
|
||||
return [
|
||||
(f"{prefix}.{e}.{proj}_proj.weight", (half, cols))
|
||||
for e in range(experts)
|
||||
for proj in ("gate", "up")
|
||||
]
|
||||
|
||||
def metadata(self) -> list[ParamMeta]:
|
||||
return [
|
||||
ParamMeta(entry_name, param.dtype, shape)
|
||||
for name, param in self._module.named_parameters()
|
||||
for entry_name, shape in self._expand(name, param)
|
||||
]
|
||||
|
||||
def __iter__(self):
|
||||
for name, param in self._module.named_parameters():
|
||||
m = _FUSED_EXPERT_RE.match(name)
|
||||
if m is None:
|
||||
yield name, materialize_full_tensor(param)
|
||||
continue
|
||||
# One gather per FUSED param, not per expert: the views below are
|
||||
# into that one tensor, and every expert of a layer rides in the
|
||||
# same gather group, so it stays resident exactly as long as needed.
|
||||
prefix, kind = m.group(1), m.group(2)
|
||||
full = materialize_full_tensor(param)
|
||||
if kind == "down_proj":
|
||||
for e in range(full.shape[0]):
|
||||
yield f"{prefix}.{e}.down_proj.weight", full[e]
|
||||
else:
|
||||
half = full.shape[1] // 2
|
||||
for e in range(full.shape[0]):
|
||||
yield f"{prefix}.{e}.gate_proj.weight", full[e, :half, :]
|
||||
yield f"{prefix}.{e}.up_proj.weight", full[e, half:, :]
|
||||
@@ -0,0 +1,270 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sharded-RDT weight sync on FOUR GPUs, one node: 2 FSDP2 trainer ranks -> 2
|
||||
vLLM DP ranks with expert parallelism.
|
||||
|
||||
The reference example for the backend, sized to fit a CI runner:
|
||||
``CheckpointNameSource`` over an FSDP2 model, one serve actor per rank, the
|
||||
HTTP control plane, and assertions so it can run unattended.
|
||||
|
||||
Why a MoE model rather than a small dense one: the point of sharded RDT is that
|
||||
each inference worker pulls only the slices it consumes, so the example is only
|
||||
meaningful when no rank holds the whole model. That needs experts on both sides.
|
||||
``Qwen/Qwen1.5-MoE-A2.7B`` is the reference shape, a real MoE of the same family
|
||||
as the 30B model the big example uses. Its WEIGHTS are never read: the trainer
|
||||
builds the config and the server runs ``--load-format dummy``, so the sync is
|
||||
what is under test and nothing is downloaded but the config and tokenizer. A
|
||||
smaller MoE via ``RDT_MODEL`` changes nothing about what is exercised, which is
|
||||
what CI runs.
|
||||
|
||||
The trainer publishes CHECKPOINT names, not the names its own modules carry:
|
||||
Transformers fuses each layer's experts into ``[E, ...]`` tensors
|
||||
(``mlp.experts.gate_up_proj``/``down_proj``) while the checkpoint stores them per
|
||||
expert, and vLLM's MoE loaders read the per-expert form. ``CheckpointNameSource``
|
||||
splits them back, which is the same conversion a real trainer does when it maps
|
||||
its internal layout onto checkpoint names. Publishing the fused names instead
|
||||
works only for the Qwen families, whose loaders happen to accept them.
|
||||
|
||||
FSDP2 shards those fused tensors on dim 0 — the expert dimension — so after
|
||||
sharding no rank stores every expert, and the inference side gets real EP from
|
||||
``--enable-expert-parallel`` over DP2. Two simplifications an example can afford
|
||||
and a real trainer cannot: each rank BUILDS the whole model before
|
||||
``fully_shard`` splits it (fine for a tiny config, so nothing here streams a
|
||||
checkpoint), and iteration all-gathers every parameter, so ownership stays
|
||||
uniform. Serving only the experts a rank holds (``held_names``) is where a real
|
||||
trainer earns its keep and is deliberately out of scope.
|
||||
|
||||
Model naming is load-bearing: names are mapped onto checkpoint keys one-to-one,
|
||||
so a model whose HF module names have drifted from its checkpoint (as
|
||||
GraniteMoe's router has) will not work here.
|
||||
|
||||
What it checks (a smoke test, not a numerical equivalence check):
|
||||
- generation CHANGES after the first sync — the weights really moved, so a
|
||||
silently-skipped sync fails instead of printing plausible text;
|
||||
- generation is UNCHANGED by a second sync — replay is stable.
|
||||
|
||||
Run:
|
||||
python examples/rl/rlhf_sharded_rdt_small_ep.py
|
||||
|
||||
Needs 4 GPUs on one node, each big enough for the whole model, since every rank
|
||||
builds it before ``fully_shard`` splits it: the default is ~29 GiB in bf16, so
|
||||
40 GiB cards and up. Set ``RDT_MODEL`` to a tiny MoE on smaller GPUs. Joins an
|
||||
existing Ray cluster if there is one and otherwise starts its own.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import ray
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from ray.util.placement_group import placement_group, placement_group_table
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
from rdt_vllm_serve import (
|
||||
http_generate,
|
||||
launch_vllm_serve,
|
||||
pause_generation,
|
||||
resume_generation,
|
||||
shutdown_server,
|
||||
wait_for_server,
|
||||
)
|
||||
from rdt_weight_source import CheckpointNameSource
|
||||
from torch.distributed.fsdp import fully_shard
|
||||
from transformers import AutoConfig, AutoModelForCausalLM
|
||||
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
ShardedRDTTrainerInitInfo,
|
||||
)
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
# Weights are built, never loaded, so this costs a config fetch -- but every rank
|
||||
# materializes the model before sharding, so the default wants a 40 GiB card.
|
||||
MODEL_NAME = os.environ.get("RDT_MODEL", "Qwen/Qwen1.5-MoE-A2.7B")
|
||||
RAY_NAMESPACE = "sharded_rdt_small_ep_example"
|
||||
VLLM_PORT = int(os.environ.get("RDT_VLLM_PORT", "8100"))
|
||||
VLLM_ENDPOINT = f"http://127.0.0.1:{VLLM_PORT}"
|
||||
|
||||
FSDP_WORLD_SIZE = 2
|
||||
INFERENCE_TP_SIZE = 1
|
||||
INFERENCE_DP_SIZE = 2
|
||||
NUM_INFERENCE_CONSUMERS = INFERENCE_TP_SIZE * INFERENCE_DP_SIZE
|
||||
|
||||
PROMPTS = ["The capital of France is", "The future of AI is"]
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
class FSDPTrainWorker:
|
||||
"""One FSDP2 rank per GPU. Identical to the 8-GPU example's worker: the
|
||||
sharded-RDT engine owns the NIXL serve surface, so this stays a plain Ray
|
||||
actor with no producer mixin."""
|
||||
|
||||
def __init__(self, model_name, rank, world_size, master_addr, master_port):
|
||||
self.rank = rank
|
||||
self.engine = None
|
||||
os.environ["MASTER_ADDR"] = master_addr
|
||||
os.environ["MASTER_PORT"] = str(master_port)
|
||||
dist.init_process_group(backend="nccl", rank=rank, world_size=world_size)
|
||||
torch.accelerator.set_device_index(0)
|
||||
|
||||
# Same seed on every rank, so the shards compose into ONE model rather
|
||||
# than a blend of independent inits -- the second sync's stability check
|
||||
# is only meaningful against coherent weights.
|
||||
torch.manual_seed(0)
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
with torch.device("cuda"):
|
||||
model = AutoModelForCausalLM.from_config(config, dtype=torch.bfloat16)
|
||||
for layer in model.model.layers:
|
||||
fully_shard(layer)
|
||||
fully_shard(model)
|
||||
self.model = model
|
||||
|
||||
def get_rank(self):
|
||||
return self.rank
|
||||
|
||||
def setup_engine(self, vllm_endpoint: str):
|
||||
self.engine = WeightTransferTrainerFactory.trainer_init(
|
||||
ShardedRDTTrainerInitInfo(
|
||||
rank=self.rank,
|
||||
num_consumers=NUM_INFERENCE_CONSUMERS,
|
||||
trainer_actor_namespace=RAY_NAMESPACE,
|
||||
num_rdt_buffers=int(os.environ.get("NUM_RDT_BUFFERS", "2")),
|
||||
buffer_presize_gb=float(os.environ.get("RDT_BUFFER_PRESIZE_GB", "0")),
|
||||
),
|
||||
client=HTTPVLLMWeightSyncClient(vllm_endpoint),
|
||||
source=CheckpointNameSource(self.model),
|
||||
)
|
||||
|
||||
def sync_weights(self):
|
||||
self.engine.send_weights()
|
||||
|
||||
|
||||
def main():
|
||||
# Ship a minimal working_dir (this example dir) so Ray actors do NOT inherit
|
||||
# a workspace snapshot that shadows the editable vLLM install.
|
||||
runtime_env: dict[str, object] = {
|
||||
"py_executable": sys.executable,
|
||||
"working_dir": os.path.dirname(os.path.abspath(__file__)),
|
||||
}
|
||||
forwarded = {
|
||||
k: os.environ[k]
|
||||
for k in (
|
||||
"NCCL_CUMEM_ENABLE",
|
||||
"VLLM_NCCL_SO_PATH",
|
||||
"LD_PRELOAD",
|
||||
"LD_LIBRARY_PATH",
|
||||
)
|
||||
if k in os.environ
|
||||
}
|
||||
if forwarded:
|
||||
runtime_env["env_vars"] = forwarded
|
||||
if not ray.is_initialized():
|
||||
try:
|
||||
ray.init(address="auto", runtime_env=runtime_env, namespace=RAY_NAMESPACE)
|
||||
except (ConnectionError, RuntimeError):
|
||||
# No cluster to join (a bare CI runner): start one on this node.
|
||||
# `launch_vllm_serve` hands the child this cluster's address, which
|
||||
# it needs -- vLLM's ray DP backend would otherwise start its own.
|
||||
ray.init(runtime_env=runtime_env, namespace=RAY_NAMESPACE)
|
||||
|
||||
print(
|
||||
f"[init] model {MODEL_NAME}, {FSDP_WORLD_SIZE} trainer + "
|
||||
f"{INFERENCE_DP_SIZE} inference GPUs"
|
||||
)
|
||||
|
||||
# Reserve the trainer's GPUs BEFORE launching the server, so the server's DP
|
||||
# workers take the node's remaining two. STRICT_PACK keeps the trainer's
|
||||
# all-gathers on one node's NVLink -- here that is the only node anyway.
|
||||
trainer_pg = placement_group(
|
||||
[{"GPU": 1, "CPU": 1}] * FSDP_WORLD_SIZE, strategy="STRICT_PACK"
|
||||
)
|
||||
ray.get(trainer_pg.ready())
|
||||
pg_node_id = next(
|
||||
iter(placement_group_table(trainer_pg)["bundles_to_node_id"].values())
|
||||
)
|
||||
master_addr = next(
|
||||
n["NodeManagerAddress"] for n in ray.nodes() if n["NodeID"] == pg_node_id
|
||||
)
|
||||
|
||||
@ray.remote(
|
||||
num_cpus=0,
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=trainer_pg
|
||||
),
|
||||
)
|
||||
def _free_port():
|
||||
return get_open_port()
|
||||
|
||||
master_port = ray.get(_free_port.remote())
|
||||
|
||||
workers = [
|
||||
FSDPTrainWorker.options(
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=trainer_pg, placement_group_bundle_index=rank
|
||||
),
|
||||
).remote(MODEL_NAME, rank, FSDP_WORLD_SIZE, master_addr, master_port)
|
||||
for rank in range(FSDP_WORLD_SIZE)
|
||||
]
|
||||
ray.get([w.get_rank.remote() for w in workers])
|
||||
print(f"[init] {FSDP_WORLD_SIZE} FSDP trainer workers ready (seeded weights).")
|
||||
|
||||
server = launch_vllm_serve(
|
||||
MODEL_NAME,
|
||||
tensor_parallel_size=INFERENCE_TP_SIZE,
|
||||
data_parallel_size=INFERENCE_DP_SIZE,
|
||||
enable_expert_parallel=True,
|
||||
port=VLLM_PORT,
|
||||
extra_args=["--max-model-len", "2048", "--max-num-seqs", "8"],
|
||||
)
|
||||
try:
|
||||
wait_for_server(VLLM_ENDPOINT, server)
|
||||
|
||||
before = http_generate(VLLM_ENDPOINT, MODEL_NAME, PROMPTS)
|
||||
print("[generate] BEFORE sync (vLLM's own dummy init):")
|
||||
for p, t in zip(PROMPTS, before):
|
||||
print(f" {p!r} -> {t!r}")
|
||||
|
||||
print("[sync] building trainer engines (bake on the sender)...")
|
||||
t0 = time.perf_counter()
|
||||
ray.get([w.setup_engine.remote(VLLM_ENDPOINT) for w in workers])
|
||||
print(f"[sync] engine setup (incl. bake) took {time.perf_counter() - t0:.2f}s")
|
||||
|
||||
pause_generation(VLLM_ENDPOINT)
|
||||
t0 = time.perf_counter()
|
||||
ray.get([w.sync_weights.remote() for w in workers])
|
||||
print(f"[sync] iter 0 took {time.perf_counter() - t0:.3f}s")
|
||||
resume_generation(VLLM_ENDPOINT)
|
||||
|
||||
after = http_generate(VLLM_ENDPOINT, MODEL_NAME, PROMPTS)
|
||||
print("[generate] AFTER sync (the trainer's weights):")
|
||||
for p, t in zip(PROMPTS, after):
|
||||
print(f" {p!r} -> {t!r}")
|
||||
|
||||
# Sync again; replay must be stable, so generation must not move.
|
||||
pause_generation(VLLM_ENDPOINT)
|
||||
t0 = time.perf_counter()
|
||||
ray.get([w.sync_weights.remote() for w in workers])
|
||||
print(f"[sync] iter 1 took {time.perf_counter() - t0:.3f}s")
|
||||
resume_generation(VLLM_ENDPOINT)
|
||||
again = http_generate(VLLM_ENDPOINT, MODEL_NAME, PROMPTS)
|
||||
|
||||
# Greedy decoding, so both comparisons are exact.
|
||||
assert after != before, (
|
||||
f"generation did not change across the weight sync, so the sync "
|
||||
f"moved nothing: {before!r}"
|
||||
)
|
||||
assert again == after, (
|
||||
f"a second sync changed generation, so replay is not stable: "
|
||||
f"{after!r} then {again!r}"
|
||||
)
|
||||
print("[ok] weights moved on sync 0 and replay was stable on sync 1")
|
||||
finally:
|
||||
shutdown_server(server)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user