diff --git a/.buildkite/test_areas/distributed.yaml b/.buildkite/test_areas/distributed.yaml index ed23e9ec4c..ab4f2c6845 100644 --- a/.buildkite/test_areas/distributed.yaml +++ b/.buildkite/test_areas/distributed.yaml @@ -17,6 +17,21 @@ steps: - pytest -v -s distributed/test_shm_buffer.py - pytest -v -s distributed/test_shm_storage.py +- label: ":nvidia: (L4) Sharded RDT Weight Transfer" + key: sharded-rdt-weight-transfer + timeout_in_minutes: 20 + working_dir: "/vllm-workspace/tests" + num_devices: 1 + source_file_dependencies: + - vllm/distributed/weight_transfer/ + - tests/distributed/test_sharded_rdt_plan.py + - tests/distributed/test_sharded_rdt_producer.py + - tests/distributed/test_sharded_rdt_trainer.py + commands: + - pytest -v -s distributed/test_sharded_rdt_plan.py + - pytest -v -s distributed/test_sharded_rdt_producer.py + - pytest -v -s distributed/test_sharded_rdt_trainer.py + - label: ":nvidia: (L4) Distributed DP Basic" key: distributed-dp-tests-2-gpus timeout_in_minutes: 35 @@ -107,7 +122,7 @@ steps: - label: ":nvidia: (L4) Distributed Torchrun + Examples" key: distributed-torchrun-examples-4-gpus - timeout_in_minutes: 30 + timeout_in_minutes: 45 working_dir: "/vllm-workspace" num_devices: 4 source_file_dependencies: @@ -137,6 +152,15 @@ steps: # rlhf examples (each launches its own `vllm serve` and tears it down) - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py - VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py + # sharded RDT: 2 FSDP2 trainer ranks -> 2 vLLM DP ranks with EP. Its data + # plane is NIXL, which ships only in the kv-connector set, and the script + # also drops the nixl-cu wheel variant that does not match this image. Kept + # last so the commands above run in an unmodified environment. RDT_MODEL swaps + # the script's reference MoE for a tiny one: no weights are downloaded either + # way, but every trainer rank builds the model before sharding it, and the + # default does not fit an L4. + - bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh + - RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py - label: ":nvidia: (L4) Distributed DP Extended" key: distributed-dp-tests-4-gpus diff --git a/docs/training/weight_transfer/README.md b/docs/training/weight_transfer/README.md index b394e41415..f26c033281 100644 --- a/docs/training/weight_transfer/README.md +++ b/docs/training/weight_transfer/README.md @@ -34,6 +34,7 @@ engine drives on your behalf: | [NCCL](nccl.md) | NCCL broadcast | Separate GPUs for training and inference | | [IPC](ipc.md) | CUDA IPC handles | Colocated training and inference on same GPU | | [sparse_nccl](nccl.md#sparse-nccl) | NCCL broadcast | Sparse flat-index weight patches (TP=1/PP=1) | +| [sharded_rdt](sharded_rdt.md) | NIXL / Ray Direct Transport (pull-based) | Very large models where each worker needs only its own slice (MoE with expert parallelism) | ## Quickstart @@ -48,7 +49,7 @@ from vllm.config import WeightTransferConfig llm = LLM( model="my-model", - weight_transfer_config=WeightTransferConfig(backend="nccl"), # or "ipc", "sparse_nccl" + weight_transfer_config=WeightTransferConfig(backend="nccl"), # or "ipc", "sparse_nccl", "sharded_rdt" ) ``` @@ -158,7 +159,7 @@ When running vLLM as an HTTP server, the following endpoints are available for w !!! note The HTTP weight transfer endpoints require `VLLM_SERVER_DEV_MODE=1` to be set. -The Rust frontend's optional gRPC `Control` service exposes the same pause, sleep, weight-transfer, and weight-version lifecycle for trusted sidecars. The `ServerInfo.rl_capabilities` response reports whether weight transfer and sleep mode were configured. Backend-specific `init_info` and `update_info` remain JSON metadata; model tensors continue to move over the configured NCCL, IPC, or sparse-NCCL transport. +The Rust frontend's optional gRPC `Control` service exposes the same pause, sleep, weight-transfer, and weight-version lifecycle for trusted sidecars. The `ServerInfo.rl_capabilities` response reports whether weight transfer and sleep mode were configured. Backend-specific `init_info` and `update_info` remain JSON metadata; model tensors continue to move over the configured NCCL, IPC, sparse-NCCL, or sharded-RDT transport. ## Extending the System diff --git a/docs/training/weight_transfer/base.md b/docs/training/weight_transfer/base.md index 1e29051aca..d73c88436a 100644 --- a/docs/training/weight_transfer/base.md +++ b/docs/training/weight_transfer/base.md @@ -20,8 +20,14 @@ vice versa, so the registries stay independent. ### WeightSource -A `WeightSource` is a **re-iterable** source of the trainer's weights with two -channels: +**This is the adapter for whatever shape your trainer's weights are in.** +`WeightSource` defines how you extract your weights for your specific framework. + +What an engine gets is always the same: HF-format parameter names, and tensors +already materialized to their full (unsharded) shape. Whatever gathering, +re-fusing, dequantizing or renaming that takes belongs inside the source. + +A `WeightSource` is **re-iterable** and has two required channels: - **`metadata() -> list[ParamMeta]`** — the name, wire dtype, and full shape of every parameter, *without transferring anything*. Cheap when shapes are known @@ -51,9 +57,7 @@ class ParamMeta: Materializing is typically a collective, so **every trainer rank must iterate the same source in the same order, in lockstep**, or ranks deadlock. `metadata()` can itself be a collective for custom producers, so it too runs on every rank — only -the sender ships the result. Under pipeline parallelism a rank may not own a -parameter at all; iterating still drives the collective, and the yielded tensor -is only meaningful on the sender. +the sender ships the result. `iter(source)` must yield a *fresh* pass each round. @@ -70,35 +74,133 @@ from vllm.distributed.weight_transfer import ModuleSource source = ModuleSource(model) ``` -#### Custom Sources +#### Custom sources -Subclass `WeightSource` when the weights you want to send require additional processing to convert to a HF compatible format. +Subclass `WeightSource` when the weights need work to reach HF format — a +framework-specific export, a re-fusing step, a dtype cast. ```python from vllm.distributed.weight_transfer import ParamMeta, WeightSource -from vllm.distributed.weight_transfer.base import materialize_full_tensor -class MyExportSource(WeightSource): - def __init__(self, model): - self._model = model +class MegatronBridgeSource(WeightSource): + """Megatron model -> HF names, via a bridge that gathers TP/PP/EP internally + and returns full tensors on every rank.""" + + def __init__(self, bridge, module, dtype): + self._bridge, self._module, self._dtype = bridge, module, dtype self._meta: list[ParamMeta] | None = None + def _export(self): + return self._bridge.export_hf_weights(self._module) + def metadata(self) -> list[ParamMeta]: # Cache: for producers that must materialize to learn shapes, this is # the expensive channel. Runs on every rank (it may be a collective). if self._meta is None: self._meta = [ - ParamMeta(name, t.dtype, tuple(t.shape)) for name, t in self._export() + ParamMeta(name, self._dtype, tuple(t.shape)) + for name, t in self._export() ] return self._meta def __iter__(self): # Must yield exactly what metadata() declared, in the same order. for name, tensor in self._export(): - yield name, materialize_full_tensor(tensor) + yield name, tensor.to(self._dtype).detach().contiguous() ``` +#### `held_names()`: partial ownership + +By default every rank is assumed to be able to produce every parameter — which is +what the source above does, since its bridge gathers across all parallelism before +yielding. That is simple and always correct, but it means the gather cost is paid +in full on every rank. + +Optionally, you can override `held_names()` when the ranks are split so each holds only part of the +model. It returns the +parameter names this rank holds (or `None`, the default, for all of them): + +```python +def held_names(self): + # This pipeline stage's layers, and within them only this EP rank's experts. + return self._my_stage_names - self._foreign_expert_names +``` + +This covers various trainer layouts — pipeline stages (a rank holds some layers), +expert parallelism (a rank holds some experts), both at once, or a shape that fits +neither. Backends that can route per parameter (see +[sharded RDT](sharded_rdt.md)) then pull each name from a rank that actually +holds it. + +Three requirements come with overriding it: + +- **`metadata()` must still describe the whole model on every rank.** Only the + sender's metadata reaches the inference side, so a rank that reported just its + own share would leave the rest silently un-transferred. Sharded RDT cross-checks + this across ranks at init. +- **Every name must be held by at least one rank**, or it can never be served. + The engine raises at init naming the first orphan. +- **Iteration yields `None` for a name this rank does not hold.** The name still + appears, in metadata order, so the order check stays aligned across ranks — only + the data is absent. Claiming a name and then yielding `None` for it is an error + the engine reports by name. + +!!! warning "Partial ownership only works with sharded RDT" + Only a backend that routes per parameter honours `held_names()`. + Broadcast backends ignore it and send every name from every rank, so + declaring partial ownership there changes nothing. + +#### Gather groups + +Some backends transfer a layer at a time rather than a model at a time, so they +partition `metadata()` into **gather groups**. `layerwise_groups` keys each name +on the **outermost index segment** it contains, so **a group is one decoder +layer**, with runs of un-indexed names (the embeddings, the final norm, +`lm_head`) forming groups of their own where they appear: + +```text +group 0 model.embed_tokens.weight +group 1 model.layers.0.* <- one decoder layer +group 2 model.layers.1.* +... +group N+1 model.norm.weight, lm_head.weight +``` + +Keying on the index rather than a literal prefix means no per-architecture table: +`model.layers.0.`, `model.language_model.layers.0.` (recent Qwen text +checkpoints), `transformer.h.0.` (GPT-2, Falcon), `backbone.layers.0.` (Mamba) +and a vision tower's `visual.blocks.0.` all partition the same way. The index +taken is the *outermost* one, which keeps a MoE layer whole — a per-expert name +like `model.layers.3.mlp.experts.7.w1` keys on the layer, not the expert. + +Group index *g* means the same layer on every rank and every consumer, because +every side derives it from one rank's `metadata()`. That agreement is what lets a +backend bound its buffers to one layer and free a layer once everyone is done +with it. + +!!! warning "A leaf module's sources must all fall in one group" + The sharded-RDT engine frees a group as soon as its last chunk lands, so a + module split across groups would park a pull until the stall watchdog fires. + The default partition guarantees this; a `groups()` override must preserve + it. + +Two hooks follow from this, both with working defaults: + +- **`groups()`** — this rank's groups, in metadata order. The default is + `layerwise_groups(metadata())` restricted to the groups holding at least one + held name. A group with nothing held here is skipped entirely. +- **`iter_groups()`** — the same stream, batched one group at a time. The default + drives `__iter__` and batches its output, checking that names arrive in metadata + order. Override it when your framework can produce a whole group in one step: + materializing is usually a collective, and driving it per group rather than per + tensor turns ~37k generator resumes into ~95 on a per-expert MoE model. + +Because `metadata()` order defines the partition, **all names sharing a layer +index must be contiguous in it**. A source whose natural export order interleaves +layers — bucketing all the MoE experts together, say — has to reorder before +returning. + ### VLLMWeightSyncClient **This is the adapter for however your RL stack reaches vLLM.** Many RL frameworks wrap @@ -372,7 +474,7 @@ Subclasses must implement five methods: The base class provides: 1. `__init__`, taking `config` (`WeightTransferConfig`), `vllm_config` (`VllmConfig`), `device` (`torch.device`), and `model` (`nn.Module`). -2. `update_weights(update_info_dict)`, a thin wrapper for `receive_weights`: it parses the dict into the typed dataclass, calls `receive_weights`, and synchronizes the device. +2. `update_weights(update_info_dict)`, a thin wrapper for `receive_weights`: it parses the dict into the typed dataclass, calls `receive_weights`, and synchronizes the device — unless the engine sets `defers_processing`, below. 3. `parse_init_info` / `parse_update_info`, which convert API-level dicts into the typed dataclasses and raise `ValueError` on a bad payload. 4. `set_weight_update_target` / `reset_weight_update_target`, used to retarget an update at the speculative draft model. @@ -383,6 +485,25 @@ The base class provides: update info carries only per-round metadata. This is what makes a trainer/worker mismatch unrepresentable. +!!! note "`defers_processing`: when a returned update means queued, not applied" + An engine that pipelines its GPU post-processing onto background threads + cannot let `update_weights` synchronize the device — that would block on + those threads and serialize the pipeline. Such an engine sets the class + attribute `defers_processing = True`, omits the per-update sync, and + guarantees completion in `finish_weight_update` instead. + + Callers that go through `finish_weight_update` need do nothing; the engine + drains there. A caller that drives the tail itself — running its own + `finalize_layerwise_reload`, say — must check the flag and call + `drain_pending()` first, because with it set a returned `update_weights` + means *queued*, not *applied*. `drain_pending()` is idempotent, and a no-op + on an engine that processes synchronously, so it is always safe to call. + + [Sharded RDT](sharded_rdt.md) is the built-in engine that sets it: it + scatters and quantizes on background threads with their own CUDA streams, so + its `drain_pending()` joins both queues and syncs both streams before + `finalize_layerwise_reload` runs. + ### Request Classes The API-level request classes provide backend-agnostic serialization using plain dictionaries. @@ -497,7 +618,7 @@ Once registered, users select your backend via `WeightTransferConfig(backend="my ### WeightTransferEngineFactory -The factory uses a registry pattern with lazy loading. Built-in engines (`nccl`, `ipc`, and `sparse_nccl`) are registered at import time but their modules are only loaded when the backend is actually requested. This avoids importing heavy dependencies (like NCCL communicators) when they aren't needed. +The factory uses a registry pattern with lazy loading. Built-in engines (`nccl`, `ipc`, `sparse_nccl` and `sharded_rdt`) are registered at import time but their modules are only loaded when the backend is actually requested. This avoids importing heavy dependencies (like NCCL communicators) when they aren't needed. ```python from vllm.distributed.weight_transfer import WeightTransferEngineFactory diff --git a/docs/training/weight_transfer/sharded_rdt.md b/docs/training/weight_transfer/sharded_rdt.md new file mode 100644 index 0000000000..8582b50ae4 --- /dev/null +++ b/docs/training/weight_transfer/sharded_rdt.md @@ -0,0 +1,126 @@ +# Sharded RDT Engine + +The sharded RDT weight transfer engine moves weights point-to-point over [NIXL](https://github.com/ai-dynamo/nixl) using Ray Direct Transport (RDT), Ray's zero-copy tensor transport between actors. It is **pull-based**: the inference workers initiate every transfer and each one asks only for the *slice* it consumes under tensor and expert parallelism. A large MoE model therefore moves roughly `total_bytes / num_workers` per worker instead of the `total_bytes` a broadcast costs. + +## When to Use Sharded RDT + +- Very large models where broadcasting whole parameters is the bottleneck — typically an MoE served with expert parallelism, where each worker owns a small fraction of the experts +- Trainer and inference on **separate GPUs**, over a fabric NIXL supports (InfiniBand, RoCE, EFA) +- Trainers that are themselves sharded, including **pipeline-parallel** ones where a rank holds only part of the model + +Requirements: + +- `distributed_executor_backend="ray"` — the workers must be Ray actors +- Ray `>= 2.56.0` on both the trainer and the workers +- `nixl` installed in the environment shared by trainer and workers +- Weight loaders that stay inside the supported op set (below) +- EPLB (`enable_eplb=true`) is rejected: it rearranges experts at runtime, which invalidates the recorded plan + +## How It Works + +### Slices are tracked through vLLM's own weight loaders + +A weight loader normally receives a full HF-format tensor and slices out the part this worker needs. Instead of sending it one, the engine hands it a `FakeRDTTensor`: a zero-storage tensor that answers `.shape` / `.dtype` / `.size()` but holds no data. Every view or slice op the loader calls returns a new fake with that op appended to a recorded chain, and `copy_` is the sink that ends it. + +That chain is the wire format. `("model.layers.0.w", (("narrow", (0, 512, 512), ()), ("t", (), ())))` tells the trainer: take this tensor, `narrow` it, transpose it, send the result. The trainer replays it with `getattr(tensor, op)(*args, **kwargs)`. + +Discovery is expensive, so it happens **once**, at `init_transfer_engine`, as a dry run over `model.load_weights` with every parameter on meta. Nothing is transferred; the engine just records, per leaf module, which slice feeds which destination region. Every later sync is pure replay. + +Anything a loader does that needs real data — arithmetic, `.to()`, `.float()`, `.item()`, `.data`, bool-mask indexing — falls outside the allowlist and raises at init. That is deliberate: failing loudly during setup beats silently transferring the wrong bytes. `SUPPORTED_OPS` in `sharded_rdt_common.py` is the single table both sides derive from, so the recorder and the replayer cannot drift. + +### Received slices land directly in the layerwise reload buffers + +The engine drives [layerwise reload](../layerwise.md) itself, in `start_weight_update` / `finish_weight_update`. Because the dry run already recorded each destination as an `as_strided` region of its parameter, an arriving slice is copied straight into the layer being reloaded — no full HF tensor is ever materialized on the worker, and no second pass over `load_weights` runs. Each layer is quantized and copied into its persistent kernel storage as soon as its last slice lands. + +### Gathers and pulls are pipelined, which is what `gather_lookahead` bounds + +The trainer usually cannot serve its parameters as they sit: FSDP shards them, and even an EP-split trainer has to assemble a whole expert. So each sync still runs gather collectives — but a layer at a time, not a model at a time. + +**A gather group is one decoder layer.** The parameter list is keyed on the outermost index segment of each name, which leaves runs of un-indexed names — the embeddings before the first layer, the final norm and `lm_head` after the last — as groups of their own: + +```text +group 0 model.embed_tokens.weight +group 1 model.layers.0.* <- one decoder layer +group 2 model.layers.1.* +... +group N+1 model.norm.weight, lm_head.weight +``` + +The layer is the unit of everything that follows: the trainer gathers a layer, publishes it (immediately pullable), and moves on to the next while the consumers pull the one it just published. Once every consumer has signalled that it is done with a layer, the trainer drops it and gains a credit to gather another. `gather_lookahead` is how far ahead of the consumers that loop may run, so at most `gather_lookahead + 1` layers are resident on the trainer at a time. The default of 1 keeps the next layer gathered and pullable while the current one is being pulled — enough to hide the handoff without doubling trainer memory. Raise it only if one layer's gather is slower than its pulls. + +Because a layer is also the unit the consumers free and the unit the receive buffers are sized against, it is what keeps memory bounded on both sides: without it the whole model would be one transfer, and both sides would have to hold their full share of it at once. Keying on the index rather than a fixed `model.layers.` prefix is what makes that hold across naming conventions — a VLM's `model.language_model.layers.`, GPT-2's `transformer.h.`, a vision tower's `visual.blocks.`. Sources can control the partition; see [gather groups](base.md#gather-groups). + +### Ownership + +A trainer rank need not hold the whole model. Each one declares what it holds through [`WeightSource.held_names()`](base.md#held_names-partial-ownership), the fleet all-gathers those declarations at `trainer_init`, and the consumers route each pull to a rank that actually holds the name. Pipeline stages, expert parallelism, and combinations of the two are all the same declaration. Consumers spread their pulls across the ranks that hold a name, so no single trainer NIC becomes the bottleneck. + +## Inference Side + +```python +from vllm import LLM +from vllm.config import WeightTransferConfig + +llm = LLM( + model="my-model", + weight_transfer_config=WeightTransferConfig(backend="sharded_rdt"), + distributed_executor_backend="ray", +) +``` + +```bash +vllm serve my-model \ + --distributed-executor-backend ray \ + --weight-transfer-config '{"backend": "sharded_rdt"}' +``` + +Everything else — which producers exist, how the model splits into layer groups, the ownership table — arrives from the trainer at the init handshake. + +!!! warning "Size the receive buffers before choosing `gpu_memory_utilization`" + Each worker holds `num_rdt_buffers` receive buffers, each large enough for the biggest single slice batch it pulls. Like NCCL and NIXL internals, they do **not** count against `gpu_memory_utilization`, so a fraction that leaves no headroom OOMs at the first sync even though the engine came up healthy. The buffer size is driven by the largest atomic slice — for an untied vocab matrix on a worker that holds it unsliced, that is the whole embedding. + +## Trainer Side + +```python +from vllm.distributed.weight_transfer import ( + ModuleSource, + HTTPVLLMWeightSyncClient, + WeightTransferTrainerFactory, +) +from vllm.distributed.weight_transfer.sharded_rdt_trainer import ( + ShardedRDTTrainerInitInfo, +) + +engine = WeightTransferTrainerFactory.trainer_init( + init_info=ShardedRDTTrainerInitInfo( + rank=rank, # rank 0 is the sender + num_consumers=8, # inference workers, fleet-wide + trainer_actor_namespace="my_namespace", # must be visible to the workers + ), + client=HTTPVLLMWeightSyncClient("http://localhost:8000"), + source=ModuleSource(model), +) + +engine.send_weights() # once per sync, on every trainer rank +``` + +`trainer_init` and `send_weights` run on **every** trainer rank: each one owns a serve actor and takes part in the gathers, while only rank 0 drives the inference-side handshake. Any [`VLLMWeightSyncClient`](base.md#vllmweightsyncclient) works. + +To adapt a trainer that is not a plain `nn.Module` — a Megatron export, a raw sharded checkpoint — subclass [`WeightSource`](base.md#weightsource). + +### `ShardedRDTTrainerInitInfo` + +| Field | Default | Description | +| ----- | ------- | ----------- | +| `rank` | — | Keyword-only. This trainer rank; **0 is the sender** | +| `num_consumers` | — | Inference workers across the whole fleet (TP × DP) | +| `trainer_actor_namespace` | `None` | Ray namespace for the serve actors; the workers resolve them by name here | +| `num_rdt_buffers` | 2 | Ring depth on both sides | +| `buffer_presize_gb` | 0.0 | Pre-size each buffer slot, in GiB. Set it to cover the largest atomic slice | +| `gather_lookahead` | 1 | Gathered-but-unfreed layers the gather loop runs ahead by | +| `stall_timeout_s` | 300.0 | Seconds without progress before the sync fails. A liveness backstop for a consumer that dies mid-sync, not a latency target | + +## Examples + +- [Small MoE on 4 GPUs](../../../examples/rl/rlhf_sharded_rdt_small_ep.py) — 2 FSDP2 trainer ranks → 2 vLLM DP ranks with expert parallelism, one node. It pairs a trainer fleet with a separate inference fleet, the only arrangement this backend supports, and asserts that the sync moved the weights and that a second sync leaves generation unchanged — so it runs unattended in CI + +It keeps the trainer deliberately small — just enough FSDP2 to make the weights real — so the file stays about the weight sync rather than about the trainer. For a full RL trainer, SkyRL integrates this backend with Megatron (PP-local gathering and expert-stack fusion for MoE) alongside FSDP: [NovaSky-AI/SkyRL#1753](https://github.com/NovaSky-AI/SkyRL/pull/1753). diff --git a/examples/rl/rdt_vllm_serve.py b/examples/rl/rdt_vllm_serve.py new file mode 100644 index 0000000000..c01ebf232b --- /dev/null +++ b/examples/rl/rdt_vllm_serve.py @@ -0,0 +1,152 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Driver-side helpers to run the inference fleet as a standard ``vllm serve`` +process for the sharded-RDT examples. + +The trainer reaches the server's weight-sync control plane over the RLHF HTTP +routes (``HTTPVLLMWeightSyncClient``); generation uses ``/v1/completions``. RDT +still needs the workers to be RayExecutorV2 tensor-transport actors sharing the +trainer's Ray cluster, so the server is launched with the Ray v2 executor and +inherits this process's env (``address=auto``). These helpers run only on the +driver — they are not shipped to or imported by the Ray actors. +""" + +import json +import os +import shutil +import subprocess +import sys +import time + +import ray +import requests + + +def launch_vllm_serve( + model: str, + *, + tensor_parallel_size: int = 1, + data_parallel_size: int = 1, + enable_expert_parallel: bool = False, + port: int = 8000, + gpu_memory_utilization: float = 0.7, + extra_args: list[str] | None = None, +) -> subprocess.Popen: + """Start ``vllm serve`` for the RDT inference fleet. + + ``VLLM_SERVER_DEV_MODE`` exposes the RLHF weight-sync routes; the Ray v2 + executor makes the workers tensor-transport actors (RDT's data plane). The + child inherits our env, so it joins the same Ray cluster (RAY_ADDRESS below) + and picks up the editable vLLM install via the venv interpreter. + """ + vllm_bin = os.path.join(os.path.dirname(sys.executable), "vllm") + if not os.path.exists(vllm_bin): + vllm_bin = shutil.which("vllm") or "vllm" + cmd = [ + vllm_bin, + "serve", + model, + "--port", + str(port), + "--enforce-eager", + "--load-format", + "dummy", + "--gpu-memory-utilization", + str(gpu_memory_utilization), + "--distributed-executor-backend", + "ray", + "--weight-transfer-config", + json.dumps({"backend": "sharded_rdt"}), + ] + if tensor_parallel_size > 1: + cmd += ["--tensor-parallel-size", str(tensor_parallel_size)] + # Only engage the DP-ray backend when actually data-parallel: DP=1 with + # data_parallel_backend=ray forces vLLM's DP-placement path, which fails. + if data_parallel_size > 1: + cmd += [ + "--data-parallel-size", + str(data_parallel_size), + "--data-parallel-backend", + "ray", + ] + if enable_expert_parallel: + cmd.append("--enable-expert-parallel") + if extra_args: + cmd += extra_args + env = dict( + os.environ, + VLLM_SERVER_DEV_MODE="1", + VLLM_USE_RAY_V2_EXECUTOR_BACKEND="1", + ) + # Pin the child to THIS Ray cluster. vLLM's ray DP backend reaches Ray + # through a bare ``ray.init()``, which STARTS a cluster rather than + # attaching, so wherever we started Ray ourselves (a bare CI runner) the + # workers would come up in a second cluster and never see the producer + # actors -- the failure is an actor lookup that cannot succeed. A bare + # ``ray.init()`` does honour RAY_ADDRESS, so hand it over explicitly. + if ray.is_initialized(): + env["RAY_ADDRESS"] = ray.get_runtime_context().gcs_address + # Let the server's Ray workers use the ambient interpreter/install rather + # than a workspace snapshot working_dir. On an Anyscale dev workspace the + # RAY_RUNTIME_ENV_HOOK injects a git snapshot as the workers' working_dir, + # which shadows an editable vLLM install (its compiled .so is gitignored, so + # the snapshot lacks it). No-op off Anyscale (the var is unset there). + env.pop("RAY_RUNTIME_ENV_HOOK", None) + print(f"[serve] launching: {' '.join(cmd)}", flush=True) + return subprocess.Popen(cmd, env=env) + + +def wait_for_server( + endpoint: str, proc: subprocess.Popen, timeout: float = 1800 +) -> None: + """Block until ``/health`` returns 200, failing fast if the server exits.""" + deadline = time.time() + timeout + while time.time() < deadline: + if proc.poll() is not None: + raise RuntimeError(f"vllm serve exited early (code {proc.returncode})") + try: + if requests.get(f"{endpoint}/health", timeout=5).status_code == 200: + print("[serve] server is healthy", flush=True) + return + except requests.RequestException: + pass + time.sleep(3) + raise RuntimeError("vllm serve did not become healthy in time") + + +def http_generate( + endpoint: str, model: str, prompts: list[str], max_tokens: int = 16 +) -> list[str]: + """Greedy ``/v1/completions`` for each prompt; returns the generated texts.""" + outs = [] + for p in prompts: + r = requests.post( + f"{endpoint}/v1/completions", + json={ + "model": model, + "prompt": p, + "max_tokens": max_tokens, + "temperature": 0, + }, + timeout=120, + ) + r.raise_for_status() + outs.append(r.json()["choices"][0]["text"]) + return outs + + +def pause_generation(endpoint: str, mode: str = "abort") -> None: + requests.post(f"{endpoint}/pause", params={"mode": mode}, timeout=60) + + +def resume_generation(endpoint: str) -> None: + requests.post(f"{endpoint}/resume", timeout=60) + + +def shutdown_server(proc: subprocess.Popen) -> None: + print("[serve] shutting down vllm serve", flush=True) + proc.terminate() + try: + proc.wait(timeout=30) + except subprocess.TimeoutExpired: + proc.kill() diff --git a/examples/rl/rdt_weight_source.py b/examples/rl/rdt_weight_source.py new file mode 100644 index 0000000000..b24253102f --- /dev/null +++ b/examples/rl/rdt_weight_source.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The ``WeightSource`` for ``rlhf_sharded_rdt_small_ep.py``. + +A trainer's producer job is to present its weights under the names the inference +side expects, which for vLLM means HF CHECKPOINT names. Transformers does not +store MoE experts that way -- it fuses each layer's into ``[E, ...]`` tensors -- +so that is the one conversion this file does. Kept beside the example rather than +inside it so the example stays about the sync itself, alongside +``rdt_vllm_serve.py``. +""" + +import regex as re +import torch + +from vllm.distributed.weight_transfer import ParamMeta, WeightSource +from vllm.distributed.weight_transfer.base import materialize_full_tensor + +# Transformers fuses a layer's experts into ``[E, ...]`` tensors; the checkpoint +# stores them per expert. This matches the fused parameter names. +_FUSED_EXPERT_RE = re.compile(r"^(.*\.experts)\.(gate_up_proj|down_proj)$") + + +class CheckpointNameSource(WeightSource): + """`WeightSource` over an FSDP2 model that publishes HF CHECKPOINT names. + + Transformers keeps each layer's experts fused as ``[E, ...]`` tensors, but + vLLM's MoE loaders are written against the per-expert checkpoint entries -- + only ``qwen{2,3}_moe`` also accept the fused names, so publishing them + straight from ``named_parameters()`` limits the example to one family and + fails the consumer's bake on ``experts.gate_up_proj`` everywhere else. + Splitting them back is also what a real trainer does, since converting an + internal layout to checkpoint names is the normal producer job. + + Every rank publishes the whole model: fused params are all-gathered and then + sliced per expert. Ownership stays uniform (no ``held_names``) -- serving + only the experts a rank holds is a real trainer's concern, not an example's. + """ + + def __init__(self, module: torch.nn.Module) -> None: + self._module = module + + @staticmethod + def _expand(name: str, param: torch.Tensor) -> list[tuple[str, tuple]]: + """The checkpoint entries ``name`` contributes, as (name, shape). + + ``gate_up_proj`` is ``[E, 2I, H]`` with gate in rows ``:I`` and up in + ``I:``; ``down_proj`` is ``[E, H, I]``. Anything else is already a + checkpoint name and passes through. + """ + m = _FUSED_EXPERT_RE.match(name) + if m is None: + return [(name, tuple(param.shape))] + prefix, kind = m.group(1), m.group(2) + experts, rows, cols = param.shape + if kind == "down_proj": + return [ + (f"{prefix}.{e}.down_proj.weight", (rows, cols)) for e in range(experts) + ] + half = rows // 2 + return [ + (f"{prefix}.{e}.{proj}_proj.weight", (half, cols)) + for e in range(experts) + for proj in ("gate", "up") + ] + + def metadata(self) -> list[ParamMeta]: + return [ + ParamMeta(entry_name, param.dtype, shape) + for name, param in self._module.named_parameters() + for entry_name, shape in self._expand(name, param) + ] + + def __iter__(self): + for name, param in self._module.named_parameters(): + m = _FUSED_EXPERT_RE.match(name) + if m is None: + yield name, materialize_full_tensor(param) + continue + # One gather per FUSED param, not per expert: the views below are + # into that one tensor, and every expert of a layer rides in the + # same gather group, so it stays resident exactly as long as needed. + prefix, kind = m.group(1), m.group(2) + full = materialize_full_tensor(param) + if kind == "down_proj": + for e in range(full.shape[0]): + yield f"{prefix}.{e}.down_proj.weight", full[e] + else: + half = full.shape[1] // 2 + for e in range(full.shape[0]): + yield f"{prefix}.{e}.gate_proj.weight", full[e, :half, :] + yield f"{prefix}.{e}.up_proj.weight", full[e, half:, :] diff --git a/examples/rl/rlhf_sharded_rdt_small_ep.py b/examples/rl/rlhf_sharded_rdt_small_ep.py new file mode 100644 index 0000000000..213eae55ef --- /dev/null +++ b/examples/rl/rlhf_sharded_rdt_small_ep.py @@ -0,0 +1,270 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Sharded-RDT weight sync on FOUR GPUs, one node: 2 FSDP2 trainer ranks -> 2 +vLLM DP ranks with expert parallelism. + +The reference example for the backend, sized to fit a CI runner: +``CheckpointNameSource`` over an FSDP2 model, one serve actor per rank, the +HTTP control plane, and assertions so it can run unattended. + +Why a MoE model rather than a small dense one: the point of sharded RDT is that +each inference worker pulls only the slices it consumes, so the example is only +meaningful when no rank holds the whole model. That needs experts on both sides. +``Qwen/Qwen1.5-MoE-A2.7B`` is the reference shape, a real MoE of the same family +as the 30B model the big example uses. Its WEIGHTS are never read: the trainer +builds the config and the server runs ``--load-format dummy``, so the sync is +what is under test and nothing is downloaded but the config and tokenizer. A +smaller MoE via ``RDT_MODEL`` changes nothing about what is exercised, which is +what CI runs. + +The trainer publishes CHECKPOINT names, not the names its own modules carry: +Transformers fuses each layer's experts into ``[E, ...]`` tensors +(``mlp.experts.gate_up_proj``/``down_proj``) while the checkpoint stores them per +expert, and vLLM's MoE loaders read the per-expert form. ``CheckpointNameSource`` +splits them back, which is the same conversion a real trainer does when it maps +its internal layout onto checkpoint names. Publishing the fused names instead +works only for the Qwen families, whose loaders happen to accept them. + +FSDP2 shards those fused tensors on dim 0 — the expert dimension — so after +sharding no rank stores every expert, and the inference side gets real EP from +``--enable-expert-parallel`` over DP2. Two simplifications an example can afford +and a real trainer cannot: each rank BUILDS the whole model before +``fully_shard`` splits it (fine for a tiny config, so nothing here streams a +checkpoint), and iteration all-gathers every parameter, so ownership stays +uniform. Serving only the experts a rank holds (``held_names``) is where a real +trainer earns its keep and is deliberately out of scope. + +Model naming is load-bearing: names are mapped onto checkpoint keys one-to-one, +so a model whose HF module names have drifted from its checkpoint (as +GraniteMoe's router has) will not work here. + +What it checks (a smoke test, not a numerical equivalence check): + - generation CHANGES after the first sync — the weights really moved, so a + silently-skipped sync fails instead of printing plausible text; + - generation is UNCHANGED by a second sync — replay is stable. + +Run: + python examples/rl/rlhf_sharded_rdt_small_ep.py + +Needs 4 GPUs on one node, each big enough for the whole model, since every rank +builds it before ``fully_shard`` splits it: the default is ~29 GiB in bf16, so +40 GiB cards and up. Set ``RDT_MODEL`` to a tiny MoE on smaller GPUs. Joins an +existing Ray cluster if there is one and otherwise starts its own. +""" + +import os +import sys +import time + +import ray +import torch +import torch.distributed as dist +from ray.util.placement_group import placement_group, placement_group_table +from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy +from rdt_vllm_serve import ( + http_generate, + launch_vllm_serve, + pause_generation, + resume_generation, + shutdown_server, + wait_for_server, +) +from rdt_weight_source import CheckpointNameSource +from torch.distributed.fsdp import fully_shard +from transformers import AutoConfig, AutoModelForCausalLM + +from vllm.distributed.weight_transfer import ( + HTTPVLLMWeightSyncClient, + WeightTransferTrainerFactory, +) +from vllm.distributed.weight_transfer.sharded_rdt_trainer import ( + ShardedRDTTrainerInitInfo, +) +from vllm.utils.network_utils import get_open_port + +# Weights are built, never loaded, so this costs a config fetch -- but every rank +# materializes the model before sharding, so the default wants a 40 GiB card. +MODEL_NAME = os.environ.get("RDT_MODEL", "Qwen/Qwen1.5-MoE-A2.7B") +RAY_NAMESPACE = "sharded_rdt_small_ep_example" +VLLM_PORT = int(os.environ.get("RDT_VLLM_PORT", "8100")) +VLLM_ENDPOINT = f"http://127.0.0.1:{VLLM_PORT}" + +FSDP_WORLD_SIZE = 2 +INFERENCE_TP_SIZE = 1 +INFERENCE_DP_SIZE = 2 +NUM_INFERENCE_CONSUMERS = INFERENCE_TP_SIZE * INFERENCE_DP_SIZE + +PROMPTS = ["The capital of France is", "The future of AI is"] + + +@ray.remote(num_gpus=1) +class FSDPTrainWorker: + """One FSDP2 rank per GPU. Identical to the 8-GPU example's worker: the + sharded-RDT engine owns the NIXL serve surface, so this stays a plain Ray + actor with no producer mixin.""" + + def __init__(self, model_name, rank, world_size, master_addr, master_port): + self.rank = rank + self.engine = None + os.environ["MASTER_ADDR"] = master_addr + os.environ["MASTER_PORT"] = str(master_port) + dist.init_process_group(backend="nccl", rank=rank, world_size=world_size) + torch.accelerator.set_device_index(0) + + # Same seed on every rank, so the shards compose into ONE model rather + # than a blend of independent inits -- the second sync's stability check + # is only meaningful against coherent weights. + torch.manual_seed(0) + config = AutoConfig.from_pretrained(model_name) + with torch.device("cuda"): + model = AutoModelForCausalLM.from_config(config, dtype=torch.bfloat16) + for layer in model.model.layers: + fully_shard(layer) + fully_shard(model) + self.model = model + + def get_rank(self): + return self.rank + + def setup_engine(self, vllm_endpoint: str): + self.engine = WeightTransferTrainerFactory.trainer_init( + ShardedRDTTrainerInitInfo( + rank=self.rank, + num_consumers=NUM_INFERENCE_CONSUMERS, + trainer_actor_namespace=RAY_NAMESPACE, + num_rdt_buffers=int(os.environ.get("NUM_RDT_BUFFERS", "2")), + buffer_presize_gb=float(os.environ.get("RDT_BUFFER_PRESIZE_GB", "0")), + ), + client=HTTPVLLMWeightSyncClient(vllm_endpoint), + source=CheckpointNameSource(self.model), + ) + + def sync_weights(self): + self.engine.send_weights() + + +def main(): + # Ship a minimal working_dir (this example dir) so Ray actors do NOT inherit + # a workspace snapshot that shadows the editable vLLM install. + runtime_env: dict[str, object] = { + "py_executable": sys.executable, + "working_dir": os.path.dirname(os.path.abspath(__file__)), + } + forwarded = { + k: os.environ[k] + for k in ( + "NCCL_CUMEM_ENABLE", + "VLLM_NCCL_SO_PATH", + "LD_PRELOAD", + "LD_LIBRARY_PATH", + ) + if k in os.environ + } + if forwarded: + runtime_env["env_vars"] = forwarded + if not ray.is_initialized(): + try: + ray.init(address="auto", runtime_env=runtime_env, namespace=RAY_NAMESPACE) + except (ConnectionError, RuntimeError): + # No cluster to join (a bare CI runner): start one on this node. + # `launch_vllm_serve` hands the child this cluster's address, which + # it needs -- vLLM's ray DP backend would otherwise start its own. + ray.init(runtime_env=runtime_env, namespace=RAY_NAMESPACE) + + print( + f"[init] model {MODEL_NAME}, {FSDP_WORLD_SIZE} trainer + " + f"{INFERENCE_DP_SIZE} inference GPUs" + ) + + # Reserve the trainer's GPUs BEFORE launching the server, so the server's DP + # workers take the node's remaining two. STRICT_PACK keeps the trainer's + # all-gathers on one node's NVLink -- here that is the only node anyway. + trainer_pg = placement_group( + [{"GPU": 1, "CPU": 1}] * FSDP_WORLD_SIZE, strategy="STRICT_PACK" + ) + ray.get(trainer_pg.ready()) + pg_node_id = next( + iter(placement_group_table(trainer_pg)["bundles_to_node_id"].values()) + ) + master_addr = next( + n["NodeManagerAddress"] for n in ray.nodes() if n["NodeID"] == pg_node_id + ) + + @ray.remote( + num_cpus=0, + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=trainer_pg + ), + ) + def _free_port(): + return get_open_port() + + master_port = ray.get(_free_port.remote()) + + workers = [ + FSDPTrainWorker.options( + scheduling_strategy=PlacementGroupSchedulingStrategy( + placement_group=trainer_pg, placement_group_bundle_index=rank + ), + ).remote(MODEL_NAME, rank, FSDP_WORLD_SIZE, master_addr, master_port) + for rank in range(FSDP_WORLD_SIZE) + ] + ray.get([w.get_rank.remote() for w in workers]) + print(f"[init] {FSDP_WORLD_SIZE} FSDP trainer workers ready (seeded weights).") + + server = launch_vllm_serve( + MODEL_NAME, + tensor_parallel_size=INFERENCE_TP_SIZE, + data_parallel_size=INFERENCE_DP_SIZE, + enable_expert_parallel=True, + port=VLLM_PORT, + extra_args=["--max-model-len", "2048", "--max-num-seqs", "8"], + ) + try: + wait_for_server(VLLM_ENDPOINT, server) + + before = http_generate(VLLM_ENDPOINT, MODEL_NAME, PROMPTS) + print("[generate] BEFORE sync (vLLM's own dummy init):") + for p, t in zip(PROMPTS, before): + print(f" {p!r} -> {t!r}") + + print("[sync] building trainer engines (bake on the sender)...") + t0 = time.perf_counter() + ray.get([w.setup_engine.remote(VLLM_ENDPOINT) for w in workers]) + print(f"[sync] engine setup (incl. bake) took {time.perf_counter() - t0:.2f}s") + + pause_generation(VLLM_ENDPOINT) + t0 = time.perf_counter() + ray.get([w.sync_weights.remote() for w in workers]) + print(f"[sync] iter 0 took {time.perf_counter() - t0:.3f}s") + resume_generation(VLLM_ENDPOINT) + + after = http_generate(VLLM_ENDPOINT, MODEL_NAME, PROMPTS) + print("[generate] AFTER sync (the trainer's weights):") + for p, t in zip(PROMPTS, after): + print(f" {p!r} -> {t!r}") + + # Sync again; replay must be stable, so generation must not move. + pause_generation(VLLM_ENDPOINT) + t0 = time.perf_counter() + ray.get([w.sync_weights.remote() for w in workers]) + print(f"[sync] iter 1 took {time.perf_counter() - t0:.3f}s") + resume_generation(VLLM_ENDPOINT) + again = http_generate(VLLM_ENDPOINT, MODEL_NAME, PROMPTS) + + # Greedy decoding, so both comparisons are exact. + assert after != before, ( + f"generation did not change across the weight sync, so the sync " + f"moved nothing: {before!r}" + ) + assert again == after, ( + f"a second sync changed generation, so replay is not stable: " + f"{after!r} then {again!r}" + ) + print("[ok] weights moved on sync 0 and replay was stable on sync 1") + finally: + shutdown_server(server) + + +if __name__ == "__main__": + main() diff --git a/requirements/test/cpu.txt b/requirements/test/cpu.txt index 02a7b2a0c5..b1de63ca32 100644 --- a/requirements/test/cpu.txt +++ b/requirements/test/cpu.txt @@ -895,7 +895,7 @@ pyzmq==27.1.0 # via -r requirements/test/../common.txt rapidfuzz==3.12.1 # via jiwer -ray==2.48.0 +ray==2.56.1 # via -r requirements/test/cuda.in redis==5.2.0 # via tensorizer diff --git a/requirements/test/cuda.in b/requirements/test/cuda.in index d11da082ad..9317a459e8 100644 --- a/requirements/test/cuda.in +++ b/requirements/test/cuda.in @@ -22,7 +22,7 @@ vector_quantize_pytorch # required for minicpmo_26 test vocos # required for minicpmo_26 test peft>=0.19.1 # required for phi-4-mm test pqdm -ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests +ray[cgraph,default]>=2.56.0,<2.57 # Ray Compiled Graph, required by pipeline parallelism tests; sharded-RDT weight transfer needs Ray Direct Transport (>=2.56) sentence-transformers>=5.2.0 # required for embedding tests soundfile # required for audio tests jiwer # required for audio tests diff --git a/requirements/test/cuda.txt b/requirements/test/cuda.txt index f28f733cd4..e59f340a95 100644 --- a/requirements/test/cuda.txt +++ b/requirements/test/cuda.txt @@ -988,7 +988,7 @@ pyzmq==27.1.0 # -r requirements/test/../common.txt rapidfuzz==3.12.1 # via jiwer -ray==2.48.0 +ray==2.56.1 # via -r requirements/test/cuda.in redis==5.2.0 # via tensorizer diff --git a/requirements/test/nightly-torch.txt b/requirements/test/nightly-torch.txt index 10c639af42..c59d40484b 100644 --- a/requirements/test/nightly-torch.txt +++ b/requirements/test/nightly-torch.txt @@ -16,7 +16,7 @@ librosa # required for audio tests vocos # required for minicpmo_26 test peft pqdm -ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests +ray[cgraph,default]>=2.56.0,<2.57 # Ray Compiled Graph, required by pipeline parallelism tests; sharded-RDT weight transfer needs Ray Direct Transport (>=2.56) sentence-transformers>=5.2.0 # required for embedding tests soundfile # required for audio tests jiwer # required for audio tests diff --git a/tests/distributed/test_sharded_rdt_plan.py b/tests/distributed/test_sharded_rdt_plan.py new file mode 100644 index 0000000000..7a75f0b7fc --- /dev/null +++ b/tests/distributed/test_sharded_rdt_plan.py @@ -0,0 +1,1753 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Consumer-side unit tests for the sharded-RDT backend. + +The consumer engine's planning core is pure — it turns the baked plan plus the +driver's gather-group partition into a static `_CallPlan` with no pulls, no +model and no Ray — so most of it is exercised here on CPU/meta tensors only. +These tests pin the values the engine produces today (chunk boundaries, packed +byte offsets, recorded op chains) so a refactor that changes them fails loudly +instead of silently shipping different bytes. + +`TestBakeOnARealModel` at the end is the exception: it needs a GPU, because the +bake only exists to be driven through a real `model.load_weights`, and the parts +that matter there — that the monkeypatched loaders are put back, and that the +recorded op chains reproduce what the loaders did — cannot be faked. + +`tests/distributed/test_sharded_rdt_trainer.py` covers the trainer (producer) +side; `test_sharded_rdt_producer.py` covers its serve-actor protocol. +""" + +from types import SimpleNamespace + +import pytest +import torch + +from vllm.distributed.weight_transfer.base import layerwise_groups +from vllm.distributed.weight_transfer.sharded_rdt_common import ( + ALLOWED_OPS, + SUPPORTED_OPS, + RdtRouter, + assign_producer_indices, + buffer_alloc_bytes, +) +from vllm.distributed.weight_transfer.sharded_rdt_engine import ( + ShardedRDTWeightTransferEngine, + ShardedRDTWeightTransferInitInfo, + _dtype_from_name, +) +from vllm.distributed.weight_transfer.sharded_rdt_fake import ( + BakeSink, + FakeRDTTensor, + _Scatter, + _UnsupportedFakeOp, +) + +META = torch.device("meta") + + +# --------------------------------------------------------------------------- +# Harness +# --------------------------------------------------------------------------- + + +class _FakeLayer: + """Stand-in for a leaf module. The planner only stores it and keys on + ``id()``; nothing in the plan path touches its attributes.""" + + def __init__(self, tag: str) -> None: + self.tag = tag + + def __repr__(self) -> str: + return f"_FakeLayer({self.tag!r})" + + +def _module(layer, copies): + """What the bake records for one leaf module: a scatter list whose entries + all carry that module as their ``layer``.""" + for c in copies: + c.layer = layer + return copies + + +def _copy( + name, + layer_param="weight", + *, + offset=0, + shape=(4,), + ops=(), + layer=None, + dtype=torch.bfloat16, +): + """A `_Scatter` as the bake would record it: source key, owning layer, and + the meta destination region.""" + stride = ( + (1,) + if len(shape) == 1 + else tuple( + int(torch.empty(shape, device=META).stride()[i]) for i in range(len(shape)) + ) + ) + return _Scatter( + layer=layer, + param_name=layer_param, + src=(name, tuple(ops)), + offset=offset, + shape=tuple(shape), + stride=stride, + dtype=dtype, + ) + + +def _planner(baked, *, name_meta, live=None, held_by=None, group_lens=None): + """A planning-only engine. + + `__init__` needs a config/model/device the planner never reads, so build the + instance without it and set exactly the state `_build_call_plan` consumes: + the baked plan, the name metadata, the live-name set, and a bound router. + + ``held_by``: name -> the ranks holding it. Names left out are held by every + producer (the replicated class 0). Default: one producer holding everything, + the degenerate router. Stand-in ranks 0-3 let plan tests assert chunk + structure; owner VALUES are pinned by TestCallPlanRouting. + """ + names = list(name_meta) + if held_by: + n_prod = max(r for owners in held_by.values() for r in owners) + 1 + owner_sets = [list(range(n_prod))] + seen = {tuple(range(n_prod)): 0} + classes = [] + for n in names: + key = tuple(sorted(held_by.get(n, range(n_prod)))) + if key not in seen: + seen[key] = len(owner_sets) + owner_sets.append(list(key)) + classes.append(seen[key]) + else: + n_prod, owner_sets, classes = 1, [[0]], [0] * len(names) + eng = object.__new__(ShardedRDTWeightTransferEngine) + eng._name_to_plan = dict(baked) + eng._name_meta = dict(name_meta) + eng._live_names = set(live or name_meta) + router = RdtRouter( + n_prod, 1, owner_sets, classes, names, group_lens or [1] * len(names) + ) + router.bind([object()] * n_prod, [object()] * n_prod, 0) + eng._router = router + return eng + + +def _one_module_per_layer(n_layers, *, dtype="bfloat16", numel=4): + """`(baked, name_meta, names, group_lens)` for a pre / N-layer / post model + where every gather group is one fully-baked leaf module.""" + names = ["embed.weight"] + names += [f"model.layers.{i}.w" for i in range(n_layers)] + names += ["norm.weight"] + baked = {} + for n in names: + layer = _FakeLayer(n) + baked[n] = _module(layer, [_copy(n, shape=(numel,))]) + name_meta = {n: (dtype, [numel]) for n in names} + group_lens = [1] * len(names) + return baked, name_meta, names, group_lens + + +# --------------------------------------------------------------------------- +# FakeRDTTensor: op-chain recording +# --------------------------------------------------------------------------- + + +class TestFakeOpChains: + """Every allowlisted op must append itself to the chain and hand back a + child whose shape/dtype PyTorch itself computed. The chain is the wire + format the producer replays, so its exact contents are load-bearing.""" + + def _fake(self, shape=(4, 6), dtype=torch.bfloat16, sink=None): + return FakeRDTTensor( + name="w", shape=torch.Size(shape), dtype=dtype, device=META, sink=sink + ) + + def test_bare_fake_has_metadata_but_no_chain(self): + t = self._fake() + assert t.shape == (4, 6) + assert t.dtype is torch.bfloat16 + assert t._key() == ("w", ()) + + @pytest.mark.parametrize( + "call,expect_op,expect_shape", + [ + (lambda t: t.narrow(0, 1, 2), ("narrow", (0, 1, 2), ()), (2, 6)), + (lambda t: t.view(6, 4), ("view", (6, 4), ()), (6, 4)), + (lambda t: t.reshape(24), ("reshape", (24,), ()), (24,)), + (lambda t: t[1], ("__getitem__", (1,), ()), (6,)), + (lambda t: t.unsqueeze(0), ("unsqueeze", (0,), ()), (1, 4, 6)), + (lambda t: t.transpose(0, 1), ("transpose", (0, 1), ()), (6, 4)), + (lambda t: t.t(), ("t", (), ()), (6, 4)), + (lambda t: t.permute(1, 0), ("permute", (1, 0), ()), (6, 4)), + (lambda t: t.flatten(), ("flatten", (), ()), (24,)), + (lambda t: t.contiguous(), ("contiguous", (), ()), (4, 6)), + ], + ) + def test_single_return_op_appends_one_spec(self, call, expect_op, expect_shape): + child = call(self._fake()) + assert isinstance(child, FakeRDTTensor) + assert child._key() == ("w", (expect_op,)) + assert tuple(child.shape) == expect_shape + + def test_squeeze_records_its_argument(self): + child = self._fake(shape=(1, 4)).squeeze(0) + assert child._key() == ("w", (("squeeze", (0,), ()),)) + assert tuple(child.shape) == (4,) + + def test_chains_compose_in_call_order(self): + child = self._fake().t().narrow(0, 0, 3).flatten() + assert child._key() == ( + "w", + ( + ("t", (), ()), + ("narrow", (0, 0, 3), ()), + ("flatten", (), ()), + ), + ) + assert tuple(child.shape) == (12,) + + def test_kwargs_are_frozen_sorted_for_hashability(self): + child = self._fake().narrow(dim=0, start=1, length=2) + (op, args, kwargs) = child._key()[1][0] + assert (op, args) == ("narrow", ()) + assert kwargs == (("dim", 0), ("length", 2), ("start", 1)) + hash(child._key()) # the chain is used as a dict key on both sides + + @pytest.mark.parametrize("op,n", [("chunk", 2), ("unbind", 4)]) + def test_multi_return_op_emits_one_child_per_output(self, op, n): + """chunk/unbind hand back a tuple; each child carries the base op plus a + trailing __getitem__(i) so the producer can index the replayed result.""" + parts = getattr(self._fake(), op)(*((n,) if op == "chunk" else ()), 0) + assert isinstance(parts, tuple) + assert len(parts) == n + for i, part in enumerate(parts): + base, index = part._key()[1] + assert base[0] == op + assert index == ("__getitem__", (i,), ()) + + def test_op_chain_is_hashable_as_a_fetch_key(self): + keys = {self._fake().t()._key(), self._fake().t()._key()} + assert len(keys) == 1, "equal chains must collapse — they dedup pull keys" + + +class TestFakeUnsupportedOps: + """Anything that needs real data must fail loudly at bake time rather than + silently transferring the wrong bytes.""" + + def _fake(self): + return FakeRDTTensor( + name="w", + shape=torch.Size((4,)), + dtype=torch.bfloat16, + device=META, + sink=None, + ) + + @pytest.mark.parametrize( + "call", + [ + lambda t: t.to(torch.float32), + lambda t: t.float(), + lambda t: t + 1, + lambda t: t * 2, + lambda t: t.sum(), + ], + ids=["to", "float", "add", "mul", "sum"], + ) + def test_data_dependent_ops_raise(self, call): + with pytest.raises(_UnsupportedFakeOp): + call(self._fake()) + + def test_error_names_the_weight_and_the_chain(self): + with pytest.raises(_UnsupportedFakeOp) as exc: + self._fake().narrow(0, 0, 2).float() + msg = str(exc.value) + assert "'w'" in msg + assert "narrow" in msg + + def test_unsupported_is_a_notimplementederror(self): + """Callers distinguish "this backend can't handle the loader" from bugs.""" + assert issubclass(_UnsupportedFakeOp, NotImplementedError) + + +class TestBakeRecording: + """During the dry run the fake's ``copy_`` is the data sink: it records the + source chain plus the meta destination's strided region and moves nothing.""" + + def _recorder_and_fake(self, shape=(4, 6)): + rec = BakeSink() + fake = FakeRDTTensor( + name="w", + shape=torch.Size(shape), + dtype=torch.bfloat16, + device=META, + sink=rec, + ) + return rec, fake + + def test_copy_records_the_destination_region(self): + rec, fake = self._recorder_and_fake() + layer = _FakeLayer("q_proj") + param = torch.empty((8, 6), dtype=torch.bfloat16, device=META) + dest = param.narrow(0, 4, 4) # the second half of a fused param + rec.current = (layer, "weight") + dest.copy_(fake) + + (recorded,) = rec.copies_by_layer[layer] + assert recorded.src == ("w", ()) + assert recorded.param_name == "weight" + assert recorded.offset == dest.storage_offset() == 24 + assert recorded.shape == (4, 6) + assert recorded.stride == (6, 1) + + def test_copy_marks_the_source_name_live(self): + rec, fake = self._recorder_and_fake() + rec.current = (_FakeLayer("l"), "weight") + torch.empty((4, 6), dtype=torch.bfloat16, device=META).copy_(fake) + assert rec.copied_names == {"w"} + + def test_unattributed_copy_is_live_but_unrecorded(self): + """A copy_ with no loader stamp cannot be attributed to a param, so its + module must fall back to the plain load — but the name still moved data.""" + rec, fake = self._recorder_and_fake() + rec.current = None + torch.empty((4, 6), dtype=torch.bfloat16, device=META).copy_(fake) + assert rec.copied_names == {"w"} + assert dict(rec.copies_by_layer) == {} + + def test_copies_are_grouped_by_module_in_call_order(self): + rec, _ = self._recorder_and_fake() + layer = _FakeLayer("gate_up") + for i, name in enumerate(("gate", "up")): + fake = FakeRDTTensor( + name=name, + shape=torch.Size((4,)), + dtype=torch.bfloat16, + device=META, + sink=rec, + ) + param = torch.empty((8,), dtype=torch.bfloat16, device=META) + rec.current = (layer, "weight") + param.narrow(0, 4 * i, 4).copy_(fake) + assert [c.src[0] for c in rec.copies_by_layer[layer]] == ["gate", "up"] + assert [c.offset for c in rec.copies_by_layer[layer]] == [0, 4] + + def test_recording_a_sliced_source(self): + """The chain on the source and the region on the dest are independent.""" + rec, fake = self._recorder_and_fake(shape=(8, 6)) + layer = _FakeLayer("k_proj") + rec.current = (layer, "weight") + param = torch.empty((4, 6), dtype=torch.bfloat16, device=META) + param.copy_(fake.narrow(0, 2, 4)) + (recorded,) = rec.copies_by_layer[layer] + assert recorded.src == ("w", (("narrow", (0, 2, 4), ()),)) + assert recorded.shape == (4, 6) + + def test_the_recorded_dtype_is_the_chains_output_not_the_sources(self): + """``view(dtype)`` is allowlisted, so a chain can reinterpret dtype. The + producer packs what the replay yields, so the record must carry the + POST-chain dtype: taking it from the source name's metadata instead sizes + the slice with the wrong itemsize and shifts every later slice in the + chunk, carving the packed blob differently on the two sides.""" + rec, fake = self._recorder_and_fake(shape=(4,)) + layer = _FakeLayer("reinterpreted") + rec.current = (layer, "weight") + viewed = fake.view(torch.float32) # 4 x bf16 -> 2 x f32 + param = torch.empty((2,), dtype=torch.float32, device=META) + param.copy_(viewed) + + (recorded,) = rec.copies_by_layer[layer] + assert recorded.dtype is torch.float32, "recorded the source dtype" + assert recorded.shape == (2,) + # What the source name alone would have said, and what the producer + # would NOT have sent. + assert 2 * torch.float32.itemsize == 8 != 2 * torch.bfloat16.itemsize + + def test_a_broadcasting_copy_is_refused_at_bake(self): + """The packed layout sizes each slice from the DESTINATION shape while + the producer packs what the chain yields, and nothing downstream can + catch a mismatch — the consumer's buffer view is exactly prod(dest.shape) + elements, so it reshapes cleanly over bytes laid out at other offsets. + So the bake refuses rather than recording a slice it cannot carve.""" + rec, fake = self._recorder_and_fake(shape=(1, 6)) + rec.current = (_FakeLayer("broadcast"), "weight") + param = torch.empty((4, 6), dtype=torch.bfloat16, device=META) + with pytest.raises(_UnsupportedFakeOp, match="broadcasting"): + param.copy_(fake) + + +# --------------------------------------------------------------------------- +# Consumer identity +# --------------------------------------------------------------------------- + + +class TestConsumerIdentity: + """Every worker in the fleet needs a DISTINCT id in 0..C-1: it selects the + worker's producer block and keys the producer's per-consumer serve ring, so + a collision silently serves two workers out of one ring.""" + + def _engine(self, *, dp_index, rank, world_size): + eng = object.__new__(ShardedRDTWeightTransferEngine) + eng.parallel_config = SimpleNamespace( + data_parallel_index=dp_index, rank=rank, world_size=world_size + ) + return eng + + def _ids(self, *, num_consumers, workers, replica_rank=0, num_replicas=1): + """Consumer ids for a DP-only engine whose workers are ``dp_index``es.""" + info = ShardedRDTWeightTransferInitInfo( + num_consumers=num_consumers, + replica_rank=replica_rank, + num_replicas=num_replicas, + ) + out = [] + for dp_index in workers: + eng = self._engine(dp_index=dp_index, rank=0, world_size=1) + eng._num_consumers_override = num_consumers + out.append(eng._resolve_consumer_id(info)) + return out + + def test_one_engine_indexes_by_dp_and_tp(self): + """dense-via-TP and MoE-via-DP+EP both flatten to 0..C-1.""" + eng = self._engine(dp_index=0, rank=3, world_size=8) + eng._num_consumers_override = 8 + info = ShardedRDTWeightTransferInitInfo(num_consumers=8) + assert eng._resolve_consumer_id(info) == 3 + + eng = self._engine(dp_index=5, rank=0, world_size=1) + eng._num_consumers_override = 8 + assert eng._resolve_consumer_id(info) == 5 + + def test_independent_engines_offset_into_distinct_ranges(self): + """A fleet of separate engines restarts _global_worker_index at 0 in + each, so without the replica offset every engine would claim 0..w-1. + The driver sets replica_rank on the payload; these are the ids it buys.""" + ids = [ + self._ids(num_consumers=8, workers=range(4), replica_rank=r, num_replicas=2) + for r in (0, 1) + ] + assert ids == [[0, 1, 2, 3], [4, 5, 6, 7]] + assert len(set(ids[0] + ids[1])) == 8, "every worker distinct fleet-wide" + + def test_the_default_is_a_single_engine_with_no_offset(self): + assert self._ids(num_consumers=4, workers=range(4)) == [0, 1, 2, 3] + + +# --------------------------------------------------------------------------- +# RdtRouter: per-name ownership -> one producer per pull +# --------------------------------------------------------------------------- + + +def _router( + num_producers, + num_consumers, + owner_sets=None, + name_owner_class=None, + names=None, + group_lens=None, +): + """A router over ``n`` single-name groups unless told otherwise, so a test + can state expectations in group indices.""" + if names is None: + n = len(name_owner_class or []) or (len(group_lens or []) or 1) + names = [f"g{i}" for i in range(n)] + if group_lens is None: + group_lens = [1] * len(names) + return RdtRouter( + num_producers, num_consumers, owner_sets, name_owner_class, names, group_lens + ) + + +class TestRdtRouter: + """Who serves each name. + + A wrong answer is not a wrong number but a hang or a loud misroute: a + consumer pulling from a producer that never gathered the name trips its + served-names guard. Freeing does NOT route this way — every consumer signals + every owner of a group and the producers count to the live total — so these + pin PULL routing, plus ``group_owners`` for the free fan-out. + """ + + def test_identity_when_fleets_match(self): + r = _router(8, 8, group_lens=[1] * 6) + assert all(r.producer_for(3, f"g{g}") == 3 for g in range(6)) + assert all(r.producer_for(c, "g0") == c for c in range(8)) + + def test_every_producer_nic_carries_traffic(self): + """16 producers / 8 consumers: the block rule spreads each consumer's + pulls over its block, alternating by group, so no producer sits idle. + Plain `consumer_id % len(owners)` would idle half of them.""" + r = _router(16, 8, group_lens=[1] * 95) + names = [f"g{i}" for i in range(95)] + for c in range(8): + block = assign_producer_indices(16, 8, c) + assert {r.producer_for(c, n) for n in names} == set(block) + assert [r.producer_for(0, f"g{g}") for g in range(4)] == [0, 1, 0, 1] + assert {r.producer_for(c, n) for c in range(8) for n in names} == set(range(16)) + + def test_fan_in_shares_one_producer(self): + r = _router(2, 8, group_lens=[1] * 5) + names = [f"g{i}" for i in range(5)] + assert [{r.producer_for(c, n) for n in names} for c in range(8)] == [ + {c // 4} for c in range(8) + ] + + def test_pipeline_stages_route_to_the_owning_stage(self): + """2 stages x 8 ranks: groups 0-2 on stage 0, 3-5 on stage 1. Two owner + sets, one per stage; each consumer reaches both.""" + r = _router( + 16, + 8, + owner_sets=[list(range(8)), list(range(8, 16))], + name_owner_class=[0, 0, 0, 1, 1, 1], + ) + for c in range(8): + picks = [r.producer_for(c, f"g{g}") for g in range(6)] + assert picks == [c] * 3 + [c + 8] * 3 + assert r.group_owners(0) == list(range(8)) + assert r.group_owners(3) == list(range(8, 16)) + r.validate() + + def test_expert_names_route_to_the_holding_ranks(self): + """A name held by only some ranks routes to those; a replicated name + keeps the full set.""" + r = RdtRouter( + 8, + 4, + [[0, 1, 2, 3], [2, 3], [4, 5, 6, 7], [6, 7]], + [0, 1, 2, 3], + ["repl0", "expert0", "repl1", "expert1"], + [2, 2], + ) + r.validate() + assert r.owners("expert0") == [2, 3] + assert r.owners("expert1") == [6, 7] + assert r.owners("repl1") == [4, 5, 6, 7] + for c in range(4): + assert r.producer_for(c, "expert0") in (2, 3) + assert r.producer_for(c, "repl1") in (4, 5, 6, 7) + + def test_consumers_spread_over_an_owner_set(self): + """Several ranks hold a name (its TP peers); the block rule must spread + consumers across them, not funnel through one.""" + r = _router( + 4, 4, owner_sets=[[0, 1]], name_owner_class=[0] * 6, group_lens=[1] * 6 + ) + served = {r.producer_for(c, f"g{g}") for c in range(4) for g in range(6)} + assert served == {0, 1} + + def test_a_group_frees_at_every_owner_of_any_of_its_names(self): + """The free barrier is per group, so its fan-out is the UNION over the + group's names: a rank holding none of them is not signalled.""" + r = RdtRouter(4, 2, [[0, 1], [2, 3]], [0, 1, 0], ["a", "b", "c"], [2, 1]) + assert r.group_owners(0) == [0, 1, 2, 3] + assert r.group_owners(1) == [0, 1] + + def test_an_unowned_name_raises(self): + r = RdtRouter(2, 2, [[]], [0], ["w"], [1]) + with pytest.raises(ValueError, match="has no owner"): + r.producer_for(0, "w") + + def test_validate_rejects_an_empty_owner_set(self): + with pytest.raises(ValueError, match="empty"): + RdtRouter( + 4, 2, [[0, 1], [], [2, 3]], [0, 1, 2], ["a", "b", "c"], [3] + ).validate() + + def test_validate_rejects_an_out_of_range_owner(self): + with pytest.raises(ValueError, match="out of range"): + RdtRouter(2, 2, [[0, 5]], [0], ["a"], [1]).validate() + + def test_validate_rejects_an_unresolvable_class(self): + with pytest.raises(ValueError, match="owner class"): + RdtRouter(4, 2, [[0, 1]], [0, 7], ["a", "b"], [2]).validate() + + +class TestReplicaOverlay: + """[RDT-SHARE-SLOTS] Several inference deployments carve the same blocks. + + The producer can serve R deployments out of ONE slot only if the R copies of + a worker meet on the same producer asking for the same bytes. Identical + plans give the second half for free; this is the first half. Carving over + the whole fleet instead spreads the multi-owner (replicated) names of + different deployments onto different producers, so a producer ends up + holding a ring per distinct worker index and nothing can merge. + """ + + def _fleet( + self, + num_producers, + workers, + replicas, + owner_sets=None, + name_owner_class=None, + n_names=6, + ): + names = [f"g{i}" for i in range(n_names)] + return ( + RdtRouter( + num_producers, + workers * replicas, + owner_sets, + name_owner_class, + names, + [1] * n_names, + workers_per_replica=workers, + ), + names, + ) + + def test_every_replica_of_a_worker_resolves_the_same_producer(self): + r, names = self._fleet(16, 8, 4) + for worker in range(8): + for name in names: + picks = {r.producer_for(worker + rep * 8, name) for rep in range(4)} + assert len(picks) == 1, ( + f"worker {worker} split across producers {picks}" + ) + + def test_adding_a_deployment_does_not_move_the_first_one(self): + """The strongest form: an existing consumer's routes are byte-identical + before and after another deployment is provisioned.""" + one, names = self._fleet(16, 8, 1) + two, _ = self._fleet(16, 8, 2) + for consumer_id in range(8): + assert [one.producer_for(consumer_id, n) for n in names] == [ + two.producer_for(consumer_id, n) for n in names + ] + + def test_one_deployment_is_the_historical_carve(self): + """Default (no width) and width == the whole fleet must agree with the + plain block rule, so a single-deployment fleet is untouched.""" + plain = _router(16, 8, group_lens=[1] * 6) + explicit, names = self._fleet(16, 8, 1) + for consumer_id in range(8): + block = assign_producer_indices(16, 8, consumer_id) + for name in names: + assert plain.producer_for(consumer_id, name) == explicit.producer_for( + consumer_id, name + ) + assert explicit.producer_for(consumer_id, name) in block + + def test_the_overlay_holds_for_partially_owned_names(self): + """Pipeline stages and experts too: two owner sets, one narrow. The + narrow one already forced agreement (one owner, no block to carve); the + wide one is what the carve decides.""" + r, names = self._fleet( + 8, + 4, + 2, + owner_sets=[[0, 1, 2, 3], [5]], + name_owner_class=[0, 1, 0, 1, 0, 1], + ) + for worker in range(4): + for name in names: + assert r.producer_for(worker, name) == r.producer_for(worker + 4, name) + assert {r.producer_for(c, "g1") for c in range(8)} == {5} + + def test_a_width_that_does_not_divide_the_fleet_raises(self): + """A uniform fleet is asserted upstream in ``get_world_size``; if it ever + is not, the overlay would map two workers of one deployment onto the same + block index, so refuse rather than serve the wrong bytes.""" + with pytest.raises(ValueError, match="does not divide"): + RdtRouter(4, 6, None, None, ["a"], [1], workers_per_replica=4) + + def test_pulls_still_carry_the_global_consumer_id(self): + """The carve uses the index WITHIN a deployment, but the wire must keep + the fleet-global id: it is what lets the producer tell the sharers of a + slot apart, and count their arrivals separately.""" + + class _Method: + def __init__(self): + self.calls = [] + + def remote(self, keys, consumer_id, seq): + self.calls.append((keys, consumer_id, seq)) + + r, _ = self._fleet(4, 2, 2) + methods = [_Method() for _ in range(4)] + r.bind([object()] * 4, methods, consumer_id=3) # replica 1, worker 1 + r.pull(2, ["k"], 7) + + assert methods[2].calls == [(["k"], 3, 7)] + + +# --------------------------------------------------------------------------- +# Packed layout: the cross-process invariant +# --------------------------------------------------------------------------- + + +def _producer_pack_offsets(slices): + """The producer's rule, transcribed from ``rdt_produce_weights_batched`` + (sharded_rdt_trainer.py): 16B-aligned offsets in specs order. + + Kept as an independent implementation on purpose: the consumer computing the + same offsets is the invariant that makes the packed blob readable, and this + is the only thing that catches the two rules drifting apart. A divergence is + silent at runtime — the bytes still arrive, they are just carved up wrong. + """ + pack_cur = 0 + offsets = [] + for numel, dtype in slices: + off = (pack_cur + 15) & ~15 + pack_cur = off + numel * dtype.itemsize + offsets.append(off) + return offsets, pack_cur + + +class TestPackedLayout: + def test_consumer_layout_matches_the_producer_rule(self): + """Mixed dtypes, sizes that do not land on 16B — i.e. the Kimi-style + group (fp8 weights + fp32 scales + bf16 norms) that exposes alignment.""" + specs = [ + ("w.fp8", "float8_e4m3fn", [17]), + ("w.scale", "float32", [3]), + ("w.norm", "bfloat16", [5]), + ("w.big", "bfloat16", [1024]), + ] + baked, name_meta = {}, {} + layer = _FakeLayer("mixed") + copies = [] + for name, dtype_name, shape in specs: + name_meta[name] = (dtype_name, shape) + copies.append( + _copy(name, shape=tuple(shape), dtype=_dtype_from_name(dtype_name)) + ) + mod_plan = _module(layer, copies) + for name, _d, _s in specs: + baked[name] = mod_plan + + eng = _planner(baked, name_meta=name_meta) + plan = eng._build_call_plan([n for n, _, _ in specs], [len(specs)]) + (chunk,) = plan.chunks + + want_offsets, want_bytes = _producer_pack_offsets( + [(s[0], _dtype_from_name(d)) for _n, d, s in specs] + ) + assert [off for off, _dt, _n, _shape in chunk.pack_layout] == want_offsets + assert chunk.pack_bytes == want_bytes + + def test_every_offset_is_16b_aligned(self): + specs = [(f"w{i}", "bfloat16", [i * 3 + 1]) for i in range(6)] + layer = _FakeLayer("l") + copies = [_copy(n, shape=tuple(s)) for n, _d, s in specs] + mod_plan = _module(layer, copies) + eng = _planner( + {n: mod_plan for n, _d, _s in specs}, + name_meta={n: (d, s) for n, d, s in specs}, + ) + (chunk,) = eng._build_call_plan([n for n, _, _ in specs], [len(specs)]).chunks + assert all(off % 16 == 0 for off, _dt, _n, _s in chunk.pack_layout) + + def test_layout_carries_dtype_numel_and_shape_for_the_view_carve(self): + baked, name_meta, names, group_lens = _one_module_per_layer(1, numel=6) + eng = _planner(baked, name_meta=name_meta) + chunk = eng._build_call_plan(names, group_lens).chunks[0] + (off, dt, numel, shape) = chunk.pack_layout[0] + assert (off, dt, numel, shape) == (0, torch.bfloat16, 6, (6,)) + + def test_duplicate_source_keys_are_packed_once(self): + """Two destinations fed by the SAME (name, chain) pull one slice.""" + layer = _FakeLayer("l") + shared = ("w", ()) + copies = [ + _copy("w", "a", shape=(4,)), + _copy("w", "b", shape=(4,)), + ] + assert copies[0].src == shared and copies[1].src == shared + eng = _planner( + {"w": _module(layer, copies)}, + name_meta={"w": ("bfloat16", [4])}, + ) + (chunk,) = eng._build_call_plan(["w"], [1]).chunks + assert len(chunk.scatters) == 2 + assert chunk.keys == [shared] + assert len(chunk.pack_layout) == 1 + + +# --------------------------------------------------------------------------- +# Chunk splitting +# --------------------------------------------------------------------------- + + +class TestChunkModuleScatters: + """The chunk cut: one chunk per distinct owner class present in the copies, + ascending by class index. Derived purely from the bake plus the ownership + table, so vLLM's expert placement needs no cases.""" + + def test_unstamped_copies_form_one_chunk(self): + layer = _FakeLayer("l") + copies = [_copy(f"w{i}", shape=(4,)) for i in range(5)] + eng = _planner({}, name_meta={f"w{i}": ("bfloat16", [4]) for i in range(5)}) + chunks = eng._chunk_module_scatters([_module(layer, copies)]) + assert [ci for ci, _s in chunks] == [0] + assert len(chunks[0][1]) == 5 + + def test_one_chunk_per_owner_class_ascending(self): + """Classes are numbered by first appearance in metadata order, so the + replicated class (0) leads and the rest follow — the ordering the old + "-1 first, then ascending stamp" rule produced, now falling out of the + table instead of a special case.""" + layer = _FakeLayer("moe") + names = ["norm", "e5", "e0", "e2"] + copies = [_copy(n, shape=(4,)) for n in names] + eng = _planner( + {}, + name_meta={n: ("bfloat16", [4]) for n in names}, + held_by={"e0": [0], "e2": [1], "e5": [2]}, + ) + chunks = eng._chunk_module_scatters([_module(layer, copies)]) + assert [ci for ci, _s in chunks] == [0, 1, 2, 3] + assert [[s.src[0] for s in sc] for _ci, sc in chunks] == [ + ["norm"], + ["e5"], + ["e0"], + ["e2"], + ] + + def test_copy_order_within_a_chunk_is_bake_order(self): + """Scatter order within a chunk must follow the bake, not the stamp + sort: the packed layout and the producer's replay agree on keys order.""" + layer = _FakeLayer("moe") + names = ["e3", "e1", "e2"] # one stamp, deliberately unsorted names + copies = [_copy(n, shape=(4,)) for n in names] + eng = _planner( + {}, + name_meta={n: ("bfloat16", [4]) for n in names}, + held_by={n: [0] for n in names}, + ) + ((_ci, scatters),) = eng._chunk_module_scatters([_module(layer, copies)]) + assert [s.src[0] for s in scatters] == names + + def test_stamps_interleave_across_modules_without_reordering_within(self): + """Two modules' copies bucket by stamp; within each bucket the order is + module-then-bake order.""" + a, b = _FakeLayer("a"), _FakeLayer("b") + mods = [ + _module(a, [_copy("a0", shape=(4,)), _copy("a1", shape=(4,))]), + _module(b, [_copy("b0", shape=(4,)), _copy("b1", shape=(4,))]), + ] + eng = _planner( + {}, + name_meta={n: ("bfloat16", [4]) for n in ("a0", "a1", "b0", "b1")}, + held_by={"a0": [0], "a1": [1], "b0": [0], "b1": [1]}, + ) + chunks = eng._chunk_module_scatters(mods) + assert [(ci, [s.src[0] for s in sc]) for ci, sc in chunks] == [ + (1, ["a0", "b0"]), + (2, ["a1", "b1"]), + ] + + def test_scatters_carry_their_own_dtype(self): + """dtype rides the record from the bake, where it is the fake's dtype + AFTER its op chain — not a plan-time lookup of the source name, which + would be wrong for any chain that reinterprets dtype.""" + layer = _FakeLayer("l") + eng = _planner({}, name_meta={"w": ("float32", [10])}) + ((_er, scatters),) = eng._chunk_module_scatters( + [_module(layer, [_copy("w", shape=(10,), dtype=torch.float32)])] + ) + (sc,) = scatters + assert sc.dtype is torch.float32 + assert sc.layer is layer + + +# --------------------------------------------------------------------------- +# _build_call_plan +# --------------------------------------------------------------------------- + + +class TestBuildCallPlan: + def test_one_chunk_per_group_when_unstamped(self): + baked, name_meta, names, group_lens = _one_module_per_layer(3) + eng = _planner(baked, name_meta=name_meta) + plan = eng._build_call_plan(names, group_lens) + assert len(plan.chunks) == len(group_lens) == 5 + assert plan.pre_free == [] + + def test_stamps_multiply_chunks_within_a_group(self): + layer = _FakeLayer("l") + names = [f"w{i}" for i in range(4)] + copies = [_copy(n, shape=(4,)) for n in names] + mod_plan = _module(layer, copies) + eng = _planner( + {n: mod_plan for n in names}, + name_meta={n: ("bfloat16", [4]) for n in names}, + held_by={"w2": [0], "w3": [1]}, + ) + plan = eng._build_call_plan(names, [4]) + assert len(plan.chunks) == 3 # replicated, w2's class, w3's class + + def test_materialize_fires_on_a_modules_first_chunk_only(self): + """Empty HF params are allocated once per module, by construction — + including a FusedMoE-like module whose copies span owner-class chunks.""" + layer = _FakeLayer("spanning") + names = [f"w{i}" for i in range(4)] + copies = [_copy(n, shape=(4,)) for n in names] + mod_plan = _module(layer, copies) + eng = _planner( + {n: mod_plan for n in names}, + name_meta={n: ("bfloat16", [4]) for n in names}, + held_by={"w0": [0], "w1": [0], "w2": [1], "w3": [1]}, + ) + plan = eng._build_call_plan(names, [4]) + assert [c.materialize for c in plan.chunks] == [[layer], []] + + def test_quant_defers_to_a_modules_last_chunk(self): + layer = _FakeLayer("spanning") + names = [f"w{i}" for i in range(4)] + copies = [_copy(n, shape=(4,)) for n in names] + mod_plan = _module(layer, copies) + eng = _planner( + {n: mod_plan for n in names}, + name_meta={n: ("bfloat16", [4]) for n in names}, + held_by={"w0": [0], "w1": [0], "w2": [1], "w3": [1]}, + ) + plan = eng._build_call_plan(names, [4]) + assert [c.quant for c in plan.chunks] == [[], [layer]] + + def test_free_signal_fires_on_each_groups_last_chunk(self): + baked, name_meta, names, group_lens = _one_module_per_layer(2) + eng = _planner(baked, name_meta=name_meta) + plan = eng._build_call_plan(names, group_lens) + assert [c.free for c in plan.chunks] == [[0], [1], [2], [3]] + + def test_free_signal_waits_for_a_groups_last_chunk_when_it_is_split(self): + """The load-bearing case: with the group cut across owner-class chunks the + signal must hang off the LAST one. Signaling earlier lets the producers + drop the gather buffers while a later chunk's RDMA is still reading.""" + layer = _FakeLayer("l") + names = [f"w{i}" for i in range(4)] + copies = [_copy(n, shape=(4,)) for n in names] + mod_plan = _module(layer, copies) + eng = _planner( + {n: mod_plan for n in names}, + name_meta={n: ("bfloat16", [4]) for n in names}, + held_by={"w2": [0], "w3": [1]}, + ) + plan = eng._build_call_plan(names, [4]) + assert len(plan.chunks) == 3 + assert [c.free for c in plan.chunks] == [[], [], [0]] + + def test_free_signal_timing_across_two_split_groups(self): + """Two groups x 2 owner-class chunks each: signals land on each group's + last chunk only.""" + names_a = [f"model.layers.0.w{i}" for i in range(4)] + names_b = [f"model.layers.1.w{i}" for i in range(4)] + baked, name_meta, stamps = {}, {}, {} + for names in (names_a, names_b): + layer = _FakeLayer(names[0]) + mod_plan = _module(layer, [_copy(n, shape=(4,)) for n in names]) + for i, n in enumerate(names): + baked[n] = mod_plan + name_meta[n] = ("bfloat16", [4]) + stamps[n] = [i // 2] # two owner classes per group + eng = _planner(baked, name_meta=name_meta, held_by=stamps) + plan = eng._build_call_plan(names_a + names_b, [4, 4]) + assert [c.free for c in plan.chunks] == [[], [0], [], [1]] + + def test_a_group_with_nothing_local_is_signaled_at_sync_start(self): + """The owners still published it, so this consumer must signal it — + immediately, before the pipeline (signal-before-publish is tolerated), + never hung off another group's chunk.""" + baked, name_meta, names, group_lens = _one_module_per_layer(2) + del baked["model.layers.1.w"] # nothing baked, and not live => no pull + eng = _planner( + baked, name_meta=name_meta, live=set(names) - {"model.layers.1.w"} + ) + plan = eng._build_call_plan(names, group_lens) + assert len(plan.chunks) == 3 + assert plan.pre_free == [2] + assert [c.free for c in plan.chunks] == [[0], [1], [3]] + + def test_leading_groups_with_nothing_local_go_to_pre_free(self): + baked, name_meta, names, group_lens = _one_module_per_layer(2) + del baked["embed.weight"] + eng = _planner(baked, name_meta=name_meta, live=set(names) - {"embed.weight"}) + plan = eng._build_call_plan(names, group_lens) + assert plan.pre_free == [0] + assert len(plan.chunks) == 3 + + def test_modules_are_deduped_within_a_group(self): + """Several names of one fused module map to the same scatter list; the + plan must not chunk it twice.""" + layer = _FakeLayer("fused") + names = ["qkv.q", "qkv.k", "qkv.v"] + copies = [_copy(n, shape=(4,)) for n in names] + mod_plan = _module(layer, copies) + eng = _planner( + {n: mod_plan for n in names}, + name_meta={n: ("bfloat16", [4]) for n in names}, + ) + plan = eng._build_call_plan(names, [3]) + (chunk,) = plan.chunks + assert chunk.materialize == [layer] + assert len(chunk.scatters) == 3 + + def test_plan_is_a_pure_function_of_its_inputs(self): + """Called twice with the same arguments it must produce the same plan — + that is what lets the engine build it once and reuse it every sync.""" + baked, name_meta, names, group_lens = _one_module_per_layer(2) + eng = _planner(baked, name_meta=name_meta) + a = eng._build_call_plan(names, group_lens) + b = eng._build_call_plan(names, group_lens) + assert [c.pack_bytes for c in a.chunks] == [c.pack_bytes for c in b.chunks] + assert [c.keys for c in a.chunks] == [c.keys for c in b.chunks] + assert [c.free for c in a.chunks] == [c.free for c in b.chunks] + assert a.pre_free == b.pre_free + + def test_single_producer_routes_every_chunk_to_local_owner_zero(self): + baked, name_meta, names, group_lens = _one_module_per_layer(2) + eng = _planner(baked, name_meta=name_meta) + plan = eng._build_call_plan(names, group_lens) + assert [c.owner for c in plan.chunks] == [0] * len(plan.chunks) + + +class TestUnbakedNamesGuard: + """A live name with no baked plan has no way to load — there is no fallback, + and its pull would target groups the pipeline already freed — so the plan + build fails at init, naming the names.""" + + def test_a_live_unbaked_name_fails_the_plan_build(self): + baked, name_meta, names, group_lens = _one_module_per_layer(2) + del baked["norm.weight"] # unbaked but live + eng = _planner(baked, name_meta=name_meta, live=set(names)) + with pytest.raises(RuntimeError, match="norm.weight"): + eng._build_call_plan(names, group_lens) + + def test_never_copied_names_are_dropped_entirely(self): + """Experts owned by another EP rank no-op in their loader: not an error, + and not a chunk — their group is pre-freed if nothing else fills it.""" + baked, name_meta, names, group_lens = _one_module_per_layer(2) + del baked["norm.weight"] + eng = _planner(baked, name_meta=name_meta, live=set(names) - {"norm.weight"}) + plan = eng._build_call_plan(names, group_lens) + assert len(names) - 1 == len(plan.chunks) + assert plan.pre_free == [len(group_lens) - 1] + + +class TestCallPlanRouting: + """Under partial ownership each chunk must be pulled from a producer that + actually holds every name in it. Consumers bind EVERY producer (signals fan + out to all owners), so local index == trainer rank.""" + + def _routed_planner( + self, held_by_group, num_consumers, consumer_id, *, held_by=None + ): + """``held_by_group[g]`` = the ranks holding group g's names, so a test + can still speak in groups; ``held_by`` overrides individual names.""" + baked, name_meta, names, group_lens = _one_module_per_layer( + len(held_by_group) - 2 + ) + per_name = {n: list(held_by_group[gi]) for gi, n in enumerate(names)} + per_name.update(held_by or {}) + n_prod = max(max(o) for o in per_name.values()) + 1 + owner_sets: list[list[int]] = [] + seen: dict[tuple, int] = {} + classes = [] + for n in names: + key = tuple(sorted(per_name[n])) + if key not in seen: + seen[key] = len(owner_sets) + owner_sets.append(list(key)) + classes.append(seen[key]) + eng = _planner(baked, name_meta=name_meta) + router = RdtRouter( + n_prod, num_consumers, owner_sets, classes, names, group_lens + ) + router.validate() + router.bind([object()] * n_prod, [object()] * n_prod, consumer_id) + eng._router = router + return eng, names, group_lens + + def test_two_stage_ownership_splits_chunks_between_owners(self): + """Two PP stages, 4 groups: stage 0 holds the first half, stage 1 the + second. Every chunk goes to a holder of its names.""" + eng, names, group_lens = self._routed_planner([[0], [0], [1], [1]], 1, 0) + plan = eng._build_call_plan(names, group_lens) + assert [c.owner for c in plan.chunks] == [0, 0, 1, 1] + + def test_a_multi_owner_group_still_resolves_to_one_producer(self): + """Every chunk is served by exactly ONE producer: splitting a pull only + multiplies produce calls, since the consumer's NIC bounds it.""" + eng, names, group_lens = self._routed_planner([[0, 1]] * 4, 1, 0) + plan = eng._build_call_plan(names, group_lens) + assert all(0 <= c.owner < 2 for c in plan.chunks) + + def test_an_expert_name_routes_to_the_rank_holding_it(self): + """PP ∩ EP: stage 0 = ranks 0-1, stage 1 = ranks 2-3, and one expert + name inside a stage-1 group is held by rank 3 alone. The name's own owner + set is what routes it.""" + eng, names, group_lens = self._routed_planner( + [[0, 1], [0, 1], [2, 3], [2, 3]], + 1, + 0, + held_by={"model.layers.1.w": [3]}, + ) + plan = eng._build_call_plan(names, group_lens) + assert plan.chunks[2].owner == 3 + assert plan.chunks[0].owner in (0, 1) + assert plan.chunks[3].owner in (2, 3) + + def test_an_unheld_name_is_rejected_at_init(self): + """A name no rank holds can never be served. The trainer raises when it + transposes the holdings; the router's validate() is the consumer-side + backstop, and either way it fails at init rather than at first pull.""" + with pytest.raises(ValueError, match="empty"): + RdtRouter(2, 1, [[]], [0], ["w"], [1]).validate() + + +class TestSignalCompleteness: + """Every gather group is signaled exactly once — after its last chunk when + the worker pulls from it, at sync start otherwise — for ANY expert + placement. A group signaled twice over-credits the barrier; a group never + signaled parks a producer credit and hangs end_sync.""" + + def _moe_planner(self, worker_experts, *, ep_size=4, n_experts=8, n_layers=2): + """pre / n_layers MoE layers / post. Each layer: one norm (its own + module, stamp -1) + n_experts expert names stamped ``e // n_local``. + The worker's bake covers only ``worker_experts`` (its placement); + foreign expert names never copied => dropped entirely, as in the real + bake.""" + n_local = n_experts // ep_size + names = ["embed.weight"] + group_lens = [1] + baked = { + "embed.weight": _module( + _FakeLayer("embed"), [_copy("embed.weight", shape=(4,))] + ) + } + name_meta = {"embed.weight": ("bfloat16", [4])} + stamps: dict = {} + live = {"embed.weight"} + for li in range(n_layers): + norm = f"model.layers.{li}.norm" + enames = [f"model.layers.{li}.experts.{e}.w" for e in range(n_experts)] + names += [norm] + enames + group_lens.append(1 + n_experts) + name_meta[norm] = ("bfloat16", [4]) + live.add(norm) + baked[norm] = _module(_FakeLayer(f"norm{li}"), [_copy(norm, shape=(4,))]) + fused = _module( + _FakeLayer(f"moe{li}"), + [ + _copy(f"model.layers.{li}.experts.{e}.w", shape=(4,)) + for e in worker_experts + ], + ) + for e in range(n_experts): + n = f"model.layers.{li}.experts.{e}.w" + name_meta[n] = ("bfloat16", [4]) + stamps[n] = [e // n_local] # held by that EP rank alone + if e in worker_experts: + baked[n] = fused + live.add(n) + names.append("lm_head.weight") + group_lens.append(1) + baked["lm_head.weight"] = _module( + _FakeLayer("head"), [_copy("lm_head.weight", shape=(4,))] + ) + name_meta["lm_head.weight"] = ("bfloat16", [4]) + live.add("lm_head.weight") + eng = _planner(baked, name_meta=name_meta, live=live, held_by=stamps) + return eng, names, group_lens + + @staticmethod + def _signals(plan): + return sorted(list(plan.pre_free) + [gi for c in plan.chunks for gi in c.free]) + + def test_linear_placement_signals_every_group_exactly_once(self): + eng, names, group_lens = self._moe_planner(worker_experts=[0, 1, 2, 3]) + plan = eng._build_call_plan(names, group_lens) + assert self._signals(plan) == list(range(len(group_lens))) + + def test_round_robin_placement_signals_every_group_exactly_once(self): + eng, names, group_lens = self._moe_planner(worker_experts=[0, 2, 4, 6]) + plan = eng._build_call_plan(names, group_lens) + assert self._signals(plan) == list(range(len(group_lens))) + + def test_placement_changes_only_the_chunk_count_never_the_signals(self): + """linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6 + hits all 4. More chunks per group, identical signal set.""" + plans = {} + for label, experts in (("linear", [0, 1, 2, 3]), ("round_robin", [0, 2, 4, 6])): + eng, names, group_lens = self._moe_planner(worker_experts=experts) + plans[label] = eng._build_call_plan(names, group_lens) + # per MoE group: the replicated chunk + one per owner class present + assert ( + len(plans["linear"].chunks) == 2 + 2 * 3 + ) # embed+head + 2 layers x (1 + 2) + assert len(plans["round_robin"].chunks) == 2 + 2 * 5 # 2 layers x (1 + 4) + assert self._signals(plans["linear"]) == self._signals(plans["round_robin"]) + + def test_the_fused_module_materializes_first_and_quants_last_across_chunks(self): + """The FusedMoE shape: one module's copies span every owner-class chunk of + its group; materialize on its first, quant on its last.""" + eng, names, group_lens = self._moe_planner( + worker_experts=[0, 2, 4, 6], n_layers=1 + ) + plan = eng._build_call_plan(names, group_lens) + # group 1 = [norm chunk, coord 0..3 chunks] at chunk indices 1..5 + fused_chunks = [ + c + for c in plan.chunks[1:6] + if any("experts" in sc.src[0] for sc in c.scatters) + ] + assert len(fused_chunks) == 4 + (fused_layer,) = fused_chunks[0].materialize + assert fused_layer.tag == "moe0" + assert fused_chunks[0].quant == [] + assert fused_chunks[-1].quant == [fused_layer] + + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + + +class TestBufferAllocBytes: + def test_rounds_up_to_a_coarse_256mb_boundary(self): + assert buffer_alloc_bytes(1) == 256 << 20 + assert buffer_alloc_bytes(256 << 20) == 256 << 20 + assert buffer_alloc_bytes((256 << 20) + 1) == 512 << 20 + + def test_presize_is_a_floor_not_a_cap(self): + assert buffer_alloc_bytes(1, presize=3 << 30) == 3 << 30 + big = 4 << 30 + assert buffer_alloc_bytes(big, presize=1 << 30) == big + + def test_never_returns_less_than_requested(self): + for nbytes in (1, 1 << 20, (700 << 20) + 3): + assert buffer_alloc_bytes(nbytes) >= nbytes + + +class TestLayerwiseGroups: + def test_partition_is_one_group_per_layer_between_unindexed_blocks(self): + names = [ + "embed.weight", + "model.layers.0.a", + "model.layers.0.b", + "model.layers.1.a", + "norm.weight", + ] + assert layerwise_groups(names) == [ + ["embed.weight"], + ["model.layers.0.a", "model.layers.0.b"], + ["model.layers.1.a"], + ["norm.weight"], + ] + + def test_flattening_a_partition_returns_the_input_order(self): + """Un-indexed names group by POSITION, not by name class — so one that + looks like a pre-block name still lands after the layers it follows. + Group index therefore means the same thing on every rank.""" + names = [ + "embed.weight", + "model.layers.0.a", + "embed.weight.tied", # after the layers => its own trailing block + ] + groups = layerwise_groups(names) + assert [n for g in groups for n in g] == names + + def test_layers_are_ordered_by_index_not_appearance(self): + names = ["model.layers.10.a", "model.layers.2.a"] + assert layerwise_groups(names) == [ + ["model.layers.2.a"], + ["model.layers.10.a"], + ] + + def test_stacks_keep_prefix_appearance_order_and_sort_within(self): + """Two stacks stay in the order they appear — the vision tower before the + text stack — while each sorts by index internally.""" + names = [ + "visual.blocks.1.w", + "visual.blocks.0.w", + "model.language_model.layers.1.w", + "model.language_model.layers.0.w", + ] + assert layerwise_groups(names) == [ + ["visual.blocks.0.w"], + ["visual.blocks.1.w"], + ["model.language_model.layers.0.w"], + ["model.language_model.layers.1.w"], + ] + + def test_the_post_block_lands_last_however_early_it_arrives(self): + """Megatron-Bridge streams the last pipeline stage's output block before + its layers. Sweeping un-indexed names after the first layer into a + trailing group is what lets the gather loop walk groups ascending.""" + names = ["model.layers.0.w", "model.norm.weight", "model.layers.1.w"] + assert layerwise_groups(names) == [ + ["model.layers.0.w"], + ["model.layers.1.w"], + ["model.norm.weight"], + ] + + def test_names_sharing_a_layer_coalesce_when_the_source_interleaves(self): + """A raw-checkpoint source may yield in shard-packing order. Coalescing on + the key keeps that from shattering one layer into many groups.""" + names = [ + "model.layers.0.a", + "model.layers.1.a", + "model.layers.0.b", + ] + assert layerwise_groups(names) == [ + ["model.layers.0.a", "model.layers.0.b"], + ["model.layers.1.a"], + ] + + def test_no_empty_groups(self): + for names in ([], ["only.pre"], ["model.layers.0.a"]): + assert all(g for g in layerwise_groups(names)) + + def test_a_model_with_no_layers_is_one_pre_group(self): + assert layerwise_groups(["a", "b"]) == [["a", "b"]] + + @pytest.mark.parametrize( + "prefix", + [ + "model.layers.", # Llama, Qwen3, GLM-4.5, DeepSeek, Mixtral + "model.language_model.layers.", # Qwen3.5/3.6, Qwen2.5-VL, Gemma3 + "language_model.layers.", # Kimi-K2.5, Nemotron-VL + "thinker.model.layers.", # Qwen2.5/3-Omni + "transformer.h.", # GPT-2, GPT-J, Bloom, Falcon + "transformer.blocks.", # MPT, DBRX + "transformer.encoder.layers.", # ChatGLM + "backbone.layers.", # Mamba, Mamba2 + "gpt_neox.layers.", # GPT-NeoX + "model.decoder.layers.", # OPT + "model.h.", # Exaone + "model.layers.layers.", # Plamo3 + "bert.encoder.layer.", # BERT, RoBERTa + "visual.blocks.", # Qwen-VL vision tower + "vision_tower.vision_model.encoder.layers.", # SigLIP/CLIP towers + "audio_tower.layers.", # Whisper-style audio towers + ], + ) + def test_every_supported_naming_convention_partitions_per_layer(self, prefix): + """One decoder layer per group under any prefix, with no per-architecture + table. A literal `model.layers.` match silently yields one whole-model + group for most of these.""" + names = [f"{prefix}{i}.self_attn.q_proj.weight" for i in range(3)] + assert layerwise_groups(names) == [[n] for n in names] + + def test_a_moe_layer_stays_whole(self): + """The OUTERMOST index wins, so per-expert names group by layer. Splitting + on the expert would cut FusedMoE — one leaf module — across groups.""" + names = [ + "model.layers.0.mlp.experts.0.w1.weight", + "model.layers.0.mlp.experts.1.w1.weight", + "model.layers.1.mlp.experts.0.w1.weight", + ] + assert layerwise_groups(names) == [ + [ + "model.layers.0.mlp.experts.0.w1.weight", + "model.layers.0.mlp.experts.1.w1.weight", + ], + ["model.layers.1.mlp.experts.0.w1.weight"], + ] + + def test_a_vlm_partitions_the_vision_tower_and_the_text_stack(self): + """Qwen-VL layout. Today's literal `model.layers.` match puts every one of + these in ONE group — the whole model — because the text stack ships as + `model.language_model.layers.`.""" + names = [ + "visual.patch_embed.proj.weight", + "visual.blocks.0.attn.qkv.weight", + "visual.blocks.1.attn.qkv.weight", + "visual.merger.mlp.weight", + "model.language_model.layers.0.self_attn.q_proj.weight", + "lm_head.weight", + ] + assert layerwise_groups(names) == [ + ["visual.patch_embed.proj.weight"], + ["visual.blocks.0.attn.qkv.weight"], + ["visual.blocks.1.attn.qkv.weight"], + ["model.language_model.layers.0.self_attn.q_proj.weight"], + ["visual.merger.mlp.weight", "lm_head.weight"], + ] + + def test_two_stacks_sharing_an_index_do_not_merge(self): + """The key carries the prefix, so vision block 0 and text layer 0 are + different groups even though both are index 0.""" + names = [ + "visual.blocks.0.attn.qkv.weight", + "model.language_model.layers.0.self_attn.q_proj.weight", + ] + assert layerwise_groups(names) == [[n] for n in names] + + +class TestDtypeFromName: + def test_resolves_wire_names(self): + assert _dtype_from_name("bfloat16") is torch.bfloat16 + assert _dtype_from_name("float8_e4m3fn") is torch.float8_e4m3fn + + def test_rejects_a_non_dtype_attribute(self): + with pytest.raises(ValueError): + _dtype_from_name("nn") # torch.nn exists but is not a dtype + + def test_rejects_an_unknown_name(self): + with pytest.raises(ValueError): + _dtype_from_name("float9") + + +class TestOpAllowlistAgreement: + """The consumer records op chains from ``SUPPORTED_OPS``; the producer + replays a chain only if every op is in ``ALLOWED_OPS``. The two must describe + the same contract, so they are derived from one table. + + A divergence is silent until first pull: an op the consumer can emit but the + producer rejects bakes fine at init, then fails a whole sync later with + "disallowed op". + """ + + def test_the_two_allowlists_describe_the_same_contract(self): + assert set(SUPPORTED_OPS.values()) == set(ALLOWED_OPS) + + def test_every_bake_recordable_op_is_replayable(self): + """Anything the bake can record must be serveable, or the failure lands a + whole sync later than the mistake.""" + assert not set(SUPPORTED_OPS.values()) - set(ALLOWED_OPS) + + def test_the_producer_allows_nothing_the_consumer_cannot_emit(self): + """The allowlist is also a guard against a misbehaving or spoofed + consumer invoking arbitrary methods on trainer tensors, so it must not be + wider than the ops the bake can actually produce. ``to`` in particular + would let a replay change dtype or device.""" + assert not set(ALLOWED_OPS) - set(SUPPORTED_OPS.values()) + assert "to" not in ALLOWED_OPS + + def test_transpose_via_t_is_serveable(self): + """The specific regression: a chain recorded from ``.t()`` must pass the + producer's guard.""" + fake = FakeRDTTensor( + name="w", + shape=torch.Size((4, 6)), + dtype=torch.bfloat16, + device=META, + sink=None, + ) + (op, _args, _kwargs) = fake.t()._key()[1][0] + assert op in ALLOWED_OPS + + +# --------------------------------------------------------------------------- +# Bake against a real model (GPU) +# --------------------------------------------------------------------------- + +TINY_QWEN = "Qwen/Qwen3-0.6B" + +# One small model per attention/FFN shape the bake has to survive. The point is +# the checkpoint->module mapping, which differs structurally between them: GQA +# fuses Q/K/V of unequal head counts into one param, MLA replaces them with the +# compressed kv_a/kv_b pair, and MoE turns per-expert checkpoint entries into +# stacked expert params. +# +# Keep them small: `load_format="dummy"` skips the download but not the +# allocation, so every parameter is still built on the device at full size. +BAKE_MODELS = [ + pytest.param(TINY_QWEN, id="gqa-dense"), + pytest.param("TitanML/tiny-mixtral", id="moe"), + # MLA (kv_lora_rank set, so `ModelConfig.use_mla`) plus routed experts. Only + # the STRUCTURE is under test -- `_bake_probe` synthesizes the checkpoint + # from the safetensors headers and never reads a weight value -- so a random + # 0.06B model does the job, as long as it has the MLA head shape the prefill + # backends accept: (qk_nope=128, qk_rope=64, v=128). + # + # Not an fp8 checkpoint: `process_weights_after_loading` swaps the vLLM + # parameter subclasses out on those, and the plain second `load_weights` + # this test compares against then cannot run. + pytest.param("hmellor/tiny-random-DeepseekV2ForCausalLM", id="mla-moe"), +] + + +class TestRequiresTheRayExecutor: + """The engine's data plane is Ray's, so a non-Ray executor cannot work. The + check is at construction because the alternative is an opaque failure during + the first handshake, long after the misconfiguration.""" + + def _construct(self, backend): + cfg = SimpleNamespace( + parallel_config=SimpleNamespace(distributed_executor_backend=backend) + ) + eng = object.__new__(ShardedRDTWeightTransferEngine) + # Only the guard is under test, so stand in for the base __init__. + with pytest.MonkeyPatch.context() as m: + m.setattr( + ShardedRDTWeightTransferEngine.__bases__[0], + "__init__", + lambda self, *a, **k: None, + ) + ShardedRDTWeightTransferEngine.__init__(eng, None, cfg, META, None) + return eng + + @pytest.mark.parametrize("backend", ["uni", "mp", "external_launcher"]) + def test_a_non_ray_executor_is_refused(self, backend): + with pytest.raises(ValueError, match="requires distributed_executor"): + self._construct(backend) + + def test_ray_is_accepted(self): + self._construct("ray") + + def test_a_custom_executor_class_is_left_alone(self): + """A `type[Executor]` override is deliberate and unjudgeable here, so it + must not be rejected for merely not being the string "ray".""" + + class _CustomExecutor: + pass + + self._construct(_CustomExecutor) + + +def _bake_probe(self, model_name): # noqa: PLR0915 (runs in the vLLM worker) + """Drive `_bake` against the worker's real model and check the three + properties. Returns a small summary; every assertion fires here, in the + worker, so a failure surfaces as the RPC's exception. + + Self-contained on purpose: this is pickled to the worker, so it imports what + it needs rather than closing over this module's globals. + """ + import torch + from huggingface_hub import get_safetensors_metadata + + from vllm.distributed.weight_transfer.sharded_rdt_common import ALLOWED_OPS + from vllm.distributed.weight_transfer.sharded_rdt_engine import ( + ShardedRDTWeightTransferEngine, + ShardedRDTWeightTransferInitInfo, + ) + from vllm.model_executor.model_loader.reload.utils import get_layer_tensors + + model = self.model_runner.model + device = next(model.parameters()).device + + # ---- the checkpoint, exactly as a trainer would describe and serve it ---- + # Only the safetensors HEADERS are fetched (a few KB); the values are + # synthesized. Nothing here depends on real weights -- both the reference + # load and the replay read this same dict, so the comparison is unaffected, + # and it keeps a multi-GB download out of CI. + st_dtype = { + "F64": torch.float64, + "F32": torch.float32, + "F16": torch.float16, + "BF16": torch.bfloat16, + "F8_E4M3": torch.float8_e4m3fn, + "F8_E5M2": torch.float8_e5m2, + "I64": torch.int64, + "I32": torch.int32, + "I8": torch.int8, + "U8": torch.uint8, + "BOOL": torch.bool, + } + meta = get_safetensors_metadata(model_name) + specs = {} + for file_meta in meta.files_metadata.values(): + for name, t in file_meta.tensors.items(): + specs[name] = (st_dtype[t.dtype], tuple(t.shape)) + + gen = torch.Generator(device="cpu").manual_seed(0) + ckpt: dict[str, torch.Tensor] = {} + for name, (dt, shape) in specs.items(): + if dt.is_floating_point: + # Small magnitudes so any narrowing cast stays finite. + v = torch.randn(shape, generator=gen, dtype=torch.float32) * 0.1 + ckpt[name] = v.to(dtype=dt) + else: + ckpt[name] = torch.zeros(shape, dtype=dt) + names = sorted(ckpt) + dtype_names = [str(ckpt[n].dtype).split(".")[-1] for n in names] + shapes = [list(ckpt[n].shape) for n in names] + + def _feed(): + return iter([(n, ckpt[n]) for n in names]) + + # ---- reference: what the model's OWN loader produces from that checkpoint -- + # Poison first, so "which params did load_weights actually write?" is + # answerable. Some params have no checkpoint source at all -- derived ones, + # or ones process_weights_after_loading builds -- and those keep their NaNs + # and are legitimately outside the plan. That set is what the plan must + # cover, not every parameter of the model. + def _poison(): + with torch.no_grad(): + for prm in model.parameters(): + if prm.is_floating_point(): + prm.fill_(float("nan")) + else: + prm.zero_() + + _poison() + model.load_weights(_feed()) + loaded = { + n + for n, prm in model.named_parameters() + if not (prm.is_floating_point() and torch.isnan(prm).any()) + } + # On the host: the weights themselves already fill the GPU. + reference = { + n: prm.detach().to("cpu", copy=True) + for n, prm in model.named_parameters() + if n in loaded + } + + # ---- pre-bake state, for the restoration check -------------------------- + param_name_of = {} + for mod_name, mod in model.named_modules(): + for pname in get_layer_tensors(mod): + param_name_of[(id(mod), pname)] = f"{mod_name}.{pname}".lstrip(".") + loaders_before = {} + for mod in model.modules(): + for pname, tensor in get_layer_tensors(mod).items(): + loaders_before[(id(mod), pname)] = getattr(tensor, "weight_loader", None) + + # ---- bake --------------------------------------------------------------- + eng = object.__new__(ShardedRDTWeightTransferEngine) + eng.model = model + eng.device = device + eng._name_to_plan = {} + eng._name_meta = {} + eng._live_names = set() + eng._bake( + ShardedRDTWeightTransferInitInfo( + names=names, dtype_names=dtype_names, shapes=shapes + ) + ) + + # (1) the bake ran and produced a plan + assert eng._name_to_plan, "bake produced no plan" + assert eng._live_names, "bake recorded no live names" + n_modules = len({id(v) for v in eng._name_to_plan.values()}) + + # (2) every intermediate change to model state was undone + for mod in model.modules(): + for pname, tensor in get_layer_tensors(mod).items(): + key = (id(mod), pname) + loader = getattr(tensor, "weight_loader", None) + assert not hasattr(loader, "_rdt_stamp_inner"), ( + f"recording stamp left on {param_name_of.get(key, key)}" + ) + assert loader is loaders_before[key], ( + f"weight_loader not restored on {param_name_of.get(key, key)}" + ) + assert not tensor.is_meta, ( + f"{param_name_of.get(key, key)} left on meta after the dry run" + ) + for pname, p in model.named_parameters(): + assert torch.equal(p.detach().cpu(), reference[pname]), f"bake mutated {pname}" + + # (3) replaying the recorded op chains reproduces that same load + scatters = {id(v): v for v in eng._name_to_plan.values()} + + # No deferred-attention layer may be a plan target. The engine's quant + # thread calls `info.reset()` on every module it processes, which makes + # finalize_layerwise_reload SKIP that module -- and finalize is the only + # thing that calls the LAYER's process_weights_after_loading, which is where + # MLA derives W_UK_T/W_UV from kv_b_proj. An attention layer in the plan + # would therefore leave those derived weights stale after a sync, silently. + from vllm.model_executor.layers.attention import is_deferred_attention_layer + + attn_in_plan = sorted( + { + type(c.layer).__name__ + for plan in scatters.values() + for c in plan + if is_deferred_attention_layer(c.layer) + } + ) + assert not attn_in_plan, ( + f"plan targets deferred-attention layer(s) {attn_in_plan}; the quant " + f"thread would reset them and finalize would skip their " + f"process_weights_after_loading, staling any derived weights" + ) + + touched = set() + _poison() + with torch.no_grad(): + for plan in scatters.values(): + for c in plan: + src_name, chain = c.src + t = ckpt[src_name] + for op, args, kw in chain: + assert op in ALLOWED_OPS, f"disallowed op {op!r}" + t = getattr(t, op)(*args, **dict(kw)) + dest = getattr(c.layer, c.param_name) + dest.as_strided(c.shape, c.stride, c.offset).copy_(t) + touched.add(param_name_of[(id(c.layer), c.param_name)]) + + assert loaded, "the reference load wrote nothing" + # The plan owns everything with a checkpoint source. Anything else the model + # ends up holding is DERIVED -- under MLA, `process_weights_after_loading` + # splits kv_b_proj into W_UK_T/W_UV via `replace_parameter` -- and belongs to + # finalize_layerwise_reload, which `finish_weight_update` runs after the + # replay. So the uncovered set must contain only params owned by a module + # that post-processes; a plain loadable weight going missing still fails. + post_processed = { + id(mod) + for mod in model.modules() + if hasattr(mod, "process_weights_after_loading") + } + modules_by_name = dict(model.named_modules()) + uncovered = sorted(loaded - touched) + derived: list[str] = [] + unexplained: list[str] = [] + for n in uncovered: + mod_name = n.rsplit(".", 1)[0] + mod = modules_by_name.get(mod_name) + is_derived = mod is not None and id(mod) in post_processed + (derived if is_derived else unexplained).append(n) + assert not unexplained, ( + f"the plan misses {len(unexplained)} param(s) load_weights wrote that " + f"nothing derives: {unexplained[:5]}" + ) + mismatched = [ + n + for n, prm in model.named_parameters() + if n in loaded and not torch.equal(prm.detach().cpu(), reference[n]) + ] + assert not mismatched, f"replay differs from load_weights for: {mismatched[:5]}" + + return { + "names": len(names), + "baked": len(eng._name_to_plan), + "live": len(eng._live_names), + "modules": n_modules, + "loaded_params": len(loaded), + "derived_not_in_plan": len(derived), + "replayed_params": len(touched), + "total_params": sum(1 for _ in model.named_parameters()), + } + + +@pytest.mark.skipif( + not torch.cuda.is_available(), reason="the bake drives a real model on GPU" +) +class TestBakeOnARealModel: + """`_bake` against a real vLLM model, which the CPU tests above cannot reach: + they drive the planner with hand-built scatters, so nothing there exercises + the dry run through a real `load_weights` — the part that monkeypatches the + model and has to put it back. + """ + + @pytest.mark.parametrize("model", BAKE_MODELS) + def test_bake_records_a_replayable_plan_and_restores_the_model( + self, vllm_runner, monkeypatch, model + ): + monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1") + with vllm_runner( + model, load_format="dummy", enforce_eager=True, max_model_len=1024 + ) as llm: + (summary,) = llm.llm.collective_rpc(_bake_probe, args=(model,)) + print(f"bake summary [{model}]: {summary}") + # Non-vacuity: the checks above are only worth something if the plan + # covers the model. Fusion means far fewer leaf modules than names -- + # QKV and gate/up for GQA, the stacked experts for MoE. + assert summary["baked"] >= 0.95 * summary["names"], summary + assert summary["live"] >= 0.95 * summary["names"], summary + assert 0 < summary["modules"] < summary["baked"], summary + # The probe already asserts the plan covers every param load_weights + # wrote; this just keeps that from being vacuously few. + assert summary["loaded_params"] > 0.5 * summary["total_params"], summary diff --git a/tests/distributed/test_sharded_rdt_producer.py b/tests/distributed/test_sharded_rdt_producer.py new file mode 100644 index 0000000000..62fc638762 --- /dev/null +++ b/tests/distributed/test_sharded_rdt_producer.py @@ -0,0 +1,1214 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Protocol tests for the sharded-RDT producer server (`_RDTProducerServer`). + +The publish -> serve -> free_group -> release lifecycle is the subsystem's +concurrency core: a group whose barrier never completes stalls `end_sync` +forever, and a group freed too early drops CUDA-IPC storage while a consumer is +still reading it. Freeing is a PER-GROUP BARRIER: every live consumer signals +`free_group(gi)` at every owner of the group exactly once per sync, and the +group is released when the count reaches the live total handed to `begin_sync` +— one uniform integer, no routed per-producer targets. + +These tests drive the real server. Everything here is CPU-clean: the IPC rebuild +is stubbed (`rebuild_cuda_tensor`) and the serve path is only entered as far as +the routing guard, which fires before any CUDA work. + +`rdt_produce_weights_batched`'s pack/copy half needs a real GPU buffer; it stays +covered by the GPU tests in `test_sharded_rdt_trainer.py`. +""" + +import contextlib +import gc +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import TimeoutError as FuturesTimeout + +import pytest +import torch + +import vllm.distributed.weight_transfer.sharded_rdt_trainer as trainer_mod +from vllm.distributed.weight_transfer.sharded_rdt_common import ( + buffer_alloc_bytes, +) +from vllm.distributed.weight_transfer.sharded_rdt_trainer import ( + DEFAULT_GATHER_LOOKAHEAD, + ShardedRDTTrainerInitInfo, + ShardedRDTTrainerWeightTransferEngine, + _RDTProducerServer, +) + +# Publish/free are keyed by GROUP INDEX; the names ride the publish entries. +GI_A, GI_B = 0, 1 +GROUP_A = ("model.layers.0.w", "model.layers.0.b") +GROUP_B = ("model.layers.1.w",) + + +@pytest.fixture +def server_factory(monkeypatch): + """Build real `_RDTProducerServer` instances without a GPU. + + Three process-global side effects in `__init__` have to be neutralized: + accelerator discovery (no driver here), the Ray NIXL monkeypatch (fail-soft + already, but pointless), and `gc.freeze()` — which would otherwise leak a + frozen GC generation into the pytest process for the rest of the session. + """ + monkeypatch.setattr( + torch.accelerator, "current_device_index", lambda *a, **kw: 0, raising=False + ) + monkeypatch.setattr(gc, "freeze", lambda: None) + + # The IPC rebuild: hand back a plain CPU uint8 storage of the requested + # size. publish_group then does the real .view(dtype) + as_strided on it, so + # the view reconstruction under test is genuine. + def _fake_rebuild(*args): + return torch.arange(args[0], dtype=torch.uint8) if args else torch.empty(0) + + monkeypatch.setattr(trainer_mod, "rebuild_cuda_tensor", _fake_rebuild) + + built = [] + + def _make(**kwargs): + kwargs.setdefault("num_rdt_buffers", 2) + kwargs.setdefault("buffer_presize_gb", 0.0) + kwargs.setdefault("gather_lookahead", 2) + server = _RDTProducerServer(**kwargs) + built.append(server) + return server + + yield _make + for server in built: + server.shutdown() + + +def _entries(names, *, nbytes=64, dtype="bfloat16"): + """The `(storages, views)` payload the engine's gather loop ships: one + storage export shared by every name, plus a per-name as_strided view spec. + + `storages` values stand in for `reduce_tensor`'s args. Only the arity + matters here: `publish_group` overwrites index 6 with the server's own + device index, so the tuple must be at least 7 long. The stubbed rebuild + reads the byte count from index 0. + """ + sid = 1 + storages = {sid: (nbytes, None, None, None, None, None, -1, None)} + itemsize = getattr(torch, dtype).itemsize + per = (nbytes // itemsize) // max(1, len(names)) + views = {name: (sid, dtype, [per], [1], i * per) for i, name in enumerate(names)} + return storages, views + + +def _publish(server, gi, names, **kw): + return server.publish_group(gi, _entries(names, **kw)) + + +class TestPublishAndRebuild: + def test_publish_makes_every_name_serveable(self, server_factory): + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + assert set(server._cache) == set(GROUP_A) + + def test_rebuilt_views_have_the_requested_dtype_and_geometry(self, server_factory): + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A, nbytes=64, dtype="bfloat16") + for name in GROUP_A: + t = server._cache[name] + assert t.dtype is torch.bfloat16 + assert tuple(t.shape) == (16,) # 64 bytes / 2 names / 2 bytes + + def test_views_of_one_storage_do_not_overlap(self, server_factory): + """One IPC export per storage, one as_strided view per name — the whole + point of the storage/view split. Overlapping views would serve the same + bytes for two different weights.""" + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A, nbytes=64) + a, b = (server._cache[n] for n in GROUP_A) + assert a.storage_offset() != b.storage_offset() + assert a.data_ptr() != b.data_ptr() + + def test_publish_records_the_groups_names_for_release(self, server_factory): + server = server_factory() + server.begin_sync(3) + _publish(server, GI_A, GROUP_A) + assert server._group_names[GI_A] == list(GROUP_A) + + def test_publish_records_the_group_as_unfreed(self, server_factory): + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + assert server._inflight_groups == [GI_A] + + def test_the_default_lookahead_is_one(self): + """1 is the sweet spot: group N+1 is gathered AND pullable while N is + being pulled, with resident memory at its floor of 2 groups — the bound + the larger-model runs size against.""" + assert DEFAULT_GATHER_LOOKAHEAD == 1 + + +class TestFreeBarrier: + def test_single_consumer_signal_releases_the_group(self, server_factory): + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + server.free_group(GI_A) + assert server._inflight_groups == [] + assert set(server._cache) == set() + + def test_group_is_held_until_the_last_live_consumer_signals(self, server_factory): + """Every live consumer signals every owner; releasing on the first + signal would drop storage other consumers are still reading.""" + server = server_factory() + server.begin_sync(3) + _publish(server, GI_A, GROUP_A) + held_after_each_signal = [] + for _ in range(3): + server.free_group(GI_A) + held_after_each_signal.append(GI_A in server._inflight_groups) + assert held_after_each_signal == [True, True, False] + + def test_intermediate_signals_do_not_release(self, server_factory): + server = server_factory() + server.begin_sync(3) + _publish(server, GI_A, GROUP_A) + server.free_group(GI_A) + assert server._inflight_groups == [GI_A] + assert set(server._cache) == set(GROUP_A) + server.free_group(GI_A) + assert server._inflight_groups == [GI_A] + server.free_group(GI_A) + assert server._inflight_groups == [] + + def test_a_signal_that_beats_its_publish_is_completed_at_publish( + self, server_factory + ): + """A consumer with nothing to pull from a group signals it at sync + start, which can precede the publish. The publish must then release the + group rather than wait for a signal that will never come again.""" + server = server_factory() + server.begin_sync(1) + server.free_group(GI_A) + _publish(server, GI_A, GROUP_A) + assert server._inflight_groups == [] + assert server._freed_pending == [GI_A] + + def test_early_signals_from_every_consumer_complete_at_publish( + self, server_factory + ): + server = server_factory() + server.begin_sync(3) + for _ in range(3): + server.free_group(GI_A) + _publish(server, GI_A, GROUP_A) + assert server._freed_pending == [GI_A] + + def test_freed_groups_are_reported_back_to_the_engine(self, server_factory): + """The engine holds the CUDA-IPC export refs and may only drop them once + the server says the group is gone — and it hears that ONLY through + ``wait_freed`` / ``end_sync``, never a publish return (a freed notice + riding an unharvested async publish while the engine blocks in + ``wait_freed`` would wedge the loop).""" + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + server.free_group(GI_A) + assert server.wait_freed() == [GI_A] + assert server._freed_pending == [] + + def test_a_signal_for_an_unpublished_group_releases_nothing(self, server_factory): + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + server.free_group(99) + assert server._inflight_groups == [GI_A] + assert set(server._cache) == set(GROUP_A) + + def test_release_drops_cache_and_count_entries(self, server_factory): + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + _publish(server, GI_B, GROUP_B) + server.free_group(GI_A) + assert set(server._cache) == set(GROUP_B) + assert server._free_counts.get(GI_A) is None + assert server._group_names.get(GI_A) is None + + +class _CountingSource: + """`iter_groups` yielding one CPU-tensor group per call, built lazily so a + group's tensors exist only once the engine's credit gate lets it gather.""" + + def __init__(self, groups): + self._groups = groups + + def iter_groups(self): + for names in self._groups: + yield list(names), [torch.zeros(16, dtype=torch.float32) for _ in names] + + +class _ResidencyDict(dict): + """The engine's `_inflight`, instrumented: `max_len` is the high-water mark + of groups whose CUDA-IPC export refs were alive at once — the trainer's + resident-memory bound in groups.""" + + def __init__(self): + super().__init__() + self.max_len = 0 + + def __setitem__(self, key, value): + super().__setitem__(key, value) + self.max_len = max(self.max_len, len(self)) + + +def _loop_engine(server, n_groups, *, lookahead): + """A real engine wired straight to a real (non-Ray) server, with only the + state `_run_gather_loop` reads. The `_rpc` seam dispatches inline; + `_publish_async` sees no `.remote` and runs inline too.""" + groups = [(f"model.layers.{gi}.w",) for gi in range(n_groups)] + e = ShardedRDTTrainerWeightTransferEngine.__new__( + ShardedRDTTrainerWeightTransferEngine + ) + e._init_info = ShardedRDTTrainerInitInfo( + rank=0, num_consumers=1, gather_lookahead=lookahead + ) + e.source = _CountingSource(groups) + e._server = server + e._rpc = lambda method, *a: getattr(server, method)(*a) + e._groups = [list(g) for g in groups] + e._owned_idx = list(range(n_groups)) + e._held_names = None + e._inflight = _ResidencyDict() + return e + + +class TestGatherCredit: + """The memory bound gates GATHERS, not publishes: a gathered group is + serveable immediately, and the ENGINE's loop stops gathering while more than + `gather_lookahead` groups are unfreed. So at most `lookahead + 1` groups are + resident — two at the default — while group N+1 is pulled the instant N's + pulls finish.""" + + @pytest.fixture + def gather_engine(self, monkeypatch): + """Neutralize the two CUDA touches in the export path so the REAL + gather loop runs on CPU: `.cuda()` becomes identity, and the storage + export ships the `(nbytes, ..., device, ...)` tuple the fixture's + stubbed `rebuild_cuda_tensor` expects.""" + monkeypatch.setattr(torch.Tensor, "cuda", lambda self: self) + monkeypatch.setattr( + trainer_mod, + "reduce_tensor", + lambda base: (None, (base.numel(), None, None, None, None, None, -1, None)), + ) + return _loop_engine + + def test_publish_never_blocks(self, server_factory): + """Publishing must not park on a credit: the whole point is that a + gathered group becomes serveable immediately. Three publishes with no + frees, inline on this thread — a block here hangs the test.""" + server = server_factory(gather_lookahead=1) + server.begin_sync(1) + for gi in range(3): + _publish(server, gi, (f"model.layers.{gi}.w",)) + assert server._inflight_groups == [0, 1, 2] + + def test_lookahead_one_holds_at_most_two_groups_resident( + self, server_factory, gather_engine + ): + """THE memory invariant: at lookahead=1 the trainer never holds more + than 2 groups of gathered weights, no matter how the consumer paces — + and the pipeline still overlaps, because the consumer here refuses to + free group N until N+1 is already published (pullable). A gate an + off-by-one too tight deadlocks this test; too loose fails max_len.""" + server = server_factory(gather_lookahead=1) + n_groups = 8 + engine = gather_engine(server, n_groups, lookahead=1) + + failures: list[str] = [] + + def _consumer(): + for gi in range(n_groups): + # N+1 must be pullable before N frees + target = min(gi + 1, n_groups - 1) + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + with server._cache_cond: + if target in server._group_names: + break + time.sleep(0.002) + else: + failures.append( + f"group {target} never published while {gi} was unfreed" + ) + return + server.free_group(gi) + + consumer = threading.Thread(target=_consumer, daemon=True) + consumer.start() + engine._run_gather_loop(update_future=None, live_count=1) + consumer.join(timeout=10) + + assert not consumer.is_alive() and not failures, failures + assert engine._inflight.max_len <= 2, ( + f"{engine._inflight.max_len} groups were resident at lookahead=1" + ) + assert engine._inflight == {} and server._cache == {}, ( + "everything must be freed by end_sync" + ) + + def test_lookahead_zero_holds_exactly_one_group( + self, server_factory, gather_engine + ): + """0 is the serialized baseline: the loop waits for group N to be freed + before gathering N+1, so exactly ONE group is ever resident and nothing + overlaps. A floor of 1 on the bound would silently pipeline instead.""" + server = server_factory(gather_lookahead=0) + engine = gather_engine(server, 4, lookahead=0) + + def _run(): + # unwound via set_gather_error below + with contextlib.suppress(Exception): + engine._run_gather_loop(update_future=None, live_count=1) + + loop = threading.Thread(target=_run, daemon=True) + loop.start() + deadline = time.monotonic() + 5 + while time.monotonic() < deadline and not server._inflight_groups: + time.sleep(0.005) + time.sleep(0.1) # a would-be second gather gets every chance to run + try: + assert engine._inflight.max_len == 1 + assert server._inflight_groups == [0] + finally: + server.set_gather_error("test over") # unblock the parked wait_freed + loop.join(timeout=5) + + def test_the_residency_bound_scales_with_the_lookahead( + self, server_factory, gather_engine + ): + """lookahead + 1, not a hardcoded 2: at lookahead=2 a free-nothing + consumer sees exactly 3 groups gathered before the loop parks.""" + server = server_factory(gather_lookahead=2) + engine = gather_engine(server, 8, lookahead=2) + + def _run(): + # unwound via set_gather_error below + with contextlib.suppress(Exception): + engine._run_gather_loop(update_future=None, live_count=1) + + loop = threading.Thread(target=_run, daemon=True) + loop.start() + deadline = time.monotonic() + 5 + while time.monotonic() < deadline and len(server._inflight_groups) < 3: + time.sleep(0.005) + time.sleep(0.1) # would-be 4th gather gets every chance to (wrongly) run + try: + assert engine._inflight.max_len == 3 + assert server._inflight_groups == [0, 1, 2] + finally: + server.set_gather_error("test over") # unblock the parked wait_freed + loop.join(timeout=10) + assert not loop.is_alive() + + def test_wait_freed_returns_the_freed_backlog(self, server_factory): + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + _publish(server, GI_B, GROUP_B) + server.free_group(GI_A) + assert server.wait_freed() == [GI_A] + server.free_group(GI_B) + assert server.wait_freed() == [GI_B] + + def test_a_gather_error_releases_a_blocked_wait_freed(self, server_factory): + """Otherwise a trainer-side failure on another rank deadlocks this one + inside its credit gate. It must RAISE, not return empty — an empty + return would spin the engine straight back into the wait.""" + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + + errors: list = [] + done = threading.Event() + + def _wait(): + try: + server.wait_freed() + except BaseException as e: # noqa: BLE001 + errors.append(e) + done.set() + + threading.Thread(target=_wait, daemon=True).start() + assert not done.wait(timeout=0.3), ( + "wait_freed must block while nothing is freed" + ) + + server.set_gather_error("rank 3 export failed") + assert done.wait(timeout=5), "set_gather_error must unblock wait_freed" + assert errors and "rank 3 export failed" in str(errors[0]) + + +class TestEndSync: + def test_end_sync_returns_immediately_when_everything_is_freed( + self, server_factory + ): + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + server.free_group(GI_A) + assert server.end_sync() == [GI_A] + + def test_end_sync_waits_for_an_outstanding_group(self, server_factory): + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + + returned = [] + done = threading.Event() + + def _end(): + returned.append(server.end_sync()) + done.set() + + t = threading.Thread(target=_end, daemon=True) + t.start() + assert not done.wait(timeout=0.3), "end_sync must wait for the barrier" + + server.free_group(GI_A) + assert done.wait(timeout=5) + t.join(timeout=5) + assert returned == [[GI_A]] + + def test_a_gather_error_releases_a_blocked_end_sync(self, server_factory): + server = server_factory() + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + + done = threading.Event() + + def _end_then_signal(): + server.end_sync() + done.set() + + threading.Thread(target=_end_then_signal, daemon=True).start() + assert not done.wait(timeout=0.3) + server.set_gather_error("boom") + assert done.wait(timeout=5) + + +class TestBeginSync: + def test_begin_sync_sets_the_barrier_target(self, server_factory): + server = server_factory() + server.begin_sync(5) + assert server._live_count == 5 + + def test_the_target_is_floored_at_one(self, server_factory): + """A bare/zero call must not make every group free instantly (or divide + the barrier by zero).""" + server = server_factory() + server.begin_sync(0) + assert server._live_count == 1 + + def test_a_degraded_sync_lowers_the_target(self, server_factory): + """FT degraded sync: the live count is the WHOLE degraded-sync + mechanism on this side — the group releases after the live consumers + alone.""" + server = server_factory() + server.begin_sync(2) # 4 provisioned, 2 alive + _publish(server, GI_A, GROUP_A) + server.free_group(GI_A) + assert server._inflight_groups == [GI_A] + server.free_group(GI_A) + assert server._inflight_groups == [] + + def test_begin_sync_clears_per_sync_state(self, server_factory): + server = server_factory() + server.begin_sync(2) + _publish(server, GI_A, GROUP_A) + server.free_group(GI_A) + + server.begin_sync(2) + assert server._inflight_groups == [] + assert server._freed_pending == [] + assert server._free_counts == {} + assert server._group_names == {} + + def test_a_straggler_signal_cannot_credit_the_next_sync(self, server_factory): + """begin_sync resets the counts, so a signal that leaked past the sync + boundary would over-credit — the reason the consumer drains its fired + signals before finishing. Pin the reset side of that contract.""" + server = server_factory() + server.begin_sync(2) + _publish(server, GI_A, GROUP_A) + server.free_group(GI_A) # one of two consumers; sync then aborts/ends + server.begin_sync(2) + _publish(server, GI_A, GROUP_A) + server.free_group(GI_A) + assert server._inflight_groups == [GI_A], ( + "one signal after the reset must not release" + ) + + def test_begin_sync_clears_a_previous_gather_error(self, server_factory): + server = server_factory() + server.set_gather_error("last sync failed") + server.begin_sync(1) + assert server._gather_error is None + + def test_the_packed_destination_cache_survives_the_sync_boundary( + self, server_factory + ): + """The packed layout repeats every sync — that is what makes caching the + destination views worth ~5ms per 384-spec group.""" + server = server_factory() + server._pack_dsts[("sentinel",)] = (0, []) + server.begin_sync(1) + assert ("sentinel",) in server._pack_dsts + + +class TestServedNamesGuard: + """A producer serves only the names it holds: its stage's groups, and + within them its own EP coordinate's experts plus the replicated names. + Without this guard a misrouted pull would block forever in the cache wait + for a name this rank never publishes.""" + + def test_a_pull_for_an_unserved_name_fails_loudly(self, server_factory): + server = server_factory(served_names=list(GROUP_A)) + server.begin_sync(1) + with pytest.raises(RuntimeError, match="wrong producer"): + server.rdt_produce_weights_batched([(GROUP_B[0], ())], consumer_id=0) + + def test_the_error_names_the_unserved_weights(self, server_factory): + server = server_factory(served_names=list(GROUP_A)) + server.begin_sync(1) + with pytest.raises(RuntimeError, match=GROUP_B[0]): + server.rdt_produce_weights_batched([(GROUP_B[0], ())], consumer_id=0) + + def test_served_names_none_accepts_anything(self, server_factory): + """Gather-to-all: every producer holds every group, so there is nothing + to guard. The call proceeds to the cache wait (not exercised here).""" + server = server_factory(served_names=None) + assert server._served_names is None + + +class TestStallWatchdog: + """The liveness bound on the producer's three blocking waits. + + Engine-death detection is generation-driven, and no generation is in flight + during a weight sync, so a consumer that dies INSIDE the sync window has no + detector. It never sends its `free_group` signals, the group's barrier never + completes, and `wait_freed` / `end_sync` / the serve cache wait block + forever — which stops this rank iterating its WeightSource, which is a + collective, which wedges every other trainer rank in NCCL with no exception + anywhere. + + The watchdog converts that into one real error. These use a tiny timeout; + the production default is 300s. + """ + + def test_a_blocked_wait_freed_fails_instead_of_hanging(self, server_factory): + server = server_factory(gather_lookahead=1, stall_timeout_s=0.3) + server.begin_sync(1) + # the engine's credit gate now waits; nobody will signal + _publish(server, GI_A, GROUP_A) + + errors: list = [] + done = threading.Event() + + def _wait(): + try: + server.wait_freed() + except BaseException as e: # noqa: BLE001 + errors.append(e) + done.set() + + threading.Thread(target=_wait, daemon=True).start() + + assert done.wait(timeout=10), ( + "wait_freed must give up rather than block forever" + ) + assert isinstance(server._gather_error, RuntimeError) + assert "RDT stall" in str(server._gather_error) + assert errors, "the engine must get an exception, not an empty credit list" + + def test_end_sync_fails_when_a_consumer_never_signals(self, server_factory): + """The exact mid-sync death signature: the group is published, one of + the two live consumers that owed a signal is gone.""" + server = server_factory(stall_timeout_s=0.3) + server.begin_sync(2) + _publish(server, GI_A, GROUP_A) + server.free_group(GI_A) # one of the two consumers signals; the other died + + server.end_sync() + assert server._gather_error is not None + assert "RDT stall" in str(server._gather_error) + + def test_the_error_reaches_every_blocked_waiter(self, server_factory): + """The watchdog fires on the existing `set_gather_error` channel, so one + stall unwinds the whole rank through one path rather than each waiter + timing out separately — here both a consumer parked in the serve cache + wait and the engine parked in its credit gate.""" + server = server_factory(gather_lookahead=1, stall_timeout_s=0.3) + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + + errors: list = [] + + def _pull(): + try: + server.rdt_produce_weights_batched([(GROUP_B[0], ())], consumer_id=0) + except BaseException as e: # noqa: BLE001 + errors.append(("pull", e)) + + def _wait(): + try: + server.wait_freed() + except BaseException as e: # noqa: BLE001 + errors.append(("credit", e)) + + puller = threading.Thread(target=_pull, daemon=True) + waiter = threading.Thread(target=_wait, daemon=True) + puller.start() + waiter.start() + puller.join(timeout=10) + waiter.join(timeout=10) + assert not puller.is_alive() and not waiter.is_alive() + assert {who for who, _ in errors} == {"pull", "credit"} + assert all("gather errored" in str(e) for _, e in errors) + + def test_progress_anywhere_keeps_a_slow_sync_alive(self, server_factory): + """A consumer that is slow but signaling must never trip it: the stamp + is global to the producer, so a steady trickle of signals holds the + whole rank open. Nine sequential publishes with a 0.3s timeout each take + longer in total than the timeout, and none may fire.""" + server = server_factory(gather_lookahead=1, stall_timeout_s=0.3) + server.begin_sync(1) + for gi in range(9): + _publish(server, gi, (f"model.layers.{gi}.w",)) + time.sleep(0.1) + server.free_group(gi) + assert server._gather_error is None + server.end_sync() + assert server._gather_error is None + + def test_begin_sync_resets_the_progress_stamp(self, server_factory): + """Syncs are minutes apart. Without the reset the first publish of sync N+1 + would measure its stall from somewhere inside sync N and fire immediately.""" + server = server_factory(gather_lookahead=1, stall_timeout_s=0.3) + server.begin_sync(1) + _publish(server, GI_A, GROUP_A) + server.free_group(GI_A) + server.end_sync() + + time.sleep(0.5) # the idle gap between syncs, > the timeout + server.begin_sync(1) + _publish(server, GI_B, GROUP_B) + assert server._gather_error is None + + def test_default_timeout_is_the_documented_one(self, server_factory): + server = server_factory() + assert server._stall_timeout == trainer_mod.DEFAULT_STALL_TIMEOUT_S + + +class TestConcurrentPublishAndFree: + def test_the_lifecycle_survives_concurrent_publishes_and_signals( + self, server_factory + ): + """Every group published must end up freed and out of the cache, with + end_sync returning cleanly — the property whose violation is a hang.""" + server = server_factory(gather_lookahead=2) + server.begin_sync(1) + groups = [(gi, (f"model.layers.{gi}.w",)) for gi in range(12)] + + def _publisher(): + for gi, names in groups: + _publish(server, gi, names) + + def _signaler(): + for gi, _names in groups: + while True: + with server._cache_cond: + published = gi in server._group_names + if published: + break + server.free_group(gi) + + pub = threading.Thread(target=_publisher, daemon=True) + free = threading.Thread(target=_signaler, daemon=True) + pub.start() + free.start() + pub.join(timeout=10) + free.join(timeout=10) + assert not pub.is_alive() and not free.is_alive() + + server.end_sync() + assert server._inflight_groups == [] + assert server._cache == {} + + def test_concurrent_signals_of_one_group_release_it_exactly_once( + self, server_factory + ): + """All live consumers signaling at once: the release must happen on the + last signal and only once, or the engine drops IPC refs twice (or + never).""" + server = server_factory() + server.begin_sync(8) + _publish(server, GI_A, GROUP_A) + + barrier = threading.Barrier(8) + + def _signal(): + barrier.wait() + server.free_group(GI_A) + + threads = [threading.Thread(target=_signal, daemon=True) for _ in range(8)] + for t in threads: + t.start() + for t in threads: + t.join(timeout=5) + assert not t.is_alive() + + assert server._inflight_groups == [] + assert server._freed_pending == [GI_A] + assert server._cache == {} + + +class TestFakeServerAgreesWithTheRealOne: + """`test_sharded_rdt_trainer.py`'s `_FakeProducerServer` is a second, + independent model of this protocol that the engine-side tests assert on. Pin + the two against each other so the fake cannot silently drift from the + semantics the real server enforces.""" + + @staticmethod + def _fake(): + from tests.distributed.test_sharded_rdt_trainer import _FakeProducerServer + + return _FakeProducerServer(auto_free=False) + + def test_the_barrier_and_early_signal_semantics_match(self, server_factory): + def publish_fake(srv, gi, names): + return srv.publish_group(gi, _entries(names)) + + for server, publish in ( + (server_factory(), _publish), + (self._fake(), publish_fake), + ): + server.begin_sync(2) + # early signal, then publish, then the closing signal + server.free_group(GI_A) + assert publish(server, GI_A, GROUP_A) is None, "publish returns nothing" + assert server._inflight_groups == [GI_A] + server.free_group(GI_A) + assert server._inflight_groups == [] + assert server.wait_freed() == [GI_A] + assert server.end_sync() == [] + + +class TestExportRing: + """The packed export path. Its one hard invariant is that the trainer packs + a group exactly the way ``publish_group`` rebuilds it — a layout + disagreement would serve the wrong bytes with nothing downstream to catch + it.""" + + @staticmethod + def _engine(): + eng = ShardedRDTTrainerWeightTransferEngine.__new__( + ShardedRDTTrainerWeightTransferEngine + ) + eng._export_ring = [None, None] + eng._export_ring_args = [None, None] + return eng + + def _roundtrip(self, held, slot=0): + eng = self._engine() + storages, views, _refs = eng._pack_group_for_export(held, slot) + assert len(storages) == 1, "a packed group must export exactly one storage" + slot_t = eng._export_ring[slot] + return { + name: torch.as_strided(slot_t.view(getattr(torch, dt)), shape, stride, off) + for name, (_sid, dt, shape, stride, off) in views.items() + } + + def test_packed_group_rebuilds_bit_identically(self): + held = [ + ("bf16.2d", torch.randn(12, 7).bfloat16()), + ("fp32.1d", torch.randn(9)), + ("bf16.3d", torch.randn(4, 5, 6).bfloat16()), + ("bf16.transposed", torch.randn(6, 11).bfloat16().t()), + ("bf16.sliced", torch.randn(8, 32).bfloat16()[:, 3:19]), + ] + got = self._roundtrip(held) + for name, t in held: + assert got[name].dtype is t.dtype + assert tuple(got[name].shape) == tuple(t.shape) + assert torch.equal(got[name], t), name + + def test_a_reused_slot_reallocates_and_still_rebuilds(self): + eng = self._engine() + eng._pack_group_for_export([("a", torch.randn(4, 4).bfloat16())], 0) + small = eng._export_ring[0].numel() + big = [("b", torch.randn(256, 128).bfloat16())] + _st, views, _rf = eng._pack_group_for_export(big, 0) + assert eng._export_ring[0].numel() > small + _name, (_sid, dt, shape, stride, off) = next(iter(views.items())) + rebuilt = torch.as_strided( + eng._export_ring[0].view(getattr(torch, dt)), shape, stride, off + ) + assert torch.equal(rebuilt, big[0][1]) + + def test_slots_are_distinct_so_adjacent_groups_cannot_alias(self): + eng = self._engine() + eng._pack_group_for_export([("a", torch.randn(8, 8).bfloat16())], 0) + eng._pack_group_for_export([("b", torch.randn(8, 8).bfloat16())], 1) + assert eng._export_ring[0].data_ptr() != eng._export_ring[1].data_ptr() + + +class TestExportRingSlotSafety: + """The export ring packs a group's small storages into one reused buffer and + ships per-name views into it, so a slot may only be rewritten once the group + that last used it is freed everywhere. The credit gate does not establish + that on its own: it bounds how many groups are unfreed, not the order they + free in, and barriers complete out of order (a consumer pre-frees a group it + pulls nothing from; a light group can finish before an earlier heavy one). + """ + + @pytest.fixture + def gather_engine(self, monkeypatch): + monkeypatch.setattr(torch.Tensor, "cuda", lambda self: self) + monkeypatch.setattr( + trainer_mod, + "reduce_tensor", + lambda base: ( + None, + (base.numel(), None, None, None, None, None, -1, None), + ), + ) + return _loop_engine + + @staticmethod + def _await_published(server, gi, timeout=10): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + with server._cache_cond: + if gi in server._group_names: + return True + time.sleep(0.002) + return False + + def test_a_slot_is_never_rewritten_while_its_group_is_still_live( + self, server_factory, gather_engine + ): + """Free the later group of each pair first and hold the earlier one live + until the NEXT group is gathered -- the window a modular slot counter got + wrong (at lookahead=1 there are 2 slots, so group 2 took group 0's).""" + server = server_factory(gather_lookahead=1) + n_groups = 8 + engine = gather_engine(server, n_groups, lookahead=1) + + violations: list[tuple[int, int]] = [] + last_holder: dict[int, int] = {} + pending: dict[str, int] = {} + orig_pack = engine._pack_group_for_export + orig_pub = engine._publish_async + + def _pack(held, slot_idx): + prev = last_holder.get(slot_idx) + if prev is not None and prev in engine._inflight: + violations.append((slot_idx, prev)) + pending["slot"] = slot_idx + return orig_pack(held, slot_idx) + + def _pub(group_idx, entries): + if "slot" in pending: + last_holder[pending.pop("slot")] = group_idx + return orig_pub(group_idx, entries) + + engine._pack_group_for_export = _pack + engine._publish_async = _pub + + def _consumer(): + for base in range(0, n_groups, 2): + if not self._await_published(server, base + 1): + return + server.free_group(base + 1) + nxt = base + 2 + if nxt < n_groups and not self._await_published(server, nxt): + return + server.free_group(base) + + consumer = threading.Thread(target=_consumer, daemon=True) + consumer.start() + engine._run_gather_loop(update_future=None, live_count=1) + consumer.join(timeout=15) + + assert last_holder, "no group was ring-packed; the test proves nothing" + assert not violations, f"slot rewritten while its group was live: {violations}" + assert engine._inflight == {} + + def test_distinct_live_groups_never_share_a_slot( + self, server_factory, gather_engine + ): + server = server_factory(gather_lookahead=1) + engine = gather_engine(server, 6, lookahead=1) + seen: list[dict] = [] + orig_pack = engine._pack_group_for_export + + def _pack(held, slot_idx): + live = { + g: s for g, s in engine._slot_of_group.items() if g in engine._inflight + } + seen.append({"slot": slot_idx, "live": live}) + return orig_pack(held, slot_idx) + + engine._pack_group_for_export = _pack + + def _consumer(): + for gi in range(6): + if not self._await_published(server, gi): + return + server.free_group(gi) + + consumer = threading.Thread(target=_consumer, daemon=True) + consumer.start() + engine._run_gather_loop(update_future=None, live_count=1) + consumer.join(timeout=15) + + assert seen, "no group was ring-packed; the test proves nothing" + for rec in seen: + assert rec["slot"] not in rec["live"].values(), rec + + +@pytest.fixture +def sharing_server(server_factory): + """A producer whose serve slots are plain CPU tensors. + + The pack is the only part of the serve path that touches a buffer, so + replacing the one allocation seam (``_new_serve_buffer``, which would + ``torch.empty(..., device="cuda")`` and register the result with NIXL) puts + the whole shared-pack path within reach of a CPU test. + """ + + def _make(**kwargs): + kwargs.setdefault("workers_per_replica", 2) + server = server_factory(**kwargs) + server._serve_device = torch.device("cpu") + server._new_serve_buffer = lambda n: torch.zeros(n, dtype=torch.uint8) + server.packs = 0 + real_pack = server._pack_shared + + def _counting_pack(gen, specs): + server.packs += 1 + return real_pack(gen, specs) + + server._pack_shared = _counting_pack + return server + + return _make + + +def _blob_bytes(server, name): + """The cached tensor's bytes, as the pack should land them in the slot.""" + t = server._cache[name] + return t.contiguous().view(torch.uint8).flatten() + + +class TestSharedServeSlots: + """One registered slot serving several inference deployments. + + Consumers whose ids differ by a multiple of ``workers_per_replica`` are the + same worker of different deployments: same parallel config, same baked plan, + same chunk sequence, byte-identical pack layout. They rendezvous per chunk, + the last arrival packs, and every one of them returns that blob, so R + deployments cost one ring and one pack rather than R of each. + """ + + def test_one_deployment_never_shares(self, sharing_server): + """With the width equal to the live count every group is a singleton, so + each call packs for itself out of its own ring.""" + server = sharing_server(workers_per_replica=2) + server.begin_sync(2, [0, 1]) + _publish(server, GI_A, GROUP_A) + specs = [(GROUP_A[0], ())] + blobs = [server.rdt_produce_weights_batched(specs, c, 0)[0] for c in (0, 1)] + + assert server.packs == 2 + assert blobs[0].data_ptr() != blobs[1].data_ptr() + assert len(server._serve_rings) == 2 + + def test_sharers_are_served_one_pack_out_of_one_buffer(self, sharing_server): + server = sharing_server(workers_per_replica=2) + server.begin_sync(4, [0, 1, 2, 3]) + _publish(server, GI_A, GROUP_A) + specs = [(GROUP_A[0], ())] + + with ThreadPoolExecutor(max_workers=2) as ex: + futures = [ + ex.submit(server.rdt_produce_weights_batched, specs, c, 0) + for c in (0, 2) + ] + blobs = [f.result(timeout=10)[0] for f in futures] + + expected = _blob_bytes(server, GROUP_A[0]) + assert server.packs == 1 + assert blobs[0].data_ptr() == blobs[1].data_ptr() + assert len(server._serve_rings) == 1 + assert torch.equal(blobs[0][: expected.numel()], expected) + + def test_a_sharer_blocks_until_its_peer_arrives(self, sharing_server): + """Without the wait the packer could repack a slot a peer had not read.""" + server = sharing_server(workers_per_replica=2, stall_timeout_s=30.0) + server.begin_sync(4, [0, 1, 2, 3]) + _publish(server, GI_A, GROUP_A) + specs = [(GROUP_A[0], ())] + + with ThreadPoolExecutor(max_workers=2) as ex: + first = ex.submit(server.rdt_produce_weights_batched, specs, 0, 0) + with pytest.raises(FuturesTimeout): + first.result(timeout=0.5) + second = ex.submit(server.rdt_produce_weights_batched, specs, 2, 0) + assert ( + first.result(timeout=10)[0].data_ptr() + == second.result(timeout=10)[0].data_ptr() + ) + + def test_the_slot_follows_the_consumers_issue_order(self, sharing_server): + """The invariant slot safety rests on. The consumer drains pull i before + issuing i+K, so choosing the slot as ``seq % K`` means a slot is only + repacked once the read of its previous contents is over. Deriving it from + a per-call counter on the producer instead (execution order) can hand a + slot to a pull whose predecessor is still being read, because Ray may + start a consumer's concurrent produce calls in any order. + """ + server = sharing_server(workers_per_replica=2, num_rdt_buffers=2) + server.begin_sync(4, [0, 1, 2, 3]) + names = ("a.w", "b.w", "c.w") + for gi, name in enumerate(names): + _publish(server, gi, (name,)) + + # Arrive OUT of issue order: seq 1 first, then 0, then 2. + with ThreadPoolExecutor(max_workers=2) as ex: + for seq in (1, 0, 2): + futures = [ + ex.submit( + server.rdt_produce_weights_batched, + [(names[seq], ())], + c, + seq, + ) + for c in (0, 2) + ] + [f.result(timeout=10) for f in futures] + + slots = {seq: gen.slot for seq, gen in server._gens[0].items()} + assert slots == {0: 0, 1: 1, 2: 0}, slots + + def test_a_seqless_call_is_refused(self, sharing_server): + """The slot comes from ``seq``, so a caller that sends none cannot be + served safely.""" + server = sharing_server(workers_per_replica=2) + server.begin_sync(1, [0]) + _publish(server, GI_A, GROUP_A) + + with pytest.raises(ValueError, match="requires the consumer's issue"): + server.rdt_produce_weights_batched([(GROUP_A[0], ())], 0) + + def test_a_failed_pack_reaches_every_sharer(self, sharing_server): + """Only the packer touches the buffer, so its failure has to be + republished: a waiter must raise rather than read a half-written slot.""" + server = sharing_server(workers_per_replica=2) + server.begin_sync(4, [0, 1, 2, 3]) + _publish(server, GI_A, GROUP_A) + specs = [(GROUP_A[0], (("mutate_", (), ()),))] # not in ALLOWED_OPS + + raised = [] + with ThreadPoolExecutor(max_workers=2) as ex: + futures = [ + ex.submit(server.rdt_produce_weights_batched, specs, c, 0) + for c in (0, 2) + ] + for future in futures: + with pytest.raises((ValueError, RuntimeError)) as excinfo: + future.result(timeout=10) + raised.append(excinfo.value) + + assert any( + isinstance(e, ValueError) and "disallowed op" in str(e) for e in raised + ) + assert any("packing this chunk failed" in str(e) for e in raised) + + def test_a_degraded_sync_narrows_the_rendezvous(self, sharing_server): + """A dead deployment must not be waited for: a survivor left alone in + its group packs for itself instead of blocking on the corpse.""" + server = sharing_server(workers_per_replica=2, stall_timeout_s=30.0) + server.begin_sync(3, [0, 1, 3]) # consumer 2 died + _publish(server, GI_A, GROUP_A) + + blob = server.rdt_produce_weights_batched([(GROUP_A[0], ())], 0, 0)[0] + + expected = _blob_bytes(server, GROUP_A[0]) + assert torch.equal(blob[: expected.numel()], expected) + assert server._sharers_of(0) == frozenset({0}) + assert server._sharers_of(1) == frozenset({1, 3}) + + def test_a_sharer_that_never_arrives_trips_the_watchdog(self, sharing_server): + """A consumer that dies INSIDE the sync window has no detector, exactly + as for the free barrier; the rendezvous waits on the same channel.""" + server = sharing_server(workers_per_replica=2, stall_timeout_s=0.3) + server.begin_sync(4, [0, 1, 2, 3]) + _publish(server, GI_A, GROUP_A) + + with pytest.raises(RuntimeError, match="gather errored|RDT stall"): + server.rdt_produce_weights_batched([(GROUP_A[0], ())], 0, 0) + assert "RDT stall" in str(server._gather_error) + + +class TestSharedReservation: + """``reserve_serve_buffer`` under sharing: one ring per GROUP, and the + init-time check that the group's members pull the same chunks.""" + + def test_matching_plans_reserve_one_ring_between_them(self, sharing_server): + server = sharing_server(workers_per_replica=2) + for consumer_id in (0, 2): + server.reserve_serve_buffer(consumer_id, 1024, "plan-a") + + assert list(server._serve_rings) == [0] + assert [s.numel() for s in server._serve_rings[0]] == [ + buffer_alloc_bytes(1024) + ] * 2 + + def test_sharing_off_reserves_a_ring_per_consumer(self, sharing_server): + server = sharing_server(workers_per_replica=0) + for consumer_id in (0, 2): + server.reserve_serve_buffer(consumer_id, 1024, "plan-a") + + assert sorted(server._serve_rings) == [0, 2] + + def test_mismatched_plans_fail_at_reservation(self, sharing_server): + """Sharing rests on the sharers pulling the same chunks. Checked here, + where it names both consumers, rather than at serve time, where it is a + rendezvous nobody completes.""" + server = sharing_server(workers_per_replica=2) + server.reserve_serve_buffer(0, 1024, "plan-a") + + with pytest.raises(RuntimeError, match="do not pull the same chunks"): + server.reserve_serve_buffer(2, 1024, "plan-b") + + def test_different_groups_may_differ(self, sharing_server): + """Only the members of ONE group have to agree; worker 0 and worker 1 of + the same deployment hold different slices and never share a slot.""" + server = sharing_server(workers_per_replica=2) + server.reserve_serve_buffer(0, 1024, "plan-a") + server.reserve_serve_buffer(1, 1024, "plan-b") + + assert sorted(server._serve_rings) == [0, 1] + + def test_a_missing_digest_skips_the_check(self, sharing_server): + """The digest is optional on the wire, so an engine that does not send + one still reserves.""" + server = sharing_server(workers_per_replica=2) + server.reserve_serve_buffer(0, 1024, None) + server.reserve_serve_buffer(2, 1024, None) + + assert list(server._serve_rings) == [0] diff --git a/tests/distributed/test_sharded_rdt_trainer.py b/tests/distributed/test_sharded_rdt_trainer.py new file mode 100644 index 0000000000..9339e503c4 --- /dev/null +++ b/tests/distributed/test_sharded_rdt_trainer.py @@ -0,0 +1,628 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the sharded-RDT trainer-side engine. + +The trainer engine's own surface: init info and factory registration, the +send_weights round trip against an in-process fake producer server, the +ownership table it resolves and ships to the workers, and the producer's packed +destination-view cache. + +The plan/replay half (including `RdtRouter`, which is consumer-side) lives in +test_sharded_rdt_plan.py; the producer server's protocol in +test_sharded_rdt_producer.py. +""" + +import itertools +from dataclasses import asdict + +import pytest +import torch + +from tests.distributed.test_weight_transfer import RecordingClient +from vllm.distributed.weight_transfer import ( + ModuleSource, + ParamMeta, + WeightSource, + WeightTransferTrainerFactory, +) +from vllm.distributed.weight_transfer.base import layerwise_groups +from vllm.distributed.weight_transfer.sharded_rdt_trainer import ( + ShardedRDTTrainerInitInfo, + ShardedRDTTrainerWeightTransferEngine, +) + + +class _ListSource(WeightSource): + """A WeightSource over an explicit ordered (name, cpu-tensor) list, so the + sharded-RDT group/order logic can be tested without a real model.""" + + def __init__(self, pairs): + self._pairs = list(pairs) + + def metadata(self): + return [ParamMeta(n, t.dtype, tuple(t.shape)) for n, t in self._pairs] + + def __iter__(self): + return iter(self._pairs) + + +class _FakeProducerServer: + """In-process stand-in for the _RDTProducerServer Ray actor. Records the + engine->server call sequence and, by default, frees each group as soon as + it is published (simulating the consumers' free_group barrier) so the + gather loop's credit gate never parks. Mirrors the real server's + per-group barrier: publish/free are keyed by GROUP INDEX, signals count + to the ``begin_sync`` live total, publish_group returns nothing, and freed + groups flow back only through wait_freed / end_sync (see + test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne).""" + + def __init__(self, auto_free=True): + self.order: list[str] = [] + self.published: list[int] = [] + self.live_count = 1 + self._inflight_groups: list[int] = [] + self.auto_free = auto_free + self.free_counts: dict[int, int] = {} + self._pending_freed: list[int] = [] + + # Alias for tests that inspect in-flight groups. + @property + def inflight(self): + return self._inflight_groups + + def begin_sync(self, live_count, live_consumer_ids=None): + self.order.append("begin") + self.live_count = max(1, int(live_count)) + self.live_ids = live_consumer_ids + + def publish_group(self, group_idx, entries): + self.order.append("publish") + self.published.append(group_idx) + self._inflight_groups.append(group_idx) + if self.auto_free or self.free_counts.get(group_idx, 0) >= self.live_count: + self._inflight_groups.remove(group_idx) + self._pending_freed.append(group_idx) + + def free_group(self, group_idx): + """Consumer back-edge; may arrive before the group's publish.""" + self.free_counts[group_idx] = self.free_counts.get(group_idx, 0) + 1 + if ( + group_idx in self._inflight_groups + and self.free_counts[group_idx] >= self.live_count + ): + self._inflight_groups.remove(group_idx) + self._pending_freed.append(group_idx) + + def wait_freed(self): + """The engine's credit gate. The real server blocks here; the fake + must already have a banked credit when the gate asks (auto_free, or a + test's own free_group) — anything else is the deadlock the real + watchdog would kill, so fail loudly. Not appended to ``order``: it is + pacing, not a lifecycle milestone.""" + assert self._pending_freed, ( + "wait_freed with nothing freed: the gather loop would deadlock" + ) + freed = self._pending_freed + self._pending_freed = [] + return freed + + def end_sync(self): + self.order.append("end") + freed = self._pending_freed + self._pending_freed = [] + return freed + + def set_gather_error(self, message): + self.order.append("error") + + +def _rdt_engine_with_fake_server( + source, *, is_sender, client, server, monkeypatch, fleet_owned=None +): + """Build a ShardedRDTTrainerWeightTransferEngine wired to an in-process fake + server (no Ray, no CUDA IPC): bypass trainer_init's spawn, set the + group-major metadata, and route _rpc to the fake.""" + import vllm.distributed.weight_transfer.sharded_rdt_trainer as mod + + # reduce_tensor needs CUDA; the fake server never rebuilds, so stub it. + monkeypatch.setattr(mod, "reduce_tensor", lambda t: (None, ("fake",))) + + init_info = ShardedRDTTrainerInitInfo(num_consumers=1, rank=0 if is_sender else 1) + engine = ShardedRDTTrainerWeightTransferEngine( + client=client, source=source, is_sender=is_sender, init_info=init_info + ) + engine._meta = list(source.metadata()) + names = [m.name for m in engine._meta] + engine._groups = layerwise_groups(names) + engine._server = server + engine._rpc = lambda method, *args: getattr(server, method)(*args) + # What trainer_init resolves from the source's held names + the fleet's + # all-gather. ``fleet_owned`` stands in for that all-gather so a partial- + # ownership rank can be tested without a real process group; the fleet must + # cover every name or the transpose rejects it (nothing would serve the rest). + if fleet_owned is None: + engine._resolve_ownership(1, 0) + else: + # Stands in for the (metadata digest, held-name bitmask) all-gather: + # each rank's group list becomes the mask its held names would set. + names = [m.name for m in engine._meta] + groups = engine._groups + + def _fake_gather(_world, mine): + out = [] + for owned in fleet_owned: + held = {n for gi in owned for n in groups[gi]} + mask = bytearray((len(names) + 7) // 8) + for i, n in enumerate(names): + if n in held: + mask[i >> 3] |= 1 << (i & 7) + out.append((mine[0], bytes(mask))) + return out + + monkeypatch.setattr(engine, "_all_gather_owned", _fake_gather) + engine._resolve_ownership(len(fleet_owned), 0) + return engine + + +def _rdt_source_two_layers(): + return _ListSource( + [ + ("embed.weight", torch.zeros(2)), + ("model.layers.0.w", torch.zeros(2)), + ("model.layers.1.w", torch.zeros(2)), + ("norm.weight", torch.zeros(2)), + ] + ) + + +class TestShardedRDTTrainerInitInfo: + def test_declares_backend(self): + assert ShardedRDTTrainerInitInfo.backend == "sharded_rdt" + + def test_rank_is_keyword_only_and_drives_is_sender(self): + assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=0).is_sender is True + assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=1).is_sender is False + with pytest.raises(TypeError): + # rank is keyword-only. + ShardedRDTTrainerInitInfo(4, 0) # type: ignore[misc] + + def test_registered_in_trainer_factory(self): + cls = WeightTransferTrainerFactory._registry["sharded_rdt"]() + assert cls is ShardedRDTTrainerWeightTransferEngine + + +def test_sharded_rdt_trainer_init_requires_source(): + with pytest.raises(ValueError, match="requires a WeightSource"): + ShardedRDTTrainerWeightTransferEngine.trainer_init( + ShardedRDTTrainerInitInfo(num_consumers=1, rank=0), + client=RecordingClient(), + source=None, + ) + + +def test_sharded_rdt_worker_init_info_is_group_major(monkeypatch): + source = _rdt_source_two_layers() + engine = _rdt_engine_with_fake_server( + source, + is_sender=True, + client=RecordingClient(), + server=_FakeProducerServer(), + monkeypatch=monkeypatch, + ) + worker_init = engine._build_worker_init_info(["srv_rk0"]) + # 4 params -> pre / layer0 / layer1 / post = 4 groups of length 1. + assert worker_init.names == [ + "embed.weight", + "model.layers.0.w", + "model.layers.1.w", + "norm.weight", + ] + assert worker_init.group_lens == [1, 1, 1, 1] + assert worker_init.trainer_actor_names == ["srv_rk0"] + assert worker_init.produce_method_name == "rdt_produce_weights_batched" + assert sum(worker_init.group_lens) == len(worker_init.names) + + +@pytest.mark.skipif( + not torch.cuda.is_available(), + reason="the gather loop's CUDA-IPC export needs a device", +) +def test_sharded_rdt_send_weights_drives_client_in_order(monkeypatch): + server = _FakeProducerServer(auto_free=True) + client = RecordingClient() + engine = _rdt_engine_with_fake_server( + _rdt_source_two_layers(), + is_sender=True, + client=client, + server=server, + monkeypatch=monkeypatch, + ) + engine.send_weights() + + assert client.order == ["start", "update", "finish"] + # begin, one publish per group (4), end. + assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"] + assert len(server.published) == 4 + # every group freed -> no engine-held refs remain. + assert engine._inflight == {} + + +def test_sharded_rdt_send_weights_group_order_mismatch_raises(monkeypatch): + # Source whose iteration order disagrees with its metadata order. + class _BadSource(_ListSource): + def __iter__(self): + reordered = list(self._pairs) + reordered[0], reordered[1] = reordered[1], reordered[0] + return iter(reordered) + + server = _FakeProducerServer() + engine = _rdt_engine_with_fake_server( + _BadSource(_rdt_source_two_layers()._pairs), + is_sender=True, + client=RecordingClient(), + server=server, + monkeypatch=monkeypatch, + ) + with pytest.raises(RuntimeError, match="iteration order must match"): + engine.send_weights() + assert "error" in server.order # gather error propagated to the server + + +@pytest.mark.skipif( + not torch.cuda.is_available(), + reason="the gather loop's CUDA-IPC export needs a device", +) +def test_sharded_rdt_non_sender_skips_client(monkeypatch): + class _RaisingClient(RecordingClient): + def start_weight_update(self): + raise AssertionError("non-sender must not touch the client") + + def update_weights(self, update_info): + raise AssertionError("non-sender must not touch the client") + + def finish_weight_update(self, weight_version: str | None = None): + raise AssertionError("non-sender must not touch the client") + + server = _FakeProducerServer(auto_free=True) + client = _RaisingClient() + engine = _rdt_engine_with_fake_server( + _rdt_source_two_layers(), + is_sender=False, + client=client, + server=server, + monkeypatch=monkeypatch, + ) + engine.send_weights() # gathers only; must not raise + assert client.order == [] + assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"] + + +@pytest.mark.skipif( + not torch.cuda.is_available(), + reason="the gather loop's CUDA-IPC export needs a device", +) +def test_sharded_rdt_send_weights_surfaces_update_error(monkeypatch): + class _FailingUpdateClient(RecordingClient): + def update_weights(self, update_info): + self.order.append("update") + raise RuntimeError("inference side rejected update") + + server = _FakeProducerServer(auto_free=True) + engine = _rdt_engine_with_fake_server( + _rdt_source_two_layers(), + is_sender=True, + client=_FailingUpdateClient(), + server=server, + monkeypatch=monkeypatch, + ) + with pytest.raises(RuntimeError, match="inference side rejected update"): + engine.send_weights() + + +class _OwnedSource(_ListSource): + """A source holding only some groups' names, like a pipeline-parallel rank. + Takes group indices for convenience and declares the names inside them.""" + + def __init__(self, pairs, owned_group_idx): + super().__init__(pairs) + self._owned = list(owned_group_idx) + groups = layerwise_groups([n for n, _ in pairs]) + self._owned_names = [n for gi in self._owned for n in groups[gi]] + + def held_names(self): + return list(self._owned_names) + + def __iter__(self): + by_name = dict(self._pairs) + return iter([(n, by_name[n]) for n in self._owned_names]) + + +@pytest.mark.skipif( + not torch.cuda.is_available(), + reason="the gather loop's CUDA-IPC export needs a device", +) +def test_sharded_rdt_publishes_only_the_groups_it_holds(monkeypatch): + server = _FakeProducerServer(auto_free=True) + engine = _rdt_engine_with_fake_server( + _OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]), + is_sender=False, + client=RecordingClient(), + server=server, + monkeypatch=monkeypatch, + fleet_owned=[[1, 2], [0, 3]], # this rank holds the layers, rank 1 the rest + ) + engine.send_weights() + + assert server.published == [1, 2] + assert server.order == ["begin", "publish", "publish", "end"] + assert engine._inflight == {} + # embed + norm on rank 1, the two layers here: two distinct owner sets, + # numbered by first appearance in metadata order. + assert engine._owner_sets == [[1], [0]] + assert engine._name_owner_class == [0, 1, 1, 0] + + +def test_sharded_rdt_owned_group_order_mismatch_raises(monkeypatch): + class _MisorderedOwned(_OwnedSource): + def __iter__(self): + return iter(list(super().__iter__())[::-1]) + + server = _FakeProducerServer(auto_free=True) + engine = _rdt_engine_with_fake_server( + _MisorderedOwned(_rdt_source_two_layers()._pairs, [1, 2]), + is_sender=False, + client=RecordingClient(), + server=server, + monkeypatch=monkeypatch, + fleet_owned=[[1, 2], [0, 3]], + ) + with pytest.raises(RuntimeError, match="iteration order must match"): + engine.send_weights() + assert "error" in server.order + + +@pytest.mark.skipif( + not torch.cuda.is_available(), + reason="the gather loop's CUDA-IPC export needs a device", +) +def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch): + """The free barrier's target is one integer per sync: the live consumer + count, defaulting to the whole provisioned fleet.""" + server = _FakeProducerServer(auto_free=True) + engine = _rdt_engine_with_fake_server( + _rdt_source_two_layers(), + is_sender=False, + client=RecordingClient(), + server=server, + monkeypatch=monkeypatch, + ) + engine.send_weights() + assert server.live_count == 1 # num_consumers=1 in the harness + engine.send_weights(live_consumer_ids=[0]) + assert server.live_count == 1 + + +class TestLiveCountPlumbing: + """``send_weights(live_consumer_ids)`` -> the barrier target. The + provisioned geometry is frozen; a degraded sync only lowers the target and + narrows each slot-sharing group's rendezvous.""" + + @staticmethod + def _engine(num_consumers, world=2, rank=0): + engine = ShardedRDTTrainerWeightTransferEngine.__new__( + ShardedRDTTrainerWeightTransferEngine + ) + engine._init_info = ShardedRDTTrainerInitInfo( + rank=rank, num_consumers=num_consumers + ) + engine.source = object() # send_weights asserts a source is present + received: list = [] + engine._send_weights_inner = lambda count, ids: received.append((count, ids)) + return engine, received + + def test_none_counts_the_whole_provisioned_fleet(self): + engine, got = self._engine(num_consumers=8) + engine.send_weights(None) + assert got == [(8, list(range(8)))] + + def test_a_live_set_counts_its_distinct_members(self): + engine, got = self._engine(num_consumers=8) + engine.send_weights([0, 1, 4, 5, 5]) + assert got == [(4, [0, 1, 4, 5])] + + def test_a_full_live_set_matches_the_provisioned_count(self): + engine, got = self._engine(num_consumers=8) + engine.send_weights(list(range(8))) + engine.send_weights(None) + assert got == [(8, list(range(8)))] * 2 + + def test_which_consumers_died_does_not_matter_only_how_many(self): + """The whole point of the barrier: no routed per-producer targets, so + the identity of the dead consumer is irrelevant to the producers.""" + counts = [] + for live in ([0, 1, 2, 3], [4, 5, 6, 7], [0, 2, 4, 6]): + engine, got = self._engine(num_consumers=8) + engine.send_weights(live) + counts += [c for c, _ids in got] + assert counts == [4, 4, 4] + + def test_the_live_ids_travel_with_the_count(self): + """The count sizes the free barrier, the ids size the slot-sharing + rendezvous, so a producer that shares slots can tell WHICH consumers it + is still waiting for. They must describe the same set.""" + engine, got = self._engine(num_consumers=8) + engine.send_weights([6, 0, 2, 2]) + ((count, ids),) = got + assert ids == [0, 2, 6] and count == len(ids) + + +def test_sharded_rdt_worker_init_info_carries_the_ownership_table(monkeypatch): + import json + + server = _FakeProducerServer(auto_free=True) + engine = _rdt_engine_with_fake_server( + _OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]), + is_sender=True, + client=RecordingClient(), + server=server, + monkeypatch=monkeypatch, + fleet_owned=[[0, 1], [2, 3]], + ) + worker_init = engine._build_worker_init_info(["srv_rk0", "srv_rk1"]) + assert worker_init.owner_sets == [[0], [1]] + assert worker_init.name_owner_class == [0, 0, 1, 1] + assert len(worker_init.name_owner_class) == len(worker_init.names) + # The payload crosses the control plane as JSON: nested lists must survive. + round_tripped = json.loads(json.dumps(asdict(worker_init))) + assert round_tripped["owner_sets"] == [[0], [1]] + assert round_tripped["name_owner_class"] == [0, 0, 1, 1] + + +def test_weight_source_holds_everything_by_default(): + """The contract's default: a source that says nothing holds the whole model.""" + src = _rdt_source_two_layers() + assert src.held_names() is None + assert ModuleSource(torch.nn.Linear(2, 2)).held_names() is None + + +def test_sharded_rdt_rejects_a_held_name_outside_metadata(monkeypatch): + class _BadHeld(_OwnedSource): + def held_names(self): + return ["embed.weight", "not.a.real.name"] + + with pytest.raises(ValueError, match="not"): + _rdt_engine_with_fake_server( + _BadHeld(_rdt_source_two_layers()._pairs, [0]), + is_sender=False, + client=RecordingClient(), + server=_FakeProducerServer(), + monkeypatch=monkeypatch, + ) + + +def test_sharded_rdt_rejects_a_rank_holding_nothing(monkeypatch): + class _HoldsNothing(_ListSource): + def held_names(self): + return [] + + with pytest.raises(ValueError, match="empty"): + _rdt_engine_with_fake_server( + _HoldsNothing(_rdt_source_two_layers()._pairs), + is_sender=False, + client=RecordingClient(), + server=_FakeProducerServer(), + monkeypatch=monkeypatch, + ) + + +def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch): + """Every name must be held somewhere or it can never be served — caught + when the holdings are transposed, naming the orphan.""" + with pytest.raises(ValueError, match="no trainer rank holds"): + _rdt_engine_with_fake_server( + _OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]), + is_sender=False, + client=RecordingClient(), + server=_FakeProducerServer(), + monkeypatch=monkeypatch, + fleet_owned=[[0, 1], [2]], # nobody holds group 3 + ) + + +def test_sharded_rdt_rejects_metadata_disagreement_across_ranks(monkeypatch): + """Only the sender's metadata reaches the consumers, so a rank describing + just its own share must fail loudly rather than silently drop the rest.""" + engine = _rdt_engine_with_fake_server( + _OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]), + is_sender=False, + client=RecordingClient(), + server=_FakeProducerServer(auto_free=True), + monkeypatch=monkeypatch, + fleet_owned=[[1, 2], [0, 3]], # a covering fleet, so construction succeeds + ) + # Now rank 1 reports a DIFFERENT metadata digest for the same model. + monkeypatch.setattr( + engine, + "_all_gather_owned", + lambda w, mine: [mine, ("deadbeefdeadbeef", mine[1])], + ) + with pytest.raises(ValueError, match="disagrees across trainer ranks"): + engine._resolve_ownership(2, 0) + + +def _serve_ring_server(src_name, src): + """A producer server with one cached tensor and a pre-seeded serve ring, so a + pull needs no Ray and no NIXL registration. Returns (server, serve) where + ``serve(chain)`` packs one spec into the SAME ring slot every time — which is + what puts the destination-view cache, and only it, under test.""" + from vllm.distributed.weight_transfer.sharded_rdt_trainer import ( + _RDTProducerServer, + ) + + srv = _RDTProducerServer( + num_rdt_buffers=2, + buffer_presize_gb=0.0, + gather_lookahead=2, + ) + srv._cache[src_name] = src + srv._serve_rings[0] = [ + torch.empty(1 << 16, dtype=torch.uint8, device="cuda") for _ in range(2) + ] + + # The slot is ``seq % nring``, so stepping seq by nring keeps every pull in + # slot 0 while still making each one its own generation -- reusing a seq + # would rendezvous with the finished pack and return it unchanged. + seqs = itertools.count(0, srv._nring) + + def serve(chain): + return srv.rdt_produce_weights_batched( + [(src_name, chain)], consumer_id=0, seq=next(seqs) + )[0] + + return srv, serve + + +@pytest.mark.skipif( + not torch.cuda.is_available(), reason="the producer server needs a CUDA device" +) +def test_serve_does_not_reuse_packed_views_of_another_shape(): + """Two requests can share a name yet pack different slices of it. + + The producer caches the destination views it carves into a serve ring slot. + Keyed by name alone, the second request is packed through the first's views. + Reachable when one name's copies split across owner-class chunks. + """ + name = "model.layers.0.w" + src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8) + _srv, serve = _serve_ring_server(name, src) + + serve((("narrow", (0, 0, 2), ()),)) # 2 rows + wide = src.narrow(0, 0, 6) # 6 rows, same name, same slot + blob = serve((("narrow", (0, 0, 6), ()),)) + + got = blob[: wide.numel() * wide.element_size()].view(wide.dtype).reshape(6, 8) + assert torch.equal(got, wide) + + +@pytest.mark.skipif( + not torch.cuda.is_available(), reason="the producer server needs a CUDA device" +) +def test_serve_does_not_reuse_packed_views_of_another_dtype(): + """The SILENT case of the same cache: two requests whose slices have the same + name and the same shape but different dtypes pack at identical offsets, so + reusing the stale views raises nothing — ``copy_`` just casts, and the blob + carries the wrong bytes with no check downstream. + + ``view(dtype)`` rather than ``to(dtype)`` because the chain is replayed under + ``ALLOWED_OPS``, which rejects ``to``: same-itemsize dtypes keep the byte + length identical, which is exactly what makes the stale view fit. + """ + name = "model.layers.0.w" + src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8) + _srv, serve = _serve_ring_server(name, src) + + serve((("view", (torch.float16,), ()),)) # same shape and bytes, fp16 + blob = serve(()) # same name and shape, bf16 + + got = blob[: src.numel() * src.element_size()].view(src.dtype).reshape(8, 8) + assert torch.equal(got, src), "packed through a cached view of the wrong dtype" diff --git a/tests/distributed/test_weight_transfer.py b/tests/distributed/test_weight_transfer.py index 05a57dbea5..44eafe0123 100644 --- a/tests/distributed/test_weight_transfer.py +++ b/tests/distributed/test_weight_transfer.py @@ -32,8 +32,10 @@ from vllm.distributed.weight_transfer import ( ) from vllm.distributed.weight_transfer.base import ( TrainerInitInfo, + WeightTransferEngine, WeightTransferInitRequest, WeightTransferUpdateRequest, + layerwise_groups, ) from vllm.distributed.weight_transfer.ipc_engine import ( IPCTrainerInitInfo, @@ -1412,6 +1414,162 @@ class TestModuleSource: assert [m.shape for m in meta] == [tuple(t.shape) for _, t in pairs] +class TestWeightSourceGroupContract: + """`groups()` / `iter_groups()` on the WeightSource ABC. Groups are what + backends gather and free by, so the default must agree with + `layerwise_groups` over `metadata()`, restricted to what this rank holds.""" + + class _Source(WeightSource): + """Minimal source over an ordered (name, tensor) list, optionally holding + only some names (in which case it iterates only their groups).""" + + def __init__(self, names, held=None, reverse=False): + self._pairs = [(n, torch.full((2,), float(i))) for i, n in enumerate(names)] + self._held = held + self._reverse = reverse + + def metadata(self): + return [ParamMeta(n, t.dtype, tuple(t.shape)) for n, t in self._pairs] + + def held_names(self): + return self._held + + def __iter__(self): + pairs = self._pairs + if self._held is not None: + keep = set(self._held) + pairs = [(n, t) for n, t in pairs if n in keep] + return iter(list(reversed(pairs)) if self._reverse else pairs) + + def _source(self, names, held=None, reverse=False): + return self._Source(names, held, reverse) + + def test_groups_defaults_to_the_layerwise_partition(self): + names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"] + assert self._source(names).groups() == layerwise_groups(names) + + def test_groups_keeps_only_groups_holding_a_held_name(self): + names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"] + held = ["model.layers.0.a", "model.layers.1.a"] + assert self._source(names, held=held).groups() == [ + ["model.layers.0.a"], + ["model.layers.1.a"], + ] + + def test_groups_keeps_a_partially_held_group_whole(self): + """One held name selects the WHOLE group, unheld members included. + + A group is the collective unit, so its membership has to be + rank-uniform: a rank that narrowed it to what it holds would disagree + with its peers about which names group *g* covers, and the gather would + desynchronize. Selection is per group, but only whole groups; the + per-name split is the backend's, via owner sets. This is the + foreign-expert shape under expert parallelism — a rank holds some + experts of a layer, not all of them. + """ + names = ["model.layers.0.a", "model.layers.0.b", "model.layers.1.a"] + held = ["model.layers.0.a"] + assert self._source(names, held=held).groups() == [ + ["model.layers.0.a", "model.layers.0.b"], + ] + + def test_groups_order_follows_metadata_not_the_declaration(self): + """The declaration is a SET of names; the groups it selects still come + out in metadata order, so each pairs with the right ``iter_groups()`` + batch however the source listed them.""" + names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"] + source = self._source( + names, held=["model.layers.1.a", "model.layers.0.a", "model.layers.1.a"] + ) + assert source.groups() == [["model.layers.0.a"], ["model.layers.1.a"]] + assert [ns for ns, _ in source.iter_groups()] == [ + ["model.layers.0.a"], + ["model.layers.1.a"], + ] + + def test_iter_groups_batches_the_stream_per_group(self): + names = ["embed.w", "model.layers.0.a", "model.layers.0.b", "norm.w"] + batches = list(self._source(names).iter_groups()) + assert [ns for ns, _ in batches] == [ + ["embed.w"], + ["model.layers.0.a", "model.layers.0.b"], + ["norm.w"], + ] + assert all(len(ns) == len(ts) for ns, ts in batches) + + def test_iter_groups_yields_the_tensors_iteration_produced(self): + names = ["model.layers.0.a", "model.layers.0.b"] + (batch,) = list(self._source(names).iter_groups()) + _names, tensors = batch + assert [float(t[0]) for t in tensors] == [0.0, 1.0] + + def test_iter_groups_yields_only_held_groups(self): + names = ["embed.w", "model.layers.0.a", "model.layers.1.a"] + batches = list(self._source(names, held=["model.layers.1.a"]).iter_groups()) + assert [ns for ns, _ in batches] == [["model.layers.1.a"]] + + def test_out_of_order_iteration_raises(self): + """Materializing is usually a collective, so a rank that iterates out of + order deadlocks its peers -- fail loudly instead.""" + source = self._source(["model.layers.0.a", "model.layers.0.b"], reverse=True) + with pytest.raises(RuntimeError, match="iteration order must match"): + list(source.iter_groups()) + + def test_a_source_may_override_iter_groups(self): + """The extension point: materialize a whole group in one step instead of + one generator resume per tensor.""" + calls = [] + + class _Batched(TestWeightSourceGroupContract._Source): + def iter_groups(self): + for group in self.groups(): + calls.append(len(group)) + yield group, [torch.zeros(2) for _ in group] + + source = _Batched(["model.layers.0.a", "model.layers.0.b"]) + assert [ns for ns, _ in source.iter_groups()] == [ + ["model.layers.0.a", "model.layers.0.b"] + ] + assert calls == [2] + + +class TestDeferredProcessingContract: + """`defers_processing` and `drain_pending` are two halves of one contract: a + caller that takes over the update tail (running its own + `finalize_layerwise_reload` instead of going through `finish_weight_update`) + reads the flag and calls the method. Both must be answerable on any engine, or + that caller ends up reaching through a getattr.""" + + def _engines(self): + return { + name: loader() + for name, loader in WeightTransferEngineFactory._registry.items() + } + + def test_every_engine_declares_whether_it_defers(self): + for name, cls in self._engines().items(): + assert isinstance(cls.defers_processing, bool), name + + def test_every_engine_can_be_drained(self): + """The default is a no-op, so a caller never has to check whether the + method exists before calling it.""" + for name, cls in self._engines().items(): + assert callable(cls.drain_pending), name + + def test_the_default_is_not_to_defer(self): + assert WeightTransferEngine.defers_processing is False + + def test_a_synchronous_engine_drains_as_a_no_op(self): + engine = object.__new__(WeightTransferEngineFactory._registry["nccl"]()) + engine.drain_pending() # must not raise, and must not need any state + + def test_the_rdt_engine_defers_and_overrides_the_drain(self): + """The one engine the contract exists for.""" + cls = WeightTransferEngineFactory._registry["sharded_rdt"]() + assert cls.defers_processing is True + assert cls.drain_pending is not WeightTransferEngine.drain_pending + + class TestTrainerFactory: """WeightTransferTrainerFactory registry mechanics.""" diff --git a/tools/pre_commit/check_torch_cuda.py b/tools/pre_commit/check_torch_cuda.py index aec7b85d59..f444da1998 100644 --- a/tools/pre_commit/check_torch_cuda.py +++ b/tools/pre_commit/check_torch_cuda.py @@ -20,6 +20,7 @@ ALLOWED_FILES = { "vllm/platforms/", "vllm/device_allocator/", "vllm/distributed/weight_transfer/ipc_engine.py", + "vllm/distributed/weight_transfer/sharded_rdt_engine.py", "tests/distributed/test_packed_tensor.py", } diff --git a/vllm/config/weight_transfer.py b/vllm/config/weight_transfer.py index c1add61411..3e849753c2 100644 --- a/vllm/config/weight_transfer.py +++ b/vllm/config/weight_transfer.py @@ -9,7 +9,7 @@ from vllm.config.utils import config class WeightTransferConfig: """Configuration for weight transfer during RL training.""" - backend: Literal["nccl", "ipc", "sparse_nccl"] | str = "nccl" + backend: Literal["nccl", "ipc", "sparse_nccl", "sharded_rdt"] | str = "nccl" """The backend to use for weight transfer. Validated against the `WeightTransferEngineFactory` registry at engine creation time. """ diff --git a/vllm/distributed/weight_transfer/base.py b/vllm/distributed/weight_transfer/base.py index ddefb77da0..ed997a4529 100644 --- a/vllm/distributed/weight_transfer/base.py +++ b/vllm/distributed/weight_transfer/base.py @@ -3,7 +3,7 @@ """Base class for weight transfer engines.""" from abc import ABC, abstractmethod -from collections.abc import Iterator +from collections.abc import Collection, Iterator from dataclasses import dataclass, field from typing import ( TYPE_CHECKING, @@ -33,6 +33,76 @@ TTrainerInitInfo = TypeVar("TTrainerInitInfo", bound="TrainerInitInfo") # channel. The built-in `ModuleSource` uses `materialize_full_tensor`. +def _stack_key(name: str) -> tuple[str, int] | None: + """``(prefix, index)`` of the OUTERMOST integer segment, or None if there is + none. + + Outermost is what keeps a MoE layer whole: + ``model.layers.3.mlp.experts.7.w1`` keys on the layer, not the expert. + """ + parts = name.split(".") + for i, part in enumerate(parts): + if part.isdigit(): + return ".".join(parts[:i]), int(part) + return None + + +def layerwise_groups(names: list[str]) -> list[list[str]]: + """Partition flat parameter names into one group per decoder layer, keyed on + the outermost index segment of each name. + + This defines what a *group index* means for `WeightSource.groups` and + `WeightSource.iter_groups`: index *g* names the same group on every trainer + rank and every consumer, because it is derived from one rank's `metadata()` + order. + + Keying on the index rather than a literal prefix needs no per-architecture + naming table: ``model.layers.0.``, ``model.language_model.layers.0.``, + ``transformer.h.0.``, ``backbone.layers.0.`` and a vision tower's + ``visual.blocks.0.`` all partition alike. Matching one fixed prefix does not, + and its failure is silent — every name lands in a single group holding the + whole model, which defeats the per-layer bound below. + + Un-indexed names split by POSITION relative to the first indexed one: the pre + block (embeddings) and the post block (the final norm, `lm_head`, and any + inter-stack projector). Post lands last however early it arrived, which is + what a pipeline-parallel source needs — Megatron-Bridge streams the last + stage's output block *before* its layers. + + Stacks come out in first-appearance order of their prefix and ascending index + within it, whatever order the source yielded them, so a source can normalize + an arbitrary export order by flattening this partition. + + Backends that gather and free per group (sharded RDT) also use it as the unit + of transfer, which bounds their buffer sizes: without it a whole model becomes + one chunk. + """ + pre: list[str] = [] + post: list[str] = [] + stacks: dict[tuple[str, int], list[str]] = {} + order: list[tuple[str, int]] = [] + for name in names: + key = _stack_key(name) + if key is None: + (post if order else pre).append(name) + continue + if key not in stacks: + stacks[key] = [] + order.append(key) + stacks[key].append(name) + + prefix_rank: dict[str, int] = {} + for key in order: + prefix_rank.setdefault(key[0], len(prefix_rank)) + order.sort(key=lambda key: (prefix_rank[key[0]], key[1])) + + groups: list[list[str]] = [pre] if pre else [] + groups += [stacks[key] for key in order] + if post: + groups.append(post) + return groups + + def materialize_full_tensor(tensor: torch.Tensor) -> torch.Tensor: """Return a full, locally-materialized tensor ready to send. @@ -66,10 +136,13 @@ class WeightSource(ABC): case it should cache. * iteration — yields fully-materialized `(name, tensor)` pairs, one at a time. Materializing is typically a collective (FSDP `full_tensor()`, a - Megatron export), so every trainer rank must iterate the same source in the - same order in lockstep, or ranks deadlock. Under pipeline parallelism a - rank may not own a parameter at all — iterating still drives the collective - and the yielded tensor is only meaningful on the sender. + Megatron export), so the ranks that share a parameter must iterate it in + the same order in lockstep, or they deadlock. + * `held_names()` — which parameters this rank holds, for producers that are + split so each rank holds only part of the model. Defaults to all. + * `iter_groups()` — the same stream batched per gather group (see + `layerwise_groups`). Defaults to batching `__iter__`; override to + materialize a whole group in one step. `iter(source)` must yield a *fresh* pass each round. Backends with custom producer logic (Megatron export, RDT plans, MoE re-fusing) subclass this. @@ -92,6 +165,79 @@ class WeightSource(ABC): def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]: raise NotImplementedError + def held_names(self) -> "Collection[str] | None": + """The parameters this rank holds, or None for all of them. + + This is the whole ownership contract. Override it when producers are + split so each holds only part of the model — pipeline parallelism (a rank + holds some layers), expert parallelism (a rank holds some experts), or + any combination, including layouts that fit neither. A consumer routes + each name to a rank that holds it, so per-name is the granularity that + matters; the engine derives everything else from this. + + Three requirements come with overriding it: + + * `metadata()` must still describe the WHOLE model on every rank. The + group partition, the iteration checks and the consumers' pull plans are + all built from one rank's metadata, so a rank that reported only its own + share would leave the rest of the model silently un-transferred. The + sharded-RDT engine cross-checks this across ranks at init. + * Every name must be held by at least one rank, or it can never be + served. The engine raises at init naming the first orphan. + * Iteration must cover exactly `groups()` in metadata order, yielding a + real tensor for each held name and `None` for the rest. A group's + gather is a collective among the ranks that hold part of it, so the + name must still appear (to keep the order check aligned) while the + data is absent. + + Returns: + The held parameter names, or None to hold every one. + """ + return None + + def groups(self) -> list[list[str]]: + """This rank's gather groups, in metadata order: `layerwise_groups` over + `metadata()`, restricted to the groups holding at least one held name. + + A group with nothing held here is not iterated at all — its gather is a + collective among the ranks that do hold part of it. + """ + groups = layerwise_groups([m.name for m in self.metadata()]) + held = self.held_names() + if held is None: + return groups + held = set(held) + return [g for g in groups if any(n in held for n in g)] + + def iter_groups(self) -> Iterator[tuple[list[str], list[torch.Tensor]]]: + """Yield one `(names, tensors)` batch per group from `groups()`. + + The default drives `__iter__` and batches its output, checking as it goes + that the names arrive in metadata order — ranks sharing a parameter + materialize it with a collective, so a rank that iterates out of order + deadlocks its peers rather than returning wrong data. + + Override when a backend can produce a whole group at once. Materializing + is usually a collective, and driving it per group instead of per tensor + turns ~37k generator resumes into ~95 on a per-expert MoE model (worth + ~0.9s per sync there). An override must yield the same batches in the same + order as this default. + """ + it = iter(self) + for group in self.groups(): + names: list[str] = [] + tensors: list[torch.Tensor] = [] + for expected in group: + name, tensor = next(it) + if name != expected: + raise RuntimeError( + f"WeightSource yielded {name!r} but expected " + f"{expected!r}; iteration order must match metadata()." + ) + names.append(name) + tensors.append(tensor) + yield names, tensors + class ModuleSource(WeightSource): """`WeightSource` over `module.named_parameters()` — the common case. @@ -207,6 +353,31 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]): supports_draft_weight_update: bool = True + defers_processing: bool = False + """Whether `update_weights` returns before the weights are on the device. + + An engine that pipelines its GPU post-processing onto background threads + cannot let `update_weights` synchronize the device — that would block on those + threads and serialize the pipeline. Such an engine sets this True, omits the + per-update sync, and guarantees completion in `finish_weight_update` instead. + + Callers that go through `finish_weight_update` need do nothing: the engine + drains there. A caller that instead drives the tail itself — running its own + `finalize_layerwise_reload`, say — must read this flag and call + `drain_pending()` first, because with it set a returned `update_weights` means + "queued", not "applied". + """ + + def drain_pending(self) -> None: + """Block until every deferred update has been applied to the model. + + The companion to `defers_processing`: a caller that has taken over the + update tail calls this to re-establish the guarantee that + `finish_weight_update` would otherwise have given it. Idempotent, and a + no-op by default — an engine that processes synchronously has nothing to + drain, so this is always safe to call. + """ + def __init__( self, config: WeightTransferConfig, @@ -385,7 +556,11 @@ class TrainerWeightTransferEngine(ABC, Generic[TTrainerInitInfo]): Symmetric to `WeightTransferEngine` but lives in the training process. Constructed via the `trainer_init` factory classmethod; carries any backend-specific state (NCCL communicators, IPC device info, transfer - plans) on `self`. + plans) on `self`. Full-resync backends (NCCL, IPC) take a `WeightSource` at + `trainer_init` and replay it each round via the no-argument + `send_weights()`. Backends that push per-round deltas instead (e.g. sparse + patches) leave `source` as `None` and take their payload as a `send_weights` + argument. Unlike the worker engine, the trainer side does not take a `WeightTransferConfig`: the backend is selected from the init info's diff --git a/vllm/distributed/weight_transfer/factory.py b/vllm/distributed/weight_transfer/factory.py index fe2ab1bc9e..ba48642296 100644 --- a/vllm/distributed/weight_transfer/factory.py +++ b/vllm/distributed/weight_transfer/factory.py @@ -234,6 +234,12 @@ WeightTransferEngineFactory.register_engine( "SparseNCCLWeightTransferEngine", ) +WeightTransferEngineFactory.register_engine( + "sharded_rdt", + "vllm.distributed.weight_transfer.sharded_rdt_engine", + "ShardedRDTWeightTransferEngine", +) + # Trainer-side engines, parallel to the worker registry above. WeightTransferTrainerFactory.register_engine( @@ -253,3 +259,9 @@ WeightTransferTrainerFactory.register_engine( "vllm.distributed.weight_transfer.sparse_nccl_engine", "SparseNCCLTrainerWeightTransferEngine", ) + +WeightTransferTrainerFactory.register_engine( + "sharded_rdt", + "vllm.distributed.weight_transfer.sharded_rdt_trainer", + "ShardedRDTTrainerWeightTransferEngine", +) diff --git a/vllm/distributed/weight_transfer/sharded_rdt_common.py b/vllm/distributed/weight_transfer/sharded_rdt_common.py new file mode 100644 index 0000000000..a5a7ed9233 --- /dev/null +++ b/vllm/distributed/weight_transfer/sharded_rdt_common.py @@ -0,0 +1,339 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Shared pieces of the sharded-RDT backend: the op-chain allowlist, buffer +sizing and the minimum Ray version, plus ``RdtRouter`` — consumer-only, since +routing is a consumer decision. + +The gather-group partition itself is ``base.layerwise_groups``: it defines what a +group index means for any ``WeightSource``, not just this transport. +""" + +from collections.abc import Callable +from typing import Any + +import torch + +# The op-chain contract, with two enforcers. The consumer's ``FakeRDTTensor`` +# intercepts exactly these methods during the bake and records the string name; +# anything else reaches ``__torch_dispatch__`` and raises, so a loader needing +# real data fails at init rather than transferring wrong bytes. The producer +# replays a chain as ``getattr(tensor, op)(...)`` and refuses anything outside +# ``ALLOWED_OPS``, so a spoofed consumer cannot invoke arbitrary methods. +# +# Every entry must be a pure view; ``to`` in particular must never be added, as +# it is the dtype/device escape the bake exists to reject. Both sets derive from +# this one table so they cannot disagree. +SUPPORTED_OPS: dict[Callable, str] = { + torch.Tensor.narrow: "narrow", + torch.Tensor.view: "view", + torch.Tensor.reshape: "reshape", + torch.Tensor.__getitem__: "__getitem__", + torch.Tensor.unsqueeze: "unsqueeze", + torch.Tensor.squeeze: "squeeze", + torch.Tensor.transpose: "transpose", + torch.Tensor.t: "t", + torch.Tensor.permute: "permute", + torch.Tensor.flatten: "flatten", + torch.Tensor.contiguous: "contiguous", + torch.Tensor.chunk: "chunk", + # Multi-return like chunk: one child per output, trailing __getitem__(i). + torch.Tensor.unbind: "unbind", +} + +ALLOWED_OPS = frozenset(SUPPORTED_OPS.values()) + +# Ray Direct Transport landed in halves: the actor opt-in +# ``enable_tensor_transport`` in 2.49, and ``ray.experimental``'s +# ``register_nixl_memory`` / ``set_target_for_ref`` — which this transport calls +# directly — in 2.55. The floor is the tested version rather than the earliest +# one that could import; 2.55 has never been run against. +RDT_MIN_RAY_VERSION = "2.56.0" + + +def check_ray_rdt_version() -> None: + """Refuse an installed Ray older than the one this backend is tested on. + + vLLM does not depend on Ray, so there is no pin to carry this. Without the + check the failure is an opaque option-validation error out of + ``.options(enable_tensor_transport=True)`` (below 2.49) or an ImportError + raised deep in the first pull, long after init reported success (below 2.55). + + Raises: + ValueError: the installed Ray predates ``RDT_MIN_RAY_VERSION``. + """ + import importlib.metadata + + from packaging import version + + required = version.parse(RDT_MIN_RAY_VERSION) + current = version.parse(importlib.metadata.version("ray")) + if current < required: + raise ValueError( + f"The 'sharded_rdt' weight transfer backend requires Ray " + f">= {required}, the version it is tested against; Ray Direct " + f"Transport (ray.experimental.register_nixl_memory / " + f"set_target_for_ref) needs at least 2.55. Found {current}. " + f"Run `pip install -U 'ray>={required}'`." + ) + + +def assign_producer_indices( + num_producers: int, num_consumers: int, consumer_idx: int +) -> list[int]: + """Producers (global indices) that consumer ``consumer_idx`` binds.""" + p = max(1, num_producers) + c = max(1, num_consumers) + if p >= c: + return list(range(consumer_idx * p // c, (consumer_idx + 1) * p // c)) + return [consumer_idx * p // c] + + +class RdtRouter: + """Decides which producer serves each weight name, and — once bound — sends. + + Ownership is per NAME. ``owner_sets`` holds the few distinct producer sets + that occur, and ``name_owner_class[i]`` indexes it for ``names[i]``; a name + determines both its owner set and its gather group, so neither appears in the + routing API. Empty tables mean every producer holds everything. + + Any placement is expressible this way: a pipeline stage is a set of names + sharing one owner set, an expert is a name whose owner set is a single rank, + and a group produced by two stages is just two classes inside one group. + + Both engines derive the same tables from the same wire data, so they agree on + who serves what. Disagreement is not a wrong answer but a hang or a loud + misroute: a pull sent to a producer that never gathered the name trips its + served-names guard. + + Routing is consumer-only. The trainer publishes what it holds and answers + whatever arrives; it never asks who serves what. + """ + + def __init__( + self, + num_producers: int, + num_consumers: int, + owner_sets: list[list[int]] | None = None, + name_owner_class: list[int] | None = None, + names: list[str] | None = None, + group_lens: list[int] | None = None, + workers_per_replica: int = 0, + ) -> None: + """Build the routing tables from the wire data both engines receive. + + The four table arguments travel together and default together: with all + of them empty the router degrades to "every producer holds everything". + + Args: + num_producers: Trainer ranks. Owner indices are positions in this + range, and ``validate`` rejects any that fall outside it. + num_consumers: Inference workers across the whole fleet. Fixes + the id space; the block carve uses ``workers_per_replica`` + of it. + owner_sets: The distinct producer sets that occur, one row per + owner class; each row is deduplicated and sorted here. Empty + means a single class owning every producer. + name_owner_class: Parallel to ``names`` — ``name_owner_class[i]`` + indexes ``owner_sets`` for ``names[i]``. A name with no entry + falls back to class 0. + names: Every parameter name, in group-major order: concatenating + the gather groups reproduces this list exactly. The other two + tables are keyed by this order. + group_lens: Length of each gather group, consecutive over + ``names``, so they sum to ``len(names)``. Fixes name -> group + and the per-group owner union the free barrier fans out to. + workers_per_replica: Consumers per inference DEPLOYMENT. The block + carve spreads this many consumers over an owner set and every + deployment reuses that carve, so the same worker of each + deployment resolves one producer (see ``producer_for``). 0 + means one deployment: carve over the whole fleet. + + Raises: + ValueError: ``workers_per_replica`` does not divide + ``num_consumers``, so the deployments are not uniform and two + workers of one deployment would share a block index. + """ + self.num_producers = max(1, num_producers) + self.num_consumers = max(1, num_consumers) + # Width of the block carve: one deployment's worth of consumers, so + # deployments overlay rather than spread. See ``producer_for``. + self._block_consumers = min( + self.num_consumers, max(1, workers_per_replica or self.num_consumers) + ) + if self.num_consumers % self._block_consumers: + raise ValueError( + f"workers_per_replica={workers_per_replica} does not divide " + f"num_consumers={self.num_consumers}; the inference deployments " + f"must be uniform for the per-deployment block carve to be " + f"well defined." + ) + self._owner_sets = ( + [sorted(set(row)) for row in owner_sets] + if owner_sets + else [list(range(self.num_producers))] + ) + names = list(names or []) + classes = list(name_owner_class or []) + self._class_of = { + n: (classes[i] if i < len(classes) else 0) for i, n in enumerate(names) + } + # name -> gather group, and the per-group owner union the free barrier + # fans out to. Both are pure functions of the tables, so precompute once. + self._group_of: dict[str, int] = {} + self._group_owners: list[list[int]] = [] + pos = 0 + for gi, glen in enumerate(group_lens or []): + union: set[int] = set() + for n in names[pos : pos + glen]: + self._group_of[n] = gi + c = self._class_of.get(n, 0) + # Bounds-checked: a bad class must surface from validate() as a + # named ValueError, not as an IndexError from this precompute. + if 0 <= c < len(self._owner_sets): + union.update(self._owner_sets[c]) + pos += glen + self._group_owners.append(sorted(union)) + # Bound at init on the consumer; the rule methods never touch these. + self._actors: list[Any] = [] + self._produce_methods: list[Any] = [] + self.consumer_id = 0 + + # ---------------- rules (pure; safe to use unbound) ---------------- + + def class_of(self, name: str) -> int: + """The name's owner class — the planner's bucketing key, since all names + of a chunk must share one producer.""" + return self._class_of.get(name, 0) + + def owners(self, name: str) -> list[int]: + """Every producer holding ``name``.""" + return list(self._owner_sets[self.class_of(name)]) + + def group_owners(self, group_idx: int) -> list[int]: + """Every producer holding any name of ``group_idx``. + + The one group-keyed rule, because the free barrier is per group: a + consumer signals ``free_group(gi)`` at each of these and the producer + counts signals against its live-consumer total. Names route; groups free. + """ + if not self._group_owners: + return list(range(self.num_producers)) + return list(self._group_owners[group_idx]) + + def producer_for(self, consumer_id: int, name: str) -> int: + """The single producer ``consumer_id`` pulls ``name`` from. + + Blocks consumers across the name's owner set with the same rule that + binds producers globally, then rotates by the name's GROUP index so a + consumer spreads its groups over its block instead of hammering one NIC. + Rotating per group (not per name) keeps every name of a chunk on one + producer, which is what lets a chunk be a single pull. + + The carve is over ONE DEPLOYMENT (this consumer's index within its own + deployment, over ``workers_per_replica`` of them), so the same worker of + every deployment resolves the same producer for every name. Carving over + the whole fleet instead spreads the multi-owner names of different + deployments onto different producers while single-owner names still + route by ownership, so a producer serves several distinct worker + indices, each needing its own serve ring, and none of them can share a + slot. With one deployment the width is the fleet, so this is the plain + block rule. + """ + own = self.owners(name) + if not own: + raise ValueError(f"{name!r} has no owner") + block = assign_producer_indices( + len(own), self._block_consumers, consumer_id % self._block_consumers + ) + return own[block[self._group_of.get(name, 0) % len(block)]] + + def validate(self) -> None: + """Check the ownership tables can be served. + + Raises: + ValueError: an owner set is empty or out of range, or a class index + does not resolve. + """ + for c, row in enumerate(self._owner_sets): + if not row: + raise ValueError(f"owner set {c} is empty: no producer holds it") + bad = [p for p in row if not 0 <= p < self.num_producers] + if bad: + raise ValueError(f"owner set {c} out of range: {bad}") + for name, c in self._class_of.items(): + if not 0 <= c < len(self._owner_sets): + raise ValueError( + f"{name!r} has owner class {c}, but only " + f"{len(self._owner_sets)} owner set(s) were shipped" + ) + + # ---------------- send surface (needs the binding) ---------------- + + def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None: + """Attach this consumer's producer handles. Rebuilt wholesale per init, + never appended to: every owner index is a position in these lists, so a + rejoining engine that re-inits must not shift them.""" + if len(actors) != len(produce_methods): + raise ValueError("actors and produce_methods must be parallel") + self._actors = list(actors) + self._produce_methods = list(produce_methods) + self.consumer_id = consumer_id + + def _bound(self) -> None: + if not self._produce_methods: + raise RuntimeError("RdtRouter has no producers bound; call bind() first.") + + def pull(self, owner: int, keys: list, seq: int) -> Any: + """Issue one packed pull to ``owner``. + + The consumer id keys the producer's serve ring; without it every worker + is served out of ring 0 and concurrent pulls overwrite each other's blob. + ``seq`` is this call's index in the stream to ``owner`` (see ``_Chunk``): + it selects the serve slot in ISSUE order, so a slot is never repacked + while the read of its previous contents is still in flight. + """ + self._bound() + return self._produce_methods[owner].remote( + keys, consumer_id=self.consumer_id, seq=seq + ) + + def free_group(self, group_idx: int) -> list: + """Signal the group done at every producer holding any of its names.""" + self._bound() + return [ + self._actors[p].free_group.remote(group_idx) + for p in self.group_owners(group_idx) + ] + + def reserve_serve_buffers( + self, bytes_by_producer: list[int], plan_digests: list[str] | None = None + ) -> list: + """Ask each producer to pre-register a serve ring sized to the most this + consumer will pull from it. + + ``plan_digests[p]`` describes the chunks this consumer pulls from + producer ``p``, in pull order. A producer that shares one serve ring + across the consumers of several deployments compares it across them, so + a fleet whose deployments are not identical fails at init instead of + stalling mid-sync. A producer that shares nothing ignores it. + """ + self._bound() + digests: list = ( + list(plan_digests) if plan_digests else [None] * len(bytes_by_producer) + ) + return [ + self._actors[p].reserve_serve_buffer.remote( + self.consumer_id, nb, digests[p] + ) + for p, nb in enumerate(bytes_by_producer) + if nb > 0 + ] + + +def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int: + """Size a NIXL buffer / ring slot for ``nbytes``: the max of the request, an + optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is + allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a + perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers + and the producer's serve rings.""" + return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20)) diff --git a/vllm/distributed/weight_transfer/sharded_rdt_engine.py b/vllm/distributed/weight_transfer/sharded_rdt_engine.py new file mode 100644 index 0000000000..2b8187007b --- /dev/null +++ b/vllm/distributed/weight_transfer/sharded_rdt_engine.py @@ -0,0 +1,1479 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Sharded Ray Direct Transport (RDT) weight transfer engine (consumer side). + +Pulls only the slice each vLLM worker consumes under tensor/expert parallelism, +not the full HF-format tensor. + +Two phases. BAKE, once at ``init_transfer_engine``: drive ``model.load_weights`` +against ``FakeRDTTensor`` placeholders and record, per leaf module, how each +destination slice is fetched (an op chain) and where it lands (an ``as_strided`` +descriptor). REPLAY, every sync: no ``load_weights``, no ``FakeRDTTensor`` +dispatch, no discovery — pull the recorded slices in packed chunks over a ring +of receive buffers, scatter them into freshly materialized params, then quant +and kernel-copy. A live name with no recorded plan fails the plan build: there +is no fallback load. + +Weights arrive in checkpoint format: the engine drives layerwise reload itself, +in ``start_weight_update`` / ``finish_weight_update``. + +Data flow +--------- +One thing at four resolutions, over three lifetimes. ``FetchKey`` -- +``(name, op_chain)``, "which slice of which trainer tensor" -- is the atom; +everything else is bookkeeping around it. + + BAKE once at init, kept for the engine's life + FakeRDTTensor intercepts the model's loaders; each copy_ records a + _Scatter (sharded_rdt_fake): src FetchKey, owning layer, the destination + as_strided region, and the produced dtype/nbytes. + -> _name_to_plan: name -> that module's scatter list + + PLAN once, cached, _build_call_plan + _Chunk one packed pull: its scatters, deduped keys, the byte-exact + pack_layout, which producer serves it, and what to run after + (materialize / quant / free). One per (group, owner class). + _CallPlan all chunks + pre_free. + + RUN per chunk, per sync, _run_chunk_pipeline + _Chunk -> _PendingPull issued, not yet landed: Ray ref, buffer views, + ring slot. [RPC thread] + -> _ProcItem chunk + results + slot. [hand-off to the + background scatter thread] + +The RUN pair stays split on purpose: it is the thread boundary, and only +``targets`` should outlive the get -- carrying the Ray ref and the whole-buffer +blob into the queue would keep both alive for the scatter's lifetime. + +See docs/training/weight_transfer/sharded_rdt.md for the design and the measured +results behind the choices here. +""" + +import time +from dataclasses import dataclass, field +from math import prod +from typing import TYPE_CHECKING, Any, cast + +import torch + +from vllm.config.weight_transfer import WeightTransferConfig +from vllm.distributed.weight_transfer.base import ( + WeightTransferEngine, + WeightTransferInitInfo, + WeightTransferUpdateInfo, +) +from vllm.distributed.weight_transfer.sharded_rdt_common import ( + RdtRouter, + buffer_alloc_bytes, + check_ray_rdt_version, +) +from vllm.distributed.weight_transfer.sharded_rdt_fake import ( + BakeSink, + FakeRDTTensor, + FetchKey, + _Scatter, +) +from vllm.logger import init_logger + +if TYPE_CHECKING: + from vllm.config import VllmConfig + +logger = init_logger(__name__) + + +@dataclass +class _Chunk: + """One packed pull plus its post-processing (see Data flow). + + A module's copies span chunks when its experts sit in several owner classes, + so ``materialize`` fires on its FIRST chunk and ``quant`` on its LAST -- + materialize-once by construction, not by a runtime counter. ``pack_layout`` + mirrors the producer's rule byte-exactly. + """ + + scatters: "list[_Scatter]" + keys: "list[FetchKey]" + pack_layout: "list[tuple[int, torch.dtype, int, tuple[int, ...]]]" # off,dt,n,shape + pack_bytes: int + materialize: "list[Any]" + quant: "list[Any]" + free: "list[int]" + # Trainer rank serving this chunk, resolved at plan time. One per chunk: + # every name in it shares an owner class, so one producer holds them all. + owner: int + # This chunk's index among the chunks this worker pulls from ``owner``, in + # issue order. It picks the producer's serve slot (``seq % ring_depth``), + # which is what makes slot reuse safe: the pipeline drains pull i before + # issuing i+K, so the slot seq+K takes was last used by seq, whose read is + # finished. Choosing the slot in EXECUTION order instead -- as a per-call + # counter on the producer would -- can hand a slot to a pull whose + # predecessor is still being read, because Ray may start a consumer's + # concurrent produce calls in any order. Static, since the plan is. + seq: int = 0 + + +@dataclass +class _CallPlan: + """The static plan for one sync (see Data flow). Pure, so it is built once + and reused; runtime is then execution only. + + ``pre_free`` = groups with NO chunk on this worker, signaled at sync start + (owners tolerate a signal preceding its publish). With the last-chunk signals + this keeps the completeness invariant consumer-local: every group is signaled + exactly once. + """ + + chunks: "list[_Chunk]" + pre_free: "list[int]" + + +@dataclass +class _PendingPull: + """A dispatched pull whose blocking ``ray.get`` has not run (see Data flow). + + ``targets``/``blob`` must stay strongly referenced until it completes: + ``set_target_for_ref`` stores WEAKREFS, so dropping them silently reroutes the + transfer into a fallback buffer.""" + + ref: "Any" + keys: "list[FetchKey]" + targets: "list[torch.Tensor]" + blob: "torch.Tensor" + slot: int + + +@dataclass +class _ProcItem: + """A landed pull handed to the background scatter thread (see Data flow). + + ``results`` alias the ring buffer ``slot``, held as strong refs so they + outlive the RPC-thread frame until the scatter consumes them. + """ + + chunk: "_Chunk" + results: "dict[FetchKey, torch.Tensor]" + slot: int + + +@dataclass +class ShardedRDTWeightTransferInitInfo(WeightTransferInitInfo): + """Initialization info for the sharded RDT backend.""" + + trainer_actor_names: list[str] = field(default_factory=list) + """Names of all trainer Ray actors exposing the producer method (set via + ``.options(name=...)``), ordered by trainer rank. ``RdtRouter`` picks one of + them per pull, out of the name's owner set. Every actor is bound regardless, + because ``free_group`` fans out to every owner. Must be non-empty; a + single-producer trainer passes a one-element list.""" + + trainer_actor_namespace: str | None = None + """Optional Ray namespace the trainer actor(s) live in.""" + + produce_method_name: str = "rdt_produce_weights_batched" + """Name of the trainer-side producer method. It and the rest of the serve + surface (``free_group``, ``reserve_serve_buffer``) are documented where they + are implemented, on ``_RDTProducerServer`` in ``sharded_rdt_trainer.py``.""" + + names: list[str] = field(default_factory=list) + """The trainer's complete, flat param name list. The bake drives + ``model.load_weights`` over all of them once and keys the plan by source + name.""" + + dtype_names: list[str] = field(default_factory=list) + """Dtype name (e.g. 'bfloat16') for each entry of ``names``.""" + + shapes: list[list[int]] = field(default_factory=list) + """Full HF shape for each entry of ``names``.""" + + group_lens: list[int] = field(default_factory=list) + """Partition of ``names`` into gather groups, in the SAME order the trainers + gather and publish them (group-major; ``sum(group_lens) == len(names)``, and + ``names`` must be ordered to match). Required: the engine pre-builds the whole + static chunk/signal plan from it at init, and fires ``free_group`` at every + owner as each group's last chunk completes.""" + + owner_sets: list[list[int]] = field(default_factory=list) + """The distinct producer sets that occur, each a sorted list of trainer ranks + (indices into ``trainer_actor_names``). Indexed by ``name_owner_class``. + Empty means every producer holds every name.""" + + name_owner_class: list[int] = field(default_factory=list) + """Per-name index into ``owner_sets``, parallel to ``names``: which producers + hold each name. Empty means every producer holds every name. + + This one table expresses every layout the trainer can have — pipeline stages + (the names of these groups have this owner set), expert parallelism (this + expert name has this one-rank owner set), and combinations of the two — so + the engine cuts each group's baked copies into one chunk per distinct class + present and routes each chunk to that class's owner. Derived by the trainer + from ``WeightSource.held_names()``.""" + + num_consumers: int = 0 + """Total consumer count across the fleet, for M:N routing. Authoritative when + > 0; at 0 the engine infers it from ``parallel_config``, which is correct for + the supported serving modes but worth setting explicitly under M:N. Each + worker's distinct index comes from ``_global_worker_index``.""" + + num_rdt_buffers: int = 2 + """Depth of the consumer receive-buffer ring. Must match the + producer's — ``_run_chunk_pipeline``'s slot-safety argument rests on it. 2 = + double buffer: chunk i+1's serve overlaps chunk i's RDMA, and scatter(i-1) + overlaps RDMA(i) in the other slot. Keep depth x chunk_bytes under the + fabric's address-translation reach (~2-3 GB/flow on the reference 8xB200 RoCE + cluster, where K=3 measurably hurt).""" + + buffer_presize_gb: float = 0.0 + """Pre-size each packed receive-buffer slot to this many GiB + (0 = size to the first chunk + coarse 256MB round-up). Set it to cover the + model's largest atomic chunk (e.g. an untied lm_head). Sizing buffers ONCE + matters beyond perf -- see the doc's "Sizing buffers once matters beyond + throughput".""" + + replica_rank: int = 0 + """This inference engine's ordinal in the fleet (0..``num_replicas``-1). + + Multi-engine deployments run several INDEPENDENT inference engines, each with + its own self-contained parallel config, so every engine's + ``_global_worker_index`` restarts at 0 and would collide across engines. The + driver gives each engine a distinct ``replica_rank`` (with identical + ``num_replicas``) so the engine offsets its consumers into a globally distinct + range for the M:N block assignment.Default 0/1 (single + engine) needs no override.""" + + num_replicas: int = 1 + """Number of independent inference engines in the fleet. Default 1 => the + per-replica offset is 0 and consumer identity is exactly + ``_global_worker_index`` (preserves single-engine and single-DP-deployment + behavior). When > 1, ``workers_per_replica = num_consumers // num_replicas`` + and this engine's consumers occupy + ``replica_rank * workers_per_replica + _global_worker_index()``. Assumes a + uniform fleet (every replica has the same worker count). Set alongside + ``replica_rank``, by the driver.""" + + +@dataclass +class ShardedRDTWeightTransferUpdateInfo(WeightTransferUpdateInfo): + """Update info for the sharded RDT backend: intentionally EMPTY. + + The chunk/free plan is a pure function of the baked plan and the driver's + gather-group partition, both fixed for the engine's lifetime, so it is built + once at ``init_transfer_engine`` from ``ShardedRDTWeightTransferInitInfo``'s + ``names`` + ``group_lens``. ONE ``update_weights`` per sync then just re-runs + that plan; there is nothing per-sync to carry. + """ + + +class ShardedRDTWeightTransferEngine( + WeightTransferEngine[ + ShardedRDTWeightTransferInitInfo, + ShardedRDTWeightTransferUpdateInfo, + ] +): + """Pull-based RDT/NIXL backend that transports only the slice each worker + consumes. + + Requires ``distributed_executor_backend="ray"``, ``nixl`` in the shared env, + a named trainer actor exposing a ``@ray.method(tensor_transport="nixl")`` + producer, and weight loaders that stay inside ``SUPPORTED_OPS`` — anything + needing real data (``.to``, ``.item``, arithmetic, bool-mask indexing) + raises during the bake. + + The plan is baked once at ``init_transfer_engine`` into one scatter list + per fully-loaded leaf module, indexed by source name; every + ``update_weights`` replays the modules its gathered names cover. + """ + + init_info_cls = ShardedRDTWeightTransferInitInfo + update_info_cls = ShardedRDTWeightTransferUpdateInfo + # receive_weights pulls synchronously but defers GPU post-processing to a + # background thread so it overlaps the next chunk's pull, so ``update_weights`` + # skips the base's device sync and ``finish_weight_update`` drains the + # deferred work before finalize. + defers_processing = True + # The baked replay plan is a function of one concrete model's parameter + # layout, so a separate draft model cannot reuse it. + supports_draft_weight_update = False + + def __init__( + self, + config: WeightTransferConfig, + vllm_config: "VllmConfig", + device: torch.device, + model: torch.nn.Module, + ) -> None: + super().__init__(config, vllm_config, device, model) + # This engine's data plane is Ray's: the consumer resolves each trainer's + # serve actor with ``ray.get_actor`` and pulls over NIXL through the + # workers' tensor transport, neither of which exists unless the workers + # ARE Ray actors. Caught here, at construction, because the alternative + # is an opaque failure during the first handshake. A single-worker + # deployment defaults to "uni", so "ray" has to be asked for explicitly. + backend = vllm_config.parallel_config.distributed_executor_backend + # A custom Executor subclass is a deliberate override we cannot judge; + # only reject the named backends that are definitely not Ray. + if isinstance(backend, str) and backend != "ray": + raise ValueError( + f"The 'sharded_rdt' weight transfer backend requires " + f"distributed_executor_backend='ray', got {backend!r}. Pass " + f"--distributed-executor-backend ray (and keep " + f"VLLM_USE_RAY_V2_EXECUTOR_BACKEND=1, the default, so the " + f"workers are RayExecutorV2 actors with tensor transport)." + ) + # Ownership table, routing rules, and (once bound) every producer's + # handle — pulls go to one owner per chunk, but free_group fans out to + # every owner of the group, so all handles are bound. + self._router: RdtRouter | None = None + # Driver-supplied total consumer count (init_info.num_consumers); 0 => infer + # from parallel_config. See _num_consumers. + self._num_consumers_override: int = 0 + # Source name -> the plan for the module consuming it. Several names of a + # fused module share one entry; replay dedups. A live name absent here + # fails the plan build. + self._name_to_plan: dict[str, list[_Scatter]] = {} + # name -> (dtype_name, shape) for every init name; the bake builds its + # lazies from this. + self._name_meta: dict[str, tuple[str, list[int]]] = {} + # Names whose copy_ fired during the bake. Unbaked names not here never + # move data (e.g. non-local EP experts), so receive_weights skips them. + self._live_names: set[str] = set() + + # ---- Consumer-side pre-registered receive buffers ---------------------- + # One persistent 1-D buffer per dtype per ring slot, STORAGE-registered with + # NIXL once, so every per-slice view hits Ray's registration cache and the + # recv path does zero registration in steady state. See the doc's "Sizing + # buffers once matters beyond throughput" for why they rarely grow. + # + # Ring depth is both the slot count and the in-flight pull count, one + # quantity from ``num_rdt_buffers``. Decoupling them with a spare slot + # measurably slows the sync. + self._ring_depth = 1 + self._dest_buffers: list[dict[torch.dtype, torch.Tensor]] = [{}] + self._slot_read_done: list[Any] = [] # one torch.cuda.Event per slot + # Generation counters guarding the events (see _ensure_proc_worker). + self._slot_queued: list[int] = [] + self._slot_done: list[int] = [] + self._slot_cv: Any = None + self._pull_slot = 0 + # (id(chunk), slot) -> (buffer data_ptr, per-key views). + # Lives on the engine, not on the _Chunk: the plan is static and shared + # across syncs, so a per-slot runtime cache has no business inside it. + self._targets_cache: dict[tuple[int, int], tuple[int, list[torch.Tensor]]] = {} + self._buffer_presize = 0 # bytes; set from init_info + self._pending_frees: list[Any] = [] # free_group signal refs, drained per sync + # The STATIC plan: built once — at init from init_info.group_lens, else + # lazily on the first update_weights — and reused. Non-None means + # update_weights ignores its per-sync names. + self._cached_plan: _CallPlan | None = None + # Completed syncs. The chunk pipeline runs SERIAL during sync 0, when both + # sides still register buffers, and pipelines from sync 1. See the doc. + self._completed_syncs = 0 + + # ---- Background post-processing thread (pull/process pipelining) ------- + # The RPC thread hands pulled buffer views to this worker, which runs + # materialize/scatter/quant/kernel_copy on its own stream while the next + # pull proceeds. drain_pending() joins it before finalize. + self._proc_queue: Any | None = None + self._proc_thread: Any | None = None + self._proc_stream: Any | None = None + # Dedicated quant thread (pass 2): own queue + stream so quant never + # sits between two items' scatters on the scatter thread (measured to + # stall the RPC thread's slot handshake ~0.5-0.75s/iter). + self._quant_queue: Any | None = None + self._quant_thread: Any | None = None + self._quant_stream: Any | None = None + self._proc_error: BaseException | None = None + + def init_transfer_engine(self, init_info: ShardedRDTWeightTransferInitInfo) -> None: + """Configure the ring, bind the producers, bake the replay plan, and + pre-register every NIXL buffer -- in that order, because each step depends + on the previous one. + + The bake drives ``model.load_weights`` and the pre-registration blocks on + RPCs to the producers, so this is a heavyweight one-off; every later + ``update_weights`` is pure replay. + """ + # Read the attribute directly: if a vLLM bump renames it, this guard + # must fail loudly at init rather than silently disappear. + if self.parallel_config.enable_eplb: + raise RuntimeError( + "sharded_rdt does not support EPLB (enable_eplb=true): dynamic " + "expert rearrangement invalidates the baked replay plan, which " + "records each expert's destination slot once at init — after a " + "rearrangement the replay would silently load weights into the " + "wrong expert slots. Disable EPLB or use another weight-transfer " + "backend." + ) + # The worker is often a different install from whatever spawned it, so + # check here too and not only where the actor options are set. + check_ray_rdt_version() + self._configure_ring(init_info) + self._resolve_producers(init_info) + + # A pure dry run: the trainer's gather cache is empty at init, so nothing + # can (or does) get pulled -- we only record how each slice is fetched and + # where it lands, then restore the model. + self._bake(init_info) + self._build_static_plan(init_info) + # Register ALL NIXL memory now, while the fabric is idle: dma-buf GPUDirect + # registration concurrent with in-flight RDMA intermittently fails + # (ibv_reg_mr 'Bad address'), which bites under M:N fan-in. Sizes come from + # the static plan, so this is exact. + self._preregister_at_init() + # Start the background post-processing worker (pull/process pipelining). + self._ensure_proc_worker() + + def _configure_ring(self, init_info: ShardedRDTWeightTransferInitInfo) -> None: + """Ring depth K. + + Must run before ``_ensure_proc_worker`` creates the per-slot events and + counters, and before any buffer is grown (both happen on the first pull). + """ + self._num_consumers_override = int(init_info.num_consumers or 0) + k = max(1, int(init_info.num_rdt_buffers)) + self._ring_depth = k + self._dest_buffers = [{} for _ in range(k)] + self._targets_cache = {} + self._buffer_presize = int(float(init_info.buffer_presize_gb) * (1 << 30)) + logger.info( + "[RDT-RING] active_pulls=%d slots=%d presize=%.2fGiB", + k, + k, + self._buffer_presize / (1 << 30), + ) + + def _resolve_producers(self, init_info: ShardedRDTWeightTransferInitInfo) -> None: + """Work out this worker's consumer identity, build the router, and bind + EVERY producer actor. + + Pull routing is M:N: each chunk goes to ONE producer holding every name + in it (see ``RdtRouter``), so one producer serves a whole pull. All + producers are bound regardless, because the per-group ``free_group`` + signal fans out to every owner of a group, including producers this + worker never pulls from. + """ + try: + import ray + except ImportError as e: + raise RuntimeError( + "Ray is required for the 'sharded_rdt' weight transfer " + "backend. Install Ray and run workers as Ray actors " + "(distributed_executor_backend='ray')." + ) from e + + producer_names = list(init_info.trainer_actor_names) + if not producer_names: + raise RuntimeError( + "Sharded RDT engine requires a trainer producer: set " + "init_info.trainer_actor_names." + ) + + consumer_id = self._resolve_consumer_id(init_info) + name_owner_class = list(init_info.name_owner_class or []) + if name_owner_class and len(name_owner_class) != len(init_info.names): + raise RuntimeError( + f"Sharded RDT engine: {len(name_owner_class)} owner-class entries " + f"for {len(init_info.names)} names." + ) + router = RdtRouter( + len(producer_names), + self._num_consumers(), + list(init_info.owner_sets) or None, + name_owner_class or None, + list(init_info.names), + list(init_info.group_lens), + # Carve the block per deployment so every replica's worker w pulls + # each name from the same producer, which is what lets that producer + # serve all of them from one slot. + workers_per_replica=self._workers_per_replica(init_info), + ) + router.validate() + + # A fresh router per init, never appended to: every owner index is a + # position in these lists, so a rejoining engine's second + # init_transfer_engine must not shift them. + actors: list[Any] = [] + methods: list[Any] = [] + + for chosen_name in producer_names: + try: + actor = ray.get_actor( + chosen_name, + namespace=init_info.trainer_actor_namespace, + ) + except ValueError as e: + raise RuntimeError( + f"Sharded RDT engine could not find trainer actor " + f"{chosen_name!r} (namespace=" + f"{init_info.trainer_actor_namespace!r})." + ) from e + # ``ray.get_actor`` resolves by NAME, in a process that never + # imported the producer's class, and ``enable_tensor_transport`` + # lives in that class's metadata: the creating process passes + # ``meta.enable_tensor_transport``, and Ray infers it from the + # class's ``@ray.method(tensor_transport=...)`` decorators. A + # name-resolved handle therefore always reports False and the + # dispatch guard rejects the pull, even though the trainer set the + # option. Inherent to resolving by name across processes, not a + # version bug; it fails on Ray 2.56.0. + # + # Forcing it skips no validation. Ray's next guard, + # ``actor_has_tensor_transport``, asks the LIVE actor whether it can + # build a NIXL agent and still runs, and ``_spawn_server`` hardcodes + # the actor option, so a misconfigured producer cannot reach here. + actor._ray_enable_tensor_transport = True + actors.append(actor) + methods.append(getattr(actor, init_info.produce_method_name)) + router.bind(actors, methods, consumer_id) + self._router = router + logger.info( + "Sharded RDT engine (consumer %d) bound to all %d producers " + "(batched method %r, %d owner class(es))", + consumer_id, + len(producer_names), + init_info.produce_method_name, + len(init_info.owner_sets) or 1, + ) + + def _workers_per_replica(self, init_info: ShardedRDTWeightTransferInitInfo) -> int: + """Consumers per inference deployment, assuming a uniform fleet. + + Read by this worker's consumer id and by the router's block carve, which + must not disagree, so it is derived once here. + """ + return max(1, self._num_consumers() // max(1, int(init_info.num_replicas or 1))) + + def _resolve_consumer_id(self, init_info: ShardedRDTWeightTransferInitInfo) -> int: + """This worker's DISTINCT index in 0..C-1 across the whole fleet. + + Within one engine that is ``_global_worker_index()``. But a fleet of + INDEPENDENT engines (each with its own parallel config) restarts that + index at 0 per engine, so each engine offsets into its own range using + ``replica_rank``: with a uniform fleet, + ``workers_per_replica = C // num_replicas``. ``num_replicas`` defaults to + 1 (offset 0), preserving single-engine and single-DP-deployment behaviour. + """ + replica_rank = max(0, int(init_info.replica_rank or 0)) + return ( + replica_rank * self._workers_per_replica(init_info) + + self._global_worker_index() + ) + + def _build_static_plan(self, init_info: ShardedRDTWeightTransferInitInfo) -> None: + """Build the chunk/free plan once. It never changes across syncs, so + ``update_weights`` needs no per-sync names.""" + if not init_info.group_lens: + raise ValueError( + "Sharded RDT engine requires init_info.group_lens (the gather-" + "group partition of init_info.names)." + ) + if sum(init_info.group_lens) != len(init_info.names): + raise ValueError( + f"init_info.group_lens sums to {sum(init_info.group_lens)} " + f"but {len(init_info.names)} names were given." + ) + self._cached_plan = self._build_call_plan(init_info.names, init_info.group_lens) + logger.info( + "[RDT-PLAN] pre-built static call plan at init: %d chunks", + len(self._cached_plan.chunks), + ) + + def _preregister_at_init(self) -> None: + """Register every NIXL buffer this worker will use at init, before any + transfer runs, so nothing registers during the sync-0 RDMA churn. + + Both sides are sized from the static plan: receive buffers are + ``ring_depth`` slots at the largest chunk's ``pack_bytes``, and each bound + producer is asked to pre-register a serve ring at the max bytes this + consumer will pull from it. A no-op when this worker has no chunks.""" + plan = self._cached_plan + if plan is None or not plan.chunks: + return + from ray.experimental import register_nixl_memory + + # (a) consumer receive buffers — one per ring slot, at the largest chunk. + max_pack = max(c.pack_bytes for c in plan.chunks) + alloc = buffer_alloc_bytes(max_pack, self._buffer_presize) + for slot in range(self._ring_depth): + buffer = self._dest_buffers[slot].get(torch.uint8) + if buffer is None or buffer.numel() < alloc: + buffer = torch.empty(alloc, dtype=torch.uint8, device=self.device) + register_nixl_memory(buffer) + self._dest_buffers[slot][torch.uint8] = buffer + + # (b) producer serve rings — max bytes this consumer pulls from each + # bound producer, plus the per-producer plan digest a producer sharing + # one ring across deployments checks them against. + assert self._router is not None + serve_bytes = [0] * self._router.num_producers + per_owner_keys: list[list] = [[] for _ in range(self._router.num_producers)] + for c in plan.chunks: + serve_bytes[c.owner] = max(serve_bytes[c.owner], c.pack_bytes) + per_owner_keys[c.owner].append(c.keys) + digests = [_plan_digest(keys) for keys in per_owner_keys] + import ray + + refs = self._router.reserve_serve_buffers(serve_bytes, digests) + if refs: + ray.get(refs) # block until every serve ring is registered + logger.info( + "[RDT-PLAN] pre-registered %d receive slots (%.0f MiB each) + serve " + "rings on %d producer(s) %s", + self._ring_depth, + alloc / (1 << 20), + len(refs), + [nb // (1 << 20) for nb in serve_bytes], + ) + + def _global_worker_index(self) -> int: + """This worker's stable, distinct global index across the inference fleet: + ``data_parallel_index * world_size + rank`` over the TP*PP world. + + ``data_parallel_index``, not ``data_parallel_rank``: vLLM resets the + latter to 0 in a dense worker but keeps the former as the distinct global + DP rank. Same formula as the sibling ``nccl_engine``, so dense-via-TP and + MoE-via-DP+EP both yield distinct 0..C-1.""" + pc = self.parallel_config + return pc.data_parallel_index * pc.world_size + pc.rank # world_size = TP*PP + + def _num_consumers(self) -> int: + """Total inference-worker count. Prefers the driver-supplied + ``init_info.num_consumers`` (authoritative -- the driver knows the whole + fleet); else ``world_size_across_dp``, the same stride + ``_global_worker_index`` indexes with, so the two agree at any pp.""" + if self._num_consumers_override > 0: + return self._num_consumers_override + return self.parallel_config.world_size_across_dp + + def start_weight_update(self) -> None: + """Put the model's params on meta so layerwise reload streams them in + as each layer's slices land. Baked replay uses checkpoint format.""" + from vllm.model_executor.model_loader.reload import ( + initialize_layerwise_reload, + ) + + initialize_layerwise_reload(self.model) + + def finish_weight_update(self) -> None: + """Drain the deferred pull/process pipeline (so every layer is fully + loaded) before finalizing the layerwise reload.""" + from vllm.model_executor.model_loader.reload import ( + finalize_layerwise_reload, + ) + + self.drain_pending() + finalize_layerwise_reload(self.model, self.model_config) + + def update_weights(self, update_info: dict[str, Any]) -> None: + """Receive one update. Unlike the base, does NOT issue a per-update + device sync: post-processing is deferred to background threads and a + sync here would block on them and serialize the pull/process pipeline. + Completion is guaranteed by ``drain_pending`` in + ``finish_weight_update``.""" + self.receive_weights(self.parse_update_info(update_info)) + + def receive_weights( + self, + update_info: ShardedRDTWeightTransferUpdateInfo, + ) -> None: + """Pull + replay the baked leaf modules the sync covers. + + The chunk/free plan is STATIC across syncs — a pure function of the baked + plan and the driver's group partition — so it was built once at init and + every sync just re-runs the pipeline over its self-describing chunks, with + no per-sync bookkeeping and an empty ``update_info``. + + Assumes each baked module's source names fall within one gather group, + which the per-layer / pre / post partition guarantees (a leaf module's + sources all live in one decoder layer). A module that did span groups + would be planned once per group and could pull a name whose group the + pipeline already freed, which parks the pull until the producer's stall + watchdog fires. + """ + del update_info # the plan is static; nothing arrives per sync + if self._router is None: + raise RuntimeError( + "Sharded RDT engine not initialized. Call init_transfer_engine() first." + ) + # Surface any error the background thread hit on a prior item promptly. + self._raise_proc_error() + if self._cached_plan is None: + raise RuntimeError( + "Sharded RDT engine has no call plan: init_info.group_lens must " + "be supplied at init_transfer_engine()." + ) + self._run_chunk_pipeline(self._cached_plan) + + def _build_fake_weights( + self, + names: list[str], + sink: "BakeSink", + device: torch.device, + ) -> list[tuple[str, torch.Tensor]]: + """Zero-storage lazies for ``names``, dtype/shape from the init metadata, + all feeding the bake's recording sink.""" + return [ + ( + name, + FakeRDTTensor( + name=name, + shape=torch.Size(self._name_meta[name][1]), + dtype=_dtype_from_name(self._name_meta[name][0]), + device=device, + sink=sink, + ), + ) + for name in names + ] + + def _issue_pull(self, chunk: "_Chunk", slot: int) -> "_PendingPull": + """Reserve ``slot``, lay the targets out in its buffer, dispatch the + produce RPC and point the transfer at the buffer — WITHOUT the blocking + ``ray.get`` (that is ``_complete_pull``). The chunked pipeline issues + chunk i+1 before completing chunk i, so the producer serves the next + chunk while the in-flight RDMA streams. + + Slot-reuse guard, both stages required: a generation wait (the CUDA event + binds only to its LAST record, so synchronizing before the background + thread recorded this item's event passes silently — observed as + nondeterministic weight corruption), then the event synchronize. It must + precede ``set_target_for_ref``, not just the get: the transfer may start + any time after the metadata push. See the doc's "slot generation + handshake". + """ + from ray.experimental import register_nixl_memory, set_target_for_ref + + if self._slot_read_done: + with self._slot_cv: + while self._slot_done[slot] < self._slot_queued[slot]: + self._slot_cv.wait(timeout=1.0) + self._slot_read_done[slot].synchronize() + + # Every slice is byte-packed into ONE uint8 buffer, 16B-aligned in + # keys order, under one NIXL descriptor. ``chunk.pack_layout`` mirrors the + # producer's rule byte-exactly; ``targets`` carves the dtype views back out. + keys = chunk.keys + cur = chunk.pack_bytes + buffers = self._dest_buffers[slot] + buffer = buffers.get(torch.uint8) + if buffer is None or buffer.numel() < cur: + # Size ONCE with headroom: Ray's desc cache is keyed by data_ptr and + # outlives its tensor, so a regrowth can false-hit a recycled pointer + # and skip registering the new extent -> NIXL_ERR_NOT_FOUND. + alloc = buffer_alloc_bytes(cur, self._buffer_presize) + buffer = torch.empty(alloc, dtype=torch.uint8, device=self.device) + register_nixl_memory(buffer) + buffers[torch.uint8] = buffer + targets = self._pull_targets(chunk, slot, buffer) + # One produce RPC serves the whole packed chunk, since the group's owner + # holds every slice. The buffer view must stay strongly referenced through + # the get: set_target_for_ref stores WEAKREFS, and a dropped target + # reroutes the transfer into a fallback buffer. + blob = buffer[:cur] + assert self._router is not None + ref = self._router.pull(chunk.owner, keys, chunk.seq) + set_target_for_ref(ref, [blob]) + return _PendingPull( + ref=ref, + keys=keys, + targets=targets, + blob=blob, + slot=slot, + ) + + def _pull_targets( + self, chunk: "_Chunk", slot: int, buffer: torch.Tensor + ) -> "list[torch.Tensor]": + """Per-key dtype views into ``slot``'s buffer. + + The packed layout is static, so the views are built once per (chunk, slot) + rather than once per pull -- rebuilding them cost ~1150 Python ops per pull + at 235B. Keyed on the buffer pointer as well, so a regrow invalidates + instead of handing back views into a freed buffer. + """ + cached = self._targets_cache.get((id(chunk), slot)) + if cached is not None and cached[0] == buffer.data_ptr(): + return cached[1] + targets = [ + buffer[off : off + n * dt.itemsize].view(dt).reshape(shape) + for off, dt, n, shape in chunk.pack_layout + ] + self._targets_cache[(id(chunk), slot)] = (buffer.data_ptr(), targets) + return targets + + def _complete_pull(self, pending: "_PendingPull") -> "dict[FetchKey, torch.Tensor]": + """Blocking half of a pull: the NIXL read lands during this ``ray.get``.""" + import ray + + ray.get(pending.ref) + return dict(zip(pending.keys, pending.targets)) + + # ---------------- Bake (dry run, at init) / replay ---------------- + + def _bake(self, init_info: ShardedRDTWeightTransferInitInfo) -> None: + """Bake the replay plan once, as a self-driven meta dry run. + + Puts the params on meta, then drives ``model.load_weights`` over + ``init_info.names`` through the model's ORIGINAL loaders (the stamps + bypass ``online_process_loader``, so ``_layerwise_process`` is never in + the path). Nothing materializes or pulls; the fake's ``copy_`` records + the source op chain and the meta destination's geometry. Afterwards one + scatter list per fully-loaded leaf module (copied numel == loadable + size) is indexed by source name; a partial or unrecordable module fails + the plan build. The model is restored. + + This leans on layerwise internals a public API should expose first-class: + a currently-loading hook instead of monkeypatched stamps, a dry-run mode + instead of bypassing ``online_process_loader``, and an + ``abort_layerwise_reload`` instead of ``_restore_after_dry_run``. + """ + from vllm.model_executor.model_loader.reload.layerwise import ( + initialize_layerwise_reload, + ) + from vllm.model_executor.model_loader.reload.utils import get_layer_size + + names, dtype_names, shapes = ( + init_info.names, + init_info.dtype_names, + init_info.shapes, + ) + self._name_meta = {n: (d, s) for n, d, s in zip(names, dtype_names, shapes)} + if not names: + return + + model = self.model + recorder = BakeSink() + + _t0 = time.perf_counter() + with torch.device(self.device): + # Meta-restore params + save kernel tensors (we bypass the loader + # wrapping it installs, below). + initialize_layerwise_reload(model) + # Stamp the *original* loaders (bypassing online_process_loader), so + # the single load pass runs the loaders on meta and records via the + # fake's copy_ — with no inline _layerwise_process, no deferral. + self._install_recording_stamps(model, recorder) + model.load_weights(self._build_fake_weights(names, recorder, self.device)) + # Keep only fully-loaded modules (copied numel >= loadable size, the + # test online_process_loader uses): a partial module leaves unwritten + # regions that finalize would init, so baking it scatters garbage. + for module, recorded in recorder.copies_by_layer.items(): + if not recorded or any(c is None for c in recorded): + continue # unrecordable copy_ -> slow path + # Guard above guarantees every entry is a real _Scatter. + copies = cast("list[_Scatter]", recorded) + copied = sum(prod(c.shape) for c in copies) + if copied < get_layer_size(module): + continue # partial -> slow path + for c in copies: + # Every name of the module shares ONE list, so identity + # dedups them back to one module at plan time. + self._name_to_plan[c.src[0]] = copies + self._restore_after_dry_run(model) + + # Names whose copy_ fired during the bake. Names not in here no-op for + # this worker (e.g. foreign-EP experts) and are skipped; live names that + # did not bake fail the plan build. + self._live_names = set(recorder.copied_names) + + n_groups = len({id(g) for g in self._name_to_plan.values()}) + logger.info( + "Sharded RDT dry-run baked %d/%d names into %d leaf modules " + "(%d live) in %.3fs", + len(self._name_to_plan), + len(names), + n_groups, + len(self._live_names), + time.perf_counter() - _t0, + ) + + def _install_recording_stamps( + self, model: torch.nn.Module, recorder: "BakeSink" + ) -> None: + """Wrap each loadable param's ``weight_loader`` to stamp + ``recorder.current = (leaf_module, param_name)`` before delegating to the + original loader, so the fake's ``copy_`` can attribute each recorded copy. + ``functools.wraps`` keeps the loader's real signature (so vLLM's + ``_layerwise_process`` ``param`` redirect still works if a stamp leaks), + and ``_rdt_stamp_inner`` tags it so ``_restore_after_dry_run`` can unwrap it. + """ + import functools + + from vllm.model_executor.model_loader.reload.layerwise import ( + _get_original_loader, + ) + from vllm.model_executor.model_loader.reload.utils import get_layer_tensors + from vllm.model_executor.model_loader.weight_utils import default_weight_loader + + def _make_stamp(layer, name, inner, added=False): + @functools.wraps(inner) # keep ``inner``'s signature (incl. ``param``) + def stamp(*args, **kwargs): + recorder.current = (layer, name) + try: + return inner(*args, **kwargs) + finally: + recorder.current = None + + # Tag so _restore_after_dry_run can detect and unwrap leaked stamps, + # and so a second bake doesn't double-wrap. + stamp._rdt_stamp_inner = inner # type: ignore[attr-defined] + stamp._rdt_stamp_added = added # type: ignore[attr-defined] + return stamp + + for module in model.modules(): + for name, tensor in get_layer_tensors(module).items(): + if getattr(tensor, "weight_loader", None) is None: + # A param with NO loader (e.g. GLM's router bias, a plain + # nn.Parameter) is still loaded by the model's load_weights + # through the getattr(param, "weight_loader", + # default_weight_loader) fallback — unstamped, its bake copy + # would be unattributable, failing the module's coverage + # gate and the plan build. Stamp the same default loader + # the fallback would pick; the restore deletes it again. + tensor.weight_loader = _make_stamp( + module, name, default_weight_loader, added=True + ) + continue + # Bypass online_process_loader: stamp the *original* loader. + original = _get_original_loader(tensor) + tensor.weight_loader = _make_stamp(module, name, original) + + def _restore_after_dry_run(self, model: torch.nn.Module) -> None: + """Restore each layerwise layer's saved kernel tensors without pulling + (a real ``finalize_layerwise_reload`` would materialize/load) and reset + its info. Also unwrap any recording ``stamp`` left on the params, since a + leaked stamp would sit under the next sync's ``online_process_loader`` and + silently break ``_layerwise_process``'s ``param`` redirect. + """ + from vllm.model_executor.model_loader.reload.layerwise import ( + LAYERWISE_INFO, + _place_kernel_tensors, + ) + from vllm.model_executor.model_loader.reload.utils import get_layer_tensors + + for layer in model.modules(): + info = LAYERWISE_INFO.get(layer) + if info is not None and info.can_load(): + if info.kernel_tensors is not None: + _place_kernel_tensors(layer, info) + info.reset() + # Unwrap any recording stamps left on the (now-restored) params so they + # never leak into a later update_weights. ``_rdt_stamp_inner`` is set by + # ``_install_recording_stamps``; unwrap repeatedly in case of nesting. + for module in model.modules(): + for _name, tensor in get_layer_tensors(module).items(): + loader = getattr(tensor, "weight_loader", None) + added = False + while loader is not None and hasattr(loader, "_rdt_stamp_inner"): + added = added or getattr(loader, "_rdt_stamp_added", False) + loader = loader._rdt_stamp_inner + tensor.weight_loader = loader + if added: + # The stamp was ATTACHED to a param that had no loader + # (see _install_recording_stamps); leave none behind. + del tensor.weight_loader + if hasattr(model, "_original_do_torchao_reload"): + model._do_torchao_reload = model._original_do_torchao_reload + + # ---------------- Background post-processing (pull/process pipeline) ------- + + def _ensure_proc_worker(self) -> None: + """Lazily create the per-slot events, the background CUDA stream, the work + queue, and the single processing thread. Idempotent.""" + if self._proc_thread is not None: + return + import queue + import threading + + self._slot_read_done = [torch.cuda.Event() for _ in range(self._ring_depth)] + # Generation handshake for the events above: the RPC thread counts items + # queued per slot, this thread counts records, and a pull may synchronize() + # only once done[slot] has caught up with queued[slot]. Without it the + # synchronize can bind to a stale record and pass, letting the next RDMA + # overwrite a slot under a pending scatter. See the doc's "The slot + # generation handshake". + self._slot_queued = [0] * self._ring_depth + self._slot_done = [0] * self._ring_depth + self._slot_cv = threading.Condition() + self._proc_stream = torch.cuda.Stream(device=self.device) + self._proc_queue = queue.Queue() + self._proc_error = None + t = threading.Thread( + target=self._proc_worker_loop, name="rdt-postprocess", daemon=True + ) + self._proc_thread = t + t.start() + self._quant_stream = torch.cuda.Stream(device=self.device) + self._quant_queue = queue.Queue() + qt = threading.Thread( + target=self._quant_worker_loop, name="rdt-quant", daemon=True + ) + self._quant_thread = qt + qt.start() + + def _proc_worker_loop(self) -> None: + """Single persistent thread: run each queued item's process phase on the + background stream. Exits on the ``None`` sentinel (shutdown). An item that + raises is recorded in ``_proc_error`` and re-raised on the RPC thread / + at drain, so a failed sync fails loudly rather than corrupting silently. + """ + torch.cuda.set_device(self.device) + q = self._proc_queue + assert q is not None + while True: + item = q.get() + try: + if item is None: + return + # _process_item publishes the slot generation internally (in a + # finally around its scatter pass), so an error here still + # unblocks the RPC thread's generation wait. + self._process_item(item) + except BaseException as e: # noqa: BLE001 - surfaced on the RPC thread + self._proc_error = e + logger.exception("RDT background post-processing failed") + finally: + q.task_done() + + def _mark_slot_done(self, slot: int) -> None: + """Publish that a queued item's read-done event has been recorded (or the + item failed) so a pull waiting to reuse ``slot`` can proceed to its + CUDA-event synchronize.""" + with self._slot_cv: + self._slot_done[slot] += 1 + self._slot_cv.notify_all() + + def _raise_proc_error(self) -> None: + """Re-raise (once) any error captured by the background thread.""" + if self._proc_error is not None: + err = self._proc_error + self._proc_error = None + raise RuntimeError("RDT background post-processing failed") from err + + def drain_pending(self) -> None: + """Block until the background thread has processed every queued item and + its stream work is complete, then re-raise any error it hit. Called from + the worker's ``finish_weight_update`` before ``finalize_layerwise_reload`` + so every baked layer is fully loaded (and ``info.reset()``-ed) first.""" + if self._proc_queue is not None: + self._proc_queue.join() # every put() item task_done()'d + # The scatter thread feeds the quant thread, so join it SECOND (all + # completed-group batches have been put by now), then sync both streams + # so finalize sees fully-materialized, quanted, reset layers. + if self._quant_queue is not None: + self._quant_queue.join() + if self._proc_stream is not None: + self._proc_stream.synchronize() + if self._quant_stream is not None: + self._quant_stream.synchronize() + self._raise_proc_error() + # Ensure every fired free_group signal has EXECUTED on + # the producer before the sync ends: ``begin_sync`` resets the producer's + # per-group signal counts, so a signal landing after the next sync + # started would credit a group it does not belong to, over-crediting the + # gather-lookahead backpressure (extra gather resident -> trainer OOM + # risk). + if self._pending_frees: + import ray + + try: + ray.get(self._pending_frees) + finally: + self._pending_frees.clear() + # One sync iteration fully drained; buffers/registrations are at (or + # nearer) high-water — the chunk pipeline may issue ahead from now on. + self._completed_syncs += 1 + + def _dispatch_item(self, item: "_ProcItem") -> None: + """Hand one chunk item to the background scatter thread. + + Counts the item against its slot BEFORE dispatch: the next pull into + that slot must wait until the background thread has processed (and + RECORDED the read-done event for) every item ever queued on it. + """ + with self._slot_cv: + self._slot_queued[item.slot] += 1 + assert self._proc_queue is not None + self._proc_queue.put(item) + + def _chunk_module_scatters( + self, modules: "list[list[_Scatter]]" + ) -> "list[tuple[int, list[_Scatter]]]": + """Cut the modules' copies into one chunk per distinct owner class + present, ascending by class index, as ``(class_idx, scatters)`` pairs. + + A chunk is one packed pull, so every name in it must share a producer — + which is exactly what an owner class is. The cut is a pure function of + the bake and the ownership table. Copy order within a chunk is bake + order, and a module's copies may span chunks (materialize/quant fire on + its first/last chunk; see ``_build_call_plan``).""" + assert self._router is not None + by_class: dict[int, list[_Scatter]] = {} + for copies in modules: + for c in copies: + by_class.setdefault(self._router.class_of(c.src[0]), []).append(c) + return [(ci, by_class[ci]) for ci in sorted(by_class)] + + def _build_call_plan(self, names: list[str], group_lens: list[int]) -> "_CallPlan": + """Build the STATIC plan for one whole-sync call. + + Pure — no pulls, no engine state touched — so the result is cached and + reused every sync. Three passes: + 1. Split ``names`` into gather groups, one chunk per owner class + present in this worker's baked copies, recording each group's last + chunk for its ``free_group`` signal (or ``pre_free`` when this worker + pulls nothing from it). Every group is signaled exactly once by + construction. The stream has no per-group call boundaries, so group + L+1's first chunk issues while L's still stream. + 2. Per leaf module, find its FIRST and LAST chunk — materialize on the + first, quant/kernel/reset on the last, correct by construction + instead of by runtime counters. + 3. Assemble ``_Chunk``s: dedup keys and precompute the packed layout. + """ + router = self._router + assert router is not None, "init_transfer_engine() must run before planning" + + # --- pass 1: gather groups -> per-owner-class scatter chunks ----------- + raw_chunks: list[list[_Scatter]] = [] + free_at: dict[int, list[int]] = {} # chunk idx -> groups to signal after it + pre_free: list[int] = [] # groups with no chunk here (signal at start) + unbaked: list[str] = [] + pos = 0 + for gi, glen in enumerate(group_lens): + gnames = names[pos : pos + glen] + pos += glen + modules: list[list[_Scatter]] = [] + seen: set[int] = set() + for n in gnames: + mod = self._name_to_plan.get(n) + if mod is None: + if n in self._live_names: + unbaked.append(n) + elif id(mod) not in seen: + seen.add(id(mod)) + modules.append(mod) + if not modules: + # Nothing to pull for this group on this worker; its owners + # still published it, so signal it done at sync start (they + # tolerate a signal that arrives before the publish). + pre_free.append(gi) + continue + for _ci, scatters in self._chunk_module_scatters(modules): + raw_chunks.append(scatters) + free_at.setdefault(len(raw_chunks) - 1, []).append(gi) + if unbaked: + # A live name with no baked plan cannot be loaded: there is no + # fallback, and its pull would target groups the pipeline has + # already freed. Fail here, where the names are known. + raise RuntimeError( + f"Sharded RDT: {len(unbaked)} live name(s) did not bake, " + f"first: {unbaked[:3]}. Fix the loader so these names bake " + "(fully load their module in one pass), or exclude them from " + "the synced set." + ) + + # --- pass 2: per-module first/last chunk -> materialize/quant ---------- + first_at: dict[int, int] = {} + last_at: dict[int, int] = {} + layer_by_id: dict[int, Any] = {} + for ci, scatters in enumerate(raw_chunks): + for sc in scatters: + lid = id(sc.layer) + layer_by_id[lid] = sc.layer + first_at.setdefault(lid, ci) + last_at[lid] = ci + materialize_at: dict[int, list[Any]] = {} + quant_at: dict[int, list[Any]] = {} + for lid, ci in first_at.items(): + materialize_at.setdefault(ci, []).append(layer_by_id[lid]) + for lid, ci in last_at.items(): + quant_at.setdefault(ci, []).append(layer_by_id[lid]) + + # --- pass 3: assemble _Chunks (dedup keys + precompute pack layout) ---- + from collections import defaultdict + + chunks: list[_Chunk] = [] + per_owner_seq: dict[int, int] = defaultdict(int) + for ci, scatters in enumerate(raw_chunks): + keys: list[FetchKey] = [] + kmeta: dict[FetchKey, tuple[torch.dtype, tuple[int, ...]]] = {} + for sc in scatters: + if sc.src not in kmeta: + kmeta[sc.src] = (sc.dtype, sc.shape) + keys.append(sc.src) + pack_layout: list[tuple[int, torch.dtype, int, tuple[int, ...]]] = [] + cur = 0 + for k in keys: + dt, shape = kmeta[k] + numel = prod(shape) or 1 # type: ignore[arg-type] + off = (cur + 15) & ~15 + pack_layout.append((off, dt, numel, shape)) + cur = off + numel * dt.itemsize + owner = router.producer_for(router.consumer_id, scatters[0].src[0]) + chunks.append( + _Chunk( + seq=per_owner_seq[owner], + scatters=scatters, + keys=keys, + pack_layout=pack_layout, + pack_bytes=cur, + materialize=materialize_at.get(ci, []), + quant=quant_at.get(ci, []), + free=free_at.get(ci, []), + # Every name of a chunk shares an owner class and a group, + # so any of them resolves the same producer. + owner=owner, + ) + ) + per_owner_seq[owner] += 1 + return _CallPlan(chunks=chunks, pre_free=pre_free) + + def _signal_group_done(self, group_idx: int) -> None: + """Fire-and-forget ``free_group`` signal at EVERY owner of the group. + + The per-group barrier: each owner counts one signal per live consumer + and frees the group (releasing its lookahead credit) on the last one, so + every owner must hear from every consumer — including owners this worker + pulls nothing from. Refs are held and drained in ``drain_pending`` so + every signal has EXECUTED before the sync ends: ``begin_sync`` resets + the counters, and a straggler landing in the next sync would credit a + group it does not belong to.""" + assert self._router is not None + self._pending_frees.extend(self._router.free_group(group_idx)) + + def _run_chunk_pipeline(self, plan: "_CallPlan") -> None: + """Pipelined chunk pulls over the ring of receive slots. + + Issues up to ``ring_depth`` produce RPCs ahead of the blocking gets, so + while chunk i's RDMA streams the producer serves i+1 into its own ring + slot and the background thread scatters i-1 out of another. Reads stay + serialized on the shared NIC — the bandwidth floor, not a loss. + + Slot safety rests on two arguments spelled out in the doc: the producer's + ring is no shallower than this one and drain-before-issue orders its + reuse; the consumer's slots are held by ``_issue_pull``'s generation + handshake. + """ + from collections import deque + + inflight: deque[tuple[_PendingPull, _Chunk]] = deque() + + # Gather groups with no chunk on this worker (plan.pre_free): signal + # them done before the pipeline — their owners still published them, and + # a signal that arrives before its publish is counted (fire-and-forget). + for gi in plan.pre_free: + self._signal_group_done(gi) + + def drain_one() -> None: + pending, chunk = inflight.popleft() + results = self._complete_pull(pending) + self._dispatch_item( + _ProcItem(chunk=chunk, results=results, slot=pending.slot) + ) + # Each gather group whose LAST chunk this is: its read is done, so + # this consumer is finished with the group -> signal every owner + # (fire-and-forget, off the critical path). + for gi in chunk.free: + self._signal_group_done(gi) + + for chunk in plan.chunks: + if not chunk.keys: + # _chunk_module_scatters never emits empty chunks; keep the + # signals safe anyway if this ever changes. + for gi in chunk.free: + self._signal_group_done(gi) + continue + # Drain BEFORE issue once the ring is full: frees this chunk's slot + # (generation-wise) and guarantees the producer-slot invariant above. + # Sync 0 runs SERIAL (max 1 in flight): both sides still grow and + # register buffers, and a producer registration mid-flight churns its + # agent-metadata version under an in-flight pull (see __init__). + depth = 1 if self._completed_syncs == 0 else self._ring_depth + if len(inflight) >= depth: + drain_one() + slot = self._pull_slot + self._pull_slot = (slot + 1) % self._ring_depth + inflight.append((self._issue_pull(chunk, slot), chunk)) + while inflight: + drain_one() + + def _process_item(self, item: "_ProcItem") -> None: + """Scatter-thread half: materialize this chunk's first-seen modules, + scatter its slices on the process stream, publish the slot, then hand the + modules it COMPLETES to the quant thread. + + Mirrors ``_layerwise_process`` minus the loader replay. Once every scatter + reading ``item.slot`` is enqueued, records the slot's read-done event so + the RPC thread can block on it before overwriting the slot. + """ + from vllm.model_executor.model_loader.reload.layerwise import ( + LAYERWISE_INFO, + ) + from vllm.model_executor.model_loader.reload.meta import materialize_layer + + results = item.results + chunk = item.chunk + with ( + torch.cuda.device(self.device), + torch.cuda.stream(self._proc_stream), + torch.device(self.device), + ): + # PASS 1 — slot readers: materialize the modules whose FIRST scatter + # is in this chunk (empty HF params, once per module by construction) + # then scatter this chunk's copies. The scatter copies are the ONLY + # reads of the receive buffer; quant and kernel-copy operate on the + # scattered params. Releasing the slot right after the scatters lets + # the NEXT chunk's RDMA overwrite the buffer while quant still runs. + try: + for layer in chunk.materialize: + info = LAYERWISE_INFO.get(layer) + if info is None or not info.can_load(): + raise RuntimeError( + f"Baked replay: layer {type(layer).__name__} " + "was not set up for reload this sync " + "(start_weight_update must run before " + "update_weights)." + ) + materialize_layer(layer, info) + for sc in chunk.scatters: + param = getattr(sc.layer, sc.param_name) + # Through .data, exactly like default_weight_loader: params + # that keep requires_grad=True (plain nn.Parameters like + # GLM's router bias) reject an in-place copy through an + # autograd view of the leaf. + dst = param.data.as_strided(sc.shape, sc.stride, sc.offset) + with torch._C.DisableTorchFunctionSubclass(): + dst.copy_(results[sc.src]) + # All reads of this slot's buffer are now enqueued on the process + # stream; record + publish so the RPC thread can reuse the slot. + self._slot_read_done[item.slot].record(self._proc_stream) + finally: + # Publish even on error so the RPC thread's generation wait + # unblocks (the failure itself surfaces via _raise_proc_error). + self._mark_slot_done(item.slot) + + # PASS 2 — param readers only: quant / kernel-copy / reset for the + # modules whose LAST scatter is in this chunk. Handed to the + # DEDICATED quant thread (own CUDA stream, event-chained after this + # chunk's scatters) so it never delays this thread's next pass-1. + # Running it in order here stalls the RPC thread's slot handshake by + # ~0.5-0.75s/iter, since every group's quant delays the next publish. + if chunk.quant: + ready = torch.cuda.Event() + ready.record(self._proc_stream) + if self._quant_queue is None: + self._run_quant(chunk.quant, ready) + else: + self._quant_queue.put((chunk.quant, ready)) + + def _run_quant(self, layers: "list[Any]", ready: "torch.cuda.Event") -> None: + """Quant/kernel-copy/reset the given COMPLETED leaf modules, exactly as + _layerwise_process. Runs on the quant thread's own stream, ordered after + the modules' scatters via ``ready``; touches only the scattered params + (never a receive slot), so it can overlap subsequent chunks' RDMA and + scatters. ``info.reset()`` is what makes finalize skip the layer — + drain_pending joins the quant queue before finalize runs.""" + from vllm.model_executor.layers.quantization.base_config import ( + QuantizeMethodBase, + ) + from vllm.model_executor.model_loader.reload.layerwise import ( + LAYERWISE_INFO, + _copy_and_restore_kernel_tensors, + ) + + stream = self._quant_stream or self._proc_stream + assert stream is not None # created in _ensure_proc_worker before use + with ( + torch.cuda.device(self.device), + torch.cuda.stream(stream), + torch.device(self.device), + ): + stream.wait_event(ready) + for layer in layers: + info = LAYERWISE_INFO.get(layer) + assert info is not None # completed leaf module is set up for reload + quant_method = getattr(layer, "quant_method", None) + if isinstance(quant_method, QuantizeMethodBase): + if hasattr(layer, "_already_called_process_weights_after_loading"): + delattr(layer, "_already_called_process_weights_after_loading") + quant_method.process_weights_after_loading(layer) + # Copy into persistent kernel storage (preserves cudagraph refs). + if info.kernel_tensors is not None: + _copy_and_restore_kernel_tensors(layer, info) + # Reset so finalize_layerwise_reload skips this (loaded) layer. + info.reset() + + def _quant_worker_loop(self) -> None: + """Dedicated quant thread: drains (completed_modules, scatter-done event) + batches. Errors surface via _proc_error like the scatter thread's.""" + torch.cuda.set_device(self.device) + q = self._quant_queue + assert q is not None + while True: + batch = q.get() + try: + if batch is None: + return + self._run_quant(*batch) + except BaseException as e: # noqa: BLE001 - surfaced on the RPC thread + self._proc_error = e + logger.exception("RDT quant thread failed") + finally: + q.task_done() + + def shutdown(self) -> None: + # Stop the background post-processing thread (drain, then sentinel + join) + # before dropping the state it touches. + if self._proc_thread is not None: + try: + self.drain_pending() + except Exception: + logger.exception("RDT drain during shutdown failed") + assert self._proc_queue is not None + self._proc_queue.put(None) # sentinel + self._proc_thread.join(timeout=30) + self._proc_thread = None + self._proc_queue = None + self._proc_stream = None + if self._quant_thread is not None: + assert self._quant_queue is not None + self._quant_queue.put(None) # sentinel + self._quant_thread.join(timeout=30) + self._quant_thread = None + self._quant_queue = None + self._quant_stream = None + self._slot_read_done = [] + self._router = None + # Drop strong references to baked modules so the model can be freed. + self._name_to_plan.clear() + self._name_meta.clear() + self._live_names.clear() + # Drop the cached plan (holds _Scatter refs to the baked layers). + self._cached_plan = None + # Release the receive buffers (their NIXL registration is pinned for the + # process lifetime; freeing the tensors just drops our strong refs). + self._dest_buffers = [{} for _ in range(self._ring_depth)] + + +def _plan_digest(keys_per_chunk: list) -> str: + """Digest of the chunks one consumer pulls from one producer, in pull order. + + Two consumers a producer serves out of ONE shared serve ring must agree on + this, since sharing rests on their plans being identical; the producer + compares it at init. Over the whole (name, op-chain) list rather than the + names, because the chains decide the bytes each pull returns. + """ + import hashlib + + h = hashlib.sha256() + for keys in keys_per_chunk: + h.update(repr(keys).encode()) + h.update(b"\n") + return h.hexdigest()[:32] + + +def _dtype_from_name(name: str) -> torch.dtype: + """Resolve a string like 'bfloat16' to torch.bfloat16.""" + dtype = getattr(torch, name, None) + if not isinstance(dtype, torch.dtype): + raise ValueError(f"Unknown torch dtype name: {name!r}") + return dtype diff --git a/vllm/distributed/weight_transfer/sharded_rdt_fake.py b/vllm/distributed/weight_transfer/sharded_rdt_fake.py new file mode 100644 index 0000000000..5e3613e9d5 --- /dev/null +++ b/vllm/distributed/weight_transfer/sharded_rdt_fake.py @@ -0,0 +1,300 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Op-chain recording for the sharded-RDT backend. + +The consumer asks the trainer for the exact slice a worker consumes, described as +an op chain replayed on the trainer's live tensor. ``FakeRDTTensor`` builds the +chain by intercepting the model's own weight loaders; ``copy_`` is its data sink, +``BakeSink`` — the dry run records how each slice would be fetched and where it +lands. + +Chains are built against meta tensors, so nothing here touches Ray or a GPU. +""" + +from collections import defaultdict +from collections.abc import Callable +from dataclasses import dataclass, field +from typing import Any + +import torch + +from vllm.distributed.weight_transfer.sharded_rdt_common import SUPPORTED_OPS + +# ("op_name", positional_args, sorted_kwargs_items). Entries must be hashable so +# the chain can serve as a FetchKey. +OpSpec = tuple[str, tuple[Any, ...], tuple[tuple[str, Any], ...]] +OpChain = tuple[OpSpec, ...] +FetchKey = tuple[str, OpChain] + + +def _freeze_kwargs(kwargs: dict[str, Any]) -> tuple[tuple[str, Any], ...]: + """Sort kwargs into a tuple of items for hashable storage in OpSpec.""" + return tuple(sorted(kwargs.items())) + + +class _UnsupportedFakeOp(NotImplementedError): + """Raised when a weight loader does something to a FakeRDTTensor that cannot + be expressed as a slice request. + + Surfaced as NotImplementedError so callers can distinguish "this backend + can't handle this loader" from genuine bugs. + """ + + +@dataclass +class _Scatter: + """One recorded scatter: pull ``src``, copy it into ``layer``'s + ``param_name`` at the recorded strided region. The bake's output unit; see + the engine module's Data flow. + + Self-contained so replay needs no lookups. Destination geometry is read off + the meta view and rebuilt each sync as ``as_strided(shape, stride, offset)`` + -- ``layer`` is resolved to a param at replay time, never baked, since every + sync re-materializes fresh tensors. + + ``shape`` and ``dtype`` also size the slice ON THE WIRE, so ``dtype`` is the + PRODUCED dtype (the fake's after its op chain), not the source name's: + ``view(dtype)`` is allowlisted, and taking the source's would size the slice + with the wrong itemsize and carve the packed blob differently on the two + sides. + """ + + layer: Any + param_name: str + src: FetchKey + offset: int + shape: tuple[int, ...] + stride: tuple[int, ...] + dtype: torch.dtype + + +def _meta_copy_(dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor: + """Fire ``dest.copy_`` from a zero-storage meta source of ``src``'s geometry. + + Moves no data but still counts against the layer's loaded numel, which drives + ``_layerwise_process``; skipping it would leave the layer looking unloaded + forever. + + A non-meta ``dest`` is one the layerwise reload never moved there (the + ``SKIP_LOAD_TENSORS`` set, e.g. GLM's router bias). Torch forbids + ``real.copy_(meta)``, and skipping the count is consistent -- ``get_layer_size`` + excludes the same set. The caller still RECORDS the copy, so the replay writes + the real param. + """ + if not dest.is_meta: + return dest + meta_src = torch.empty(src.shape, dtype=src.dtype, device="meta") + with torch._C.DisableTorchFunctionSubclass(): + return dest.copy_(meta_src) + + +@dataclass +class BakeSink: + """``copy_`` sink for the dry-run bake: record how each slice would be fetched + and where it would land, and move nothing. + + ``_install_recording_stamps`` sets ``current = (leaf_module, param_name)`` + around each loader so ``accept_copy`` can attribute the copy. An unstamped + ``copy_`` cannot be attributed and stays unrecorded, so its module fails the + coverage gate and takes the plain load. ``copies_by_layer`` is keyed by module + object, so iterating it yields each leaf module once. + """ + + copies_by_layer: "dict[Any, list[_Scatter | None]]" = field( + default_factory=lambda: defaultdict(list) + ) + current: "tuple[Any, str] | None" = None + # Source names whose ``copy_`` fired during the bake. Names not here never + # moved data (e.g. experts owned by another EP rank) and are skipped; names + # here but unbaked fail the plan build. + copied_names: "set[str]" = field(default_factory=set) + + def accept_copy(self, dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor: + self.copied_names.add(src._name) + if self.current is not None: + layer, param_name = self.current + if dest.numel() != src.numel(): + # The packed layout sizes each slice from the DESTINATION shape, + # while the producer packs what the replayed chain produces. A + # broadcasting copy_ makes those two lengths differ, and neither + # side can notice: the consumer's buffer view is exactly + # prod(dest.shape) elements, so it reshapes cleanly over bytes + # the producer laid out at different offsets. Refuse at bake. + raise _UnsupportedFakeOp( + f"FakeRDTTensor: broadcasting copy_ into {param_name!r} of " + f"{type(layer).__name__} ({src.numel()} elements from " + f"{src._name!r} into {dest.numel()}). The sharded RDT " + "backend sends exactly the slice the loader asks for, so " + "the source and destination must have equal numel." + ) + self.copies_by_layer[layer].append( + _Scatter( + layer=layer, + param_name=param_name, + src=src._key(), + offset=dest.storage_offset(), + shape=tuple(dest.shape), + stride=tuple(dest.stride()), + dtype=src.dtype, + ) + ) + return _meta_copy_(dest, src) + + +class FakeRDTTensor(torch.Tensor): + """Zero-storage tensor that records how to fetch a weight slice. + + ``_make_wrapper_subclass`` gives it shape/dtype/device without storage. Every + op in ``SUPPORTED_OPS`` returns a child with the spec appended; ``copy_`` + delegates to the installed sink. Anything else reaches ``__torch_dispatch__`` + and raises ``_UnsupportedFakeOp``, so failures are loud rather than silently + fetching the wrong bytes. + """ + + # Declared at class scope so mypy can infer attribute types on + # instances built via ``_make_wrapper_subclass`` (where ``__new__`` + # returns a tensor it can't annotate as ``self``). + _name: str + _ops: OpChain + # Handles this fake's copy_. ``None`` only on bare construction. + _sink: "BakeSink | None" + + @staticmethod + def __new__( + cls, + name: str, + shape: torch.Size, + dtype: torch.dtype, + device: torch.device, + ops: OpChain = (), + sink: "BakeSink | None" = None, + ) -> "FakeRDTTensor": + t = torch.Tensor._make_wrapper_subclass( + cls, + shape, + dtype=dtype, + device=device, + requires_grad=False, + ) + t._name = name + t._ops = tuple(ops) + t._sink = sink + return t + + def _key(self) -> FetchKey: + return (self._name, self._ops) + + def _make_child( + self, + new_shape: torch.Size, + new_dtype: torch.dtype, + *new_ops: OpSpec, + ) -> "FakeRDTTensor": + """Append one or more ops to the chain and return a fresh child. + + Variadic so multi-return ops (e.g. chunk) can append both the base + op and an indexing op in a single call. + """ + return FakeRDTTensor( + name=self._name, + shape=new_shape, + dtype=new_dtype, + device=self.device, + ops=self._ops + new_ops, + sink=self._sink, + ) + + def _meta(self) -> torch.Tensor: + """A zero-storage meta tensor of this fake's shape/dtype, so PyTorch + itself computes post-op geometry rather than us reimplementing it.""" + return torch.empty(self.shape, dtype=self.dtype, device="meta") + + @classmethod + def __torch_function__( + cls, + func, + types, + args=(), + kwargs=None, + ): + kwargs = kwargs or {} + + # Checked before the allowlist: copy_ must not be recorded into the chain. + if func is torch.Tensor.copy_: + dest = args[0] + src = args[1] if len(args) > 1 else kwargs.get("src") + if isinstance(src, cls) and src._sink is not None: + return src._sink.accept_copy(dest, src) + + # Allowlisted slice/view/shape ops: append to chain and return child. + op_name = SUPPORTED_OPS.get(func) + if op_name is not None: + self_ = args[0] + if isinstance(self_, cls): + rest = tuple(args[1:]) + return cls._intercept(self_, func, op_name, rest, kwargs) + + # Metadata reads are answered by the wrapper subclass and stop here; ops + # that need data fall through to __torch_dispatch__, which raises. + with torch._C.DisableTorchFunctionSubclass(): + return func(*args, **kwargs) + + @classmethod + def _intercept( + cls, + self_: "FakeRDTTensor", + func: Callable, + op_name: str, + args: tuple, + kwargs: dict, + ): + """Append the op and return a child, or a tuple of children for + multi-return ops. Each child's geometry comes from running the op on a + meta tensor.""" + meta = self_._meta() + with torch._C.DisableTorchFunctionSubclass(): + meta_result = func(meta, *args, **kwargs) + + base_op: OpSpec = (op_name, tuple(args), _freeze_kwargs(kwargs)) + + # Single-tensor result: one child carrying base_op alone. + if isinstance(meta_result, torch.Tensor): + return self_._make_child(meta_result.shape, meta_result.dtype, base_op) + + # Multi-return: one child per output, each carrying base_op plus a + # trailing __getitem__(i) so the replay can index the result. + if isinstance(meta_result, tuple | list): + return tuple( + self_._make_child( + m.shape, + m.dtype, + base_op, + ("__getitem__", (i,), ()), + ) + for i, m in enumerate(meta_result) + ) + + # Non-tensor result (e.g. .item() snuck into the allowlist). + raise _UnsupportedFakeOp( + f"FakeRDTTensor: {op_name!r} returned a non-tensor " + f"({type(meta_result).__name__}); cannot defer." + ) + + @classmethod + def __torch_dispatch__(cls, func, types, args=(), kwargs=None): + kwargs = kwargs or {} + + # A fake at the aten level means an unsupported loader op. Name the op and + # the chain; materializing instead would mask a correctness bug. + for arg in (*args, *kwargs.values()): + if isinstance(arg, cls): + raise _UnsupportedFakeOp( + f"FakeRDTTensor: unsupported op {func} reached " + f"__torch_dispatch__ on fake {arg._name!r} " + f"(chain={arg._ops}). Supported ops are: " + f"{sorted(SUPPORTED_OPS.values())}, plus copy_. " + "Loaders that need .to(), .float(), .item(), arithmetic, " + "bool-mask indexing, or .data access are not supported by " + "the sharded RDT backend." + ) + return func(*args, **kwargs) diff --git a/vllm/distributed/weight_transfer/sharded_rdt_trainer.py b/vllm/distributed/weight_transfer/sharded_rdt_trainer.py new file mode 100644 index 0000000000..60d0e60fb0 --- /dev/null +++ b/vllm/distributed/weight_transfer/sharded_rdt_trainer.py @@ -0,0 +1,1521 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Trainer-side engine for the sharded-RDT (pull-based NIXL) backend. + +RDT is pull-based, so unlike NCCL this engine broadcasts nothing. It owns a +per-rank producer server (an internal Ray actor exposing the NIXL serve surface +the worker engine dials by name), and on each ``send_weights`` gathers this +rank's weights group-by-group from the ``WeightSource``, shares each group into +the server over CUDA IPC, and — on the sender — drives the inference-side +start/update/finish handshake, whose single empty ``update_weights`` unblocks the +workers to pull. + +All serve-side state lives on the server actor, so trainer processes need no +mixin, no named actors and no special actor options. + +See docs/training/weight_transfer/sharded_rdt.md for the publish -> serve -> +free_group -> release lifecycle and the ownership model. +""" + +import contextlib +import threading +import time +import uuid +from collections.abc import Callable, Collection +from concurrent.futures import ThreadPoolExecutor +from dataclasses import asdict, dataclass, field +from typing import Any, ClassVar + +import ray +import torch +from torch.multiprocessing.reductions import ( + StorageWeakRef, + rebuild_cuda_tensor, + reduce_tensor, +) +from typing_extensions import Self + +from vllm.distributed.weight_transfer.base import ( + ParamMeta, + TrainerInitInfo, + TrainerWeightTransferEngine, + VLLMWeightSyncClient, + WeightSource, + layerwise_groups, +) +from vllm.distributed.weight_transfer.sharded_rdt_common import ( + ALLOWED_OPS, + buffer_alloc_bytes, + check_ray_rdt_version, +) +from vllm.logger import init_logger + +logger = init_logger(__name__) + +# Gathered-but-unfreed groups the gather loop may run ahead by before it stops +# gathering. A gathered group is published immediately (publish_group never +# blocks), so at most ``lookahead + 1`` groups are resident on the trainer — the +# memory bound larger models size against, pinned by the tests. +# +# At 1, group N+1 is gathered and serveable while the consumers pull N, so the +# boundary's free-barrier latency hides behind live pulls. Raise it only if one +# group's gather is slower than its pulls. +DEFAULT_GATHER_LOOKAHEAD = 1 + +# Seconds with no publish, produce or free before the +# producer declares the sync dead. A consumer that dies mid-sync never signals +# ``free_group``, so the group is never released and the waits below block +# forever — which stops this rank iterating its WeightSource, a collective, and +# wedges every other trainer rank with no exception anywhere. This converts that +# into one real error. +# +# A liveness backstop, not a latency target: nominal gaps are sub-second, so this +# is ~100x margin. It rides the init info because the producer is a Ray actor and +# does not inherit the trainer's environment. +DEFAULT_STALL_TIMEOUT_S = 300.0 +# How often a blocked waiter re-checks the progress stamp. +_STALL_POLL_S = 5.0 + +# Tensors whose STORAGE is at most this are packed into a ring slot and exported +# as one CUDA-IPC handle; larger storages are exported directly. Batching the +# small ones is what keeps sync time steady -- a per-tensor handle for each of +# ~37k names dominated the variance -- and `_pack_group_for_export` is also the +# only publish path that orders the pack against the sidecar's IPC read, so the +# packing is not optional. +_EXPORT_RING_MAX_BYTES = 64 << 20 + +# The actor method the worker engine dials for the NIXL pull; fixed by contract. +PRODUCE_METHOD_NAME = "rdt_produce_weights_batched" + + +# Cross-deployment serve-slot sharing. +# +# Consumers whose ids differ by a multiple of ``workers_per_replica`` are the +# same worker of different inference deployments, so they share a baked plan and +# pull byte-identical chunks. A serve ring each would cost one full ring per +# deployment of the producer's GPU and repack the same bytes once per deployment. +# +# NIXL reads are one-sided, so R readers can read ONE registered slot; what the +# producer cannot see is when a reader has FINISHED. The release edge is +# therefore the consumer's ISSUE order, sent as ``seq``: the pipeline drains pull +# i before issuing i+K, so slot ``seq % K`` was last packed for ``seq - K``, +# whose read is over. A counter on THIS side would follow execution order +# instead, which is wrong -- Ray may start a consumer's K concurrent produce +# calls in any order, so a call that executes K-before another can be a pull +# still being read, and its slot gets repacked underneath the reader. Silent, +# showing up only as a logprob drift. +# +# Hence one rendezvous per generation, keyed by ``seq``: the group's live sharers +# meet there, the LAST to arrive packs, and all return that one blob. It carries +# the release edge too, since ``seq`` is packed only once every live sharer has +# arrived, so each has drained ``seq - K``. Mismatched plans would rendezvous on +# differing bytes, so they are rejected at init by ``reserve_serve_buffer``'s +# ``plan_digest``. A sharer that dies mid-sync stalls its group, exactly as a +# dead consumer already stalls the per-group free barrier; the next +# ``begin_sync`` narrows the live set. +# +# LIMITATION: the rendezvous synchronizes deployments that a pull-based transfer +# would otherwise leave independent -- per chunk every sharer waits for the +# slowest, so skew between their DP pause/resume brackets lands in every +# deployment's sync time. Nothing about P2P requires this. It is the price of +# serving out of GPU memory, which is too scarce for a ring per consumer, so +# slots must be reused, and reuse needs a release edge. +# +# TODO: explore staging packed chunks in HOST memory. Host buffers are cheap +# enough for a ring per consumer, which would drop the rendezvous, the +# cross-replica coupling and the slot-reuse hazard together. The cost is a D2H +# bounce on the serve path, which has been the bottleneck before, so it needs +# measuring rather than assuming. + + +@dataclass +class _SharedPack: + """One generation of one slot-sharing group: the rendezvous state for a + single chunk, identified by the consumers' issue index.""" + + group: int + slot: int + """``seq % ring_depth``. Fixed at creation, so slot reuse follows the + consumers' issue order rather than this side's execution order.""" + arrived: set[int] = field(default_factory=set) + blob: Any = None + packing: bool = False + done: bool = False + error: BaseException | None = None + + +@dataclass +class ShardedRDTTrainerInitInfo(TrainerInitInfo): + """Trainer init info for the sharded-RDT backend. + + Identical on every rank except ``rank`` (rank 0 is the sender). Carries only + the must-agree wire params; the sender forwards them verbatim onto the + worker-side init info so the two cannot drift. Server-actor names are + generated per rank and all-gathered by the engine, not supplied here. + """ + + backend: ClassVar[str] = "sharded_rdt" + + num_consumers: int + """Total inference-worker (consumer) count across the whole fleet + (DP*TP*PP*PCP), for the M:N block assignment / free ref-count.""" + workers_per_replica: int = 0 + """Consumers per inference DEPLOYMENT (``num_consumers // num_replicas``). + + Fixes the slot-sharing groups: consumers whose ids differ by a multiple of + this are the same worker of different deployments, so they bake identical + plans and pull byte-identical chunks, and ONE registered serve slot can + serve all of them. 0 disables sharing, and so does the single-deployment + value ``num_consumers``, which makes every group a singleton and the serve + path identical to the unshared one. + """ + trainer_actor_namespace: str | None = None + """Ray namespace the engine spawns its serve actors in. The inference + workers (which run in their own EngineCore subprocess with its own + ``ray.init``) resolve those actors by name, so this must be the namespace + they can see. Forwarded to the worker-side init info.""" + num_rdt_buffers: int = 2 + """Serve/receive ring depth K (must match the worker).""" + buffer_presize_gb: float = 0.0 + """Serve-buffer pre-size floor in GiB (avoids NIXL desc-cache churn).""" + gather_lookahead: int = DEFAULT_GATHER_LOOKAHEAD + """Gathered-but-unfreed groups the gather loop may run ahead by. Bounds + trainer-resident memory at ``gather_lookahead + 1`` groups.""" + stall_timeout_s: float = DEFAULT_STALL_TIMEOUT_S + """Seconds of no publish/serve/free progress before the producer fails the + sync (see ``DEFAULT_STALL_TIMEOUT_S``).""" + + +class _RDTProducerServer: + """Per-rank NIXL serve surface: an internal Ray actor sharing the trainer + rank's GPU over CUDA IPC. Holds the gather cache, per-consumer serve rings, + the per-group free barrier and the packed serve. The engine feeds it with + ``publish_group``; workers pull with ``rdt_produce_weights_batched`` and + signal with ``free_group``. + + A plain class — the engine wraps it with ``ray.remote(...)`` at spawn so the + actor options live in one place. + """ + + def __init__( + self, + *, + num_rdt_buffers: int, + buffer_presize_gb: float, + gather_lookahead: int, + served_names: list[str] | None = None, + stall_timeout_s: float = DEFAULT_STALL_TIMEOUT_S, + workers_per_replica: int = 0, + ) -> None: + import gc + + self._device_index = torch.accelerator.current_device_index() + # name -> rebuilt CUDA-IPC tensor (or view); guarded by _cache_cond. + self._cache: dict[str, torch.Tensor] = {} + self._cache_cond = threading.Condition() + self._gather_error: BaseException | None = None + + # Monotonic stamp of the last forward step of the + # publish -> serve -> free loop. The waits below poll it instead of + # blocking forever, so a dead consumer fails the sync with a real error. + self._stall_timeout = float(stall_timeout_s) + self._last_progress = time.monotonic() + + # Names this producer publishes, so a misrouted pull fails loudly + # instead of blocking forever in the cache wait. None = serve anything. + self._served_names = set(served_names) if served_names is not None else None + + # Every live consumer signals free_group(gi) at every + # owner, once per sync; the group frees when the count reaches + # begin_sync's live total — one uniform target, no routed per-producer + # ones. Signals may precede the publish, which completes them. + self._live_count = 1 + self._free_counts: dict[int, int] = {} + # gi -> the names this producer published for it (what release drops). + self._group_names: dict[int, list[str]] = {} + + # Published-but-unfreed groups, plus freed ones not + # yet handed back. The memory gate lives in the ENGINE's gather loop; this + # side only accounts — free_group moves a group from _inflight_groups to + # _freed_pending, which the engine collects via wait_freed / end_sync to + # drop its storage refs. _lookahead is engine-enforced, unused here. + self._lookahead = max(0, gather_lookahead) + self._inflight_groups: list[int] = [] + self._freed_pending: list[int] = [] + + # Ring of packed serve buffers, one ring per SLOT-SHARING GROUP (one + # consumer per group unless several deployments share); the slot within + # it is the consumer's ``seq % nring``. + self._nring = max(1, num_rdt_buffers) + self._serve_rings: dict[int, list[torch.Tensor | None]] = {} + self._serve_lock = threading.Lock() + # registerMem on a shared NIXL agent is not concurrency-safe; serialize. + self._reg_lock = threading.Lock() + self._buffer_presize = int(buffer_presize_gb * (1 << 30)) + self._serve_device = torch.device("cuda", self._device_index) + + # Slot-sharing state, all guarded by _cache_cond, so the rendezvous + # waits ride the stall watchdog like every other wait here. + # ``_share_width`` = consumers per deployment; 0 means every group is a + # singleton and this degenerates to the unshared serve path. + self._share_width = max(0, int(workers_per_replica)) + self._live_ids: set[int] | None = None + self._sharing_active = False + self._sharers: dict[int, frozenset] = {} # group -> live members + self._gens: dict[int, dict] = {} # group -> {seq: _SharedPack} + # group -> {cid: plan digest}. Init-time state, not cleared per sync: it + # is what proves the sharers of a group pull the same chunks. + self._plan_digests: dict[int, dict[int, str]] = {} + + # (sharing group, ring idx, packed layout) -> + # (buffer data_ptr, destination views). Keyed on the buffer pointer too, so + # a ring regrow invalidates rather than writing into a freed buffer. See + # the serve path for why the layout, not the spec names, is the key. + self._pack_dsts: dict[tuple, tuple[int, list[torch.Tensor]]] = {} + # [RDT-EXPORT-RING] handle -> rebuilt base. The trainer reuses buffers + # across groups, so the same handle arrives on many publishes; without + # this we reopen it every group. Cleared per sync so no IPC mapping into + # trainer memory outlives the sync that made it. + self._base_cache: dict[tuple, torch.Tensor] = {} + + # Freeze the static post-init object graph so gen-2 GC never stops the + # world mid-serve, which shows up as serve stragglers. + gc.collect() + gc.freeze() + + def ping(self) -> int: + return self._device_index + + # ---------------- stall watchdog ---------------- + + def _note_progress_locked(self) -> None: + """Record that the credit loop moved. Caller holds ``_cache_cond``.""" + self._last_progress = time.monotonic() + + def _wait_for(self, blocked: Callable[[], bool], what: str) -> None: + """``_cache_cond.wait()`` with a liveness bound. + + Waits while ``blocked()`` holds, returning early on a gather error. If + nothing on this producer progresses for ``_stall_timeout``, self-fires + ``set_gather_error``, which every waiter here already checks — so the rank + unwinds through one path and the driver gets a real exception. + + The progress stamp is global to the producer, not per-waiter: a merely + slow waiter is kept alive by its peers' progress. Caller holds + ``_cache_cond``. + """ + while blocked(): + if self._gather_error is not None: + return + self._cache_cond.wait(_STALL_POLL_S) + if not blocked() or self._gather_error is not None: + return + stalled = time.monotonic() - self._last_progress + if stalled >= self._stall_timeout: + msg = ( + f"RDT stall: no progress for {stalled:.0f}s while waiting for " + f"{what} (timeout {self._stall_timeout:.0f}s). A consumer most " + f"likely died mid-sync: {len(self._inflight_groups)} group(s) " + f"published and unfreed." + ) + logger.error("[rdt-stall] %s", msg) + # Same channel a gather failure uses: wakes every waiter here, and + # each raises rather than returning a half-served result. + self._gather_error = RuntimeError(msg) + self._cache_cond.notify_all() + return + + def warmup_nixl(self) -> None: + """Create this server's NIXL agent now, while the rank's GPU is quiet. + + Called at spawn, before the server-name all-gather, so no rank can be + spinning in a collective. Creating the agent lazily instead deadlocks on + EFA-class fabrics (see "warmup_nixl breaks a startup deadlock" in the + doc). The warmup buffer stays registered so the agent's CUDA-HMEM path + stays initialized. + """ + from ray.experimental import register_nixl_memory + + self._nixl_warmup_buf = torch.zeros(1 << 20, dtype=torch.uint8, device="cuda") + with self._reg_lock: + register_nixl_memory(self._nixl_warmup_buf) + + # ---------------- engine-facing (per sync) ---------------- + + def begin_sync( + self, live_count: int, live_consumer_ids: list | None = None + ) -> None: + """Reset per-sync free/backpressure state and set this sync's barrier + target. + + ``live_count`` is how many consumers take part in THIS sync. Required, no + default: a forgotten argument silently targeting 1 would free groups after + the FIRST signal while others still pull — use-after-free, not an error. + The driver awaits the previous sync's finish before the next begins, so + nothing is in flight; a straggler signal would otherwise credit the wrong + sync, which is why the consumer drains its signals before finishing. + + ``live_consumer_ids`` is that same live set enumerated, which is what + sizes each slot-sharing group's rendezvous; a count cannot, since a group + is a specific set of ids. ``None`` leaves every group a singleton. + + The packed-destination cache deliberately survives: the layout repeats + every sync. So does ``_plan_digests``, which is init-time state. + """ + with self._cache_cond: + self._gather_error = None + self._live_count = max(1, int(live_count)) + self._live_ids = ( + set(int(c) for c in live_consumer_ids) + if live_consumer_ids is not None + else None + ) + # Sharing is on only when the geometry shows several deployments; + # one deployment takes the singleton path. + self._sharing_active = ( + self._share_width > 0 + and self._live_ids is not None + and len(self._live_ids) > self._share_width + ) + self._sharers.clear() + self._gens.clear() + self._inflight_groups.clear() + self._freed_pending.clear() + self._free_counts.clear() + self._group_names.clear() + self._base_cache.clear() + self._note_progress_locked() + + def _release_group_locked(self, group_idx: int) -> None: + """Drop a freed group's cache entries and queue it for the engine + (whose gather-credit gate blocks in ``wait_freed`` on exactly this). + + Shared by the last ``free_group`` and by ``publish_group`` completing an + early-signaled group. Caller must hold ``_cache_cond``. + """ + for name in self._group_names.pop(group_idx, ()): + self._cache.pop(name, None) + self._free_counts.pop(group_idx, None) + if group_idx in self._inflight_groups: + self._inflight_groups.remove(group_idx) + self._freed_pending.append(group_idx) + + def publish_group(self, group_idx: int, entries: tuple) -> None: + """Rebuild one gather group's CUDA-IPC tensors into the serve cache. + + NEVER blocks: a gathered group is serveable immediately. The memory bound + lives in the engine's gather loop, which stops GATHERING (not publishing) + past ``gather_lookahead`` unfreed groups. + + ``entries`` is ``(storages, views)``: one CUDA-IPC export per storage, + plus per-name ``(sid, dtype_name, shape, stride, storage_offset)`` rebuilt + here as ``as_strided`` views. + + Signals can arrive BEFORE their publish — a consumer pulling nothing of a + group signals it as its plan starts — so a group whose barrier is already + satisfied is released here. Freed groups reach the engine only through + ``wait_freed`` / ``end_sync``: a freed notice riding an unharvested async + publish result would wedge the loop. + """ + storages, views = entries + bases: dict[int, torch.Tensor] = {} + for sid, reduce_args in storages.items(): + cached = self._base_cache.get(reduce_args) + if cached is not None: + bases[sid] = cached + continue + list_args = list(reduce_args) + # Index 6 of reduce_tensor's args is the exporter's device index; + # rebuild on this server's device (same physical GPU as the rank). + list_args[6] = self._device_index + bases[sid] = self._base_cache[reduce_args] = rebuild_cuda_tensor(*list_args) + rebuilt: dict[str, torch.Tensor] = {} + for name, (sid, dtype_name, shape, stride, storage_offset) in views.items(): + typed = bases[sid].view(getattr(torch, dtype_name)) + rebuilt[name] = torch.as_strided(typed, shape, stride, storage_offset) + del bases + + with self._cache_cond: + self._cache.update(rebuilt) + self._inflight_groups.append(group_idx) + self._group_names[group_idx] = list(rebuilt) + if self._free_counts.get(group_idx, 0) >= self._live_count: + self._release_group_locked(group_idx) + self._note_progress_locked() + self._cache_cond.notify_all() + + def wait_freed(self) -> list[int]: + """Block until at least one published group has been freed; return and + clear the freed backlog. The engine's gather-credit gate calls this once + its loop is ``gather_lookahead`` groups ahead, so this wait is where the + trainer paces to the consumers' pull rate. + + Raises rather than returning empty when the sync errors or stalls: the + engine is blocked here inside its gather loop, and an empty return would + spin it straight back in. + """ + with self._cache_cond: + self._wait_for(lambda: not self._freed_pending, "a freed-group credit") + if not self._freed_pending: + raise RuntimeError( + "gather errored while waiting for a freed-group credit: " + f"{self._gather_error!r}" + ) + freed = self._freed_pending + self._freed_pending = [] + return freed + + def end_sync(self) -> list[int]: + """Block until every published group has been freed by its consumers; + return the remaining freed keys so the engine drops its last refs.""" + with self._cache_cond: + self._wait_for( + lambda: bool(self._inflight_groups), + "every published group to be freed", + ) + freed = self._freed_pending + self._freed_pending = [] + self._base_cache.clear() + return freed + + def set_gather_error(self, message: str) -> None: + """Record a trainer-side gather failure so blocked serves / publishes + stop waiting and surface it.""" + with self._cache_cond: + self._gather_error = RuntimeError(message) + self._cache_cond.notify_all() + + # ---------------- consumer-facing (called by name over Ray) ---------------- + + def free_group(self, group_idx: int) -> None: + """Consumer back-edge: one consumer is done with group ``group_idx``, + either because its last chunk landed or because it had nothing to pull. + + The per-group barrier counts one signal per live consumer against the + ``begin_sync`` count; every consumer signals every owner, so the target is + the same integer everywhere. The last signal drops the cache entries and + queues the group as gather credit for ``wait_freed``. A signal arriving + before its publish is only counted — ``publish_group`` completes it. + """ + gi = int(group_idx) + with self._cache_cond: + count = self._free_counts.get(gi, 0) + 1 + self._free_counts[gi] = count + if gi in self._group_names and count >= self._live_count: + self._release_group_locked(gi) + self._note_progress_locked() + self._cache_cond.notify_all() + + def _new_serve_buffer(self, nbytes: int) -> torch.Tensor: + """Allocate + NIXL-register one serve slot: the single allocation seam, + so registration cannot be skipped on either of the two paths that make + buffers (the init-time reservation and the serve-path backstop).""" + from ray.experimental import register_nixl_memory + + t = torch.empty(nbytes, dtype=torch.uint8, device=self._serve_device) + with self._reg_lock: + register_nixl_memory(t) + return t + + def reserve_serve_buffer( + self, consumer_id: int, nbytes: int, plan_digest: str | None = None + ) -> None: + """Pre-allocate + NIXL-register this consumer's serve ring before any + pull, while the fabric is idle (avoids registration races during the + sync-0 RDMA churn under M:N fan-in). Idempotent; grows only if needed. + + The ring is keyed by SLOT-SHARING GROUP, so the sharers of one group + reserve ONE ring between them. They all call this, same group and same + size, so the whole body runs under ``_serve_lock``, or two concurrent + first-callers would each allocate a ring and one would be dropped while + still registered. + + ``plan_digest`` is the caller's ordered chunk plan for THIS producer. + Sharers must agree on it, and this is where a disagreement is caught + rather than at serve time, where it is a rendezvous nobody completes. + + Raises: + RuntimeError: two consumers of one sharing group disagree on the + chunks they pull from this producer. + """ + sg = self._share_group(consumer_id) + if plan_digest is not None: + with self._cache_cond: + seen = self._plan_digests.setdefault(sg, {}) + clash = next( + ((c, d) for c, d in seen.items() if d != plan_digest), None + ) + if clash is None: + seen[consumer_id] = plan_digest + if clash is not None: + raise RuntimeError( + f"consumers {clash[0]} and {consumer_id} are in slot-sharing " + f"group {sg} but do not pull the same chunks from this " + f"producer ({clash[1]} != {plan_digest}). Sharing a serve " + f"slot requires the deployments to be identical; check that " + f"workers_per_replica matches the inference fleet's " + f"geometry." + ) + # This is where the producer's NIXL serve memory is committed: one ring + # of K slots per SHARING GROUP that pulls from this rank -- with a single + # deployment, one ring per consumer, since every group is a singleton. + # Routing only sends a consumer here for names this rank owns, so the + # count is the fan-in, not the fleet. + # + # Every slot is sized by the LARGEST chunk that consumer pulls from us, + # rounded up to a 256 MB multiple, so the whole ring pays for the worst + # chunk in the plan while most chunks are far smaller. + # + # TODO: that over-allocates. Sizing slots per chunk instead of per plan, + # or capping chunk bytes so no chunk is an outlier, would shrink it; so + # would dropping the 256 MB round-up, which is slack once a buffer never + # regrows. Row-splitting the vocab matrix shrinks the outlier itself. + # The host-staging TODO above would remove these rings altogether. + alloc = buffer_alloc_bytes(nbytes, self._buffer_presize) + with self._serve_lock: + rings = self._serve_rings.setdefault(sg, [None] * self._nring) + for i in range(self._nring): + slot = rings[i] + if slot is None or slot.numel() < alloc: + rings[i] = self._new_serve_buffer(alloc) + + def _share_group(self, consumer_id: int) -> int: + """The slot-sharing group ``consumer_id`` belongs to: its index within + its own deployment, since that is what fixes the plan. Without a width + the group is the consumer itself and nothing is shared.""" + return consumer_id % self._share_width if self._share_width > 0 else consumer_id + + def _sharers_of(self, sg: int) -> frozenset: + """The LIVE consumers of group ``sg``: the rendezvous width. Derived from + ``begin_sync``'s live set, so a degraded sync narrows the barrier instead + of waiting forever on a dead deployment. Caller holds ``_cache_cond``.""" + cached = self._sharers.get(sg) + if cached is None: + if not self._sharing_active or self._live_ids is None: + cached = frozenset((sg,)) + else: + cached = frozenset( + c for c in self._live_ids if c % self._share_width == sg + ) or frozenset((sg,)) + self._sharers[sg] = cached + return cached + + def _join_share_group(self, consumer_id: int, seq: int) -> tuple: + """Register this call's arrival in its group's rendezvous for ``seq`` and + return ``(generation, is_packer)``. + + Exactly one caller per generation gets ``is_packer=True``, whichever + completes the arrival set, and it is the only one that touches the GPU. + The slot is ``seq % nring``: fixed by the consumers' issue order, so it + needs no release accounting on this side. + """ + sg = self._share_group(consumer_id) + with self._cache_cond: + gens = self._gens.setdefault(sg, {}) + gen = gens.get(seq) + if gen is None: + gen = gens[seq] = _SharedPack(group=sg, slot=seq % self._nring) + gen.arrived.add(consumer_id) + if gen.packing or gen.done or not gen.arrived >= self._sharers_of(sg): + return gen, False + gen.packing = True + return gen, True + + def _await_shared_pack(self, gen: _SharedPack) -> torch.Tensor: + """Block until this generation's packer published its blob, and return + it. A failed pack is re-raised here, so every sharer of a bad pack fails + instead of reading a half-written slot.""" + with self._cache_cond: + self._wait_for( + lambda: not gen.done and gen.error is None, + "a co-replica's shared serve pack", + ) + if gen.error is not None: + raise RuntimeError( + f"the sharer packing this chunk failed: {gen.error!r}" + ) from gen.error + if not gen.done: + raise RuntimeError( + f"gather errored while awaiting a shared pack: " + f"{self._gather_error!r}" + ) + return gen.blob + + def _serve_slot(self, sg: int, idx: int, need: int) -> torch.Tensor: + """Group ``sg``'s ring slot ``idx``, grown if this chunk outgrew the + reservation. Growing registers memory while the fabric is busy, the + hazard ``reserve_serve_buffer`` exists to avoid, so it is a backstop: the + reservation is sized from the same static plan as this pack.""" + with self._serve_lock: + buffer = self._serve_rings.setdefault(sg, [None] * self._nring)[idx] + if buffer is not None and buffer.numel() >= need: + return buffer + buffer = self._new_serve_buffer(buffer_alloc_bytes(need, self._buffer_presize)) + with self._serve_lock: + self._serve_rings.setdefault(sg, [None] * self._nring)[idx] = buffer + return buffer + + def _fail_shared_pack(self, gen: _SharedPack, exc: BaseException) -> None: + """Publish a pack failure so waiting sharers raise instead of hanging on + a generation that will never complete.""" + with self._cache_cond: + gen.error = exc + self._cache_cond.notify_all() + + @ray.method(tensor_transport="nixl") + def rdt_produce_weights_batched( + self, specs: list, consumer_id: int = 0, seq: int = -1 + ): + """Serve one batched slice request over NIXL. + + Waits until the specs' names are cached, then rendezvouses with the + other live sharers of this consumer's slot-sharing group. The last to + arrive replays each spec's op chain (pure views into cached tensors, + guarded by ALLOWED_OPS) and byte-packs the slices 16B-aligned into the + group's ring slot ``seq % nring``, mirroring the consumer's identical + layout; every sharer then returns that one packed blob, so R deployments + cost one pack and one slot instead of R. With a single deployment every + group is a singleton, so the arriving call is always its own packer and + this is the unshared serve path exactly. + + ``seq`` is the caller's index in its pull stream to this producer, which + is what fixes the slot. Callers that want a single slice pass one spec + and read the blob back with that slice's dtype/shape. + + Raises: + ValueError: ``seq`` was not supplied. + """ + needed = sorted({n for n, _ in specs}) + if self._served_names is not None: + unserved = [n for n in needed if n not in self._served_names] + if unserved: + # Without this the cache wait below would block forever: this + # producer never gathers these names, so they can never arrive. + raise RuntimeError( + f"pull routed to the wrong producer: {unserved[:3]} " + f"({len(unserved)} names) are not served here" + ) + with self._cache_cond: + self._wait_for( + lambda: not all(n in self._cache for n in needed), + f"{len(needed)} name(s) to be published", + ) + if not all(n in self._cache for n in needed): + # _wait_for only gives up on a gather error (its own stall + # included), so the names are never coming. + raise RuntimeError( + f"gather errored before {needed}: {self._gather_error!r}" + ) + + if seq < 0: + # The slot is derived from it, so a caller that does not send one + # cannot be served safely. + raise ValueError( + "rdt_produce_weights_batched requires the consumer's issue index `seq`" + ) + gen, is_packer = self._join_share_group(consumer_id, seq) + if not is_packer: + blob = self._await_shared_pack(gen) + else: + try: + blob = self._pack_shared(gen, specs) + except BaseException as e: + self._fail_shared_pack(gen, e) + raise + # A served pull is the third of the four progress signals the stall + # watchdog reads (publish / produce / free / begin_sync): a long sync whose + # consumers pull steadily but slowly must never trip it. + with self._cache_cond: + self._note_progress_locked() + return [blob] + + def _pack_shared(self, gen: _SharedPack, specs: list) -> torch.Tensor: + """Replay the op chains into this generation's slot and publish the blob + to the sharers waiting on it. + + Runs on exactly one call per generation and OUTSIDE ``_cache_cond``: the + pack is GPU work and must not block publishes, frees or other groups. + """ + sliced: list = [] # (byte_off, tensor) + pack_cur = 0 + for name, chain in specs: + t = self._cache[name] + for op, args, kw in chain: + if op not in ALLOWED_OPS: + raise ValueError(f"{name!r}: disallowed op {op!r}") + t = getattr(t, op)(*args, **dict(kw)) + off = (pack_cur + 15) & ~15 + pack_cur = off + t.numel() * t.element_size() + sliced.append((off, t)) + + buffer = self._serve_slot(gen.group, gen.slot, pack_cur) + + # The destination views are a pure function of the packed + # layout, which is byte-identical every sync, so build them once per + # (sharing group, ring slot, layout) and reuse — rebuilding per call cost + # 5.2ms of a 7.5ms 384-spec 235B group. + # + # The key is the LAYOUT, not the spec names: a name can appear in two + # requests with different op chains (one name's copies can split across + # owner-class chunks), and serving the second through the first's views + # would write the wrong bytes with nothing downstream to catch it. + dst_key = ( + gen.group, + gen.slot, + tuple((off, t.dtype, t.shape) for off, t in sliced), + ) + cached = self._pack_dsts.get(dst_key) + if cached is None or cached[0] != buffer.data_ptr(): + dsts = [] + for off, t in sliced: + nb = t.numel() * t.element_size() + dsts.append(buffer[off : off + nb].view(t.dtype).reshape(t.shape)) + self._pack_dsts[dst_key] = (buffer.data_ptr(), dsts) + else: + dsts = cached[1] + torch._foreach_copy_(dsts, [t for _off, t in sliced]) + + blob = buffer[:pack_cur] + with self._cache_cond: + gen.blob = blob + gen.done = True + self._note_progress_locked() + self._cache_cond.notify_all() + return blob + + def shutdown(self) -> None: + with self._cache_cond: + self._cache.clear() + with self._cache_cond: + # Generations hold blob views into the rings, so they go first. + self._gens.clear() + self._sharers.clear() + with self._serve_lock: + self._serve_rings.clear() + # Must go with the rings: these are views INTO them, and the + # data_ptr guard that normally invalidates them cannot tell a freed + # buffer from a new one recycled at the same address. + self._pack_dsts.clear() + + +class ShardedRDTTrainerWeightTransferEngine( + TrainerWeightTransferEngine[ShardedRDTTrainerInitInfo] +): + """Trainer-side engine for the pull-based sharded-RDT backend. + + Lives on every trainer rank. Owns a per-rank `_RDTProducerServer` actor + (the NIXL serve surface). `send_weights` gathers this rank's weights + group-by-group from the `WeightSource`, shares each group into the server + over CUDA IPC, and — on the sender — drives the inference-side handshake so + the workers pull. Non-sender ranks only gather (staying in the collective). + """ + + init_info_cls = ShardedRDTTrainerInitInfo + + def __init__( + self, + *, + client: VLLMWeightSyncClient, + source: WeightSource, + is_sender: bool = True, + init_info: ShardedRDTTrainerInitInfo, + ) -> None: + super().__init__(client=client, source=source, is_sender=is_sender) + self._init_info = init_info + self._server: Any = None # the _RDTProducerServer actor handle + self._server_name: str | None = None + # Group-major metadata / partition, computed at trainer_init. + self._meta: list[ParamMeta] = [] + self._groups: list[list[str]] = [] + # Ownership resolved at trainer_init from the fleet's held names: the + # distinct owner sets, the per-name index into them, this rank's groups + # and its held set. Routing itself is consumer-side; the trainer only + # ships the table. + self._owner_sets: list[list[int]] = [] + self._name_owner_class: list[int] = [] + self._owned_idx: list[int] = [] + self._held_names: set[str] | None = None + # Strong refs to gathered tensors we've shared into the server, keyed by + # group index. CUDA-IPC exports must outlive the importer, so we hold + # them until the server reports the group freed. See send_weights. + self._inflight: dict[int, dict[str, torch.Tensor]] = {} + # [RDT-EXPORT-RING] Reusable packed slots, one per resident group. See + # _pack_group_for_export. Persistent across syncs: freeing and + # reallocating them each sync fragmented the caching allocator badly + # enough to grow reserved memory ~1.5 GB per sync. + self._export_ring: list[torch.Tensor | None] = [] + self._export_ring_args: list[tuple | None] = [] + # Which live group holds which slot, and the slots free to hand out. A + # modular counter is unsafe here: the credit gate bounds the NUMBER of + # unfreed groups, not their order, so it can hand a later group the slot + # a still-live group is being served from. + self._slot_of_group: dict[int, int] = {} + self._free_slots: list[int] = [] + # Set by trainer_init from the server-name all-gather; retained so a + # restarted consumer can be re-initialized without another collective. + self._server_names: list[str] | None = None + + def _rpc(self, method: str, *args: Any) -> Any: + """Call one of the server actor's methods and block for the result. + The single seam through which the engine talks to its server, so tests + can inject a local (non-Ray) fake server.""" + import ray + + return ray.get(getattr(self._server, method).remote(*args)) + + @staticmethod + def _contig_stride(shape) -> tuple: + stride: list[int] = [] + acc = 1 + for s in reversed(list(shape)): + stride.append(acc) + acc *= int(s) + return tuple(reversed(stride)) + + def _pack_group_for_export(self, held: list, slot_idx: int) -> tuple: + """Copy ``held`` into ring slot ``slot_idx``; return ``(storages, views, + refs)`` in the same shape the per-storage path returns, but with ONE + storage. Views are built exactly as ``publish_group`` rebuilds them, so + the two sides cannot disagree about layout. + + Slot safety is not the credit gate alone: that bounds how MANY groups + are unfreed, not the order they free in, and barriers do complete out of + order. The caller takes ``slot_idx`` from a pool keyed by live group + (``_slot_of_group`` / ``_free_slots``); ``_drop_inflight`` returns a slot + only once its group is freed everywhere. + """ + offsets: list[int] = [] + cur = 0 + for _name, t in held: + cur = (cur + 15) & ~15 # 16B keeps every element size we ship aligned + offsets.append(cur) + cur += t.numel() * t.element_size() + need = max(16, (cur + 15) & ~15) + + slot = self._export_ring[slot_idx] + device = held[0][1].device + if slot is None or slot.numel() < need or slot.device != device: + self._export_ring[slot_idx] = self._export_ring_args[slot_idx] = None + slot = torch.empty(need, dtype=torch.uint8, device=device) + self._export_ring[slot_idx] = slot + + ust = slot.untyped_storage() + sid = ust.data_ptr() + reduce_args = self._export_ring_args[slot_idx] + if reduce_args is None: + base = torch.empty(0, dtype=torch.uint8, device=device) + base.set_(ust, 0, (ust.nbytes(),)) + _rebuild, reduce_args = reduce_tensor(base) + self._export_ring_args[slot_idx] = reduce_args + + views: dict[str, tuple] = {} + refs: dict[str, torch.Tensor] = {} + for (name, t), off in zip(held, offsets): + shape, esz = tuple(t.shape), t.element_size() + stride = self._contig_stride(shape) + dst = torch.as_strided(slot.view(t.dtype), shape, stride, off // esz) + dst.copy_(t) + refs[name] = dst + views[name] = ( + sid, + str(t.dtype).split(".")[-1], + list(shape), + list(stride), + off // esz, + ) + # The producer sidecar reads this slot from another process over CUDA + # IPC, while these copies are merely enqueued on this rank's stream; + # nothing orders the two. Without this sync it packs bytes the copy has + # not written yet -- silent corruption, and only for ring-packed tensors. + if slot.is_cuda: + torch.cuda.current_stream(slot.device).synchronize() + return {sid: reduce_args}, views, refs + + def _publish_async(self, group_idx: int, entries): + """Fire publish_group WITHOUT waiting on the RPC (the gather loop + overlaps the publish's server-side rebuild with the next group's + gather) and return a handle that ``_await_publish`` resolves. Ray actor + handle in production; a plain (non-Ray) fake server runs inline.""" + method = self._server.publish_group + remote = getattr(method, "remote", None) + if remote is not None: + return remote(group_idx, entries) + return method(group_idx, entries) + + def _await_publish(self, ref) -> None: + """Resolve one async publish so a server-side rebuild error surfaces at + window depth instead of at end_sync. Publishes carry nothing back — + freed groups flow only through wait_freed/end_sync (one channel; see + publish_group).""" + import ray + + if isinstance(ref, ray.ObjectRef): + ray.get(ref) + + # ---------------- construction ---------------- + + @classmethod + def trainer_init( + cls, + init_info: ShardedRDTTrainerInitInfo, + *, + client: VLLMWeightSyncClient, + source: WeightSource | None = None, + ) -> Self: + if source is None: + raise ValueError( + "Sharded RDT trainer weight transfer requires a WeightSource." + ) + engine = cls( + client=client, + source=source, + is_sender=init_info.is_sender, + init_info=init_info, + ) + + # The VMM-based expandable_segments allocator makes IPC storage opens ~9x + # slower on both export and rebuild, and CUDA-IPC publish is this engine's + # hot path. Frameworks enabling it should disable it around send_weights. + # Only the env var is visible; runtime changes are not introspectable. + import os as _os + + if "expandable_segments:True" in _os.environ.get("PYTORCH_CUDA_ALLOC_CONF", ""): + logger.warning( + "Sharded-RDT trainer: PYTORCH_CUDA_ALLOC_CONF enables " + "expandable_segments; CUDA-IPC weight publishing will be " + "several times slower. Disable expandable segments around " + "weight sync (allocate gather buffers in classic segments)." + ) + + engine._meta = list(source.metadata()) + names = [m.name for m in engine._meta] + engine._groups = layerwise_groups(names) + flat = [n for g in engine._groups for n in g] + if flat != names: + raise ValueError( + "Sharded RDT requires a WeightSource whose metadata order is " + "group-contiguous. Reorder the source so all names sharing a " + "layer index are adjacent." + ) + + world, rank = engine._world_and_rank() + engine._resolve_ownership(world, rank) + engine._spawn_server(sorted(engine._held_names or [])) + + # Every rank's server must exist before the sender's init RPC (the worker + # init calls reserve_serve_buffer back on ALL producer servers). The + # all-gather of server names doubles as that barrier. + server_names = engine._all_gather_server_names(world, rank) + # Retained so a RESTARTED consumer can be re-initialized without another + # all-gather (see get_worker_init_payload). The uuid actor names stay valid + # for the run, because producers never restart. + engine._server_names = server_names + + if engine.is_sender: + worker_init = engine._build_worker_init_info(server_names) + engine.client.init_weight_transfer_engine(asdict(worker_init)) + return engine + + def get_worker_init_payload(self) -> dict: + """The consumer-side init payload, rebuilt on demand. Pure — no collective. + + Reads only retained state, so a restarted inference engine can rejoin at a + sync boundary: the driver can ask any time, including mid-run with every + rank in its own training step. A collective here would deadlock, since the + ranks are not at a matching one. + + Raises: + RuntimeError: called before ``trainer_init`` cached the server names. + """ + if self._server_names is None: + raise RuntimeError( + "get_worker_init_payload requires trainer_init to have run (the " + "producer server names are gathered there)." + ) + return asdict(self._build_worker_init_info(self._server_names)) + + def _world_and_rank(self) -> tuple[int, int]: + if torch.distributed.is_available() and torch.distributed.is_initialized(): + return torch.distributed.get_world_size(), torch.distributed.get_rank() + return 1, self._init_info.rank + + def _resolve_ownership(self, world: int, rank: int) -> None: + """Resolve which rank holds which name, and this rank's publish plan. + + A source may hold only part of the model — pipeline stages, expert + parallelism, or any mix — so each rank declares its held names and the + fleet all-gathers them. The consumers route per name, so the wire carries + the transposed result: the distinct owner sets, and a per-name index into + them. + + The masks are positional over metadata order, which is why the metadata + digest is checked first: a rank whose names disagree would transpose into + the wrong owners entirely. + """ + assert self.source is not None # guaranteed by trainer_init + num_groups = len(self._groups) + names = [m.name for m in self._meta] + held = self.source.held_names() + if held is None: + held_set = set(names) + else: + held_set = {str(n) for n in held} + unknown = sorted(held_set - set(names)) + if unknown: + raise ValueError( + f"WeightSource.held_names() lists {len(unknown)} name(s) not " + f"in metadata(), e.g. {unknown[:3]}." + ) + if not held_set: + raise ValueError( + "WeightSource.held_names() is empty; a rank with nothing to " + "serve cannot take part in the gather" + ) + + # One collective carries this rank's holdings as a bitmask over metadata + # order, plus the digest of the metadata it indexes into. The digest is + # what makes partial ownership safe: only the sender's metadata reaches + # the consumers, so a rank describing just its own share would silently + # serve the wrong weights. + digest = self._meta_digest() + mask = bytearray((len(names) + 7) // 8) + for i, n in enumerate(names): + if n in held_set: + mask[i >> 3] |= 1 << (i & 7) + per_rank = self._all_gather_owned(world, (digest, bytes(mask))) + mismatched = [r for r, (d, *_rest) in enumerate(per_rank) if d != digest] + if mismatched: + raise ValueError( + f"WeightSource.metadata() disagrees across trainer ranks " + f"(rank {rank} digest {digest}, differing ranks {mismatched[:4]}). " + "Every rank must describe the WHOLE model, even when it holds " + "only some of it." + ) + masks = [m for _d, m in per_rank] + + # Transpose to per-name owners, then dedup into classes. Numbering by + # FIRST APPEARANCE in metadata order keeps it a pure function of + # rank-identical inputs, which is what lets a rejoining consumer rebuild + # the identical table from get_worker_init_payload. + owner_sets: list[list[int]] = [] + class_of_owners: dict[tuple[int, ...], int] = {} + name_owner_class: list[int] = [] + for i, n in enumerate(names): + owners = tuple(r for r, m in enumerate(masks) if m[i >> 3] & (1 << (i & 7))) + if not owners: + raise ValueError( + f"no trainer rank holds {n!r}; every name in metadata() must " + "be held by at least one rank or it can never be served." + ) + ci = class_of_owners.get(owners) + if ci is None: + ci = len(owner_sets) + class_of_owners[owners] = ci + owner_sets.append(list(owners)) + name_owner_class.append(ci) + + self._owner_sets = owner_sets + self._name_owner_class = name_owner_class + # Groups holding anything here: exactly what iteration must cover. + self._owned_idx = [ + gi + for gi in range(num_groups) + if any(n in held_set for n in self._groups[gi]) + ] + self._held_names = held_set + + def _validate_held_yields(self, gi: int, names, tensors) -> None: + """Check ``held_names()`` against what the source actually yields — the + one place both sit side by side. + + Without it, a source that claims a name but yields ``None`` for it dies + by stall watchdog 300s later: consumers route pulls here, the pull + passes the served-names guard, and the cache wait never completes. With + it, that is an immediate error naming the weight. One set lookup per + name per sync.""" + if self._held_names is None: + return + for name, tensor in zip(names, tensors): + if (tensor is None) == (name in self._held_names): + claim = "does not hold" if tensor is not None else "holds" + have = "a real tensor" if tensor is not None else "None" + raise RuntimeError( + f"held_names() disagrees with the yielded tensors: group {gi} " + f"name {name!r} yielded {have} but this rank {claim} it. A " + "WeightSource must yield a real tensor for every held name " + "and None for the rest (see WeightSource.held_names)." + ) + + def _meta_digest(self) -> str: + """Stable digest of this rank's metadata (name order + count).""" + import hashlib + + h = hashlib.sha256() + h.update(f"{len(self._meta)}\n".encode()) + for m in self._meta: + h.update(m.name.encode()) + h.update(b"\n") + return h.hexdigest()[:16] + + def _all_gather_owned(self, world: int, mine: tuple) -> list[tuple]: + """All-gather each rank's (metadata digest, held-name bitmask).""" + if world <= 1 or not ( + torch.distributed.is_available() and torch.distributed.is_initialized() + ): + return [mine] + gathered: list[Any] = [None] * world + torch.distributed.all_gather_object(gathered, mine) + return gathered + + def _all_gather_server_names(self, world: int, rank: int) -> list[str]: + assert self._server_name is not None + if world <= 1 or not ( + torch.distributed.is_available() and torch.distributed.is_initialized() + ): + return [self._server_name] + gathered: list[str | None] = [None] * world + torch.distributed.all_gather_object(gathered, self._server_name) + return [n for n in gathered if n is not None] + + def _spawn_server(self, served_names: list[str]) -> None: + import ray + from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy + + # The trainer is usually a separate install from the workers, so it gets + # its own check rather than trusting the consumer side's. + check_ray_rdt_version() + + ii = self._init_info + self._server_name = f"vllm_rdt_producer_{uuid.uuid4().hex[:12]}_rk{ii.rank}" + node_id = ray.get_runtime_context().get_node_id() + # Pin the server to this rank's physical GPU: num_gpus=0 so Ray does not + # allocate a second, CUDA_VISIBLE_DEVICES so CUDA IPC to the rank's + # gathered tensors works. max_concurrency > 1 serves pulls while control + # calls are in flight; enable_tensor_transport gives the NIXL serve. + gpu_ids = ray.get_gpu_ids() + # The server is the trainer rank's process twin: forward the env the + # rank runs under (library paths etc.) so it imports torch/vllm the + # same way, then pin it to the rank's physical GPU for CUDA IPC. + import os + + env_vars = { + k: os.environ[k] + for k in ( + "LD_LIBRARY_PATH", + "LD_PRELOAD", + "NCCL_CUMEM_ENABLE", + "VLLM_NCCL_SO_PATH", + "PATH", + ) + if k in os.environ + } + if gpu_ids: + # A num_gpus=0 actor sharing the rank's GPU: Ray would set + # CUDA_VISIBLE_DEVICES="" and hide every GPU, so tell it not to touch + # the var and pin the device ourselves. + env_vars["RAY_EXPERIMENTAL_NOSET_CUDA_VISIBLE_DEVICES"] = "1" + env_vars["CUDA_VISIBLE_DEVICES"] = str(gpu_ids[0]) + runtime_env = {"env_vars": env_vars} if env_vars else {} + server_cls = ray.remote(_RDTProducerServer).options( + name=self._server_name, + namespace=ii.trainer_actor_namespace, + num_cpus=0, + num_gpus=0, + # Thread budget: under EP-local routing every consumer pulls from every + # producer, and with issue-ahead each can park K produce calls in the + # cache-wait — C*K blocked calls. A smaller pool queues the control + # plane behind them and deadlocks: the parked produces wait for a + # publish that can never get a thread (the sync-2 wedge at 235B tp8, + # 16 parked vs 8 threads). Size for the worst case plus control-plane + # slack. + max_concurrency=ii.num_consumers * ii.num_rdt_buffers + 4, + enable_tensor_transport=True, + scheduling_strategy=NodeAffinitySchedulingStrategy( + node_id=node_id, soft=False + ), + runtime_env=runtime_env, + ) + self._server = server_cls.remote( + served_names=served_names, + num_rdt_buffers=ii.num_rdt_buffers, + buffer_presize_gb=ii.buffer_presize_gb, + gather_lookahead=ii.gather_lookahead, + stall_timeout_s=ii.stall_timeout_s, + workers_per_replica=ii.workers_per_replica, + ) + ray.get(self._server.ping.remote()) + # Pre-barrier NIXL warmup: must complete before the server-name + # all-gather so no rank can be inside a collective yet (see warmup_nixl). + ray.get(self._server.warmup_nixl.remote()) + + def _build_worker_init_info(self, server_names: list[str]): + from vllm.distributed.weight_transfer.sharded_rdt_engine import ( + ShardedRDTWeightTransferInitInfo, + ) + + group_lens = [len(g) for g in self._groups] + names = [m.name for m in self._meta] + dtype_names = [str(m.dtype).split(".")[-1] for m in self._meta] + shapes = [list(m.shape) for m in self._meta] + return ShardedRDTWeightTransferInitInfo( + trainer_actor_names=server_names, + trainer_actor_namespace=self._init_info.trainer_actor_namespace, + produce_method_name=PRODUCE_METHOD_NAME, + names=names, + dtype_names=dtype_names, + shapes=shapes, + group_lens=group_lens, + owner_sets=self._owner_sets, + name_owner_class=self._name_owner_class, + num_consumers=self._init_info.num_consumers, + num_rdt_buffers=self._init_info.num_rdt_buffers, + buffer_presize_gb=self._init_info.buffer_presize_gb, + ) + + # ---------------- per-round ---------------- + + def send_weights(self, live_consumer_ids: Collection[int] | None = None) -> None: + """Gather this rank's weights and publish them for the consumers to pull. + + ``live_consumer_ids`` restricts the sync to the consumers still alive; + ``None`` serves the whole provisioned set. The provisioned geometry is + FROZEN for the run — a degraded sync only lowers the live count handed to + ``begin_sync``. Every rank must get the SAME live set, since they share + the gather collectives, so the caller computes it once for all of them. + """ + assert self.source is not None + if live_consumer_ids is None: + live_ids = list(range(self._init_info.num_consumers)) + live_count = len(live_ids) + else: + live_ids = sorted(set(int(c) for c in live_consumer_ids)) + live_count = len(live_ids) + logger.warning( + "[rdt-degraded] serving %d/%d live consumers; every group's " + "free barrier counts to the live total", + live_count, + self._init_info.num_consumers, + ) + self._send_weights_inner(live_count, live_ids) + + def _send_weights_inner(self, live_count: int, live_ids: list[int]) -> None: + if not self.is_sender: + self._run_gather_loop( + update_future=None, live_count=live_count, live_ids=live_ids + ) + return + + self.client.start_weight_update() + + from vllm.distributed.weight_transfer.sharded_rdt_engine import ( + ShardedRDTWeightTransferUpdateInfo, + ) + + empty_update = asdict(ShardedRDTWeightTransferUpdateInfo()) + with ThreadPoolExecutor(max_workers=1) as exe: + # The workers block inside update_weights until they've pulled every + # group, so it runs concurrently with the gather/publish loop. + future = exe.submit(self.client.update_weights, empty_update) + self._run_gather_loop( + update_future=future, live_count=live_count, live_ids=live_ids + ) + future.result() # surface inference-side errors + + self.client.finish_weight_update() + + def _run_gather_loop( + self, update_future, live_count: int, live_ids: list[int] | None = None + ) -> None: + """Gather this rank's weights group-by-group and publish each into the + server over CUDA IPC. A gathered group is published — serveable — + immediately; the loop gates BEFORE the next gather while more than + `gather_lookahead` groups are unfreed (the per-group free barrier: a + credit releases when every live consumer has signaled the group). So + the loop self-paces to the consumers' pull rate with at most + `gather_lookahead + 1` groups resident. Runs on every rank; only the + sender has an `update_future` to fail fast on.""" + assert self.source is not None # guaranteed by trainer_init + # The live IDS ride along with the count: the free barrier needs only + # the count, but the slot-sharing rendezvous needs to know WHICH + # consumers are live. + self._rpc("begin_sync", live_count, live_ids) + # One generator resume per GROUP: `iter_groups` yields (names, tensors) + # per owned group in metadata order. Every owner must reach a group in the + # same order or their shared gather collective mismatches. + groups = self.source.iter_groups() + # Publishes fire without an inline ray.get and are harvested this window + # deep, so the RPC and server-side rebuild overlap the NEXT group's + # gather/export. The window also surfaces a server-side error at depth 2 + # rather than at end_sync. + _PUBLISH_WINDOW = 2 + pending_publish: list = [] + # The memory bound: gate BEFORE gathering while more + # than `bound` groups are unfreed, so at most bound + 1 are resident. The + # count is exact — `_inflight` gains its entry right after each gather and + # only wait_freed/end_sync shrinks it, so a group freed server-side but not + # yet collected still holds its refs and still counts. The publish window + # drains first, so every resident group is pullable while we wait. + # + # 0 is legal and means NO pipelining: the loop waits for group N to be + # freed by every live consumer before gathering N+1, so exactly one group + # is resident and the sync costs the full serialized + # gather + publish + pull + free RTT per group. 1 (the default) overlaps + # N+1's gather with N's pulls. + bound = max(0, self._init_info.gather_lookahead) + + # [RDT-EXPORT-RING] `bound + 1` slots is exactly the residency the credit + # gate enforces. Small storages are packed into a slot and exported once + # per slot instead of once per storage (518 -> 7 cudaIpcGetMemHandle + # calls per rank per sync at 235B, each 0.5-1.5ms). + nslots = bound + 1 + # data_ptr -> (storage weakref, reduce_args), this sync only. `storages` + # is per-group, so a source that reuses a buffer across groups would + # otherwise re-export it every group. The weakref is load-bearing: a + # freed allocation can be reissued at the same address, and serving that + # from a stale handle is silently wrong bytes. + handle_cache: dict[int, tuple] = {} + if len(getattr(self, "_export_ring", [])) != nslots: + self._export_ring = [None] * nslots + self._export_ring_args = [None] * nslots + # Slots are handed out per group and returned by _drop_inflight, so the + # pool starts full every sync. + self._slot_of_group = {} + self._free_slots = list(range(nslots)) + + try: + for gi in self._owned_idx: + group = self._groups[gi] + while len(self._inflight) > bound: + while pending_publish: + self._await_publish(pending_publish.pop(0)) + self._drop_inflight(self._rpc("wait_freed")) + names, tensors = next(groups) + if list(names) != list(group): + raise RuntimeError( + f"WeightSource group yielded {len(names)} names starting " + f"{names[:2]!r} but expected {len(group)} starting " + f"{group[:2]!r}; iteration order must match metadata." + ) + self._validate_held_yields(gi, names, tensors) + # Share each unique STORAGE once and describe every name as an + # as_strided view onto it. ``None`` means a name this rank does not + # hold (a foreign expert); the source keeps it in the list so the + # order check stays rank-uniform, and it is dropped here, before + # the IPC export, matching the sidecar's served_names. + storages: dict[int, tuple] = {} + views: dict[str, tuple] = {} + refs: dict[str, torch.Tensor] = {} + small: list = [] + for name, tensor in zip(names, tensors): + if tensor is None: + continue + tensor = tensor.detach() + if not tensor.is_cuda: + tensor = tensor.cuda() + # Threshold on the STORAGE, not the tensor: a source may hand + # out many small views of one large stacked buffer, and a + # per-tensor test would pack the whole thing, costing +1 + # group of peak memory for a storage already coalesced. + if tensor.untyped_storage().nbytes() <= _EXPORT_RING_MAX_BYTES: + small.append((name, tensor)) + continue + tensor = tensor.contiguous() + refs[name] = tensor # keep the export alive + ust = tensor.untyped_storage() + sid = ust.data_ptr() + if sid not in storages: + cached = handle_cache.get(sid) + if cached is not None and not cached[0].expired(): + reduce_args = cached[1] + else: + base = torch.empty( + 0, dtype=torch.uint8, device=tensor.device + ) + base.set_(ust, 0, (ust.nbytes(),)) + _rebuild, reduce_args = reduce_tensor(base) + handle_cache[sid] = (StorageWeakRef(ust), reduce_args) + storages[sid] = reduce_args + views[name] = ( + sid, + str(tensor.dtype).split(".")[-1], + list(tensor.shape), + list(tensor.stride()), + tensor.storage_offset(), + ) + if small: + # Take a slot no live group holds. The gate above already + # guarantees one exists, so this is a backstop -- but it + # must drain publishes first, like the credit gate. + while not self._free_slots: + while pending_publish: + self._await_publish(pending_publish.pop(0)) + self._drop_inflight(self._rpc("wait_freed")) + slot_idx = self._free_slots.pop(0) + self._slot_of_group[gi] = slot_idx + st, vw, rf = self._pack_group_for_export(small, slot_idx) + storages.update(st) + views.update(vw) + refs.update(rf) + del small + del tensors + # The loop leaves its last `tensor` / `ust` / `base` bound + # through the credit gate and into the next gather, and `base` + # is a whole-storage view -- so one tensor and its entire + # allocation stay alive past `_drop_inflight`, unseen by the + # `_inflight` accounting the residency bound rests on. + # Assignment, not `del`: unbound if no direct-path tensor. + tensor = ust = base = None + if not refs: + # A group with nothing held here cannot occur (every group + # carries replicated names), but publishing an empty group + # would park a credit nobody's pull is waiting on; skip. + continue + # Hold our refs before publishing; drop them only when the + # server reports the group freed (IPC export must outlive import). + self._inflight[gi] = refs + pending_publish.append(self._publish_async(gi, (storages, views))) + while len(pending_publish) >= _PUBLISH_WINDOW: + self._await_publish(pending_publish.pop(0)) + if update_future is not None and update_future.done(): + # update_weights returned/failed early — surface now instead + # of blocking further gathers. + update_future.result() + while pending_publish: + self._await_publish(pending_publish.pop(0)) + freed = self._rpc("end_sync") + self._drop_inflight(freed) + except BaseException as e: + with contextlib.suppress(Exception): + self._rpc("set_gather_error", repr(e)) + self._inflight.clear() + raise + finally: + # Keep the slots, drop the handles: the sidecar's base cache is + # cleared per sync, so a handle must not outlive the sync either. + self._export_ring_args = [None] * len(getattr(self, "_export_ring", [])) + + def _drop_inflight(self, freed_keys: list) -> None: + """Release a freed group's refs and return its export-ring slot.""" + free_slots = getattr(self, "_free_slots", None) + slot_of_group = getattr(self, "_slot_of_group", None) + for k in freed_keys: + gi = int(k) + self._inflight.pop(gi, None) + if slot_of_group is None: + continue + slot = slot_of_group.pop(gi, None) + if slot is not None and free_slots is not None and slot not in free_slots: + free_slots.append(slot) + + # ---------------- misc ---------------- + + def shutdown(self) -> None: + if self._server is None: + return + import ray + + with contextlib.suppress(Exception): + ray.get(self._server.shutdown.remote()) + ray.kill(self._server) + self._server = None + self._inflight.clear() diff --git a/vllm/v1/executor/ray_executor_v2.py b/vllm/v1/executor/ray_executor_v2.py index 33c3865157..4867757167 100644 --- a/vllm/v1/executor/ray_executor_v2.py +++ b/vllm/v1/executor/ray_executor_v2.py @@ -361,6 +361,19 @@ class RayExecutorV2(MultiprocExecutor): runtime_env = self._build_runtime_env() resource_kwargs = self._get_actor_resource_kwargs() + # The sharded_rdt weight transfer backend pulls weights via Ray's tensor + # transport (@ray.method(tensor_transport=...)). Ray requires the + # *calling* actor (the vLLM worker) to opt in via enable_tensor_transport. + wt_cfg = self.vllm_config.weight_transfer_config + extra_actor_options: dict[str, object] = {} + if wt_cfg is not None and wt_cfg.backend == "sharded_rdt": + from vllm.distributed.weight_transfer.sharded_rdt_common import ( + check_ray_rdt_version, + ) + + check_ray_rdt_version() + extra_actor_options["enable_tensor_transport"] = True + for bundle_idx in range(self.world_size): bundle = bundle_assignments[bundle_idx] is_driver_worker = self._is_driver_worker(bundle["rank"]) @@ -383,6 +396,7 @@ class RayExecutorV2(MultiprocExecutor): **resource_kwargs, scheduling_strategy=scheduling_strategy, runtime_env=runtime_env, + **extra_actor_options, ) .remote( vllm_config=self.vllm_config,