[RL] P2P RDT weight sync (#43375)
This commit is contained in:
@@ -17,6 +17,21 @@ steps:
|
||||
- pytest -v -s distributed/test_shm_buffer.py
|
||||
- pytest -v -s distributed/test_shm_storage.py
|
||||
|
||||
- label: ":nvidia: (L4) Sharded RDT Weight Transfer"
|
||||
key: sharded-rdt-weight-transfer
|
||||
timeout_in_minutes: 20
|
||||
working_dir: "/vllm-workspace/tests"
|
||||
num_devices: 1
|
||||
source_file_dependencies:
|
||||
- vllm/distributed/weight_transfer/
|
||||
- tests/distributed/test_sharded_rdt_plan.py
|
||||
- tests/distributed/test_sharded_rdt_producer.py
|
||||
- tests/distributed/test_sharded_rdt_trainer.py
|
||||
commands:
|
||||
- pytest -v -s distributed/test_sharded_rdt_plan.py
|
||||
- pytest -v -s distributed/test_sharded_rdt_producer.py
|
||||
- pytest -v -s distributed/test_sharded_rdt_trainer.py
|
||||
|
||||
- label: ":nvidia: (L4) Distributed DP Basic"
|
||||
key: distributed-dp-tests-2-gpus
|
||||
timeout_in_minutes: 35
|
||||
@@ -107,7 +122,7 @@ steps:
|
||||
|
||||
- label: ":nvidia: (L4) Distributed Torchrun + Examples"
|
||||
key: distributed-torchrun-examples-4-gpus
|
||||
timeout_in_minutes: 30
|
||||
timeout_in_minutes: 45
|
||||
working_dir: "/vllm-workspace"
|
||||
num_devices: 4
|
||||
source_file_dependencies:
|
||||
@@ -137,6 +152,15 @@ steps:
|
||||
# rlhf examples (each launches its own `vllm serve` and tears it down)
|
||||
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_nccl.py
|
||||
- VLLM_ALLOW_INSECURE_SERIALIZATION=1 python3 examples/rl/rlhf_http_ipc.py
|
||||
# sharded RDT: 2 FSDP2 trainer ranks -> 2 vLLM DP ranks with EP. Its data
|
||||
# plane is NIXL, which ships only in the kv-connector set, and the script
|
||||
# also drops the nixl-cu wheel variant that does not match this image. Kept
|
||||
# last so the commands above run in an unmodified environment. RDT_MODEL swaps
|
||||
# the script's reference MoE for a tiny one: no weights are downloaded either
|
||||
# way, but every trainer rank builds the model before sharding it, and the
|
||||
# default does not fit an L4.
|
||||
- bash /vllm-workspace/.buildkite/scripts/install-kv-connectors.sh
|
||||
- RDT_MODEL=hmellor/tiny-random-DeepseekV2ForCausalLM python3 examples/rl/rlhf_sharded_rdt_small_ep.py
|
||||
|
||||
- label: ":nvidia: (L4) Distributed DP Extended"
|
||||
key: distributed-dp-tests-4-gpus
|
||||
|
||||
@@ -34,6 +34,7 @@ engine drives on your behalf:
|
||||
| [NCCL](nccl.md) | NCCL broadcast | Separate GPUs for training and inference |
|
||||
| [IPC](ipc.md) | CUDA IPC handles | Colocated training and inference on same GPU |
|
||||
| [sparse_nccl](nccl.md#sparse-nccl) | NCCL broadcast | Sparse flat-index weight patches (TP=1/PP=1) |
|
||||
| [sharded_rdt](sharded_rdt.md) | NIXL / Ray Direct Transport (pull-based) | Very large models where each worker needs only its own slice (MoE with expert parallelism) |
|
||||
|
||||
## Quickstart
|
||||
|
||||
@@ -48,7 +49,7 @@ from vllm.config import WeightTransferConfig
|
||||
|
||||
llm = LLM(
|
||||
model="my-model",
|
||||
weight_transfer_config=WeightTransferConfig(backend="nccl"), # or "ipc", "sparse_nccl"
|
||||
weight_transfer_config=WeightTransferConfig(backend="nccl"), # or "ipc", "sparse_nccl", "sharded_rdt"
|
||||
)
|
||||
```
|
||||
|
||||
@@ -158,7 +159,7 @@ When running vLLM as an HTTP server, the following endpoints are available for w
|
||||
!!! note
|
||||
The HTTP weight transfer endpoints require `VLLM_SERVER_DEV_MODE=1` to be set.
|
||||
|
||||
The Rust frontend's optional gRPC `Control` service exposes the same pause, sleep, weight-transfer, and weight-version lifecycle for trusted sidecars. The `ServerInfo.rl_capabilities` response reports whether weight transfer and sleep mode were configured. Backend-specific `init_info` and `update_info` remain JSON metadata; model tensors continue to move over the configured NCCL, IPC, or sparse-NCCL transport.
|
||||
The Rust frontend's optional gRPC `Control` service exposes the same pause, sleep, weight-transfer, and weight-version lifecycle for trusted sidecars. The `ServerInfo.rl_capabilities` response reports whether weight transfer and sleep mode were configured. Backend-specific `init_info` and `update_info` remain JSON metadata; model tensors continue to move over the configured NCCL, IPC, sparse-NCCL, or sharded-RDT transport.
|
||||
|
||||
## Extending the System
|
||||
|
||||
|
||||
@@ -20,8 +20,14 @@ vice versa, so the registries stay independent.
|
||||
|
||||
### WeightSource
|
||||
|
||||
A `WeightSource` is a **re-iterable** source of the trainer's weights with two
|
||||
channels:
|
||||
**This is the adapter for whatever shape your trainer's weights are in.**
|
||||
`WeightSource` defines how you extract your weights for your specific framework.
|
||||
|
||||
What an engine gets is always the same: HF-format parameter names, and tensors
|
||||
already materialized to their full (unsharded) shape. Whatever gathering,
|
||||
re-fusing, dequantizing or renaming that takes belongs inside the source.
|
||||
|
||||
A `WeightSource` is **re-iterable** and has two required channels:
|
||||
|
||||
- **`metadata() -> list[ParamMeta]`** — the name, wire dtype, and full shape of
|
||||
every parameter, *without transferring anything*. Cheap when shapes are known
|
||||
@@ -51,9 +57,7 @@ class ParamMeta:
|
||||
Materializing is typically a collective, so **every trainer rank must iterate the
|
||||
same source in the same order, in lockstep**, or ranks deadlock. `metadata()` can
|
||||
itself be a collective for custom producers, so it too runs on every rank — only
|
||||
the sender ships the result. Under pipeline parallelism a rank may not own a
|
||||
parameter at all; iterating still drives the collective, and the yielded tensor
|
||||
is only meaningful on the sender.
|
||||
the sender ships the result.
|
||||
|
||||
`iter(source)` must yield a *fresh* pass each round.
|
||||
|
||||
@@ -70,35 +74,133 @@ from vllm.distributed.weight_transfer import ModuleSource
|
||||
source = ModuleSource(model)
|
||||
```
|
||||
|
||||
#### Custom Sources
|
||||
#### Custom sources
|
||||
|
||||
Subclass `WeightSource` when the weights you want to send require additional processing to convert to a HF compatible format.
|
||||
Subclass `WeightSource` when the weights need work to reach HF format — a
|
||||
framework-specific export, a re-fusing step, a dtype cast.
|
||||
|
||||
```python
|
||||
from vllm.distributed.weight_transfer import ParamMeta, WeightSource
|
||||
from vllm.distributed.weight_transfer.base import materialize_full_tensor
|
||||
|
||||
|
||||
class MyExportSource(WeightSource):
|
||||
def __init__(self, model):
|
||||
self._model = model
|
||||
class MegatronBridgeSource(WeightSource):
|
||||
"""Megatron model -> HF names, via a bridge that gathers TP/PP/EP internally
|
||||
and returns full tensors on every rank."""
|
||||
|
||||
def __init__(self, bridge, module, dtype):
|
||||
self._bridge, self._module, self._dtype = bridge, module, dtype
|
||||
self._meta: list[ParamMeta] | None = None
|
||||
|
||||
def _export(self):
|
||||
return self._bridge.export_hf_weights(self._module)
|
||||
|
||||
def metadata(self) -> list[ParamMeta]:
|
||||
# Cache: for producers that must materialize to learn shapes, this is
|
||||
# the expensive channel. Runs on every rank (it may be a collective).
|
||||
if self._meta is None:
|
||||
self._meta = [
|
||||
ParamMeta(name, t.dtype, tuple(t.shape)) for name, t in self._export()
|
||||
ParamMeta(name, self._dtype, tuple(t.shape))
|
||||
for name, t in self._export()
|
||||
]
|
||||
return self._meta
|
||||
|
||||
def __iter__(self):
|
||||
# Must yield exactly what metadata() declared, in the same order.
|
||||
for name, tensor in self._export():
|
||||
yield name, materialize_full_tensor(tensor)
|
||||
yield name, tensor.to(self._dtype).detach().contiguous()
|
||||
```
|
||||
|
||||
#### `held_names()`: partial ownership
|
||||
|
||||
By default every rank is assumed to be able to produce every parameter — which is
|
||||
what the source above does, since its bridge gathers across all parallelism before
|
||||
yielding. That is simple and always correct, but it means the gather cost is paid
|
||||
in full on every rank.
|
||||
|
||||
Optionally, you can override `held_names()` when the ranks are split so each holds only part of the
|
||||
model. It returns the
|
||||
parameter names this rank holds (or `None`, the default, for all of them):
|
||||
|
||||
```python
|
||||
def held_names(self):
|
||||
# This pipeline stage's layers, and within them only this EP rank's experts.
|
||||
return self._my_stage_names - self._foreign_expert_names
|
||||
```
|
||||
|
||||
This covers various trainer layouts — pipeline stages (a rank holds some layers),
|
||||
expert parallelism (a rank holds some experts), both at once, or a shape that fits
|
||||
neither. Backends that can route per parameter (see
|
||||
[sharded RDT](sharded_rdt.md)) then pull each name from a rank that actually
|
||||
holds it.
|
||||
|
||||
Three requirements come with overriding it:
|
||||
|
||||
- **`metadata()` must still describe the whole model on every rank.** Only the
|
||||
sender's metadata reaches the inference side, so a rank that reported just its
|
||||
own share would leave the rest silently un-transferred. Sharded RDT cross-checks
|
||||
this across ranks at init.
|
||||
- **Every name must be held by at least one rank**, or it can never be served.
|
||||
The engine raises at init naming the first orphan.
|
||||
- **Iteration yields `None` for a name this rank does not hold.** The name still
|
||||
appears, in metadata order, so the order check stays aligned across ranks — only
|
||||
the data is absent. Claiming a name and then yielding `None` for it is an error
|
||||
the engine reports by name.
|
||||
|
||||
!!! warning "Partial ownership only works with sharded RDT"
|
||||
Only a backend that routes per parameter honours `held_names()`.
|
||||
Broadcast backends ignore it and send every name from every rank, so
|
||||
declaring partial ownership there changes nothing.
|
||||
|
||||
#### Gather groups
|
||||
|
||||
Some backends transfer a layer at a time rather than a model at a time, so they
|
||||
partition `metadata()` into **gather groups**. `layerwise_groups` keys each name
|
||||
on the **outermost index segment** it contains, so **a group is one decoder
|
||||
layer**, with runs of un-indexed names (the embeddings, the final norm,
|
||||
`lm_head`) forming groups of their own where they appear:
|
||||
|
||||
```text
|
||||
group 0 model.embed_tokens.weight
|
||||
group 1 model.layers.0.* <- one decoder layer
|
||||
group 2 model.layers.1.*
|
||||
...
|
||||
group N+1 model.norm.weight, lm_head.weight
|
||||
```
|
||||
|
||||
Keying on the index rather than a literal prefix means no per-architecture table:
|
||||
`model.layers.0.`, `model.language_model.layers.0.` (recent Qwen text
|
||||
checkpoints), `transformer.h.0.` (GPT-2, Falcon), `backbone.layers.0.` (Mamba)
|
||||
and a vision tower's `visual.blocks.0.` all partition the same way. The index
|
||||
taken is the *outermost* one, which keeps a MoE layer whole — a per-expert name
|
||||
like `model.layers.3.mlp.experts.7.w1` keys on the layer, not the expert.
|
||||
|
||||
Group index *g* means the same layer on every rank and every consumer, because
|
||||
every side derives it from one rank's `metadata()`. That agreement is what lets a
|
||||
backend bound its buffers to one layer and free a layer once everyone is done
|
||||
with it.
|
||||
|
||||
!!! warning "A leaf module's sources must all fall in one group"
|
||||
The sharded-RDT engine frees a group as soon as its last chunk lands, so a
|
||||
module split across groups would park a pull until the stall watchdog fires.
|
||||
The default partition guarantees this; a `groups()` override must preserve
|
||||
it.
|
||||
|
||||
Two hooks follow from this, both with working defaults:
|
||||
|
||||
- **`groups()`** — this rank's groups, in metadata order. The default is
|
||||
`layerwise_groups(metadata())` restricted to the groups holding at least one
|
||||
held name. A group with nothing held here is skipped entirely.
|
||||
- **`iter_groups()`** — the same stream, batched one group at a time. The default
|
||||
drives `__iter__` and batches its output, checking that names arrive in metadata
|
||||
order. Override it when your framework can produce a whole group in one step:
|
||||
materializing is usually a collective, and driving it per group rather than per
|
||||
tensor turns ~37k generator resumes into ~95 on a per-expert MoE model.
|
||||
|
||||
Because `metadata()` order defines the partition, **all names sharing a layer
|
||||
index must be contiguous in it**. A source whose natural export order interleaves
|
||||
layers — bucketing all the MoE experts together, say — has to reorder before
|
||||
returning.
|
||||
|
||||
### VLLMWeightSyncClient
|
||||
|
||||
**This is the adapter for however your RL stack reaches vLLM.** Many RL frameworks wrap
|
||||
@@ -372,7 +474,7 @@ Subclasses must implement five methods:
|
||||
The base class provides:
|
||||
|
||||
1. `__init__`, taking `config` (`WeightTransferConfig`), `vllm_config` (`VllmConfig`), `device` (`torch.device`), and `model` (`nn.Module`).
|
||||
2. `update_weights(update_info_dict)`, a thin wrapper for `receive_weights`: it parses the dict into the typed dataclass, calls `receive_weights`, and synchronizes the device.
|
||||
2. `update_weights(update_info_dict)`, a thin wrapper for `receive_weights`: it parses the dict into the typed dataclass, calls `receive_weights`, and synchronizes the device — unless the engine sets `defers_processing`, below.
|
||||
3. `parse_init_info` / `parse_update_info`, which convert API-level dicts into the typed dataclasses and raise `ValueError` on a bad payload.
|
||||
4. `set_weight_update_target` / `reset_weight_update_target`, used to retarget an update at the speculative draft model.
|
||||
|
||||
@@ -383,6 +485,25 @@ The base class provides:
|
||||
update info carries only per-round metadata. This is what makes a
|
||||
trainer/worker mismatch unrepresentable.
|
||||
|
||||
!!! note "`defers_processing`: when a returned update means queued, not applied"
|
||||
An engine that pipelines its GPU post-processing onto background threads
|
||||
cannot let `update_weights` synchronize the device — that would block on
|
||||
those threads and serialize the pipeline. Such an engine sets the class
|
||||
attribute `defers_processing = True`, omits the per-update sync, and
|
||||
guarantees completion in `finish_weight_update` instead.
|
||||
|
||||
Callers that go through `finish_weight_update` need do nothing; the engine
|
||||
drains there. A caller that drives the tail itself — running its own
|
||||
`finalize_layerwise_reload`, say — must check the flag and call
|
||||
`drain_pending()` first, because with it set a returned `update_weights`
|
||||
means *queued*, not *applied*. `drain_pending()` is idempotent, and a no-op
|
||||
on an engine that processes synchronously, so it is always safe to call.
|
||||
|
||||
[Sharded RDT](sharded_rdt.md) is the built-in engine that sets it: it
|
||||
scatters and quantizes on background threads with their own CUDA streams, so
|
||||
its `drain_pending()` joins both queues and syncs both streams before
|
||||
`finalize_layerwise_reload` runs.
|
||||
|
||||
### Request Classes
|
||||
|
||||
The API-level request classes provide backend-agnostic serialization using plain dictionaries.
|
||||
@@ -497,7 +618,7 @@ Once registered, users select your backend via `WeightTransferConfig(backend="my
|
||||
|
||||
### WeightTransferEngineFactory
|
||||
|
||||
The factory uses a registry pattern with lazy loading. Built-in engines (`nccl`, `ipc`, and `sparse_nccl`) are registered at import time but their modules are only loaded when the backend is actually requested. This avoids importing heavy dependencies (like NCCL communicators) when they aren't needed.
|
||||
The factory uses a registry pattern with lazy loading. Built-in engines (`nccl`, `ipc`, `sparse_nccl` and `sharded_rdt`) are registered at import time but their modules are only loaded when the backend is actually requested. This avoids importing heavy dependencies (like NCCL communicators) when they aren't needed.
|
||||
|
||||
```python
|
||||
from vllm.distributed.weight_transfer import WeightTransferEngineFactory
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
# Sharded RDT Engine
|
||||
|
||||
The sharded RDT weight transfer engine moves weights point-to-point over [NIXL](https://github.com/ai-dynamo/nixl) using Ray Direct Transport (RDT), Ray's zero-copy tensor transport between actors. It is **pull-based**: the inference workers initiate every transfer and each one asks only for the *slice* it consumes under tensor and expert parallelism. A large MoE model therefore moves roughly `total_bytes / num_workers` per worker instead of the `total_bytes` a broadcast costs.
|
||||
|
||||
## When to Use Sharded RDT
|
||||
|
||||
- Very large models where broadcasting whole parameters is the bottleneck — typically an MoE served with expert parallelism, where each worker owns a small fraction of the experts
|
||||
- Trainer and inference on **separate GPUs**, over a fabric NIXL supports (InfiniBand, RoCE, EFA)
|
||||
- Trainers that are themselves sharded, including **pipeline-parallel** ones where a rank holds only part of the model
|
||||
|
||||
Requirements:
|
||||
|
||||
- `distributed_executor_backend="ray"` — the workers must be Ray actors
|
||||
- Ray `>= 2.56.0` on both the trainer and the workers
|
||||
- `nixl` installed in the environment shared by trainer and workers
|
||||
- Weight loaders that stay inside the supported op set (below)
|
||||
- EPLB (`enable_eplb=true`) is rejected: it rearranges experts at runtime, which invalidates the recorded plan
|
||||
|
||||
## How It Works
|
||||
|
||||
### Slices are tracked through vLLM's own weight loaders
|
||||
|
||||
A weight loader normally receives a full HF-format tensor and slices out the part this worker needs. Instead of sending it one, the engine hands it a `FakeRDTTensor`: a zero-storage tensor that answers `.shape` / `.dtype` / `.size()` but holds no data. Every view or slice op the loader calls returns a new fake with that op appended to a recorded chain, and `copy_` is the sink that ends it.
|
||||
|
||||
That chain is the wire format. `("model.layers.0.w", (("narrow", (0, 512, 512), ()), ("t", (), ())))` tells the trainer: take this tensor, `narrow` it, transpose it, send the result. The trainer replays it with `getattr(tensor, op)(*args, **kwargs)`.
|
||||
|
||||
Discovery is expensive, so it happens **once**, at `init_transfer_engine`, as a dry run over `model.load_weights` with every parameter on meta. Nothing is transferred; the engine just records, per leaf module, which slice feeds which destination region. Every later sync is pure replay.
|
||||
|
||||
Anything a loader does that needs real data — arithmetic, `.to()`, `.float()`, `.item()`, `.data`, bool-mask indexing — falls outside the allowlist and raises at init. That is deliberate: failing loudly during setup beats silently transferring the wrong bytes. `SUPPORTED_OPS` in `sharded_rdt_common.py` is the single table both sides derive from, so the recorder and the replayer cannot drift.
|
||||
|
||||
### Received slices land directly in the layerwise reload buffers
|
||||
|
||||
The engine drives [layerwise reload](../layerwise.md) itself, in `start_weight_update` / `finish_weight_update`. Because the dry run already recorded each destination as an `as_strided` region of its parameter, an arriving slice is copied straight into the layer being reloaded — no full HF tensor is ever materialized on the worker, and no second pass over `load_weights` runs. Each layer is quantized and copied into its persistent kernel storage as soon as its last slice lands.
|
||||
|
||||
### Gathers and pulls are pipelined, which is what `gather_lookahead` bounds
|
||||
|
||||
The trainer usually cannot serve its parameters as they sit: FSDP shards them, and even an EP-split trainer has to assemble a whole expert. So each sync still runs gather collectives — but a layer at a time, not a model at a time.
|
||||
|
||||
**A gather group is one decoder layer.** The parameter list is keyed on the outermost index segment of each name, which leaves runs of un-indexed names — the embeddings before the first layer, the final norm and `lm_head` after the last — as groups of their own:
|
||||
|
||||
```text
|
||||
group 0 model.embed_tokens.weight
|
||||
group 1 model.layers.0.* <- one decoder layer
|
||||
group 2 model.layers.1.*
|
||||
...
|
||||
group N+1 model.norm.weight, lm_head.weight
|
||||
```
|
||||
|
||||
The layer is the unit of everything that follows: the trainer gathers a layer, publishes it (immediately pullable), and moves on to the next while the consumers pull the one it just published. Once every consumer has signalled that it is done with a layer, the trainer drops it and gains a credit to gather another. `gather_lookahead` is how far ahead of the consumers that loop may run, so at most `gather_lookahead + 1` layers are resident on the trainer at a time. The default of 1 keeps the next layer gathered and pullable while the current one is being pulled — enough to hide the handoff without doubling trainer memory. Raise it only if one layer's gather is slower than its pulls.
|
||||
|
||||
Because a layer is also the unit the consumers free and the unit the receive buffers are sized against, it is what keeps memory bounded on both sides: without it the whole model would be one transfer, and both sides would have to hold their full share of it at once. Keying on the index rather than a fixed `model.layers.` prefix is what makes that hold across naming conventions — a VLM's `model.language_model.layers.`, GPT-2's `transformer.h.`, a vision tower's `visual.blocks.`. Sources can control the partition; see [gather groups](base.md#gather-groups).
|
||||
|
||||
### Ownership
|
||||
|
||||
A trainer rank need not hold the whole model. Each one declares what it holds through [`WeightSource.held_names()`](base.md#held_names-partial-ownership), the fleet all-gathers those declarations at `trainer_init`, and the consumers route each pull to a rank that actually holds the name. Pipeline stages, expert parallelism, and combinations of the two are all the same declaration. Consumers spread their pulls across the ranks that hold a name, so no single trainer NIC becomes the bottleneck.
|
||||
|
||||
## Inference Side
|
||||
|
||||
```python
|
||||
from vllm import LLM
|
||||
from vllm.config import WeightTransferConfig
|
||||
|
||||
llm = LLM(
|
||||
model="my-model",
|
||||
weight_transfer_config=WeightTransferConfig(backend="sharded_rdt"),
|
||||
distributed_executor_backend="ray",
|
||||
)
|
||||
```
|
||||
|
||||
```bash
|
||||
vllm serve my-model \
|
||||
--distributed-executor-backend ray \
|
||||
--weight-transfer-config '{"backend": "sharded_rdt"}'
|
||||
```
|
||||
|
||||
Everything else — which producers exist, how the model splits into layer groups, the ownership table — arrives from the trainer at the init handshake.
|
||||
|
||||
!!! warning "Size the receive buffers before choosing `gpu_memory_utilization`"
|
||||
Each worker holds `num_rdt_buffers` receive buffers, each large enough for the biggest single slice batch it pulls. Like NCCL and NIXL internals, they do **not** count against `gpu_memory_utilization`, so a fraction that leaves no headroom OOMs at the first sync even though the engine came up healthy. The buffer size is driven by the largest atomic slice — for an untied vocab matrix on a worker that holds it unsliced, that is the whole embedding.
|
||||
|
||||
## Trainer Side
|
||||
|
||||
```python
|
||||
from vllm.distributed.weight_transfer import (
|
||||
ModuleSource,
|
||||
HTTPVLLMWeightSyncClient,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
ShardedRDTTrainerInitInfo,
|
||||
)
|
||||
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=ShardedRDTTrainerInitInfo(
|
||||
rank=rank, # rank 0 is the sender
|
||||
num_consumers=8, # inference workers, fleet-wide
|
||||
trainer_actor_namespace="my_namespace", # must be visible to the workers
|
||||
),
|
||||
client=HTTPVLLMWeightSyncClient("http://localhost:8000"),
|
||||
source=ModuleSource(model),
|
||||
)
|
||||
|
||||
engine.send_weights() # once per sync, on every trainer rank
|
||||
```
|
||||
|
||||
`trainer_init` and `send_weights` run on **every** trainer rank: each one owns a serve actor and takes part in the gathers, while only rank 0 drives the inference-side handshake. Any [`VLLMWeightSyncClient`](base.md#vllmweightsyncclient) works.
|
||||
|
||||
To adapt a trainer that is not a plain `nn.Module` — a Megatron export, a raw sharded checkpoint — subclass [`WeightSource`](base.md#weightsource).
|
||||
|
||||
### `ShardedRDTTrainerInitInfo`
|
||||
|
||||
| Field | Default | Description |
|
||||
| ----- | ------- | ----------- |
|
||||
| `rank` | — | Keyword-only. This trainer rank; **0 is the sender** |
|
||||
| `num_consumers` | — | Inference workers across the whole fleet (TP × DP) |
|
||||
| `trainer_actor_namespace` | `None` | Ray namespace for the serve actors; the workers resolve them by name here |
|
||||
| `num_rdt_buffers` | 2 | Ring depth on both sides |
|
||||
| `buffer_presize_gb` | 0.0 | Pre-size each buffer slot, in GiB. Set it to cover the largest atomic slice |
|
||||
| `gather_lookahead` | 1 | Gathered-but-unfreed layers the gather loop runs ahead by |
|
||||
| `stall_timeout_s` | 300.0 | Seconds without progress before the sync fails. A liveness backstop for a consumer that dies mid-sync, not a latency target |
|
||||
|
||||
## Examples
|
||||
|
||||
- [Small MoE on 4 GPUs](../../../examples/rl/rlhf_sharded_rdt_small_ep.py) — 2 FSDP2 trainer ranks → 2 vLLM DP ranks with expert parallelism, one node. It pairs a trainer fleet with a separate inference fleet, the only arrangement this backend supports, and asserts that the sync moved the weights and that a second sync leaves generation unchanged — so it runs unattended in CI
|
||||
|
||||
It keeps the trainer deliberately small — just enough FSDP2 to make the weights real — so the file stays about the weight sync rather than about the trainer. For a full RL trainer, SkyRL integrates this backend with Megatron (PP-local gathering and expert-stack fusion for MoE) alongside FSDP: [NovaSky-AI/SkyRL#1753](https://github.com/NovaSky-AI/SkyRL/pull/1753).
|
||||
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Driver-side helpers to run the inference fleet as a standard ``vllm serve``
|
||||
process for the sharded-RDT examples.
|
||||
|
||||
The trainer reaches the server's weight-sync control plane over the RLHF HTTP
|
||||
routes (``HTTPVLLMWeightSyncClient``); generation uses ``/v1/completions``. RDT
|
||||
still needs the workers to be RayExecutorV2 tensor-transport actors sharing the
|
||||
trainer's Ray cluster, so the server is launched with the Ray v2 executor and
|
||||
inherits this process's env (``address=auto``). These helpers run only on the
|
||||
driver — they are not shipped to or imported by the Ray actors.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import ray
|
||||
import requests
|
||||
|
||||
|
||||
def launch_vllm_serve(
|
||||
model: str,
|
||||
*,
|
||||
tensor_parallel_size: int = 1,
|
||||
data_parallel_size: int = 1,
|
||||
enable_expert_parallel: bool = False,
|
||||
port: int = 8000,
|
||||
gpu_memory_utilization: float = 0.7,
|
||||
extra_args: list[str] | None = None,
|
||||
) -> subprocess.Popen:
|
||||
"""Start ``vllm serve`` for the RDT inference fleet.
|
||||
|
||||
``VLLM_SERVER_DEV_MODE`` exposes the RLHF weight-sync routes; the Ray v2
|
||||
executor makes the workers tensor-transport actors (RDT's data plane). The
|
||||
child inherits our env, so it joins the same Ray cluster (RAY_ADDRESS below)
|
||||
and picks up the editable vLLM install via the venv interpreter.
|
||||
"""
|
||||
vllm_bin = os.path.join(os.path.dirname(sys.executable), "vllm")
|
||||
if not os.path.exists(vllm_bin):
|
||||
vllm_bin = shutil.which("vllm") or "vllm"
|
||||
cmd = [
|
||||
vllm_bin,
|
||||
"serve",
|
||||
model,
|
||||
"--port",
|
||||
str(port),
|
||||
"--enforce-eager",
|
||||
"--load-format",
|
||||
"dummy",
|
||||
"--gpu-memory-utilization",
|
||||
str(gpu_memory_utilization),
|
||||
"--distributed-executor-backend",
|
||||
"ray",
|
||||
"--weight-transfer-config",
|
||||
json.dumps({"backend": "sharded_rdt"}),
|
||||
]
|
||||
if tensor_parallel_size > 1:
|
||||
cmd += ["--tensor-parallel-size", str(tensor_parallel_size)]
|
||||
# Only engage the DP-ray backend when actually data-parallel: DP=1 with
|
||||
# data_parallel_backend=ray forces vLLM's DP-placement path, which fails.
|
||||
if data_parallel_size > 1:
|
||||
cmd += [
|
||||
"--data-parallel-size",
|
||||
str(data_parallel_size),
|
||||
"--data-parallel-backend",
|
||||
"ray",
|
||||
]
|
||||
if enable_expert_parallel:
|
||||
cmd.append("--enable-expert-parallel")
|
||||
if extra_args:
|
||||
cmd += extra_args
|
||||
env = dict(
|
||||
os.environ,
|
||||
VLLM_SERVER_DEV_MODE="1",
|
||||
VLLM_USE_RAY_V2_EXECUTOR_BACKEND="1",
|
||||
)
|
||||
# Pin the child to THIS Ray cluster. vLLM's ray DP backend reaches Ray
|
||||
# through a bare ``ray.init()``, which STARTS a cluster rather than
|
||||
# attaching, so wherever we started Ray ourselves (a bare CI runner) the
|
||||
# workers would come up in a second cluster and never see the producer
|
||||
# actors -- the failure is an actor lookup that cannot succeed. A bare
|
||||
# ``ray.init()`` does honour RAY_ADDRESS, so hand it over explicitly.
|
||||
if ray.is_initialized():
|
||||
env["RAY_ADDRESS"] = ray.get_runtime_context().gcs_address
|
||||
# Let the server's Ray workers use the ambient interpreter/install rather
|
||||
# than a workspace snapshot working_dir. On an Anyscale dev workspace the
|
||||
# RAY_RUNTIME_ENV_HOOK injects a git snapshot as the workers' working_dir,
|
||||
# which shadows an editable vLLM install (its compiled .so is gitignored, so
|
||||
# the snapshot lacks it). No-op off Anyscale (the var is unset there).
|
||||
env.pop("RAY_RUNTIME_ENV_HOOK", None)
|
||||
print(f"[serve] launching: {' '.join(cmd)}", flush=True)
|
||||
return subprocess.Popen(cmd, env=env)
|
||||
|
||||
|
||||
def wait_for_server(
|
||||
endpoint: str, proc: subprocess.Popen, timeout: float = 1800
|
||||
) -> None:
|
||||
"""Block until ``/health`` returns 200, failing fast if the server exits."""
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
if proc.poll() is not None:
|
||||
raise RuntimeError(f"vllm serve exited early (code {proc.returncode})")
|
||||
try:
|
||||
if requests.get(f"{endpoint}/health", timeout=5).status_code == 200:
|
||||
print("[serve] server is healthy", flush=True)
|
||||
return
|
||||
except requests.RequestException:
|
||||
pass
|
||||
time.sleep(3)
|
||||
raise RuntimeError("vllm serve did not become healthy in time")
|
||||
|
||||
|
||||
def http_generate(
|
||||
endpoint: str, model: str, prompts: list[str], max_tokens: int = 16
|
||||
) -> list[str]:
|
||||
"""Greedy ``/v1/completions`` for each prompt; returns the generated texts."""
|
||||
outs = []
|
||||
for p in prompts:
|
||||
r = requests.post(
|
||||
f"{endpoint}/v1/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"prompt": p,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": 0,
|
||||
},
|
||||
timeout=120,
|
||||
)
|
||||
r.raise_for_status()
|
||||
outs.append(r.json()["choices"][0]["text"])
|
||||
return outs
|
||||
|
||||
|
||||
def pause_generation(endpoint: str, mode: str = "abort") -> None:
|
||||
requests.post(f"{endpoint}/pause", params={"mode": mode}, timeout=60)
|
||||
|
||||
|
||||
def resume_generation(endpoint: str) -> None:
|
||||
requests.post(f"{endpoint}/resume", timeout=60)
|
||||
|
||||
|
||||
def shutdown_server(proc: subprocess.Popen) -> None:
|
||||
print("[serve] shutting down vllm serve", flush=True)
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The ``WeightSource`` for ``rlhf_sharded_rdt_small_ep.py``.
|
||||
|
||||
A trainer's producer job is to present its weights under the names the inference
|
||||
side expects, which for vLLM means HF CHECKPOINT names. Transformers does not
|
||||
store MoE experts that way -- it fuses each layer's into ``[E, ...]`` tensors --
|
||||
so that is the one conversion this file does. Kept beside the example rather than
|
||||
inside it so the example stays about the sync itself, alongside
|
||||
``rdt_vllm_serve.py``.
|
||||
"""
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer import ParamMeta, WeightSource
|
||||
from vllm.distributed.weight_transfer.base import materialize_full_tensor
|
||||
|
||||
# Transformers fuses a layer's experts into ``[E, ...]`` tensors; the checkpoint
|
||||
# stores them per expert. This matches the fused parameter names.
|
||||
_FUSED_EXPERT_RE = re.compile(r"^(.*\.experts)\.(gate_up_proj|down_proj)$")
|
||||
|
||||
|
||||
class CheckpointNameSource(WeightSource):
|
||||
"""`WeightSource` over an FSDP2 model that publishes HF CHECKPOINT names.
|
||||
|
||||
Transformers keeps each layer's experts fused as ``[E, ...]`` tensors, but
|
||||
vLLM's MoE loaders are written against the per-expert checkpoint entries --
|
||||
only ``qwen{2,3}_moe`` also accept the fused names, so publishing them
|
||||
straight from ``named_parameters()`` limits the example to one family and
|
||||
fails the consumer's bake on ``experts.gate_up_proj`` everywhere else.
|
||||
Splitting them back is also what a real trainer does, since converting an
|
||||
internal layout to checkpoint names is the normal producer job.
|
||||
|
||||
Every rank publishes the whole model: fused params are all-gathered and then
|
||||
sliced per expert. Ownership stays uniform (no ``held_names``) -- serving
|
||||
only the experts a rank holds is a real trainer's concern, not an example's.
|
||||
"""
|
||||
|
||||
def __init__(self, module: torch.nn.Module) -> None:
|
||||
self._module = module
|
||||
|
||||
@staticmethod
|
||||
def _expand(name: str, param: torch.Tensor) -> list[tuple[str, tuple]]:
|
||||
"""The checkpoint entries ``name`` contributes, as (name, shape).
|
||||
|
||||
``gate_up_proj`` is ``[E, 2I, H]`` with gate in rows ``:I`` and up in
|
||||
``I:``; ``down_proj`` is ``[E, H, I]``. Anything else is already a
|
||||
checkpoint name and passes through.
|
||||
"""
|
||||
m = _FUSED_EXPERT_RE.match(name)
|
||||
if m is None:
|
||||
return [(name, tuple(param.shape))]
|
||||
prefix, kind = m.group(1), m.group(2)
|
||||
experts, rows, cols = param.shape
|
||||
if kind == "down_proj":
|
||||
return [
|
||||
(f"{prefix}.{e}.down_proj.weight", (rows, cols)) for e in range(experts)
|
||||
]
|
||||
half = rows // 2
|
||||
return [
|
||||
(f"{prefix}.{e}.{proj}_proj.weight", (half, cols))
|
||||
for e in range(experts)
|
||||
for proj in ("gate", "up")
|
||||
]
|
||||
|
||||
def metadata(self) -> list[ParamMeta]:
|
||||
return [
|
||||
ParamMeta(entry_name, param.dtype, shape)
|
||||
for name, param in self._module.named_parameters()
|
||||
for entry_name, shape in self._expand(name, param)
|
||||
]
|
||||
|
||||
def __iter__(self):
|
||||
for name, param in self._module.named_parameters():
|
||||
m = _FUSED_EXPERT_RE.match(name)
|
||||
if m is None:
|
||||
yield name, materialize_full_tensor(param)
|
||||
continue
|
||||
# One gather per FUSED param, not per expert: the views below are
|
||||
# into that one tensor, and every expert of a layer rides in the
|
||||
# same gather group, so it stays resident exactly as long as needed.
|
||||
prefix, kind = m.group(1), m.group(2)
|
||||
full = materialize_full_tensor(param)
|
||||
if kind == "down_proj":
|
||||
for e in range(full.shape[0]):
|
||||
yield f"{prefix}.{e}.down_proj.weight", full[e]
|
||||
else:
|
||||
half = full.shape[1] // 2
|
||||
for e in range(full.shape[0]):
|
||||
yield f"{prefix}.{e}.gate_proj.weight", full[e, :half, :]
|
||||
yield f"{prefix}.{e}.up_proj.weight", full[e, half:, :]
|
||||
@@ -0,0 +1,270 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Sharded-RDT weight sync on FOUR GPUs, one node: 2 FSDP2 trainer ranks -> 2
|
||||
vLLM DP ranks with expert parallelism.
|
||||
|
||||
The reference example for the backend, sized to fit a CI runner:
|
||||
``CheckpointNameSource`` over an FSDP2 model, one serve actor per rank, the
|
||||
HTTP control plane, and assertions so it can run unattended.
|
||||
|
||||
Why a MoE model rather than a small dense one: the point of sharded RDT is that
|
||||
each inference worker pulls only the slices it consumes, so the example is only
|
||||
meaningful when no rank holds the whole model. That needs experts on both sides.
|
||||
``Qwen/Qwen1.5-MoE-A2.7B`` is the reference shape, a real MoE of the same family
|
||||
as the 30B model the big example uses. Its WEIGHTS are never read: the trainer
|
||||
builds the config and the server runs ``--load-format dummy``, so the sync is
|
||||
what is under test and nothing is downloaded but the config and tokenizer. A
|
||||
smaller MoE via ``RDT_MODEL`` changes nothing about what is exercised, which is
|
||||
what CI runs.
|
||||
|
||||
The trainer publishes CHECKPOINT names, not the names its own modules carry:
|
||||
Transformers fuses each layer's experts into ``[E, ...]`` tensors
|
||||
(``mlp.experts.gate_up_proj``/``down_proj``) while the checkpoint stores them per
|
||||
expert, and vLLM's MoE loaders read the per-expert form. ``CheckpointNameSource``
|
||||
splits them back, which is the same conversion a real trainer does when it maps
|
||||
its internal layout onto checkpoint names. Publishing the fused names instead
|
||||
works only for the Qwen families, whose loaders happen to accept them.
|
||||
|
||||
FSDP2 shards those fused tensors on dim 0 — the expert dimension — so after
|
||||
sharding no rank stores every expert, and the inference side gets real EP from
|
||||
``--enable-expert-parallel`` over DP2. Two simplifications an example can afford
|
||||
and a real trainer cannot: each rank BUILDS the whole model before
|
||||
``fully_shard`` splits it (fine for a tiny config, so nothing here streams a
|
||||
checkpoint), and iteration all-gathers every parameter, so ownership stays
|
||||
uniform. Serving only the experts a rank holds (``held_names``) is where a real
|
||||
trainer earns its keep and is deliberately out of scope.
|
||||
|
||||
Model naming is load-bearing: names are mapped onto checkpoint keys one-to-one,
|
||||
so a model whose HF module names have drifted from its checkpoint (as
|
||||
GraniteMoe's router has) will not work here.
|
||||
|
||||
What it checks (a smoke test, not a numerical equivalence check):
|
||||
- generation CHANGES after the first sync — the weights really moved, so a
|
||||
silently-skipped sync fails instead of printing plausible text;
|
||||
- generation is UNCHANGED by a second sync — replay is stable.
|
||||
|
||||
Run:
|
||||
python examples/rl/rlhf_sharded_rdt_small_ep.py
|
||||
|
||||
Needs 4 GPUs on one node, each big enough for the whole model, since every rank
|
||||
builds it before ``fully_shard`` splits it: the default is ~29 GiB in bf16, so
|
||||
40 GiB cards and up. Set ``RDT_MODEL`` to a tiny MoE on smaller GPUs. Joins an
|
||||
existing Ray cluster if there is one and otherwise starts its own.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import ray
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from ray.util.placement_group import placement_group, placement_group_table
|
||||
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
|
||||
from rdt_vllm_serve import (
|
||||
http_generate,
|
||||
launch_vllm_serve,
|
||||
pause_generation,
|
||||
resume_generation,
|
||||
shutdown_server,
|
||||
wait_for_server,
|
||||
)
|
||||
from rdt_weight_source import CheckpointNameSource
|
||||
from torch.distributed.fsdp import fully_shard
|
||||
from transformers import AutoConfig, AutoModelForCausalLM
|
||||
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
ShardedRDTTrainerInitInfo,
|
||||
)
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
# Weights are built, never loaded, so this costs a config fetch -- but every rank
|
||||
# materializes the model before sharding, so the default wants a 40 GiB card.
|
||||
MODEL_NAME = os.environ.get("RDT_MODEL", "Qwen/Qwen1.5-MoE-A2.7B")
|
||||
RAY_NAMESPACE = "sharded_rdt_small_ep_example"
|
||||
VLLM_PORT = int(os.environ.get("RDT_VLLM_PORT", "8100"))
|
||||
VLLM_ENDPOINT = f"http://127.0.0.1:{VLLM_PORT}"
|
||||
|
||||
FSDP_WORLD_SIZE = 2
|
||||
INFERENCE_TP_SIZE = 1
|
||||
INFERENCE_DP_SIZE = 2
|
||||
NUM_INFERENCE_CONSUMERS = INFERENCE_TP_SIZE * INFERENCE_DP_SIZE
|
||||
|
||||
PROMPTS = ["The capital of France is", "The future of AI is"]
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
class FSDPTrainWorker:
|
||||
"""One FSDP2 rank per GPU. Identical to the 8-GPU example's worker: the
|
||||
sharded-RDT engine owns the NIXL serve surface, so this stays a plain Ray
|
||||
actor with no producer mixin."""
|
||||
|
||||
def __init__(self, model_name, rank, world_size, master_addr, master_port):
|
||||
self.rank = rank
|
||||
self.engine = None
|
||||
os.environ["MASTER_ADDR"] = master_addr
|
||||
os.environ["MASTER_PORT"] = str(master_port)
|
||||
dist.init_process_group(backend="nccl", rank=rank, world_size=world_size)
|
||||
torch.accelerator.set_device_index(0)
|
||||
|
||||
# Same seed on every rank, so the shards compose into ONE model rather
|
||||
# than a blend of independent inits -- the second sync's stability check
|
||||
# is only meaningful against coherent weights.
|
||||
torch.manual_seed(0)
|
||||
config = AutoConfig.from_pretrained(model_name)
|
||||
with torch.device("cuda"):
|
||||
model = AutoModelForCausalLM.from_config(config, dtype=torch.bfloat16)
|
||||
for layer in model.model.layers:
|
||||
fully_shard(layer)
|
||||
fully_shard(model)
|
||||
self.model = model
|
||||
|
||||
def get_rank(self):
|
||||
return self.rank
|
||||
|
||||
def setup_engine(self, vllm_endpoint: str):
|
||||
self.engine = WeightTransferTrainerFactory.trainer_init(
|
||||
ShardedRDTTrainerInitInfo(
|
||||
rank=self.rank,
|
||||
num_consumers=NUM_INFERENCE_CONSUMERS,
|
||||
trainer_actor_namespace=RAY_NAMESPACE,
|
||||
num_rdt_buffers=int(os.environ.get("NUM_RDT_BUFFERS", "2")),
|
||||
buffer_presize_gb=float(os.environ.get("RDT_BUFFER_PRESIZE_GB", "0")),
|
||||
),
|
||||
client=HTTPVLLMWeightSyncClient(vllm_endpoint),
|
||||
source=CheckpointNameSource(self.model),
|
||||
)
|
||||
|
||||
def sync_weights(self):
|
||||
self.engine.send_weights()
|
||||
|
||||
|
||||
def main():
|
||||
# Ship a minimal working_dir (this example dir) so Ray actors do NOT inherit
|
||||
# a workspace snapshot that shadows the editable vLLM install.
|
||||
runtime_env: dict[str, object] = {
|
||||
"py_executable": sys.executable,
|
||||
"working_dir": os.path.dirname(os.path.abspath(__file__)),
|
||||
}
|
||||
forwarded = {
|
||||
k: os.environ[k]
|
||||
for k in (
|
||||
"NCCL_CUMEM_ENABLE",
|
||||
"VLLM_NCCL_SO_PATH",
|
||||
"LD_PRELOAD",
|
||||
"LD_LIBRARY_PATH",
|
||||
)
|
||||
if k in os.environ
|
||||
}
|
||||
if forwarded:
|
||||
runtime_env["env_vars"] = forwarded
|
||||
if not ray.is_initialized():
|
||||
try:
|
||||
ray.init(address="auto", runtime_env=runtime_env, namespace=RAY_NAMESPACE)
|
||||
except (ConnectionError, RuntimeError):
|
||||
# No cluster to join (a bare CI runner): start one on this node.
|
||||
# `launch_vllm_serve` hands the child this cluster's address, which
|
||||
# it needs -- vLLM's ray DP backend would otherwise start its own.
|
||||
ray.init(runtime_env=runtime_env, namespace=RAY_NAMESPACE)
|
||||
|
||||
print(
|
||||
f"[init] model {MODEL_NAME}, {FSDP_WORLD_SIZE} trainer + "
|
||||
f"{INFERENCE_DP_SIZE} inference GPUs"
|
||||
)
|
||||
|
||||
# Reserve the trainer's GPUs BEFORE launching the server, so the server's DP
|
||||
# workers take the node's remaining two. STRICT_PACK keeps the trainer's
|
||||
# all-gathers on one node's NVLink -- here that is the only node anyway.
|
||||
trainer_pg = placement_group(
|
||||
[{"GPU": 1, "CPU": 1}] * FSDP_WORLD_SIZE, strategy="STRICT_PACK"
|
||||
)
|
||||
ray.get(trainer_pg.ready())
|
||||
pg_node_id = next(
|
||||
iter(placement_group_table(trainer_pg)["bundles_to_node_id"].values())
|
||||
)
|
||||
master_addr = next(
|
||||
n["NodeManagerAddress"] for n in ray.nodes() if n["NodeID"] == pg_node_id
|
||||
)
|
||||
|
||||
@ray.remote(
|
||||
num_cpus=0,
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=trainer_pg
|
||||
),
|
||||
)
|
||||
def _free_port():
|
||||
return get_open_port()
|
||||
|
||||
master_port = ray.get(_free_port.remote())
|
||||
|
||||
workers = [
|
||||
FSDPTrainWorker.options(
|
||||
scheduling_strategy=PlacementGroupSchedulingStrategy(
|
||||
placement_group=trainer_pg, placement_group_bundle_index=rank
|
||||
),
|
||||
).remote(MODEL_NAME, rank, FSDP_WORLD_SIZE, master_addr, master_port)
|
||||
for rank in range(FSDP_WORLD_SIZE)
|
||||
]
|
||||
ray.get([w.get_rank.remote() for w in workers])
|
||||
print(f"[init] {FSDP_WORLD_SIZE} FSDP trainer workers ready (seeded weights).")
|
||||
|
||||
server = launch_vllm_serve(
|
||||
MODEL_NAME,
|
||||
tensor_parallel_size=INFERENCE_TP_SIZE,
|
||||
data_parallel_size=INFERENCE_DP_SIZE,
|
||||
enable_expert_parallel=True,
|
||||
port=VLLM_PORT,
|
||||
extra_args=["--max-model-len", "2048", "--max-num-seqs", "8"],
|
||||
)
|
||||
try:
|
||||
wait_for_server(VLLM_ENDPOINT, server)
|
||||
|
||||
before = http_generate(VLLM_ENDPOINT, MODEL_NAME, PROMPTS)
|
||||
print("[generate] BEFORE sync (vLLM's own dummy init):")
|
||||
for p, t in zip(PROMPTS, before):
|
||||
print(f" {p!r} -> {t!r}")
|
||||
|
||||
print("[sync] building trainer engines (bake on the sender)...")
|
||||
t0 = time.perf_counter()
|
||||
ray.get([w.setup_engine.remote(VLLM_ENDPOINT) for w in workers])
|
||||
print(f"[sync] engine setup (incl. bake) took {time.perf_counter() - t0:.2f}s")
|
||||
|
||||
pause_generation(VLLM_ENDPOINT)
|
||||
t0 = time.perf_counter()
|
||||
ray.get([w.sync_weights.remote() for w in workers])
|
||||
print(f"[sync] iter 0 took {time.perf_counter() - t0:.3f}s")
|
||||
resume_generation(VLLM_ENDPOINT)
|
||||
|
||||
after = http_generate(VLLM_ENDPOINT, MODEL_NAME, PROMPTS)
|
||||
print("[generate] AFTER sync (the trainer's weights):")
|
||||
for p, t in zip(PROMPTS, after):
|
||||
print(f" {p!r} -> {t!r}")
|
||||
|
||||
# Sync again; replay must be stable, so generation must not move.
|
||||
pause_generation(VLLM_ENDPOINT)
|
||||
t0 = time.perf_counter()
|
||||
ray.get([w.sync_weights.remote() for w in workers])
|
||||
print(f"[sync] iter 1 took {time.perf_counter() - t0:.3f}s")
|
||||
resume_generation(VLLM_ENDPOINT)
|
||||
again = http_generate(VLLM_ENDPOINT, MODEL_NAME, PROMPTS)
|
||||
|
||||
# Greedy decoding, so both comparisons are exact.
|
||||
assert after != before, (
|
||||
f"generation did not change across the weight sync, so the sync "
|
||||
f"moved nothing: {before!r}"
|
||||
)
|
||||
assert again == after, (
|
||||
f"a second sync changed generation, so replay is not stable: "
|
||||
f"{after!r} then {again!r}"
|
||||
)
|
||||
print("[ok] weights moved on sync 0 and replay was stable on sync 1")
|
||||
finally:
|
||||
shutdown_server(server)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -895,7 +895,7 @@ pyzmq==27.1.0
|
||||
# via -r requirements/test/../common.txt
|
||||
rapidfuzz==3.12.1
|
||||
# via jiwer
|
||||
ray==2.48.0
|
||||
ray==2.56.1
|
||||
# via -r requirements/test/cuda.in
|
||||
redis==5.2.0
|
||||
# via tensorizer
|
||||
|
||||
@@ -22,7 +22,7 @@ vector_quantize_pytorch # required for minicpmo_26 test
|
||||
vocos # required for minicpmo_26 test
|
||||
peft>=0.19.1 # required for phi-4-mm test
|
||||
pqdm
|
||||
ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests
|
||||
ray[cgraph,default]>=2.56.0,<2.57 # Ray Compiled Graph, required by pipeline parallelism tests; sharded-RDT weight transfer needs Ray Direct Transport (>=2.56)
|
||||
sentence-transformers>=5.2.0 # required for embedding tests
|
||||
soundfile # required for audio tests
|
||||
jiwer # required for audio tests
|
||||
|
||||
@@ -988,7 +988,7 @@ pyzmq==27.1.0
|
||||
# -r requirements/test/../common.txt
|
||||
rapidfuzz==3.12.1
|
||||
# via jiwer
|
||||
ray==2.48.0
|
||||
ray==2.56.1
|
||||
# via -r requirements/test/cuda.in
|
||||
redis==5.2.0
|
||||
# via tensorizer
|
||||
|
||||
@@ -16,7 +16,7 @@ librosa # required for audio tests
|
||||
vocos # required for minicpmo_26 test
|
||||
peft
|
||||
pqdm
|
||||
ray[cgraph,default]>=2.48.0 # Ray Compiled Graph, required by pipeline parallelism tests
|
||||
ray[cgraph,default]>=2.56.0,<2.57 # Ray Compiled Graph, required by pipeline parallelism tests; sharded-RDT weight transfer needs Ray Direct Transport (>=2.56)
|
||||
sentence-transformers>=5.2.0 # required for embedding tests
|
||||
soundfile # required for audio tests
|
||||
jiwer # required for audio tests
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,628 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the sharded-RDT trainer-side engine.
|
||||
|
||||
The trainer engine's own surface: init info and factory registration, the
|
||||
send_weights round trip against an in-process fake producer server, the
|
||||
ownership table it resolves and ships to the workers, and the producer's packed
|
||||
destination-view cache.
|
||||
|
||||
The plan/replay half (including `RdtRouter`, which is consumer-side) lives in
|
||||
test_sharded_rdt_plan.py; the producer server's protocol in
|
||||
test_sharded_rdt_producer.py.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
from dataclasses import asdict
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.distributed.test_weight_transfer import RecordingClient
|
||||
from vllm.distributed.weight_transfer import (
|
||||
ModuleSource,
|
||||
ParamMeta,
|
||||
WeightSource,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import layerwise_groups
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
ShardedRDTTrainerInitInfo,
|
||||
ShardedRDTTrainerWeightTransferEngine,
|
||||
)
|
||||
|
||||
|
||||
class _ListSource(WeightSource):
|
||||
"""A WeightSource over an explicit ordered (name, cpu-tensor) list, so the
|
||||
sharded-RDT group/order logic can be tested without a real model."""
|
||||
|
||||
def __init__(self, pairs):
|
||||
self._pairs = list(pairs)
|
||||
|
||||
def metadata(self):
|
||||
return [ParamMeta(n, t.dtype, tuple(t.shape)) for n, t in self._pairs]
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._pairs)
|
||||
|
||||
|
||||
class _FakeProducerServer:
|
||||
"""In-process stand-in for the _RDTProducerServer Ray actor. Records the
|
||||
engine->server call sequence and, by default, frees each group as soon as
|
||||
it is published (simulating the consumers' free_group barrier) so the
|
||||
gather loop's credit gate never parks. Mirrors the real server's
|
||||
per-group barrier: publish/free are keyed by GROUP INDEX, signals count
|
||||
to the ``begin_sync`` live total, publish_group returns nothing, and freed
|
||||
groups flow back only through wait_freed / end_sync (see
|
||||
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne)."""
|
||||
|
||||
def __init__(self, auto_free=True):
|
||||
self.order: list[str] = []
|
||||
self.published: list[int] = []
|
||||
self.live_count = 1
|
||||
self._inflight_groups: list[int] = []
|
||||
self.auto_free = auto_free
|
||||
self.free_counts: dict[int, int] = {}
|
||||
self._pending_freed: list[int] = []
|
||||
|
||||
# Alias for tests that inspect in-flight groups.
|
||||
@property
|
||||
def inflight(self):
|
||||
return self._inflight_groups
|
||||
|
||||
def begin_sync(self, live_count, live_consumer_ids=None):
|
||||
self.order.append("begin")
|
||||
self.live_count = max(1, int(live_count))
|
||||
self.live_ids = live_consumer_ids
|
||||
|
||||
def publish_group(self, group_idx, entries):
|
||||
self.order.append("publish")
|
||||
self.published.append(group_idx)
|
||||
self._inflight_groups.append(group_idx)
|
||||
if self.auto_free or self.free_counts.get(group_idx, 0) >= self.live_count:
|
||||
self._inflight_groups.remove(group_idx)
|
||||
self._pending_freed.append(group_idx)
|
||||
|
||||
def free_group(self, group_idx):
|
||||
"""Consumer back-edge; may arrive before the group's publish."""
|
||||
self.free_counts[group_idx] = self.free_counts.get(group_idx, 0) + 1
|
||||
if (
|
||||
group_idx in self._inflight_groups
|
||||
and self.free_counts[group_idx] >= self.live_count
|
||||
):
|
||||
self._inflight_groups.remove(group_idx)
|
||||
self._pending_freed.append(group_idx)
|
||||
|
||||
def wait_freed(self):
|
||||
"""The engine's credit gate. The real server blocks here; the fake
|
||||
must already have a banked credit when the gate asks (auto_free, or a
|
||||
test's own free_group) — anything else is the deadlock the real
|
||||
watchdog would kill, so fail loudly. Not appended to ``order``: it is
|
||||
pacing, not a lifecycle milestone."""
|
||||
assert self._pending_freed, (
|
||||
"wait_freed with nothing freed: the gather loop would deadlock"
|
||||
)
|
||||
freed = self._pending_freed
|
||||
self._pending_freed = []
|
||||
return freed
|
||||
|
||||
def end_sync(self):
|
||||
self.order.append("end")
|
||||
freed = self._pending_freed
|
||||
self._pending_freed = []
|
||||
return freed
|
||||
|
||||
def set_gather_error(self, message):
|
||||
self.order.append("error")
|
||||
|
||||
|
||||
def _rdt_engine_with_fake_server(
|
||||
source, *, is_sender, client, server, monkeypatch, fleet_owned=None
|
||||
):
|
||||
"""Build a ShardedRDTTrainerWeightTransferEngine wired to an in-process fake
|
||||
server (no Ray, no CUDA IPC): bypass trainer_init's spawn, set the
|
||||
group-major metadata, and route _rpc to the fake."""
|
||||
import vllm.distributed.weight_transfer.sharded_rdt_trainer as mod
|
||||
|
||||
# reduce_tensor needs CUDA; the fake server never rebuilds, so stub it.
|
||||
monkeypatch.setattr(mod, "reduce_tensor", lambda t: (None, ("fake",)))
|
||||
|
||||
init_info = ShardedRDTTrainerInitInfo(num_consumers=1, rank=0 if is_sender else 1)
|
||||
engine = ShardedRDTTrainerWeightTransferEngine(
|
||||
client=client, source=source, is_sender=is_sender, init_info=init_info
|
||||
)
|
||||
engine._meta = list(source.metadata())
|
||||
names = [m.name for m in engine._meta]
|
||||
engine._groups = layerwise_groups(names)
|
||||
engine._server = server
|
||||
engine._rpc = lambda method, *args: getattr(server, method)(*args)
|
||||
# What trainer_init resolves from the source's held names + the fleet's
|
||||
# all-gather. ``fleet_owned`` stands in for that all-gather so a partial-
|
||||
# ownership rank can be tested without a real process group; the fleet must
|
||||
# cover every name or the transpose rejects it (nothing would serve the rest).
|
||||
if fleet_owned is None:
|
||||
engine._resolve_ownership(1, 0)
|
||||
else:
|
||||
# Stands in for the (metadata digest, held-name bitmask) all-gather:
|
||||
# each rank's group list becomes the mask its held names would set.
|
||||
names = [m.name for m in engine._meta]
|
||||
groups = engine._groups
|
||||
|
||||
def _fake_gather(_world, mine):
|
||||
out = []
|
||||
for owned in fleet_owned:
|
||||
held = {n for gi in owned for n in groups[gi]}
|
||||
mask = bytearray((len(names) + 7) // 8)
|
||||
for i, n in enumerate(names):
|
||||
if n in held:
|
||||
mask[i >> 3] |= 1 << (i & 7)
|
||||
out.append((mine[0], bytes(mask)))
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(engine, "_all_gather_owned", _fake_gather)
|
||||
engine._resolve_ownership(len(fleet_owned), 0)
|
||||
return engine
|
||||
|
||||
|
||||
def _rdt_source_two_layers():
|
||||
return _ListSource(
|
||||
[
|
||||
("embed.weight", torch.zeros(2)),
|
||||
("model.layers.0.w", torch.zeros(2)),
|
||||
("model.layers.1.w", torch.zeros(2)),
|
||||
("norm.weight", torch.zeros(2)),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class TestShardedRDTTrainerInitInfo:
|
||||
def test_declares_backend(self):
|
||||
assert ShardedRDTTrainerInitInfo.backend == "sharded_rdt"
|
||||
|
||||
def test_rank_is_keyword_only_and_drives_is_sender(self):
|
||||
assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=0).is_sender is True
|
||||
assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=1).is_sender is False
|
||||
with pytest.raises(TypeError):
|
||||
# rank is keyword-only.
|
||||
ShardedRDTTrainerInitInfo(4, 0) # type: ignore[misc]
|
||||
|
||||
def test_registered_in_trainer_factory(self):
|
||||
cls = WeightTransferTrainerFactory._registry["sharded_rdt"]()
|
||||
assert cls is ShardedRDTTrainerWeightTransferEngine
|
||||
|
||||
|
||||
def test_sharded_rdt_trainer_init_requires_source():
|
||||
with pytest.raises(ValueError, match="requires a WeightSource"):
|
||||
ShardedRDTTrainerWeightTransferEngine.trainer_init(
|
||||
ShardedRDTTrainerInitInfo(num_consumers=1, rank=0),
|
||||
client=RecordingClient(),
|
||||
source=None,
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_rdt_worker_init_info_is_group_major(monkeypatch):
|
||||
source = _rdt_source_two_layers()
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
source,
|
||||
is_sender=True,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(),
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
worker_init = engine._build_worker_init_info(["srv_rk0"])
|
||||
# 4 params -> pre / layer0 / layer1 / post = 4 groups of length 1.
|
||||
assert worker_init.names == [
|
||||
"embed.weight",
|
||||
"model.layers.0.w",
|
||||
"model.layers.1.w",
|
||||
"norm.weight",
|
||||
]
|
||||
assert worker_init.group_lens == [1, 1, 1, 1]
|
||||
assert worker_init.trainer_actor_names == ["srv_rk0"]
|
||||
assert worker_init.produce_method_name == "rdt_produce_weights_batched"
|
||||
assert sum(worker_init.group_lens) == len(worker_init.names)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_send_weights_drives_client_in_order(monkeypatch):
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
client = RecordingClient()
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
is_sender=True,
|
||||
client=client,
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
engine.send_weights()
|
||||
|
||||
assert client.order == ["start", "update", "finish"]
|
||||
# begin, one publish per group (4), end.
|
||||
assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"]
|
||||
assert len(server.published) == 4
|
||||
# every group freed -> no engine-held refs remain.
|
||||
assert engine._inflight == {}
|
||||
|
||||
|
||||
def test_sharded_rdt_send_weights_group_order_mismatch_raises(monkeypatch):
|
||||
# Source whose iteration order disagrees with its metadata order.
|
||||
class _BadSource(_ListSource):
|
||||
def __iter__(self):
|
||||
reordered = list(self._pairs)
|
||||
reordered[0], reordered[1] = reordered[1], reordered[0]
|
||||
return iter(reordered)
|
||||
|
||||
server = _FakeProducerServer()
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_BadSource(_rdt_source_two_layers()._pairs),
|
||||
is_sender=True,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="iteration order must match"):
|
||||
engine.send_weights()
|
||||
assert "error" in server.order # gather error propagated to the server
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_non_sender_skips_client(monkeypatch):
|
||||
class _RaisingClient(RecordingClient):
|
||||
def start_weight_update(self):
|
||||
raise AssertionError("non-sender must not touch the client")
|
||||
|
||||
def update_weights(self, update_info):
|
||||
raise AssertionError("non-sender must not touch the client")
|
||||
|
||||
def finish_weight_update(self, weight_version: str | None = None):
|
||||
raise AssertionError("non-sender must not touch the client")
|
||||
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
client = _RaisingClient()
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
is_sender=False,
|
||||
client=client,
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
engine.send_weights() # gathers only; must not raise
|
||||
assert client.order == []
|
||||
assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_send_weights_surfaces_update_error(monkeypatch):
|
||||
class _FailingUpdateClient(RecordingClient):
|
||||
def update_weights(self, update_info):
|
||||
self.order.append("update")
|
||||
raise RuntimeError("inference side rejected update")
|
||||
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
is_sender=True,
|
||||
client=_FailingUpdateClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="inference side rejected update"):
|
||||
engine.send_weights()
|
||||
|
||||
|
||||
class _OwnedSource(_ListSource):
|
||||
"""A source holding only some groups' names, like a pipeline-parallel rank.
|
||||
Takes group indices for convenience and declares the names inside them."""
|
||||
|
||||
def __init__(self, pairs, owned_group_idx):
|
||||
super().__init__(pairs)
|
||||
self._owned = list(owned_group_idx)
|
||||
groups = layerwise_groups([n for n, _ in pairs])
|
||||
self._owned_names = [n for gi in self._owned for n in groups[gi]]
|
||||
|
||||
def held_names(self):
|
||||
return list(self._owned_names)
|
||||
|
||||
def __iter__(self):
|
||||
by_name = dict(self._pairs)
|
||||
return iter([(n, by_name[n]) for n in self._owned_names])
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_publishes_only_the_groups_it_holds(monkeypatch):
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[1, 2], [0, 3]], # this rank holds the layers, rank 1 the rest
|
||||
)
|
||||
engine.send_weights()
|
||||
|
||||
assert server.published == [1, 2]
|
||||
assert server.order == ["begin", "publish", "publish", "end"]
|
||||
assert engine._inflight == {}
|
||||
# embed + norm on rank 1, the two layers here: two distinct owner sets,
|
||||
# numbered by first appearance in metadata order.
|
||||
assert engine._owner_sets == [[1], [0]]
|
||||
assert engine._name_owner_class == [0, 1, 1, 0]
|
||||
|
||||
|
||||
def test_sharded_rdt_owned_group_order_mismatch_raises(monkeypatch):
|
||||
class _MisorderedOwned(_OwnedSource):
|
||||
def __iter__(self):
|
||||
return iter(list(super().__iter__())[::-1])
|
||||
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_MisorderedOwned(_rdt_source_two_layers()._pairs, [1, 2]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[1, 2], [0, 3]],
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="iteration order must match"):
|
||||
engine.send_weights()
|
||||
assert "error" in server.order
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
|
||||
"""The free barrier's target is one integer per sync: the live consumer
|
||||
count, defaulting to the whole provisioned fleet."""
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
engine.send_weights()
|
||||
assert server.live_count == 1 # num_consumers=1 in the harness
|
||||
engine.send_weights(live_consumer_ids=[0])
|
||||
assert server.live_count == 1
|
||||
|
||||
|
||||
class TestLiveCountPlumbing:
|
||||
"""``send_weights(live_consumer_ids)`` -> the barrier target. The
|
||||
provisioned geometry is frozen; a degraded sync only lowers the target and
|
||||
narrows each slot-sharing group's rendezvous."""
|
||||
|
||||
@staticmethod
|
||||
def _engine(num_consumers, world=2, rank=0):
|
||||
engine = ShardedRDTTrainerWeightTransferEngine.__new__(
|
||||
ShardedRDTTrainerWeightTransferEngine
|
||||
)
|
||||
engine._init_info = ShardedRDTTrainerInitInfo(
|
||||
rank=rank, num_consumers=num_consumers
|
||||
)
|
||||
engine.source = object() # send_weights asserts a source is present
|
||||
received: list = []
|
||||
engine._send_weights_inner = lambda count, ids: received.append((count, ids))
|
||||
return engine, received
|
||||
|
||||
def test_none_counts_the_whole_provisioned_fleet(self):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights(None)
|
||||
assert got == [(8, list(range(8)))]
|
||||
|
||||
def test_a_live_set_counts_its_distinct_members(self):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights([0, 1, 4, 5, 5])
|
||||
assert got == [(4, [0, 1, 4, 5])]
|
||||
|
||||
def test_a_full_live_set_matches_the_provisioned_count(self):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights(list(range(8)))
|
||||
engine.send_weights(None)
|
||||
assert got == [(8, list(range(8)))] * 2
|
||||
|
||||
def test_which_consumers_died_does_not_matter_only_how_many(self):
|
||||
"""The whole point of the barrier: no routed per-producer targets, so
|
||||
the identity of the dead consumer is irrelevant to the producers."""
|
||||
counts = []
|
||||
for live in ([0, 1, 2, 3], [4, 5, 6, 7], [0, 2, 4, 6]):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights(live)
|
||||
counts += [c for c, _ids in got]
|
||||
assert counts == [4, 4, 4]
|
||||
|
||||
def test_the_live_ids_travel_with_the_count(self):
|
||||
"""The count sizes the free barrier, the ids size the slot-sharing
|
||||
rendezvous, so a producer that shares slots can tell WHICH consumers it
|
||||
is still waiting for. They must describe the same set."""
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights([6, 0, 2, 2])
|
||||
((count, ids),) = got
|
||||
assert ids == [0, 2, 6] and count == len(ids)
|
||||
|
||||
|
||||
def test_sharded_rdt_worker_init_info_carries_the_ownership_table(monkeypatch):
|
||||
import json
|
||||
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
|
||||
is_sender=True,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[0, 1], [2, 3]],
|
||||
)
|
||||
worker_init = engine._build_worker_init_info(["srv_rk0", "srv_rk1"])
|
||||
assert worker_init.owner_sets == [[0], [1]]
|
||||
assert worker_init.name_owner_class == [0, 0, 1, 1]
|
||||
assert len(worker_init.name_owner_class) == len(worker_init.names)
|
||||
# The payload crosses the control plane as JSON: nested lists must survive.
|
||||
round_tripped = json.loads(json.dumps(asdict(worker_init)))
|
||||
assert round_tripped["owner_sets"] == [[0], [1]]
|
||||
assert round_tripped["name_owner_class"] == [0, 0, 1, 1]
|
||||
|
||||
|
||||
def test_weight_source_holds_everything_by_default():
|
||||
"""The contract's default: a source that says nothing holds the whole model."""
|
||||
src = _rdt_source_two_layers()
|
||||
assert src.held_names() is None
|
||||
assert ModuleSource(torch.nn.Linear(2, 2)).held_names() is None
|
||||
|
||||
|
||||
def test_sharded_rdt_rejects_a_held_name_outside_metadata(monkeypatch):
|
||||
class _BadHeld(_OwnedSource):
|
||||
def held_names(self):
|
||||
return ["embed.weight", "not.a.real.name"]
|
||||
|
||||
with pytest.raises(ValueError, match="not"):
|
||||
_rdt_engine_with_fake_server(
|
||||
_BadHeld(_rdt_source_two_layers()._pairs, [0]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(),
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_rdt_rejects_a_rank_holding_nothing(monkeypatch):
|
||||
class _HoldsNothing(_ListSource):
|
||||
def held_names(self):
|
||||
return []
|
||||
|
||||
with pytest.raises(ValueError, match="empty"):
|
||||
_rdt_engine_with_fake_server(
|
||||
_HoldsNothing(_rdt_source_two_layers()._pairs),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(),
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
|
||||
"""Every name must be held somewhere or it can never be served — caught
|
||||
when the holdings are transposed, naming the orphan."""
|
||||
with pytest.raises(ValueError, match="no trainer rank holds"):
|
||||
_rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(),
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[0, 1], [2]], # nobody holds group 3
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_rdt_rejects_metadata_disagreement_across_ranks(monkeypatch):
|
||||
"""Only the sender's metadata reaches the consumers, so a rank describing
|
||||
just its own share must fail loudly rather than silently drop the rest."""
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(auto_free=True),
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[1, 2], [0, 3]], # a covering fleet, so construction succeeds
|
||||
)
|
||||
# Now rank 1 reports a DIFFERENT metadata digest for the same model.
|
||||
monkeypatch.setattr(
|
||||
engine,
|
||||
"_all_gather_owned",
|
||||
lambda w, mine: [mine, ("deadbeefdeadbeef", mine[1])],
|
||||
)
|
||||
with pytest.raises(ValueError, match="disagrees across trainer ranks"):
|
||||
engine._resolve_ownership(2, 0)
|
||||
|
||||
|
||||
def _serve_ring_server(src_name, src):
|
||||
"""A producer server with one cached tensor and a pre-seeded serve ring, so a
|
||||
pull needs no Ray and no NIXL registration. Returns (server, serve) where
|
||||
``serve(chain)`` packs one spec into the SAME ring slot every time — which is
|
||||
what puts the destination-view cache, and only it, under test."""
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
_RDTProducerServer,
|
||||
)
|
||||
|
||||
srv = _RDTProducerServer(
|
||||
num_rdt_buffers=2,
|
||||
buffer_presize_gb=0.0,
|
||||
gather_lookahead=2,
|
||||
)
|
||||
srv._cache[src_name] = src
|
||||
srv._serve_rings[0] = [
|
||||
torch.empty(1 << 16, dtype=torch.uint8, device="cuda") for _ in range(2)
|
||||
]
|
||||
|
||||
# The slot is ``seq % nring``, so stepping seq by nring keeps every pull in
|
||||
# slot 0 while still making each one its own generation -- reusing a seq
|
||||
# would rendezvous with the finished pack and return it unchanged.
|
||||
seqs = itertools.count(0, srv._nring)
|
||||
|
||||
def serve(chain):
|
||||
return srv.rdt_produce_weights_batched(
|
||||
[(src_name, chain)], consumer_id=0, seq=next(seqs)
|
||||
)[0]
|
||||
|
||||
return srv, serve
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="the producer server needs a CUDA device"
|
||||
)
|
||||
def test_serve_does_not_reuse_packed_views_of_another_shape():
|
||||
"""Two requests can share a name yet pack different slices of it.
|
||||
|
||||
The producer caches the destination views it carves into a serve ring slot.
|
||||
Keyed by name alone, the second request is packed through the first's views.
|
||||
Reachable when one name's copies split across owner-class chunks.
|
||||
"""
|
||||
name = "model.layers.0.w"
|
||||
src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8)
|
||||
_srv, serve = _serve_ring_server(name, src)
|
||||
|
||||
serve((("narrow", (0, 0, 2), ()),)) # 2 rows
|
||||
wide = src.narrow(0, 0, 6) # 6 rows, same name, same slot
|
||||
blob = serve((("narrow", (0, 0, 6), ()),))
|
||||
|
||||
got = blob[: wide.numel() * wide.element_size()].view(wide.dtype).reshape(6, 8)
|
||||
assert torch.equal(got, wide)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="the producer server needs a CUDA device"
|
||||
)
|
||||
def test_serve_does_not_reuse_packed_views_of_another_dtype():
|
||||
"""The SILENT case of the same cache: two requests whose slices have the same
|
||||
name and the same shape but different dtypes pack at identical offsets, so
|
||||
reusing the stale views raises nothing — ``copy_`` just casts, and the blob
|
||||
carries the wrong bytes with no check downstream.
|
||||
|
||||
``view(dtype)`` rather than ``to(dtype)`` because the chain is replayed under
|
||||
``ALLOWED_OPS``, which rejects ``to``: same-itemsize dtypes keep the byte
|
||||
length identical, which is exactly what makes the stale view fit.
|
||||
"""
|
||||
name = "model.layers.0.w"
|
||||
src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8)
|
||||
_srv, serve = _serve_ring_server(name, src)
|
||||
|
||||
serve((("view", (torch.float16,), ()),)) # same shape and bytes, fp16
|
||||
blob = serve(()) # same name and shape, bf16
|
||||
|
||||
got = blob[: src.numel() * src.element_size()].view(src.dtype).reshape(8, 8)
|
||||
assert torch.equal(got, src), "packed through a cached view of the wrong dtype"
|
||||
@@ -32,8 +32,10 @@ from vllm.distributed.weight_transfer import (
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import (
|
||||
TrainerInitInfo,
|
||||
WeightTransferEngine,
|
||||
WeightTransferInitRequest,
|
||||
WeightTransferUpdateRequest,
|
||||
layerwise_groups,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.ipc_engine import (
|
||||
IPCTrainerInitInfo,
|
||||
@@ -1412,6 +1414,162 @@ class TestModuleSource:
|
||||
assert [m.shape for m in meta] == [tuple(t.shape) for _, t in pairs]
|
||||
|
||||
|
||||
class TestWeightSourceGroupContract:
|
||||
"""`groups()` / `iter_groups()` on the WeightSource ABC. Groups are what
|
||||
backends gather and free by, so the default must agree with
|
||||
`layerwise_groups` over `metadata()`, restricted to what this rank holds."""
|
||||
|
||||
class _Source(WeightSource):
|
||||
"""Minimal source over an ordered (name, tensor) list, optionally holding
|
||||
only some names (in which case it iterates only their groups)."""
|
||||
|
||||
def __init__(self, names, held=None, reverse=False):
|
||||
self._pairs = [(n, torch.full((2,), float(i))) for i, n in enumerate(names)]
|
||||
self._held = held
|
||||
self._reverse = reverse
|
||||
|
||||
def metadata(self):
|
||||
return [ParamMeta(n, t.dtype, tuple(t.shape)) for n, t in self._pairs]
|
||||
|
||||
def held_names(self):
|
||||
return self._held
|
||||
|
||||
def __iter__(self):
|
||||
pairs = self._pairs
|
||||
if self._held is not None:
|
||||
keep = set(self._held)
|
||||
pairs = [(n, t) for n, t in pairs if n in keep]
|
||||
return iter(list(reversed(pairs)) if self._reverse else pairs)
|
||||
|
||||
def _source(self, names, held=None, reverse=False):
|
||||
return self._Source(names, held, reverse)
|
||||
|
||||
def test_groups_defaults_to_the_layerwise_partition(self):
|
||||
names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"]
|
||||
assert self._source(names).groups() == layerwise_groups(names)
|
||||
|
||||
def test_groups_keeps_only_groups_holding_a_held_name(self):
|
||||
names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"]
|
||||
held = ["model.layers.0.a", "model.layers.1.a"]
|
||||
assert self._source(names, held=held).groups() == [
|
||||
["model.layers.0.a"],
|
||||
["model.layers.1.a"],
|
||||
]
|
||||
|
||||
def test_groups_keeps_a_partially_held_group_whole(self):
|
||||
"""One held name selects the WHOLE group, unheld members included.
|
||||
|
||||
A group is the collective unit, so its membership has to be
|
||||
rank-uniform: a rank that narrowed it to what it holds would disagree
|
||||
with its peers about which names group *g* covers, and the gather would
|
||||
desynchronize. Selection is per group, but only whole groups; the
|
||||
per-name split is the backend's, via owner sets. This is the
|
||||
foreign-expert shape under expert parallelism — a rank holds some
|
||||
experts of a layer, not all of them.
|
||||
"""
|
||||
names = ["model.layers.0.a", "model.layers.0.b", "model.layers.1.a"]
|
||||
held = ["model.layers.0.a"]
|
||||
assert self._source(names, held=held).groups() == [
|
||||
["model.layers.0.a", "model.layers.0.b"],
|
||||
]
|
||||
|
||||
def test_groups_order_follows_metadata_not_the_declaration(self):
|
||||
"""The declaration is a SET of names; the groups it selects still come
|
||||
out in metadata order, so each pairs with the right ``iter_groups()``
|
||||
batch however the source listed them."""
|
||||
names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"]
|
||||
source = self._source(
|
||||
names, held=["model.layers.1.a", "model.layers.0.a", "model.layers.1.a"]
|
||||
)
|
||||
assert source.groups() == [["model.layers.0.a"], ["model.layers.1.a"]]
|
||||
assert [ns for ns, _ in source.iter_groups()] == [
|
||||
["model.layers.0.a"],
|
||||
["model.layers.1.a"],
|
||||
]
|
||||
|
||||
def test_iter_groups_batches_the_stream_per_group(self):
|
||||
names = ["embed.w", "model.layers.0.a", "model.layers.0.b", "norm.w"]
|
||||
batches = list(self._source(names).iter_groups())
|
||||
assert [ns for ns, _ in batches] == [
|
||||
["embed.w"],
|
||||
["model.layers.0.a", "model.layers.0.b"],
|
||||
["norm.w"],
|
||||
]
|
||||
assert all(len(ns) == len(ts) for ns, ts in batches)
|
||||
|
||||
def test_iter_groups_yields_the_tensors_iteration_produced(self):
|
||||
names = ["model.layers.0.a", "model.layers.0.b"]
|
||||
(batch,) = list(self._source(names).iter_groups())
|
||||
_names, tensors = batch
|
||||
assert [float(t[0]) for t in tensors] == [0.0, 1.0]
|
||||
|
||||
def test_iter_groups_yields_only_held_groups(self):
|
||||
names = ["embed.w", "model.layers.0.a", "model.layers.1.a"]
|
||||
batches = list(self._source(names, held=["model.layers.1.a"]).iter_groups())
|
||||
assert [ns for ns, _ in batches] == [["model.layers.1.a"]]
|
||||
|
||||
def test_out_of_order_iteration_raises(self):
|
||||
"""Materializing is usually a collective, so a rank that iterates out of
|
||||
order deadlocks its peers -- fail loudly instead."""
|
||||
source = self._source(["model.layers.0.a", "model.layers.0.b"], reverse=True)
|
||||
with pytest.raises(RuntimeError, match="iteration order must match"):
|
||||
list(source.iter_groups())
|
||||
|
||||
def test_a_source_may_override_iter_groups(self):
|
||||
"""The extension point: materialize a whole group in one step instead of
|
||||
one generator resume per tensor."""
|
||||
calls = []
|
||||
|
||||
class _Batched(TestWeightSourceGroupContract._Source):
|
||||
def iter_groups(self):
|
||||
for group in self.groups():
|
||||
calls.append(len(group))
|
||||
yield group, [torch.zeros(2) for _ in group]
|
||||
|
||||
source = _Batched(["model.layers.0.a", "model.layers.0.b"])
|
||||
assert [ns for ns, _ in source.iter_groups()] == [
|
||||
["model.layers.0.a", "model.layers.0.b"]
|
||||
]
|
||||
assert calls == [2]
|
||||
|
||||
|
||||
class TestDeferredProcessingContract:
|
||||
"""`defers_processing` and `drain_pending` are two halves of one contract: a
|
||||
caller that takes over the update tail (running its own
|
||||
`finalize_layerwise_reload` instead of going through `finish_weight_update`)
|
||||
reads the flag and calls the method. Both must be answerable on any engine, or
|
||||
that caller ends up reaching through a getattr."""
|
||||
|
||||
def _engines(self):
|
||||
return {
|
||||
name: loader()
|
||||
for name, loader in WeightTransferEngineFactory._registry.items()
|
||||
}
|
||||
|
||||
def test_every_engine_declares_whether_it_defers(self):
|
||||
for name, cls in self._engines().items():
|
||||
assert isinstance(cls.defers_processing, bool), name
|
||||
|
||||
def test_every_engine_can_be_drained(self):
|
||||
"""The default is a no-op, so a caller never has to check whether the
|
||||
method exists before calling it."""
|
||||
for name, cls in self._engines().items():
|
||||
assert callable(cls.drain_pending), name
|
||||
|
||||
def test_the_default_is_not_to_defer(self):
|
||||
assert WeightTransferEngine.defers_processing is False
|
||||
|
||||
def test_a_synchronous_engine_drains_as_a_no_op(self):
|
||||
engine = object.__new__(WeightTransferEngineFactory._registry["nccl"]())
|
||||
engine.drain_pending() # must not raise, and must not need any state
|
||||
|
||||
def test_the_rdt_engine_defers_and_overrides_the_drain(self):
|
||||
"""The one engine the contract exists for."""
|
||||
cls = WeightTransferEngineFactory._registry["sharded_rdt"]()
|
||||
assert cls.defers_processing is True
|
||||
assert cls.drain_pending is not WeightTransferEngine.drain_pending
|
||||
|
||||
|
||||
class TestTrainerFactory:
|
||||
"""WeightTransferTrainerFactory registry mechanics."""
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ ALLOWED_FILES = {
|
||||
"vllm/platforms/",
|
||||
"vllm/device_allocator/",
|
||||
"vllm/distributed/weight_transfer/ipc_engine.py",
|
||||
"vllm/distributed/weight_transfer/sharded_rdt_engine.py",
|
||||
"tests/distributed/test_packed_tensor.py",
|
||||
}
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from vllm.config.utils import config
|
||||
class WeightTransferConfig:
|
||||
"""Configuration for weight transfer during RL training."""
|
||||
|
||||
backend: Literal["nccl", "ipc", "sparse_nccl"] | str = "nccl"
|
||||
backend: Literal["nccl", "ipc", "sparse_nccl", "sharded_rdt"] | str = "nccl"
|
||||
"""The backend to use for weight transfer. Validated against the
|
||||
`WeightTransferEngineFactory` registry at engine creation time.
|
||||
"""
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
"""Base class for weight transfer engines."""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Collection, Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
@@ -33,6 +33,76 @@ TTrainerInitInfo = TypeVar("TTrainerInitInfo", bound="TrainerInitInfo")
|
||||
# channel. The built-in `ModuleSource` uses `materialize_full_tensor`.
|
||||
|
||||
|
||||
def _stack_key(name: str) -> tuple[str, int] | None:
|
||||
"""``(prefix, index)`` of the OUTERMOST integer segment, or None if there is
|
||||
none.
|
||||
|
||||
Outermost is what keeps a MoE layer whole:
|
||||
``model.layers.3.mlp.experts.7.w1`` keys on the layer, not the expert.
|
||||
"""
|
||||
parts = name.split(".")
|
||||
for i, part in enumerate(parts):
|
||||
if part.isdigit():
|
||||
return ".".join(parts[:i]), int(part)
|
||||
return None
|
||||
|
||||
|
||||
def layerwise_groups(names: list[str]) -> list[list[str]]:
|
||||
"""Partition flat parameter names into one group per decoder layer, keyed on
|
||||
the outermost index segment of each name.
|
||||
|
||||
This defines what a *group index* means for `WeightSource.groups` and
|
||||
`WeightSource.iter_groups`: index *g* names the same group on every trainer
|
||||
rank and every consumer, because it is derived from one rank's `metadata()`
|
||||
order.
|
||||
|
||||
Keying on the index rather than a literal prefix needs no per-architecture
|
||||
naming table: ``model.layers.0.``, ``model.language_model.layers.0.``,
|
||||
``transformer.h.0.``, ``backbone.layers.0.`` and a vision tower's
|
||||
``visual.blocks.0.`` all partition alike. Matching one fixed prefix does not,
|
||||
and its failure is silent — every name lands in a single group holding the
|
||||
whole model, which defeats the per-layer bound below.
|
||||
|
||||
Un-indexed names split by POSITION relative to the first indexed one: the pre
|
||||
block (embeddings) and the post block (the final norm, `lm_head`, and any
|
||||
inter-stack projector). Post lands last however early it arrived, which is
|
||||
what a pipeline-parallel source needs — Megatron-Bridge streams the last
|
||||
stage's output block *before* its layers.
|
||||
|
||||
Stacks come out in first-appearance order of their prefix and ascending index
|
||||
within it, whatever order the source yielded them, so a source can normalize
|
||||
an arbitrary export order by flattening this partition.
|
||||
|
||||
Backends that gather and free per group (sharded RDT) also use it as the unit
|
||||
of transfer, which bounds their buffer sizes: without it a whole model becomes
|
||||
one chunk.
|
||||
"""
|
||||
pre: list[str] = []
|
||||
post: list[str] = []
|
||||
stacks: dict[tuple[str, int], list[str]] = {}
|
||||
order: list[tuple[str, int]] = []
|
||||
for name in names:
|
||||
key = _stack_key(name)
|
||||
if key is None:
|
||||
(post if order else pre).append(name)
|
||||
continue
|
||||
if key not in stacks:
|
||||
stacks[key] = []
|
||||
order.append(key)
|
||||
stacks[key].append(name)
|
||||
|
||||
prefix_rank: dict[str, int] = {}
|
||||
for key in order:
|
||||
prefix_rank.setdefault(key[0], len(prefix_rank))
|
||||
order.sort(key=lambda key: (prefix_rank[key[0]], key[1]))
|
||||
|
||||
groups: list[list[str]] = [pre] if pre else []
|
||||
groups += [stacks[key] for key in order]
|
||||
if post:
|
||||
groups.append(post)
|
||||
return groups
|
||||
|
||||
|
||||
def materialize_full_tensor(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Return a full, locally-materialized tensor ready to send.
|
||||
|
||||
@@ -66,10 +136,13 @@ class WeightSource(ABC):
|
||||
case it should cache.
|
||||
* iteration — yields fully-materialized `(name, tensor)` pairs, one at a
|
||||
time. Materializing is typically a collective (FSDP `full_tensor()`, a
|
||||
Megatron export), so every trainer rank must iterate the same source in the
|
||||
same order in lockstep, or ranks deadlock. Under pipeline parallelism a
|
||||
rank may not own a parameter at all — iterating still drives the collective
|
||||
and the yielded tensor is only meaningful on the sender.
|
||||
Megatron export), so the ranks that share a parameter must iterate it in
|
||||
the same order in lockstep, or they deadlock.
|
||||
* `held_names()` — which parameters this rank holds, for producers that are
|
||||
split so each rank holds only part of the model. Defaults to all.
|
||||
* `iter_groups()` — the same stream batched per gather group (see
|
||||
`layerwise_groups`). Defaults to batching `__iter__`; override to
|
||||
materialize a whole group in one step.
|
||||
|
||||
`iter(source)` must yield a *fresh* pass each round. Backends with custom
|
||||
producer logic (Megatron export, RDT plans, MoE re-fusing) subclass this.
|
||||
@@ -92,6 +165,79 @@ class WeightSource(ABC):
|
||||
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
raise NotImplementedError
|
||||
|
||||
def held_names(self) -> "Collection[str] | None":
|
||||
"""The parameters this rank holds, or None for all of them.
|
||||
|
||||
This is the whole ownership contract. Override it when producers are
|
||||
split so each holds only part of the model — pipeline parallelism (a rank
|
||||
holds some layers), expert parallelism (a rank holds some experts), or
|
||||
any combination, including layouts that fit neither. A consumer routes
|
||||
each name to a rank that holds it, so per-name is the granularity that
|
||||
matters; the engine derives everything else from this.
|
||||
|
||||
Three requirements come with overriding it:
|
||||
|
||||
* `metadata()` must still describe the WHOLE model on every rank. The
|
||||
group partition, the iteration checks and the consumers' pull plans are
|
||||
all built from one rank's metadata, so a rank that reported only its own
|
||||
share would leave the rest of the model silently un-transferred. The
|
||||
sharded-RDT engine cross-checks this across ranks at init.
|
||||
* Every name must be held by at least one rank, or it can never be
|
||||
served. The engine raises at init naming the first orphan.
|
||||
* Iteration must cover exactly `groups()` in metadata order, yielding a
|
||||
real tensor for each held name and `None` for the rest. A group's
|
||||
gather is a collective among the ranks that hold part of it, so the
|
||||
name must still appear (to keep the order check aligned) while the
|
||||
data is absent.
|
||||
|
||||
Returns:
|
||||
The held parameter names, or None to hold every one.
|
||||
"""
|
||||
return None
|
||||
|
||||
def groups(self) -> list[list[str]]:
|
||||
"""This rank's gather groups, in metadata order: `layerwise_groups` over
|
||||
`metadata()`, restricted to the groups holding at least one held name.
|
||||
|
||||
A group with nothing held here is not iterated at all — its gather is a
|
||||
collective among the ranks that do hold part of it.
|
||||
"""
|
||||
groups = layerwise_groups([m.name for m in self.metadata()])
|
||||
held = self.held_names()
|
||||
if held is None:
|
||||
return groups
|
||||
held = set(held)
|
||||
return [g for g in groups if any(n in held for n in g)]
|
||||
|
||||
def iter_groups(self) -> Iterator[tuple[list[str], list[torch.Tensor]]]:
|
||||
"""Yield one `(names, tensors)` batch per group from `groups()`.
|
||||
|
||||
The default drives `__iter__` and batches its output, checking as it goes
|
||||
that the names arrive in metadata order — ranks sharing a parameter
|
||||
materialize it with a collective, so a rank that iterates out of order
|
||||
deadlocks its peers rather than returning wrong data.
|
||||
|
||||
Override when a backend can produce a whole group at once. Materializing
|
||||
is usually a collective, and driving it per group instead of per tensor
|
||||
turns ~37k generator resumes into ~95 on a per-expert MoE model (worth
|
||||
~0.9s per sync there). An override must yield the same batches in the same
|
||||
order as this default.
|
||||
"""
|
||||
it = iter(self)
|
||||
for group in self.groups():
|
||||
names: list[str] = []
|
||||
tensors: list[torch.Tensor] = []
|
||||
for expected in group:
|
||||
name, tensor = next(it)
|
||||
if name != expected:
|
||||
raise RuntimeError(
|
||||
f"WeightSource yielded {name!r} but expected "
|
||||
f"{expected!r}; iteration order must match metadata()."
|
||||
)
|
||||
names.append(name)
|
||||
tensors.append(tensor)
|
||||
yield names, tensors
|
||||
|
||||
|
||||
class ModuleSource(WeightSource):
|
||||
"""`WeightSource` over `module.named_parameters()` — the common case.
|
||||
@@ -207,6 +353,31 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
supports_draft_weight_update: bool = True
|
||||
|
||||
defers_processing: bool = False
|
||||
"""Whether `update_weights` returns before the weights are on the device.
|
||||
|
||||
An engine that pipelines its GPU post-processing onto background threads
|
||||
cannot let `update_weights` synchronize the device — that would block on those
|
||||
threads and serialize the pipeline. Such an engine sets this True, omits the
|
||||
per-update sync, and guarantees completion in `finish_weight_update` instead.
|
||||
|
||||
Callers that go through `finish_weight_update` need do nothing: the engine
|
||||
drains there. A caller that instead drives the tail itself — running its own
|
||||
`finalize_layerwise_reload`, say — must read this flag and call
|
||||
`drain_pending()` first, because with it set a returned `update_weights` means
|
||||
"queued", not "applied".
|
||||
"""
|
||||
|
||||
def drain_pending(self) -> None:
|
||||
"""Block until every deferred update has been applied to the model.
|
||||
|
||||
The companion to `defers_processing`: a caller that has taken over the
|
||||
update tail calls this to re-establish the guarantee that
|
||||
`finish_weight_update` would otherwise have given it. Idempotent, and a
|
||||
no-op by default — an engine that processes synchronously has nothing to
|
||||
drain, so this is always safe to call.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: WeightTransferConfig,
|
||||
@@ -385,7 +556,11 @@ class TrainerWeightTransferEngine(ABC, Generic[TTrainerInitInfo]):
|
||||
Symmetric to `WeightTransferEngine` but lives in the training process.
|
||||
Constructed via the `trainer_init` factory classmethod; carries any
|
||||
backend-specific state (NCCL communicators, IPC device info, transfer
|
||||
plans) on `self`.
|
||||
plans) on `self`. Full-resync backends (NCCL, IPC) take a `WeightSource` at
|
||||
`trainer_init` and replay it each round via the no-argument
|
||||
`send_weights()`. Backends that push per-round deltas instead (e.g. sparse
|
||||
patches) leave `source` as `None` and take their payload as a `send_weights`
|
||||
argument.
|
||||
|
||||
Unlike the worker engine, the trainer side does not take a
|
||||
`WeightTransferConfig`: the backend is selected from the init info's
|
||||
|
||||
@@ -234,6 +234,12 @@ WeightTransferEngineFactory.register_engine(
|
||||
"SparseNCCLWeightTransferEngine",
|
||||
)
|
||||
|
||||
WeightTransferEngineFactory.register_engine(
|
||||
"sharded_rdt",
|
||||
"vllm.distributed.weight_transfer.sharded_rdt_engine",
|
||||
"ShardedRDTWeightTransferEngine",
|
||||
)
|
||||
|
||||
|
||||
# Trainer-side engines, parallel to the worker registry above.
|
||||
WeightTransferTrainerFactory.register_engine(
|
||||
@@ -253,3 +259,9 @@ WeightTransferTrainerFactory.register_engine(
|
||||
"vllm.distributed.weight_transfer.sparse_nccl_engine",
|
||||
"SparseNCCLTrainerWeightTransferEngine",
|
||||
)
|
||||
|
||||
WeightTransferTrainerFactory.register_engine(
|
||||
"sharded_rdt",
|
||||
"vllm.distributed.weight_transfer.sharded_rdt_trainer",
|
||||
"ShardedRDTTrainerWeightTransferEngine",
|
||||
)
|
||||
|
||||
@@ -0,0 +1,339 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared pieces of the sharded-RDT backend: the op-chain allowlist, buffer
|
||||
sizing and the minimum Ray version, plus ``RdtRouter`` — consumer-only, since
|
||||
routing is a consumer decision.
|
||||
|
||||
The gather-group partition itself is ``base.layerwise_groups``: it defines what a
|
||||
group index means for any ``WeightSource``, not just this transport.
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
# The op-chain contract, with two enforcers. The consumer's ``FakeRDTTensor``
|
||||
# intercepts exactly these methods during the bake and records the string name;
|
||||
# anything else reaches ``__torch_dispatch__`` and raises, so a loader needing
|
||||
# real data fails at init rather than transferring wrong bytes. The producer
|
||||
# replays a chain as ``getattr(tensor, op)(...)`` and refuses anything outside
|
||||
# ``ALLOWED_OPS``, so a spoofed consumer cannot invoke arbitrary methods.
|
||||
#
|
||||
# Every entry must be a pure view; ``to`` in particular must never be added, as
|
||||
# it is the dtype/device escape the bake exists to reject. Both sets derive from
|
||||
# this one table so they cannot disagree.
|
||||
SUPPORTED_OPS: dict[Callable, str] = {
|
||||
torch.Tensor.narrow: "narrow",
|
||||
torch.Tensor.view: "view",
|
||||
torch.Tensor.reshape: "reshape",
|
||||
torch.Tensor.__getitem__: "__getitem__",
|
||||
torch.Tensor.unsqueeze: "unsqueeze",
|
||||
torch.Tensor.squeeze: "squeeze",
|
||||
torch.Tensor.transpose: "transpose",
|
||||
torch.Tensor.t: "t",
|
||||
torch.Tensor.permute: "permute",
|
||||
torch.Tensor.flatten: "flatten",
|
||||
torch.Tensor.contiguous: "contiguous",
|
||||
torch.Tensor.chunk: "chunk",
|
||||
# Multi-return like chunk: one child per output, trailing __getitem__(i).
|
||||
torch.Tensor.unbind: "unbind",
|
||||
}
|
||||
|
||||
ALLOWED_OPS = frozenset(SUPPORTED_OPS.values())
|
||||
|
||||
# Ray Direct Transport landed in halves: the actor opt-in
|
||||
# ``enable_tensor_transport`` in 2.49, and ``ray.experimental``'s
|
||||
# ``register_nixl_memory`` / ``set_target_for_ref`` — which this transport calls
|
||||
# directly — in 2.55. The floor is the tested version rather than the earliest
|
||||
# one that could import; 2.55 has never been run against.
|
||||
RDT_MIN_RAY_VERSION = "2.56.0"
|
||||
|
||||
|
||||
def check_ray_rdt_version() -> None:
|
||||
"""Refuse an installed Ray older than the one this backend is tested on.
|
||||
|
||||
vLLM does not depend on Ray, so there is no pin to carry this. Without the
|
||||
check the failure is an opaque option-validation error out of
|
||||
``.options(enable_tensor_transport=True)`` (below 2.49) or an ImportError
|
||||
raised deep in the first pull, long after init reported success (below 2.55).
|
||||
|
||||
Raises:
|
||||
ValueError: the installed Ray predates ``RDT_MIN_RAY_VERSION``.
|
||||
"""
|
||||
import importlib.metadata
|
||||
|
||||
from packaging import version
|
||||
|
||||
required = version.parse(RDT_MIN_RAY_VERSION)
|
||||
current = version.parse(importlib.metadata.version("ray"))
|
||||
if current < required:
|
||||
raise ValueError(
|
||||
f"The 'sharded_rdt' weight transfer backend requires Ray "
|
||||
f">= {required}, the version it is tested against; Ray Direct "
|
||||
f"Transport (ray.experimental.register_nixl_memory / "
|
||||
f"set_target_for_ref) needs at least 2.55. Found {current}. "
|
||||
f"Run `pip install -U 'ray>={required}'`."
|
||||
)
|
||||
|
||||
|
||||
def assign_producer_indices(
|
||||
num_producers: int, num_consumers: int, consumer_idx: int
|
||||
) -> list[int]:
|
||||
"""Producers (global indices) that consumer ``consumer_idx`` binds."""
|
||||
p = max(1, num_producers)
|
||||
c = max(1, num_consumers)
|
||||
if p >= c:
|
||||
return list(range(consumer_idx * p // c, (consumer_idx + 1) * p // c))
|
||||
return [consumer_idx * p // c]
|
||||
|
||||
|
||||
class RdtRouter:
|
||||
"""Decides which producer serves each weight name, and — once bound — sends.
|
||||
|
||||
Ownership is per NAME. ``owner_sets`` holds the few distinct producer sets
|
||||
that occur, and ``name_owner_class[i]`` indexes it for ``names[i]``; a name
|
||||
determines both its owner set and its gather group, so neither appears in the
|
||||
routing API. Empty tables mean every producer holds everything.
|
||||
|
||||
Any placement is expressible this way: a pipeline stage is a set of names
|
||||
sharing one owner set, an expert is a name whose owner set is a single rank,
|
||||
and a group produced by two stages is just two classes inside one group.
|
||||
|
||||
Both engines derive the same tables from the same wire data, so they agree on
|
||||
who serves what. Disagreement is not a wrong answer but a hang or a loud
|
||||
misroute: a pull sent to a producer that never gathered the name trips its
|
||||
served-names guard.
|
||||
|
||||
Routing is consumer-only. The trainer publishes what it holds and answers
|
||||
whatever arrives; it never asks who serves what.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
num_producers: int,
|
||||
num_consumers: int,
|
||||
owner_sets: list[list[int]] | None = None,
|
||||
name_owner_class: list[int] | None = None,
|
||||
names: list[str] | None = None,
|
||||
group_lens: list[int] | None = None,
|
||||
workers_per_replica: int = 0,
|
||||
) -> None:
|
||||
"""Build the routing tables from the wire data both engines receive.
|
||||
|
||||
The four table arguments travel together and default together: with all
|
||||
of them empty the router degrades to "every producer holds everything".
|
||||
|
||||
Args:
|
||||
num_producers: Trainer ranks. Owner indices are positions in this
|
||||
range, and ``validate`` rejects any that fall outside it.
|
||||
num_consumers: Inference workers across the whole fleet. Fixes
|
||||
the id space; the block carve uses ``workers_per_replica``
|
||||
of it.
|
||||
owner_sets: The distinct producer sets that occur, one row per
|
||||
owner class; each row is deduplicated and sorted here. Empty
|
||||
means a single class owning every producer.
|
||||
name_owner_class: Parallel to ``names`` — ``name_owner_class[i]``
|
||||
indexes ``owner_sets`` for ``names[i]``. A name with no entry
|
||||
falls back to class 0.
|
||||
names: Every parameter name, in group-major order: concatenating
|
||||
the gather groups reproduces this list exactly. The other two
|
||||
tables are keyed by this order.
|
||||
group_lens: Length of each gather group, consecutive over
|
||||
``names``, so they sum to ``len(names)``. Fixes name -> group
|
||||
and the per-group owner union the free barrier fans out to.
|
||||
workers_per_replica: Consumers per inference DEPLOYMENT. The block
|
||||
carve spreads this many consumers over an owner set and every
|
||||
deployment reuses that carve, so the same worker of each
|
||||
deployment resolves one producer (see ``producer_for``). 0
|
||||
means one deployment: carve over the whole fleet.
|
||||
|
||||
Raises:
|
||||
ValueError: ``workers_per_replica`` does not divide
|
||||
``num_consumers``, so the deployments are not uniform and two
|
||||
workers of one deployment would share a block index.
|
||||
"""
|
||||
self.num_producers = max(1, num_producers)
|
||||
self.num_consumers = max(1, num_consumers)
|
||||
# Width of the block carve: one deployment's worth of consumers, so
|
||||
# deployments overlay rather than spread. See ``producer_for``.
|
||||
self._block_consumers = min(
|
||||
self.num_consumers, max(1, workers_per_replica or self.num_consumers)
|
||||
)
|
||||
if self.num_consumers % self._block_consumers:
|
||||
raise ValueError(
|
||||
f"workers_per_replica={workers_per_replica} does not divide "
|
||||
f"num_consumers={self.num_consumers}; the inference deployments "
|
||||
f"must be uniform for the per-deployment block carve to be "
|
||||
f"well defined."
|
||||
)
|
||||
self._owner_sets = (
|
||||
[sorted(set(row)) for row in owner_sets]
|
||||
if owner_sets
|
||||
else [list(range(self.num_producers))]
|
||||
)
|
||||
names = list(names or [])
|
||||
classes = list(name_owner_class or [])
|
||||
self._class_of = {
|
||||
n: (classes[i] if i < len(classes) else 0) for i, n in enumerate(names)
|
||||
}
|
||||
# name -> gather group, and the per-group owner union the free barrier
|
||||
# fans out to. Both are pure functions of the tables, so precompute once.
|
||||
self._group_of: dict[str, int] = {}
|
||||
self._group_owners: list[list[int]] = []
|
||||
pos = 0
|
||||
for gi, glen in enumerate(group_lens or []):
|
||||
union: set[int] = set()
|
||||
for n in names[pos : pos + glen]:
|
||||
self._group_of[n] = gi
|
||||
c = self._class_of.get(n, 0)
|
||||
# Bounds-checked: a bad class must surface from validate() as a
|
||||
# named ValueError, not as an IndexError from this precompute.
|
||||
if 0 <= c < len(self._owner_sets):
|
||||
union.update(self._owner_sets[c])
|
||||
pos += glen
|
||||
self._group_owners.append(sorted(union))
|
||||
# Bound at init on the consumer; the rule methods never touch these.
|
||||
self._actors: list[Any] = []
|
||||
self._produce_methods: list[Any] = []
|
||||
self.consumer_id = 0
|
||||
|
||||
# ---------------- rules (pure; safe to use unbound) ----------------
|
||||
|
||||
def class_of(self, name: str) -> int:
|
||||
"""The name's owner class — the planner's bucketing key, since all names
|
||||
of a chunk must share one producer."""
|
||||
return self._class_of.get(name, 0)
|
||||
|
||||
def owners(self, name: str) -> list[int]:
|
||||
"""Every producer holding ``name``."""
|
||||
return list(self._owner_sets[self.class_of(name)])
|
||||
|
||||
def group_owners(self, group_idx: int) -> list[int]:
|
||||
"""Every producer holding any name of ``group_idx``.
|
||||
|
||||
The one group-keyed rule, because the free barrier is per group: a
|
||||
consumer signals ``free_group(gi)`` at each of these and the producer
|
||||
counts signals against its live-consumer total. Names route; groups free.
|
||||
"""
|
||||
if not self._group_owners:
|
||||
return list(range(self.num_producers))
|
||||
return list(self._group_owners[group_idx])
|
||||
|
||||
def producer_for(self, consumer_id: int, name: str) -> int:
|
||||
"""The single producer ``consumer_id`` pulls ``name`` from.
|
||||
|
||||
Blocks consumers across the name's owner set with the same rule that
|
||||
binds producers globally, then rotates by the name's GROUP index so a
|
||||
consumer spreads its groups over its block instead of hammering one NIC.
|
||||
Rotating per group (not per name) keeps every name of a chunk on one
|
||||
producer, which is what lets a chunk be a single pull.
|
||||
|
||||
The carve is over ONE DEPLOYMENT (this consumer's index within its own
|
||||
deployment, over ``workers_per_replica`` of them), so the same worker of
|
||||
every deployment resolves the same producer for every name. Carving over
|
||||
the whole fleet instead spreads the multi-owner names of different
|
||||
deployments onto different producers while single-owner names still
|
||||
route by ownership, so a producer serves several distinct worker
|
||||
indices, each needing its own serve ring, and none of them can share a
|
||||
slot. With one deployment the width is the fleet, so this is the plain
|
||||
block rule.
|
||||
"""
|
||||
own = self.owners(name)
|
||||
if not own:
|
||||
raise ValueError(f"{name!r} has no owner")
|
||||
block = assign_producer_indices(
|
||||
len(own), self._block_consumers, consumer_id % self._block_consumers
|
||||
)
|
||||
return own[block[self._group_of.get(name, 0) % len(block)]]
|
||||
|
||||
def validate(self) -> None:
|
||||
"""Check the ownership tables can be served.
|
||||
|
||||
Raises:
|
||||
ValueError: an owner set is empty or out of range, or a class index
|
||||
does not resolve.
|
||||
"""
|
||||
for c, row in enumerate(self._owner_sets):
|
||||
if not row:
|
||||
raise ValueError(f"owner set {c} is empty: no producer holds it")
|
||||
bad = [p for p in row if not 0 <= p < self.num_producers]
|
||||
if bad:
|
||||
raise ValueError(f"owner set {c} out of range: {bad}")
|
||||
for name, c in self._class_of.items():
|
||||
if not 0 <= c < len(self._owner_sets):
|
||||
raise ValueError(
|
||||
f"{name!r} has owner class {c}, but only "
|
||||
f"{len(self._owner_sets)} owner set(s) were shipped"
|
||||
)
|
||||
|
||||
# ---------------- send surface (needs the binding) ----------------
|
||||
|
||||
def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None:
|
||||
"""Attach this consumer's producer handles. Rebuilt wholesale per init,
|
||||
never appended to: every owner index is a position in these lists, so a
|
||||
rejoining engine that re-inits must not shift them."""
|
||||
if len(actors) != len(produce_methods):
|
||||
raise ValueError("actors and produce_methods must be parallel")
|
||||
self._actors = list(actors)
|
||||
self._produce_methods = list(produce_methods)
|
||||
self.consumer_id = consumer_id
|
||||
|
||||
def _bound(self) -> None:
|
||||
if not self._produce_methods:
|
||||
raise RuntimeError("RdtRouter has no producers bound; call bind() first.")
|
||||
|
||||
def pull(self, owner: int, keys: list, seq: int) -> Any:
|
||||
"""Issue one packed pull to ``owner``.
|
||||
|
||||
The consumer id keys the producer's serve ring; without it every worker
|
||||
is served out of ring 0 and concurrent pulls overwrite each other's blob.
|
||||
``seq`` is this call's index in the stream to ``owner`` (see ``_Chunk``):
|
||||
it selects the serve slot in ISSUE order, so a slot is never repacked
|
||||
while the read of its previous contents is still in flight.
|
||||
"""
|
||||
self._bound()
|
||||
return self._produce_methods[owner].remote(
|
||||
keys, consumer_id=self.consumer_id, seq=seq
|
||||
)
|
||||
|
||||
def free_group(self, group_idx: int) -> list:
|
||||
"""Signal the group done at every producer holding any of its names."""
|
||||
self._bound()
|
||||
return [
|
||||
self._actors[p].free_group.remote(group_idx)
|
||||
for p in self.group_owners(group_idx)
|
||||
]
|
||||
|
||||
def reserve_serve_buffers(
|
||||
self, bytes_by_producer: list[int], plan_digests: list[str] | None = None
|
||||
) -> list:
|
||||
"""Ask each producer to pre-register a serve ring sized to the most this
|
||||
consumer will pull from it.
|
||||
|
||||
``plan_digests[p]`` describes the chunks this consumer pulls from
|
||||
producer ``p``, in pull order. A producer that shares one serve ring
|
||||
across the consumers of several deployments compares it across them, so
|
||||
a fleet whose deployments are not identical fails at init instead of
|
||||
stalling mid-sync. A producer that shares nothing ignores it.
|
||||
"""
|
||||
self._bound()
|
||||
digests: list = (
|
||||
list(plan_digests) if plan_digests else [None] * len(bytes_by_producer)
|
||||
)
|
||||
return [
|
||||
self._actors[p].reserve_serve_buffer.remote(
|
||||
self.consumer_id, nb, digests[p]
|
||||
)
|
||||
for p, nb in enumerate(bytes_by_producer)
|
||||
if nb > 0
|
||||
]
|
||||
|
||||
|
||||
def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int:
|
||||
"""Size a NIXL buffer / ring slot for ``nbytes``: the max of the request, an
|
||||
optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is
|
||||
allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a
|
||||
perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers
|
||||
and the producer's serve rings."""
|
||||
return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20))
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,300 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Op-chain recording for the sharded-RDT backend.
|
||||
|
||||
The consumer asks the trainer for the exact slice a worker consumes, described as
|
||||
an op chain replayed on the trainer's live tensor. ``FakeRDTTensor`` builds the
|
||||
chain by intercepting the model's own weight loaders; ``copy_`` is its data sink,
|
||||
``BakeSink`` — the dry run records how each slice would be fetched and where it
|
||||
lands.
|
||||
|
||||
Chains are built against meta tensors, so nothing here touches Ray or a GPU.
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_common import SUPPORTED_OPS
|
||||
|
||||
# ("op_name", positional_args, sorted_kwargs_items). Entries must be hashable so
|
||||
# the chain can serve as a FetchKey.
|
||||
OpSpec = tuple[str, tuple[Any, ...], tuple[tuple[str, Any], ...]]
|
||||
OpChain = tuple[OpSpec, ...]
|
||||
FetchKey = tuple[str, OpChain]
|
||||
|
||||
|
||||
def _freeze_kwargs(kwargs: dict[str, Any]) -> tuple[tuple[str, Any], ...]:
|
||||
"""Sort kwargs into a tuple of items for hashable storage in OpSpec."""
|
||||
return tuple(sorted(kwargs.items()))
|
||||
|
||||
|
||||
class _UnsupportedFakeOp(NotImplementedError):
|
||||
"""Raised when a weight loader does something to a FakeRDTTensor that cannot
|
||||
be expressed as a slice request.
|
||||
|
||||
Surfaced as NotImplementedError so callers can distinguish "this backend
|
||||
can't handle this loader" from genuine bugs.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Scatter:
|
||||
"""One recorded scatter: pull ``src``, copy it into ``layer``'s
|
||||
``param_name`` at the recorded strided region. The bake's output unit; see
|
||||
the engine module's Data flow.
|
||||
|
||||
Self-contained so replay needs no lookups. Destination geometry is read off
|
||||
the meta view and rebuilt each sync as ``as_strided(shape, stride, offset)``
|
||||
-- ``layer`` is resolved to a param at replay time, never baked, since every
|
||||
sync re-materializes fresh tensors.
|
||||
|
||||
``shape`` and ``dtype`` also size the slice ON THE WIRE, so ``dtype`` is the
|
||||
PRODUCED dtype (the fake's after its op chain), not the source name's:
|
||||
``view(dtype)`` is allowlisted, and taking the source's would size the slice
|
||||
with the wrong itemsize and carve the packed blob differently on the two
|
||||
sides.
|
||||
"""
|
||||
|
||||
layer: Any
|
||||
param_name: str
|
||||
src: FetchKey
|
||||
offset: int
|
||||
shape: tuple[int, ...]
|
||||
stride: tuple[int, ...]
|
||||
dtype: torch.dtype
|
||||
|
||||
|
||||
def _meta_copy_(dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor:
|
||||
"""Fire ``dest.copy_`` from a zero-storage meta source of ``src``'s geometry.
|
||||
|
||||
Moves no data but still counts against the layer's loaded numel, which drives
|
||||
``_layerwise_process``; skipping it would leave the layer looking unloaded
|
||||
forever.
|
||||
|
||||
A non-meta ``dest`` is one the layerwise reload never moved there (the
|
||||
``SKIP_LOAD_TENSORS`` set, e.g. GLM's router bias). Torch forbids
|
||||
``real.copy_(meta)``, and skipping the count is consistent -- ``get_layer_size``
|
||||
excludes the same set. The caller still RECORDS the copy, so the replay writes
|
||||
the real param.
|
||||
"""
|
||||
if not dest.is_meta:
|
||||
return dest
|
||||
meta_src = torch.empty(src.shape, dtype=src.dtype, device="meta")
|
||||
with torch._C.DisableTorchFunctionSubclass():
|
||||
return dest.copy_(meta_src)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BakeSink:
|
||||
"""``copy_`` sink for the dry-run bake: record how each slice would be fetched
|
||||
and where it would land, and move nothing.
|
||||
|
||||
``_install_recording_stamps`` sets ``current = (leaf_module, param_name)``
|
||||
around each loader so ``accept_copy`` can attribute the copy. An unstamped
|
||||
``copy_`` cannot be attributed and stays unrecorded, so its module fails the
|
||||
coverage gate and takes the plain load. ``copies_by_layer`` is keyed by module
|
||||
object, so iterating it yields each leaf module once.
|
||||
"""
|
||||
|
||||
copies_by_layer: "dict[Any, list[_Scatter | None]]" = field(
|
||||
default_factory=lambda: defaultdict(list)
|
||||
)
|
||||
current: "tuple[Any, str] | None" = None
|
||||
# Source names whose ``copy_`` fired during the bake. Names not here never
|
||||
# moved data (e.g. experts owned by another EP rank) and are skipped; names
|
||||
# here but unbaked fail the plan build.
|
||||
copied_names: "set[str]" = field(default_factory=set)
|
||||
|
||||
def accept_copy(self, dest: torch.Tensor, src: "FakeRDTTensor") -> torch.Tensor:
|
||||
self.copied_names.add(src._name)
|
||||
if self.current is not None:
|
||||
layer, param_name = self.current
|
||||
if dest.numel() != src.numel():
|
||||
# The packed layout sizes each slice from the DESTINATION shape,
|
||||
# while the producer packs what the replayed chain produces. A
|
||||
# broadcasting copy_ makes those two lengths differ, and neither
|
||||
# side can notice: the consumer's buffer view is exactly
|
||||
# prod(dest.shape) elements, so it reshapes cleanly over bytes
|
||||
# the producer laid out at different offsets. Refuse at bake.
|
||||
raise _UnsupportedFakeOp(
|
||||
f"FakeRDTTensor: broadcasting copy_ into {param_name!r} of "
|
||||
f"{type(layer).__name__} ({src.numel()} elements from "
|
||||
f"{src._name!r} into {dest.numel()}). The sharded RDT "
|
||||
"backend sends exactly the slice the loader asks for, so "
|
||||
"the source and destination must have equal numel."
|
||||
)
|
||||
self.copies_by_layer[layer].append(
|
||||
_Scatter(
|
||||
layer=layer,
|
||||
param_name=param_name,
|
||||
src=src._key(),
|
||||
offset=dest.storage_offset(),
|
||||
shape=tuple(dest.shape),
|
||||
stride=tuple(dest.stride()),
|
||||
dtype=src.dtype,
|
||||
)
|
||||
)
|
||||
return _meta_copy_(dest, src)
|
||||
|
||||
|
||||
class FakeRDTTensor(torch.Tensor):
|
||||
"""Zero-storage tensor that records how to fetch a weight slice.
|
||||
|
||||
``_make_wrapper_subclass`` gives it shape/dtype/device without storage. Every
|
||||
op in ``SUPPORTED_OPS`` returns a child with the spec appended; ``copy_``
|
||||
delegates to the installed sink. Anything else reaches ``__torch_dispatch__``
|
||||
and raises ``_UnsupportedFakeOp``, so failures are loud rather than silently
|
||||
fetching the wrong bytes.
|
||||
"""
|
||||
|
||||
# Declared at class scope so mypy can infer attribute types on
|
||||
# instances built via ``_make_wrapper_subclass`` (where ``__new__``
|
||||
# returns a tensor it can't annotate as ``self``).
|
||||
_name: str
|
||||
_ops: OpChain
|
||||
# Handles this fake's copy_. ``None`` only on bare construction.
|
||||
_sink: "BakeSink | None"
|
||||
|
||||
@staticmethod
|
||||
def __new__(
|
||||
cls,
|
||||
name: str,
|
||||
shape: torch.Size,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
ops: OpChain = (),
|
||||
sink: "BakeSink | None" = None,
|
||||
) -> "FakeRDTTensor":
|
||||
t = torch.Tensor._make_wrapper_subclass(
|
||||
cls,
|
||||
shape,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
requires_grad=False,
|
||||
)
|
||||
t._name = name
|
||||
t._ops = tuple(ops)
|
||||
t._sink = sink
|
||||
return t
|
||||
|
||||
def _key(self) -> FetchKey:
|
||||
return (self._name, self._ops)
|
||||
|
||||
def _make_child(
|
||||
self,
|
||||
new_shape: torch.Size,
|
||||
new_dtype: torch.dtype,
|
||||
*new_ops: OpSpec,
|
||||
) -> "FakeRDTTensor":
|
||||
"""Append one or more ops to the chain and return a fresh child.
|
||||
|
||||
Variadic so multi-return ops (e.g. chunk) can append both the base
|
||||
op and an indexing op in a single call.
|
||||
"""
|
||||
return FakeRDTTensor(
|
||||
name=self._name,
|
||||
shape=new_shape,
|
||||
dtype=new_dtype,
|
||||
device=self.device,
|
||||
ops=self._ops + new_ops,
|
||||
sink=self._sink,
|
||||
)
|
||||
|
||||
def _meta(self) -> torch.Tensor:
|
||||
"""A zero-storage meta tensor of this fake's shape/dtype, so PyTorch
|
||||
itself computes post-op geometry rather than us reimplementing it."""
|
||||
return torch.empty(self.shape, dtype=self.dtype, device="meta")
|
||||
|
||||
@classmethod
|
||||
def __torch_function__(
|
||||
cls,
|
||||
func,
|
||||
types,
|
||||
args=(),
|
||||
kwargs=None,
|
||||
):
|
||||
kwargs = kwargs or {}
|
||||
|
||||
# Checked before the allowlist: copy_ must not be recorded into the chain.
|
||||
if func is torch.Tensor.copy_:
|
||||
dest = args[0]
|
||||
src = args[1] if len(args) > 1 else kwargs.get("src")
|
||||
if isinstance(src, cls) and src._sink is not None:
|
||||
return src._sink.accept_copy(dest, src)
|
||||
|
||||
# Allowlisted slice/view/shape ops: append to chain and return child.
|
||||
op_name = SUPPORTED_OPS.get(func)
|
||||
if op_name is not None:
|
||||
self_ = args[0]
|
||||
if isinstance(self_, cls):
|
||||
rest = tuple(args[1:])
|
||||
return cls._intercept(self_, func, op_name, rest, kwargs)
|
||||
|
||||
# Metadata reads are answered by the wrapper subclass and stop here; ops
|
||||
# that need data fall through to __torch_dispatch__, which raises.
|
||||
with torch._C.DisableTorchFunctionSubclass():
|
||||
return func(*args, **kwargs)
|
||||
|
||||
@classmethod
|
||||
def _intercept(
|
||||
cls,
|
||||
self_: "FakeRDTTensor",
|
||||
func: Callable,
|
||||
op_name: str,
|
||||
args: tuple,
|
||||
kwargs: dict,
|
||||
):
|
||||
"""Append the op and return a child, or a tuple of children for
|
||||
multi-return ops. Each child's geometry comes from running the op on a
|
||||
meta tensor."""
|
||||
meta = self_._meta()
|
||||
with torch._C.DisableTorchFunctionSubclass():
|
||||
meta_result = func(meta, *args, **kwargs)
|
||||
|
||||
base_op: OpSpec = (op_name, tuple(args), _freeze_kwargs(kwargs))
|
||||
|
||||
# Single-tensor result: one child carrying base_op alone.
|
||||
if isinstance(meta_result, torch.Tensor):
|
||||
return self_._make_child(meta_result.shape, meta_result.dtype, base_op)
|
||||
|
||||
# Multi-return: one child per output, each carrying base_op plus a
|
||||
# trailing __getitem__(i) so the replay can index the result.
|
||||
if isinstance(meta_result, tuple | list):
|
||||
return tuple(
|
||||
self_._make_child(
|
||||
m.shape,
|
||||
m.dtype,
|
||||
base_op,
|
||||
("__getitem__", (i,), ()),
|
||||
)
|
||||
for i, m in enumerate(meta_result)
|
||||
)
|
||||
|
||||
# Non-tensor result (e.g. .item() snuck into the allowlist).
|
||||
raise _UnsupportedFakeOp(
|
||||
f"FakeRDTTensor: {op_name!r} returned a non-tensor "
|
||||
f"({type(meta_result).__name__}); cannot defer."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def __torch_dispatch__(cls, func, types, args=(), kwargs=None):
|
||||
kwargs = kwargs or {}
|
||||
|
||||
# A fake at the aten level means an unsupported loader op. Name the op and
|
||||
# the chain; materializing instead would mask a correctness bug.
|
||||
for arg in (*args, *kwargs.values()):
|
||||
if isinstance(arg, cls):
|
||||
raise _UnsupportedFakeOp(
|
||||
f"FakeRDTTensor: unsupported op {func} reached "
|
||||
f"__torch_dispatch__ on fake {arg._name!r} "
|
||||
f"(chain={arg._ops}). Supported ops are: "
|
||||
f"{sorted(SUPPORTED_OPS.values())}, plus copy_. "
|
||||
"Loaders that need .to(), .float(), .item(), arithmetic, "
|
||||
"bool-mask indexing, or .data access are not supported by "
|
||||
"the sharded RDT backend."
|
||||
)
|
||||
return func(*args, **kwargs)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -361,6 +361,19 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
runtime_env = self._build_runtime_env()
|
||||
resource_kwargs = self._get_actor_resource_kwargs()
|
||||
|
||||
# The sharded_rdt weight transfer backend pulls weights via Ray's tensor
|
||||
# transport (@ray.method(tensor_transport=...)). Ray requires the
|
||||
# *calling* actor (the vLLM worker) to opt in via enable_tensor_transport.
|
||||
wt_cfg = self.vllm_config.weight_transfer_config
|
||||
extra_actor_options: dict[str, object] = {}
|
||||
if wt_cfg is not None and wt_cfg.backend == "sharded_rdt":
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_common import (
|
||||
check_ray_rdt_version,
|
||||
)
|
||||
|
||||
check_ray_rdt_version()
|
||||
extra_actor_options["enable_tensor_transport"] = True
|
||||
|
||||
for bundle_idx in range(self.world_size):
|
||||
bundle = bundle_assignments[bundle_idx]
|
||||
is_driver_worker = self._is_driver_worker(bundle["rank"])
|
||||
@@ -383,6 +396,7 @@ class RayExecutorV2(MultiprocExecutor):
|
||||
**resource_kwargs,
|
||||
scheduling_strategy=scheduling_strategy,
|
||||
runtime_env=runtime_env,
|
||||
**extra_actor_options,
|
||||
)
|
||||
.remote(
|
||||
vllm_config=self.vllm_config,
|
||||
|
||||
Reference in New Issue
Block a user