[rl] Stateful Trainer Send: New Abstractions [1/N] (#48042)
Signed-off-by: haoaaron <ahao@anyscale.com> Signed-off-by: Aaron Hao <ahao@anyscale.com> Co-authored-by: Sumanth R Hegde <39546518+SumanthRH@users.noreply.github.com>
This commit is contained in:
co-authored by
Sumanth R Hegde
parent
9354f22204
commit
fb1d8ccaf5
@@ -5,9 +5,10 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Generic, TypeVar
|
||||
from typing import TYPE_CHECKING, Any, Generic, Protocol, TypeVar, runtime_checkable
|
||||
|
||||
import torch
|
||||
from typing_extensions import Self
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
@@ -17,6 +18,85 @@ from vllm.config.weight_transfer import WeightTransferConfig
|
||||
|
||||
TInitInfo = TypeVar("TInitInfo", bound="WeightTransferInitInfo")
|
||||
TUpdateInfo = TypeVar("TUpdateInfo", bound="WeightTransferUpdateInfo")
|
||||
TConfig = TypeVar("TConfig", bound="WeightTransferConfig")
|
||||
|
||||
# A trainer supplies its parameters as a `WeightSource` (defined below): a
|
||||
# re-iterable stream of materialized `(name, tensor)` pairs plus a `metadata()`
|
||||
# channel. The built-in `ModuleSource` uses `materialize_full_tensor`.
|
||||
|
||||
|
||||
def materialize_full_tensor(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Return a full, locally-materialized tensor ready to send.
|
||||
|
||||
FSDP shards (DTensors) expose `full_tensor()`, a collective all-gather;
|
||||
regular tensors do not and are returned unchanged. Trainer engines call
|
||||
this at send time so the (potentially expensive) gather happens exactly
|
||||
once — reading `.shape`/`.dtype` for metadata does not trigger it.
|
||||
"""
|
||||
full_tensor = getattr(tensor, "full_tensor", None)
|
||||
return full_tensor() if callable(full_tensor) else tensor
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParamMeta:
|
||||
"""Name / wire dtype / full (HF) shape for one output parameter."""
|
||||
|
||||
name: str
|
||||
dtype: torch.dtype
|
||||
shape: tuple[int, ...]
|
||||
|
||||
|
||||
class WeightSource(ABC):
|
||||
"""A re-iterable source of the trainer's weights, handed to a trainer engine.
|
||||
|
||||
Two channels:
|
||||
|
||||
* `metadata()` — `(name, wire dtype, full shape)` for every parameter,
|
||||
*without* transferring. Cheap when shapes are known locally (FSDP
|
||||
`DTensor` global shape); may be expensive on first call for backends that
|
||||
must materialize to learn shapes (e.g. a Megatron-Bridge export), in which
|
||||
case it should cache.
|
||||
* iteration — yields fully-materialized `(name, tensor)` pairs, one at a
|
||||
time. Materializing is typically a collective (FSDP `full_tensor()`, a
|
||||
Megatron export), so every trainer rank must iterate the same source in the
|
||||
same order in lockstep, or ranks deadlock. Under pipeline parallelism a
|
||||
rank may not own a parameter at all — iterating still drives the collective
|
||||
and the yielded tensor is only meaningful on the sender.
|
||||
|
||||
`iter(source)` must yield a *fresh* pass each round. Backends with custom
|
||||
producer logic (Megatron export, RDT plans, MoE re-fusing) subclass this.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def metadata(self) -> list[ParamMeta]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class ModuleSource(WeightSource):
|
||||
"""`WeightSource` over `module.named_parameters()` — the common case.
|
||||
|
||||
Handles both plain dense modules and FSDP-sharded ones with no special
|
||||
casing: iteration all-gathers each `DTensor` via `full_tensor()` (a
|
||||
collective) and passes regular tensors through. `metadata()` reads the
|
||||
*global* `.shape` / `.dtype`, so it never triggers a gather.
|
||||
"""
|
||||
|
||||
def __init__(self, module: torch.nn.Module) -> None:
|
||||
self._module = module
|
||||
|
||||
def metadata(self) -> list[ParamMeta]:
|
||||
return [
|
||||
ParamMeta(name, p.dtype, tuple(p.shape))
|
||||
for name, p in self._module.named_parameters()
|
||||
]
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
for name, param in self._module.named_parameters():
|
||||
yield name, materialize_full_tensor(param)
|
||||
|
||||
|
||||
# Base protocols for backend-specific dataclasses
|
||||
@@ -27,6 +107,26 @@ class WeightTransferInitInfo(ABC): # noqa: B024
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class TrainerInitInfo(WeightTransferInitInfo):
|
||||
"""Base trainer-side init info: which trainer rank drives the transfer.
|
||||
|
||||
`rank` is this trainer process's rank, provided **explicitly** by the
|
||||
caller — the engine does not read it from a global process group, which is
|
||||
ambiguous once several groups (FSDP / TP / PP / EP) exist. Rank 0 is always
|
||||
the sender: only it opens the endpoint and drives the inference-side RPCs,
|
||||
while every rank still runs the trainer-side collectives. Backend subclasses
|
||||
add their own (positional) fields; `rank` is keyword-only so that ordering
|
||||
never conflicts.
|
||||
"""
|
||||
|
||||
rank: int = field(kw_only=True)
|
||||
|
||||
@property
|
||||
def is_sender(self) -> bool:
|
||||
return self.rank == 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class WeightTransferUpdateInfo(ABC): # noqa: B024
|
||||
"""Base class for backend-specific weight update info."""
|
||||
@@ -243,3 +343,104 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
>>> engine.trainer_send_weights(param_iter, trainer_args)
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class VLLMWeightSyncClient(Protocol):
|
||||
"""Trainer-side stub for the inference engine's weight-sync control plane.
|
||||
|
||||
Mirrors the weight-sync methods that the inference engine exposes
|
||||
(`EngineClient` / the HTTP RLHF routes / Ray actors). A
|
||||
`TrainerWeightTransferEngine` drives the full handshake through this
|
||||
protocol so trainer code never has to know the transport.
|
||||
|
||||
All methods are synchronous and accept plain dicts (matching what the
|
||||
inference side already accepts). Concurrency that some backends need
|
||||
(e.g. NCCL must run `update_weights` concurrently with the trainer-side
|
||||
broadcast) is the engine's responsibility, not the client's, so the
|
||||
protocol stays a flat four-method surface that any wrapper can implement.
|
||||
|
||||
The protocol is structural (PEP 544), so user implementations need only
|
||||
define these four methods — no import or subclassing required.
|
||||
"""
|
||||
|
||||
def init_weight_transfer_engine(self, init_info: dict[str, Any]) -> None: ...
|
||||
|
||||
def start_weight_update(self) -> None: ...
|
||||
|
||||
def update_weights(self, update_info: dict[str, Any]) -> None: ...
|
||||
|
||||
def finish_weight_update(self) -> None: ...
|
||||
|
||||
|
||||
class TrainerWeightTransferEngine(ABC, Generic[TConfig, TInitInfo]):
|
||||
"""Trainer-side weight transfer engine.
|
||||
|
||||
Symmetric to `WeightTransferEngine` but lives in the training process.
|
||||
Constructed via the `trainer_init` factory classmethod; carries any
|
||||
backend-specific state (NCCL communicators, IPC device info, transfer
|
||||
plans) on `self`. The `WeightSource` is required at `trainer_init`,
|
||||
then replayed each round by the no-argument `send_weights()`.
|
||||
|
||||
Multi-rank trainers: `trainer_init` and `send_weights` are
|
||||
called on *every* trainer rank. Rank 0 is the sender, resolved once at
|
||||
`trainer_init` into `is_sender`. Non-sender ranks still run every
|
||||
collective (iterating the source, metadata export, IPC handle all-gather) so
|
||||
the group stays aligned, but each engine explicitly guards the control-plane
|
||||
RPCs and the transmit on `self.is_sender`, so only the sender touches the
|
||||
client.
|
||||
|
||||
Subclasses should define:
|
||||
init_info_cls: Type of backend-specific trainer init info
|
||||
config_cls: Type of backend-specific config
|
||||
"""
|
||||
|
||||
# Subclasses should override these class attributes
|
||||
init_info_cls: type[TInitInfo]
|
||||
config_cls: type[TConfig]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: TConfig,
|
||||
*,
|
||||
client: "VLLMWeightSyncClient",
|
||||
source: "WeightSource",
|
||||
is_sender: bool = True,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.is_sender = is_sender
|
||||
# The real client is held on every rank; each engine only *calls* it when
|
||||
# `is_sender`, so non-sender ranks never touch the wire.
|
||||
self.client = client
|
||||
self.source = source
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
config: TConfig,
|
||||
init_info: TInitInfo,
|
||||
*,
|
||||
client: "VLLMWeightSyncClient",
|
||||
source: "WeightSource",
|
||||
) -> Self:
|
||||
"""Rendezvous with the inference side and return a ready instance.
|
||||
|
||||
Called on every trainer rank. The sender drives the full handshake via
|
||||
`client` (build the worker-side init info, call
|
||||
`client.init_weight_transfer_engine`, open the trainer-side endpoint);
|
||||
non-sender ranks skip the rendezvous and the RPC. `source` is stored on
|
||||
`self.source`; after return, `send_weights()` is callable.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def send_weights(self) -> None:
|
||||
"""Push `self.source`'s weights to inference workers and drive the full
|
||||
update round trip: `start_weight_update`, `update_weights` (run
|
||||
concurrently with the trainer-side broadcast when the backend requires
|
||||
it), then `finish_weight_update`. Called on every trainer rank."""
|
||||
raise NotImplementedError
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Tear down communicators / process groups. Default no-op."""
|
||||
|
||||
Reference in New Issue
Block a user