[rl] Stateful Trainer Send: IPC [2/N] (#48981)
Signed-off-by: haoaaron <ahao@anyscale.com>
This commit is contained in:
@@ -5,7 +5,15 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Generic, Protocol, TypeVar, runtime_checkable
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
ClassVar,
|
||||
Generic,
|
||||
Protocol,
|
||||
TypeVar,
|
||||
runtime_checkable,
|
||||
)
|
||||
|
||||
import torch
|
||||
from typing_extensions import Self
|
||||
@@ -18,7 +26,7 @@ from vllm.config.weight_transfer import WeightTransferConfig
|
||||
|
||||
TInitInfo = TypeVar("TInitInfo", bound="WeightTransferInitInfo")
|
||||
TUpdateInfo = TypeVar("TUpdateInfo", bound="WeightTransferUpdateInfo")
|
||||
TConfig = TypeVar("TConfig", bound="WeightTransferConfig")
|
||||
TTrainerInitInfo = TypeVar("TTrainerInitInfo", bound="TrainerInitInfo")
|
||||
|
||||
# A trainer supplies its parameters as a `WeightSource` (defined below): a
|
||||
# re-iterable stream of materialized `(name, tensor)` pairs plus a `metadata()`
|
||||
@@ -108,7 +116,7 @@ class WeightTransferInitInfo(ABC): # noqa: B024
|
||||
|
||||
|
||||
@dataclass
|
||||
class TrainerInitInfo(WeightTransferInitInfo):
|
||||
class TrainerInitInfo:
|
||||
"""Base trainer-side init info: which trainer rank drives the transfer.
|
||||
|
||||
`rank` is this trainer process's rank, provided **explicitly** by the
|
||||
@@ -118,10 +126,25 @@ class TrainerInitInfo(WeightTransferInitInfo):
|
||||
while every rank still runs the trainer-side collectives. Backend subclasses
|
||||
add their own (positional) fields; `rank` is keyword-only so that ordering
|
||||
never conflicts.
|
||||
|
||||
Every concrete subclass sets a class-level `backend` string (the same key it
|
||||
registers under in `WeightTransferTrainerFactory`). The factory reads it to
|
||||
dispatch, so callers pass only the init info/ It is a `ClassVar`
|
||||
(a fixed per-backend constant), so it is not an ``__init__`` field.
|
||||
"""
|
||||
|
||||
backend: ClassVar[str]
|
||||
|
||||
rank: int = field(kw_only=True)
|
||||
|
||||
def __init_subclass__(cls, **kwargs: Any) -> None:
|
||||
super().__init_subclass__(**kwargs)
|
||||
if not getattr(cls, "backend", None):
|
||||
raise TypeError(
|
||||
f"{cls.__name__} must set a class-level `backend` string "
|
||||
"(the WeightTransferTrainerFactory registry key)."
|
||||
)
|
||||
|
||||
@property
|
||||
def is_sender(self) -> bool:
|
||||
return self.rank == 0
|
||||
@@ -373,7 +396,7 @@ class VLLMWeightSyncClient(Protocol):
|
||||
def finish_weight_update(self, weight_version: str | None = None) -> None: ...
|
||||
|
||||
|
||||
class TrainerWeightTransferEngine(ABC, Generic[TConfig, TInitInfo]):
|
||||
class TrainerWeightTransferEngine(ABC, Generic[TTrainerInitInfo]):
|
||||
"""Trainer-side weight transfer engine.
|
||||
|
||||
Symmetric to `WeightTransferEngine` but lives in the training process.
|
||||
@@ -382,6 +405,13 @@ class TrainerWeightTransferEngine(ABC, Generic[TConfig, TInitInfo]):
|
||||
plans) on `self`. The `WeightSource` is required at `trainer_init`,
|
||||
then replayed each round by the no-argument `send_weights()`.
|
||||
|
||||
Unlike the worker engine, the trainer side does not take a
|
||||
`WeightTransferConfig`: the backend is selected from the init info's
|
||||
`backend` `ClassVar` (so callers pass only the init info), and the static
|
||||
wire params (packed, buffer sizes) ride the backend-specific
|
||||
`TrainerInitInfo`, which the sender also propagates to the worker at the init
|
||||
handshake.
|
||||
|
||||
Multi-rank trainers: `trainer_init` and `send_weights` are
|
||||
called on *every* trainer rank. Rank 0 is the sender, resolved once at
|
||||
`trainer_init` into `is_sender`. Non-sender ranks still run every
|
||||
@@ -392,22 +422,18 @@ class TrainerWeightTransferEngine(ABC, Generic[TConfig, TInitInfo]):
|
||||
|
||||
Subclasses should define:
|
||||
init_info_cls: Type of backend-specific trainer init info
|
||||
config_cls: Type of backend-specific config
|
||||
"""
|
||||
|
||||
# Subclasses should override these class attributes
|
||||
init_info_cls: type[TInitInfo]
|
||||
config_cls: type[TConfig]
|
||||
# Subclasses should override this class attribute
|
||||
init_info_cls: type[TTrainerInitInfo]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: TConfig,
|
||||
*,
|
||||
client: "VLLMWeightSyncClient",
|
||||
source: "WeightSource",
|
||||
is_sender: bool = True,
|
||||
) -> None:
|
||||
self.config = config
|
||||
self.is_sender = is_sender
|
||||
# The real client is held on every rank; each engine only *calls* it when
|
||||
# `is_sender`, so non-sender ranks never touch the wire.
|
||||
@@ -418,8 +444,7 @@ class TrainerWeightTransferEngine(ABC, Generic[TConfig, TInitInfo]):
|
||||
@abstractmethod
|
||||
def trainer_init(
|
||||
cls,
|
||||
config: TConfig,
|
||||
init_info: TInitInfo,
|
||||
init_info: TTrainerInitInfo,
|
||||
*,
|
||||
client: "VLLMWeightSyncClient",
|
||||
source: "WeightSource",
|
||||
|
||||
Reference in New Issue
Block a user