[RL] Support per-parameter source layouts for NCCL M2N (#60934)
Signed-off-by: pkousha <pkousha@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
OpenAI Codex
aoshen02
parent
f8112704d5
commit
6b6a6c5c98
@@ -5,21 +5,22 @@
|
||||
m2n plans a transfer from both sides' layouts, so the trainer has to say how it
|
||||
holds each parameter — which the base `WeightSource` / `ParamMeta` pair does not
|
||||
express. This module supplies the m2n flavor of both, and the DTensor-backed
|
||||
implementation that covers the common trainer.
|
||||
|
||||
A source declares its mesh once (`mesh()`) and a placement per parameter, since
|
||||
one topology describes every tensor on the side.
|
||||
implementation that covers the common trainer. Each parameter owns its source
|
||||
mesh and placements because dense and expert weights can use different
|
||||
factorizations over the same trainer ranks.
|
||||
"""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Callable, Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.base import ParamMeta, WeightSource
|
||||
from vllm.distributed.weight_transfer.base import WeightSource
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
REPLICATE,
|
||||
REPLICATED,
|
||||
M2NLayout,
|
||||
M2NMesh,
|
||||
M2NParamMeta,
|
||||
Placements,
|
||||
@@ -27,6 +28,8 @@ from vllm.distributed.weight_transfer.m2n_common import (
|
||||
|
||||
__all__ = [
|
||||
"M2NWeightSource",
|
||||
"M2NManifestEntry",
|
||||
"ManifestM2NWeightSource",
|
||||
"DTensorModuleSource",
|
||||
"mesh_from_tensor",
|
||||
"placements_from_tensor",
|
||||
@@ -40,17 +43,76 @@ class M2NWeightSource(WeightSource):
|
||||
materialized full tensor: gathering is exactly the cost m2n removes.
|
||||
"""
|
||||
|
||||
def mesh(self) -> M2NMesh:
|
||||
"""The trainer's rank topology, shared by every parameter."""
|
||||
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
|
||||
"""Name, dtype, full shape, and source layout for each parameter."""
|
||||
raise NotImplementedError
|
||||
|
||||
def metadata(self) -> list[ParamMeta]:
|
||||
"""Name, dtype, full shape, and trainer placement for each parameter."""
|
||||
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
|
||||
"""Yield `(name, local buffer)` pairs in metadata order.
|
||||
|
||||
Providers must enqueue materialization on the caller's current CUDA
|
||||
stream, or return a buffer whose writes are already visible to it.
|
||||
Buffers may be torch tensors, CUDA Array Interface objects, or raw
|
||||
device pointers accepted by nccl4py. Providers own correct
|
||||
communicator-device placement. Raw-pointer providers additionally own
|
||||
allocation lifetime, dtype, and shape correctness.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""Yield `(name, local shard)` pairs in the same order as `metadata()`."""
|
||||
raise NotImplementedError
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class M2NManifestEntry:
|
||||
"""One immutable source descriptor with a lazy rank-local tensor view."""
|
||||
|
||||
key: str
|
||||
dtype: torch.dtype
|
||||
global_shape: tuple[int, ...]
|
||||
source_layout: M2NLayout
|
||||
local_tensor: Callable[[], Any]
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
object.__setattr__(self, "global_shape", tuple(self.global_shape))
|
||||
if not self.key:
|
||||
raise ValueError("M2N manifest key must not be empty")
|
||||
if not isinstance(self.dtype, torch.dtype):
|
||||
raise TypeError(
|
||||
f"manifest entry '{self.key}' has invalid dtype {self.dtype!r}"
|
||||
)
|
||||
if not isinstance(self.source_layout, M2NLayout):
|
||||
raise TypeError(
|
||||
f"manifest entry '{self.key}' source_layout must be an M2NLayout"
|
||||
)
|
||||
if not callable(self.local_tensor):
|
||||
raise TypeError(
|
||||
f"manifest entry '{self.key}' local_tensor must be callable"
|
||||
)
|
||||
|
||||
@property
|
||||
def metadata(self) -> M2NParamMeta:
|
||||
"""Return the serializable portion consumed during initialization."""
|
||||
return M2NParamMeta(
|
||||
self.key,
|
||||
self.dtype,
|
||||
self.global_shape,
|
||||
self.source_layout,
|
||||
)
|
||||
|
||||
|
||||
class ManifestM2NWeightSource(M2NWeightSource):
|
||||
"""An ordered M2N source backed by lazy, rank-local manifest entries."""
|
||||
|
||||
def __init__(self, entries: Sequence[M2NManifestEntry]) -> None:
|
||||
self._entries = tuple(entries)
|
||||
self._metadata = tuple(entry.metadata for entry in self._entries)
|
||||
|
||||
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
|
||||
"""Return a fresh list while preserving the immutable manifest order."""
|
||||
return list(self._metadata)
|
||||
|
||||
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
|
||||
"""Materialize one rank-local buffer at a time."""
|
||||
for entry in self._entries:
|
||||
yield entry.key, entry.local_tensor()
|
||||
|
||||
|
||||
def _placement_code(placement: Any) -> int:
|
||||
@@ -69,7 +131,8 @@ def _placement_code(placement: Any) -> int:
|
||||
def mesh_from_tensor(tensor: torch.Tensor, num_trainer_ranks: int) -> M2NMesh:
|
||||
"""The mesh a parameter lives on, as an `M2NMesh`.
|
||||
|
||||
A plain tensor is identical on every trainer rank, so it spans all of them.
|
||||
Tensors without DeviceMesh metadata are treated as replicated. Implicitly
|
||||
sharded tensors require a custom source with an explicit layout.
|
||||
"""
|
||||
device_mesh = getattr(tensor, "device_mesh", None)
|
||||
if device_mesh is None:
|
||||
@@ -122,29 +185,18 @@ class DTensorModuleSource(M2NWeightSource):
|
||||
self._module = module
|
||||
self._num_trainer_ranks = num_trainer_ranks
|
||||
|
||||
def mesh(self) -> M2NMesh:
|
||||
"""The mesh every sharded parameter agrees on.
|
||||
|
||||
Replicated parameters span the trainer without a device mesh of their
|
||||
own, so they never decide this; a model whose *sharded* parameters
|
||||
disagree cannot be described by one side mesh and is rejected.
|
||||
"""
|
||||
meshes = {
|
||||
mesh_from_tensor(p, self._num_trainer_ranks)
|
||||
for _, p in self._module.named_parameters()
|
||||
if placements_from_tensor(p) is not REPLICATED
|
||||
}
|
||||
if len(meshes) > 1:
|
||||
raise ValueError(
|
||||
"nccl_m2n needs one mesh per side, but this module's sharded "
|
||||
f"parameters span several: {sorted(m.dims for m in meshes)}"
|
||||
)
|
||||
return meshes.pop() if meshes else M2NMesh((self._num_trainer_ranks, 1), 0)
|
||||
|
||||
def metadata(self) -> list[ParamMeta]:
|
||||
"""Read global shape/dtype and placements without gathering shards."""
|
||||
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
|
||||
"""Read each global shape and source layout without gathering shards."""
|
||||
return [
|
||||
M2NParamMeta(name, p.dtype, tuple(p.shape), placements_from_tensor(p))
|
||||
M2NParamMeta(
|
||||
name,
|
||||
p.dtype,
|
||||
tuple(p.shape),
|
||||
M2NLayout(
|
||||
mesh_from_tensor(p, self._num_trainer_ranks),
|
||||
placements_from_tensor(p),
|
||||
),
|
||||
)
|
||||
for name, p in self._module.named_parameters()
|
||||
]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user