[RL] Support per-parameter source layouts for NCCL M2N (#60934)

Signed-off-by: pkousha <pkousha@nvidia.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
pkousha
2026-10-16 21:41:17 -04:00
committed by GitHub
co-authored by OpenAI Codex aoshen02
parent f8112704d5
commit 6b6a6c5c98
8 changed files with 2590 additions and 695 deletions
+88 -36
View File
@@ -5,21 +5,22 @@
m2n plans a transfer from both sides' layouts, so the trainer has to say how it
holds each parameter — which the base `WeightSource` / `ParamMeta` pair does not
express. This module supplies the m2n flavor of both, and the DTensor-backed
implementation that covers the common trainer.
A source declares its mesh once (`mesh()`) and a placement per parameter, since
one topology describes every tensor on the side.
implementation that covers the common trainer. Each parameter owns its source
mesh and placements because dense and expert weights can use different
factorizations over the same trainer ranks.
"""
from collections.abc import Iterator
from collections.abc import Callable, Iterator, Sequence
from dataclasses import dataclass
from typing import Any
import torch
from vllm.distributed.weight_transfer.base import ParamMeta, WeightSource
from vllm.distributed.weight_transfer.base import WeightSource
from vllm.distributed.weight_transfer.m2n_common import (
REPLICATE,
REPLICATED,
M2NLayout,
M2NMesh,
M2NParamMeta,
Placements,
@@ -27,6 +28,8 @@ from vllm.distributed.weight_transfer.m2n_common import (
__all__ = [
"M2NWeightSource",
"M2NManifestEntry",
"ManifestM2NWeightSource",
"DTensorModuleSource",
"mesh_from_tensor",
"placements_from_tensor",
@@ -40,17 +43,76 @@ class M2NWeightSource(WeightSource):
materialized full tensor: gathering is exactly the cost m2n removes.
"""
def mesh(self) -> M2NMesh:
"""The trainer's rank topology, shared by every parameter."""
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
"""Name, dtype, full shape, and source layout for each parameter."""
raise NotImplementedError
def metadata(self) -> list[ParamMeta]:
"""Name, dtype, full shape, and trainer placement for each parameter."""
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
"""Yield `(name, local buffer)` pairs in metadata order.
Providers must enqueue materialization on the caller's current CUDA
stream, or return a buffer whose writes are already visible to it.
Buffers may be torch tensors, CUDA Array Interface objects, or raw
device pointers accepted by nccl4py. Providers own correct
communicator-device placement. Raw-pointer providers additionally own
allocation lifetime, dtype, and shape correctness.
"""
raise NotImplementedError
def __iter__(self) -> Iterator[tuple[str, torch.Tensor]]:
"""Yield `(name, local shard)` pairs in the same order as `metadata()`."""
raise NotImplementedError
@dataclass(frozen=True)
class M2NManifestEntry:
"""One immutable source descriptor with a lazy rank-local tensor view."""
key: str
dtype: torch.dtype
global_shape: tuple[int, ...]
source_layout: M2NLayout
local_tensor: Callable[[], Any]
def __post_init__(self) -> None:
object.__setattr__(self, "global_shape", tuple(self.global_shape))
if not self.key:
raise ValueError("M2N manifest key must not be empty")
if not isinstance(self.dtype, torch.dtype):
raise TypeError(
f"manifest entry '{self.key}' has invalid dtype {self.dtype!r}"
)
if not isinstance(self.source_layout, M2NLayout):
raise TypeError(
f"manifest entry '{self.key}' source_layout must be an M2NLayout"
)
if not callable(self.local_tensor):
raise TypeError(
f"manifest entry '{self.key}' local_tensor must be callable"
)
@property
def metadata(self) -> M2NParamMeta:
"""Return the serializable portion consumed during initialization."""
return M2NParamMeta(
self.key,
self.dtype,
self.global_shape,
self.source_layout,
)
class ManifestM2NWeightSource(M2NWeightSource):
"""An ordered M2N source backed by lazy, rank-local manifest entries."""
def __init__(self, entries: Sequence[M2NManifestEntry]) -> None:
self._entries = tuple(entries)
self._metadata = tuple(entry.metadata for entry in self._entries)
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
"""Return a fresh list while preserving the immutable manifest order."""
return list(self._metadata)
def __iter__(self) -> Iterator[tuple[str, Any]]: # type: ignore[override]
"""Materialize one rank-local buffer at a time."""
for entry in self._entries:
yield entry.key, entry.local_tensor()
def _placement_code(placement: Any) -> int:
@@ -69,7 +131,8 @@ def _placement_code(placement: Any) -> int:
def mesh_from_tensor(tensor: torch.Tensor, num_trainer_ranks: int) -> M2NMesh:
"""The mesh a parameter lives on, as an `M2NMesh`.
A plain tensor is identical on every trainer rank, so it spans all of them.
Tensors without DeviceMesh metadata are treated as replicated. Implicitly
sharded tensors require a custom source with an explicit layout.
"""
device_mesh = getattr(tensor, "device_mesh", None)
if device_mesh is None:
@@ -122,29 +185,18 @@ class DTensorModuleSource(M2NWeightSource):
self._module = module
self._num_trainer_ranks = num_trainer_ranks
def mesh(self) -> M2NMesh:
"""The mesh every sharded parameter agrees on.
Replicated parameters span the trainer without a device mesh of their
own, so they never decide this; a model whose *sharded* parameters
disagree cannot be described by one side mesh and is rejected.
"""
meshes = {
mesh_from_tensor(p, self._num_trainer_ranks)
for _, p in self._module.named_parameters()
if placements_from_tensor(p) is not REPLICATED
}
if len(meshes) > 1:
raise ValueError(
"nccl_m2n needs one mesh per side, but this module's sharded "
f"parameters span several: {sorted(m.dims for m in meshes)}"
)
return meshes.pop() if meshes else M2NMesh((self._num_trainer_ranks, 1), 0)
def metadata(self) -> list[ParamMeta]:
"""Read global shape/dtype and placements without gathering shards."""
def metadata(self) -> list[M2NParamMeta]: # type: ignore[override]
"""Read each global shape and source layout without gathering shards."""
return [
M2NParamMeta(name, p.dtype, tuple(p.shape), placements_from_tensor(p))
M2NParamMeta(
name,
p.dtype,
tuple(p.shape),
M2NLayout(
mesh_from_tensor(p, self._num_trainer_ranks),
placements_from_tensor(p),
),
)
for name, p in self._module.named_parameters()
]