[RL] Add sharding-aware NCCL M2N weight transfer (#51520)
Signed-off-by: Ke Wen <kwen@nvidia.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
aoshen02
parent
4bb804cc5b
commit
b6d2b4fc25
@@ -0,0 +1,261 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Destination-layout resolution for the NCCL M2N weight transfer backend.
|
||||
|
||||
For every incoming checkpoint parameter the worker needs a destination buffer
|
||||
plus its placement over the inference mesh. Two outcomes are possible:
|
||||
|
||||
* **direct** — when the model is not quantized, the parameter maps 1:1 onto a
|
||||
live vLLM parameter and its loader is a small, known-safe copy or tensor-
|
||||
parallel loader. The loader's declared input/output dimension determines the
|
||||
placement; shapes are validation, not an inference mechanism. The reshard
|
||||
writes straight into the live parameter, so each rank receives only its own
|
||||
shard and nothing is copied afterwards.
|
||||
* **fallback** — anything else. The reshard delivers the whole tensor to every
|
||||
rank and `load_weights` does the sharding, exactly as the broadcast NCCL
|
||||
backend does. Fused parameters (`qkv_proj`, `gate_up_proj`, MoE `w13`/`w2`)
|
||||
take this path: the checkpoint name does not name a vLLM parameter, so there
|
||||
is nothing to resolve against.
|
||||
|
||||
Correctness never depends on a parameter resolving — the fallback is always
|
||||
available and is the same path the existing backend uses.
|
||||
"""
|
||||
|
||||
import inspect
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import cast
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.m2n_common import (
|
||||
DESTINATION_REPLICA_AXIS,
|
||||
DESTINATION_SHARD_AXIS,
|
||||
MESH_NDIMS,
|
||||
REPLICATE,
|
||||
REPLICATED,
|
||||
Placements,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_DIRECT_WEIGHT_LOADING_BLOCKLIST = {
|
||||
"vllm.model_executor.models.gpt2.GPT2Model",
|
||||
}
|
||||
|
||||
|
||||
def _module_type_name(module: torch.nn.Module) -> str:
|
||||
cls = type(module)
|
||||
return f"{cls.__module__}.{cls.__name__}"
|
||||
|
||||
|
||||
def _destination_placements(shard_dim: int) -> Placements:
|
||||
"""Place one tensor shard on the configured destination mesh axis."""
|
||||
placements = [REPLICATE] * MESH_NDIMS
|
||||
placements[DESTINATION_REPLICA_AXIS] = REPLICATE
|
||||
placements[DESTINATION_SHARD_AXIS] = shard_dim
|
||||
return cast(Placements, tuple(placements))
|
||||
|
||||
|
||||
@dataclass
|
||||
class M2NDestination:
|
||||
"""Where one checkpoint parameter lands on this worker."""
|
||||
|
||||
name: str
|
||||
placements: Placements | None
|
||||
"""Placement over the inference mesh, or `REPLICATED` for the fallback."""
|
||||
tensor: torch.Tensor | None
|
||||
"""The live parameter view to reshard into, or None for the fallback path
|
||||
(the engine allocates a full replica per round and calls `load_weights`)."""
|
||||
|
||||
@property
|
||||
def direct(self) -> bool:
|
||||
return self.tensor is not None
|
||||
|
||||
|
||||
def _base_loader_shard_dim(
|
||||
param: torch.nn.Parameter,
|
||||
shard_axis_size: int,
|
||||
) -> int | None:
|
||||
"""Return a known-safe loader's declared shard dim, or reject it.
|
||||
|
||||
Loader identity is deliberately fail-closed. A missing loader is not
|
||||
treated as ``default_weight_loader`` because a model-level ``load_weights``
|
||||
implementation may transform the tensor before it reaches that default.
|
||||
Wrappers and partials are also rejected: recognizing their wrapped callable
|
||||
would ignore behavior added by the wrapper.
|
||||
"""
|
||||
loader = getattr(param, "weight_loader", None)
|
||||
if loader is None or isinstance(loader, partial):
|
||||
return None
|
||||
|
||||
owner = loader.__self__ if inspect.ismethod(loader) else None
|
||||
loader_fn = loader.__func__ if inspect.ismethod(loader) else loader
|
||||
|
||||
# Lazy imports avoid making the weight-transfer registry import all model
|
||||
# executor layers during normal vLLM startup.
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
replicated_loaders = {
|
||||
default_weight_loader,
|
||||
ReplicatedLinear.weight_loader,
|
||||
}
|
||||
column_loaders = {
|
||||
ColumnParallelLinear.weight_loader,
|
||||
ColumnParallelLinear.weight_loader_v2,
|
||||
}
|
||||
row_loaders = {
|
||||
RowParallelLinear.weight_loader,
|
||||
RowParallelLinear.weight_loader_v2,
|
||||
}
|
||||
|
||||
# These attributes signal packing, a transform, or pre-sharded checkpoint
|
||||
# storage. Such semantics cannot be reproduced by a plain M2N placement.
|
||||
# Check them before accepting replicated loaders too: model-level loading
|
||||
# may transform a tensor before handing it to an otherwise plain copier.
|
||||
metadata_attrs = ("packed_dim", "packed_factor", "marlin_tile_size")
|
||||
flag_attrs = ("needs_scalar_to_array", "is_sharded_weight", "is_transposed")
|
||||
if any(getattr(param, attr, None) is not None for attr in metadata_attrs):
|
||||
return None
|
||||
if any(bool(getattr(param, attr, False)) for attr in flag_attrs):
|
||||
return None
|
||||
|
||||
if loader_fn in replicated_loaders:
|
||||
return REPLICATE
|
||||
if loader_fn not in column_loaders | row_loaders:
|
||||
return None
|
||||
|
||||
if owner is None or getattr(owner, "tp_size", None) != shard_axis_size:
|
||||
return None
|
||||
|
||||
attr = "output_dim" if loader_fn in column_loaders else "input_dim"
|
||||
dim = getattr(param, attr, None)
|
||||
return dim if isinstance(dim, int) and dim >= 0 else None
|
||||
|
||||
|
||||
def _validated_shard_dim(
|
||||
global_shape: Sequence[int],
|
||||
local_shape: Sequence[int],
|
||||
declared_dim: int,
|
||||
shard_axis_size: int,
|
||||
) -> int | None:
|
||||
"""Validate a loader-declared shard dimension against both shapes."""
|
||||
if len(global_shape) != len(local_shape):
|
||||
return None
|
||||
|
||||
if declared_dim == REPLICATE:
|
||||
return REPLICATE if tuple(global_shape) == tuple(local_shape) else None
|
||||
if declared_dim >= len(global_shape):
|
||||
return None
|
||||
for dim, (whole, local) in enumerate(zip(global_shape, local_shape)):
|
||||
expected = local * shard_axis_size if dim == declared_dim else local
|
||||
if whole != expected:
|
||||
return None
|
||||
return declared_dim
|
||||
|
||||
|
||||
def resolve_parameter_destinations(
|
||||
model: torch.nn.Module,
|
||||
names: Sequence[str],
|
||||
dtypes: Sequence[torch.dtype],
|
||||
shapes: Sequence[Sequence[int]],
|
||||
*,
|
||||
num_workers: int,
|
||||
shard_axis_size: int,
|
||||
allow_direct: bool,
|
||||
) -> list[M2NDestination]:
|
||||
"""Build this worker's destination plan, one entry per checkpoint parameter.
|
||||
|
||||
Placements are relative to the inference mesh: axis 0 replicates and axis 1
|
||||
shards. `allow_direct=False` forces every parameter onto the fallback path;
|
||||
the engine sets it for pipeline-parallel or quantized deployments, where a
|
||||
parameter's local shape is not simply the checkpoint shape split across the
|
||||
shard axis. Parameters below a model type with known model-level checkpoint
|
||||
preprocessing also fall back, since bypassing `load_weights` would skip the
|
||||
transformation.
|
||||
"""
|
||||
params = dict(model.named_parameters()) if allow_direct else {}
|
||||
blocked_param_ids = {
|
||||
id(param)
|
||||
for module in model.modules()
|
||||
if _module_type_name(module) in _DIRECT_WEIGHT_LOADING_BLOCKLIST
|
||||
for param in module.parameters()
|
||||
}
|
||||
|
||||
destinations: list[M2NDestination] = []
|
||||
for name, dtype, shape in zip(names, dtypes, shapes):
|
||||
param = params.get(name)
|
||||
dim = None
|
||||
if (
|
||||
param is not None
|
||||
and id(param) not in blocked_param_ids
|
||||
and param.dtype == dtype
|
||||
and param.data.is_contiguous()
|
||||
and num_workers % shard_axis_size == 0
|
||||
):
|
||||
declared_dim = _base_loader_shard_dim(param, shard_axis_size)
|
||||
if declared_dim is not None:
|
||||
dim = _validated_shard_dim(
|
||||
shape, param.shape, declared_dim, shard_axis_size
|
||||
)
|
||||
|
||||
if dim is None:
|
||||
# A replicated parameter is identical on every rank, so it needs no
|
||||
# placement of its own — REPLICATED lets resolve_layout spread it
|
||||
# over all the workers regardless of how the mesh is factored.
|
||||
destinations.append(M2NDestination(name, REPLICATED, None))
|
||||
elif dim == REPLICATE:
|
||||
assert param is not None
|
||||
destinations.append(M2NDestination(name, REPLICATED, param.data))
|
||||
else:
|
||||
assert param is not None
|
||||
destinations.append(
|
||||
M2NDestination(name, _destination_placements(dim), param.data)
|
||||
)
|
||||
|
||||
# Layerwise reload finalizes a module as a unit. Mixing direct and fallback
|
||||
# parameters within one module can copy stale storage over the direct load.
|
||||
fallback_modules = {
|
||||
name.rpartition(".")[0]
|
||||
for name, destination in zip(names, destinations)
|
||||
if not destination.direct and name in params
|
||||
}
|
||||
destinations = [
|
||||
M2NDestination(destination.name, REPLICATED, None)
|
||||
if destination.direct
|
||||
and destination.name.rpartition(".")[0] in fallback_modules
|
||||
else destination
|
||||
for destination in destinations
|
||||
]
|
||||
|
||||
num_direct = sum(d.direct for d in destinations)
|
||||
parameter_bytes = [
|
||||
math.prod(shape) * dtype.itemsize for dtype, shape in zip(dtypes, shapes)
|
||||
]
|
||||
total_bytes = sum(parameter_bytes)
|
||||
direct_bytes = sum(
|
||||
nbytes
|
||||
for destination, nbytes in zip(destinations, parameter_bytes)
|
||||
if destination.direct
|
||||
)
|
||||
direct_byte_percentage = 100 * direct_bytes / total_bytes if total_bytes else 0
|
||||
logger.info(
|
||||
"nccl_m2n destination plan: %d/%d parameters resharded directly into "
|
||||
"the model, %d via full-tensor fallback; direct byte coverage: "
|
||||
"%d/%d bytes (%.1f%%)",
|
||||
num_direct,
|
||||
len(destinations),
|
||||
len(destinations) - num_direct,
|
||||
direct_bytes,
|
||||
total_bytes,
|
||||
direct_byte_percentage,
|
||||
)
|
||||
return destinations
|
||||
Reference in New Issue
Block a user