Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+1 -2
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Weight transfer engines for syncing model weights from trainers
"""Weight transfer engines for syncing model weights from trainers
to inference workers.
"""
+17 -20
View File
@@ -193,6 +193,7 @@ class WeightSource(ABC):
Returns:
The held parameter names, or None to hold every one.
"""
return None
@@ -329,8 +330,7 @@ class WeightTransferUpdateRequest:
class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
"""
Base class for weight transfer engines that handle transport of model weights
"""Base class for weight transfer engines that handle transport of model weights
from a trainer to inference workers.
This abstraction separates weight transfer transport logic from the worker
@@ -386,14 +386,14 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
device: torch.device,
model: torch.nn.Module,
) -> None:
"""
Initialize the weight transfer engine.
"""Initialize the weight transfer engine.
Args:
config: The configuration for the weight transfer engine
vllm_config: The full vLLM config (provides parallel/model config)
device: The device this worker's model lives on
model: The local model instance which will receive the weights
"""
self.config = config
self.vllm_config = vllm_config
@@ -419,8 +419,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
self.model_config = self._default_model_config
def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo:
"""
Construct typed init info from dict with validation.
"""Construct typed init info from dict with validation.
Args:
init_dict: Dictionary containing backend-specific initialization parameters
@@ -430,6 +429,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
Raises:
ValueError: If init_dict is invalid for this backend
"""
try:
return self.init_info_cls(**init_dict)
@@ -439,8 +439,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
) from e
def parse_update_info(self, update_dict: dict[str, Any]) -> TUpdateInfo:
"""
Construct typed update info from dict with validation.
"""Construct typed update info from dict with validation.
Args:
update_dict: Dictionary containing backend-specific update parameters
@@ -450,6 +449,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
Raises:
ValueError: If update_dict is invalid for this backend
"""
try:
return self.update_info_cls(**update_dict)
@@ -460,19 +460,18 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
@abstractmethod
def init_transfer_engine(self, init_info: TInitInfo) -> None:
"""
Initialize the weight transfer mechanism.
"""Initialize the weight transfer mechanism.
This is called once at the beginning of training.
Args:
init_info: Backend-specific initialization info
"""
raise NotImplementedError
@abstractmethod
def start_weight_update(self) -> None:
"""
Prepare the engine for a new weight update.
"""Prepare the engine for a new weight update.
Engines that receive weights in checkpoint format initialize layerwise reloading
here, else this is typically a no-op.
@@ -482,8 +481,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
@abstractmethod
def finish_weight_update(self) -> None:
"""
Finalize the current weight update.
"""Finalize the current weight update.
Checkpoint-format engines finalize layerwise reloading here; engines
that apply weights in place leave this as a no-op.
@@ -491,11 +489,11 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
raise NotImplementedError
def update_weights(self, update_info: dict[str, Any]) -> None:
"""
Receive one weight update chunk and load it into the model.
"""Receive one weight update chunk and load it into the model.
Args:
update_info: Dictionary containing backend-specific update info
"""
typed_update_info = self.parse_update_info(update_info)
self.receive_weights(typed_update_info)
@@ -505,19 +503,18 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
@abstractmethod
def receive_weights(self, update_info: TUpdateInfo) -> None:
"""
Receive weights from the trainer and load them into the model.
"""Receive weights from the trainer and load them into the model.
Args:
update_info: Backend-specific update info containing parameter metadata
and any backend-specific data
"""
raise NotImplementedError
@abstractmethod
def shutdown(self) -> None:
"""
Shutdown the weight transfer engine.
"""Shutdown the weight transfer engine.
This should be called when the worker is shutting down.
"""
raise NotImplementedError
+5 -1
View File
@@ -58,6 +58,7 @@ class WeightTransferEngineFactory:
Raises:
ValueError: If an engine with the same name is already registered
"""
if name in cls._registry:
raise ValueError(f"Weight transfer engine '{name}' is already registered.")
@@ -101,6 +102,7 @@ class WeightTransferEngineFactory:
Raises:
ValueError: If the backend is not registered
"""
backend = config.backend
if backend not in cls._registry:
@@ -139,7 +141,8 @@ class WeightTransferTrainerFactory:
class_name: str | None = None,
) -> None:
"""Register a trainer engine. Same conventions as
`WeightTransferEngineFactory.register_engine`."""
`WeightTransferEngineFactory.register_engine`.
"""
if name in cls._registry:
raise ValueError(
f"Weight transfer trainer engine '{name}' is already registered."
@@ -190,6 +193,7 @@ class WeightTransferTrainerFactory:
Raises:
ValueError: If `init_info.backend` is not registered.
"""
backend = init_info.backend
if backend not in cls._registry:
@@ -40,7 +40,8 @@ class IPCWeightTransferInitInfo(WeightTransferInitInfo):
`packed` is a must-agree wire param: the trainer ships it here at the init
handshake so the worker decodes with the same setting the trainer encoded
with. The consumer rebuilds from the IPC handle + `tensor_sizes`, so it does
not need the buffer size (producer-only)."""
not need the buffer size (producer-only).
"""
packed: bool = False
@@ -53,7 +54,8 @@ class IPCTrainerInitInfo(TrainerInitInfo):
`packed` / `packed_buffer_size_bytes` are the transfer's wire params. The
trainer propagates `packed` to the worker at `trainer_init` so the two sides
cannot disagree. `backend` is the factory dispatch key."""
cannot disagree. `backend` is the factory dispatch key.
"""
backend: ClassVar[str] = "ipc"
packed: bool = False
@@ -121,8 +123,7 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
class IPCWeightTransferEngine(
WeightTransferEngine[IPCWeightTransferInitInfo, IPCWeightTransferUpdateInfo]
):
"""
Weight transfer engine using CUDA IPC for communication between trainer and workers.
"""Weight transfer engine using CUDA IPC between trainer and workers.
This implementation uses CUDA IPC to transfer weights from the trainer (rank 0)
to all inference workers in a process group. IPC handles are used to share
@@ -149,13 +150,13 @@ class IPCWeightTransferEngine(
self._packed_importer = PackedBufferImporter()
def init_transfer_engine(self, init_info: IPCWeightTransferInitInfo) -> None:
"""
Initialize the weight transfer mechanism. No data-plane rendezvous is
"""Initialize the weight transfer mechanism. No data-plane rendezvous is
needed for IPC; this just records the trainer-supplied wire params so the
worker decodes exactly as the trainer encoded.
Args:
init_info: IPC initialization info (carries `packed`).
"""
self.packed = init_info.packed
@@ -182,8 +183,7 @@ class IPCWeightTransferEngine(
self._packed_importer.close()
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
"""
Receive weights from the trainer via CUDA IPC handles and load them.
"""Receive weights from the trainer via CUDA IPC handles and load them.
Whether the transfer is packed is read from `self.packed`, set at the
init handshake from the trainer's init info, so it is guaranteed to
@@ -193,6 +193,7 @@ class IPCWeightTransferEngine(
update_info: IPC update info containing parameter names, dtypes, shapes,
and IPC handles. Each IPC handle is a mapping between physical
GPU UUID and the rebuild_cuda_tensor args tuple.
"""
# Use the worker's assigned device rather than the ambient current
# device: the receive path is no longer wrapped in
@@ -118,7 +118,8 @@ def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
only the mode actually in use. Shared by the dense and sparse trainer
engines so the two cannot drift."""
engines so the two cannot drift.
"""
return {key: value for key, value in asdict(init_info).items() if value is not None}
@@ -142,8 +143,7 @@ def stateless_init_process_group(
world_size: int,
device,
) -> "PyNcclCommunicator":
"""
vLLM provides `StatelessProcessGroup` to create a process group
"""VLLM provides `StatelessProcessGroup` to create a process group
without considering the global process group in torch.distributed.
It is recommended to create `StatelessProcessGroup`, and then initialize
the data-plane communication (NCCL) between external (train processes)
@@ -222,8 +222,7 @@ def worker_init_process_group(
def trainer_init(
init_info: NCCLRendezvous | dict,
) -> "PyNcclCommunicator":
"""
Initialize NCCL process group for trainer-side weight transfer.
"""Initialize NCCL process group for trainer-side weight transfer.
The trainer is always rank 0 in the process group. Uses the current
CUDA device (torch.accelerator.current_device_index()).
@@ -237,6 +236,7 @@ def trainer_init(
Returns:
PyNcclCommunicator for weight transfer.
"""
if isinstance(init_info, dict):
master_address = init_info["master_address"]
@@ -69,7 +69,8 @@ class NCCLTrainerInitInfo(TrainerInitInfo):
propagates them to the worker at `trainer_init` so the two sides cannot
disagree. Note this defaults to packed, unlike the worker-side
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
ships a value. `backend` is the factory dispatch key."""
ships a value. `backend` is the factory dispatch key.
"""
backend: ClassVar[str] = "nccl"
@@ -113,8 +114,7 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
class NCCLWeightTransferEngine(
WeightTransferEngine[NCCLWeightTransferInitInfo, NCCLWeightTransferUpdateInfo]
):
"""
Weight transfer engine using NCCL for communication between trainer and workers.
"""Weight transfer engine using NCCL for communication between trainer and workers.
This implementation uses NCCL broadcast operations to transfer dense
checkpoint-format weights from the trainer (rank 0) to all inference workers
@@ -142,14 +142,14 @@ class NCCLWeightTransferEngine(
self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
"""
Initialize NCCL process group with the trainer and record the
"""Initialize NCCL process group with the trainer and record the
trainer-supplied wire params so the worker decodes exactly as the
trainer encodes.
Args:
init_info: NCCL initialization info containing master address, port,
rank offset, world size, and the packed wire params
"""
self.packed = init_info.packed
self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes
@@ -175,8 +175,7 @@ class NCCLWeightTransferEngine(
finalize_layerwise_reload(self.model, self.model_config)
def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None:
"""
Receive weights from trainer via NCCL broadcast.
"""Receive weights from trainer via NCCL broadcast.
Whether to use packed broadcasting (and the buffer geometry) is read
from `self.packed` / `self.packed_*`, set at the init handshake from the
@@ -186,6 +185,7 @@ class NCCLWeightTransferEngine(
Args:
update_info: NCCL update info containing parameter names, dtypes,
and shapes
"""
if self.model_update_group is None:
raise RuntimeError(
@@ -365,7 +365,8 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
"""Iterate the source (materializing each tensor — a collective on all
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
Non-sender ranks only replay the iteration to stay in the collective."""
Non-sender ranks only replay the iteration to stay in the collective.
"""
if not self.is_sender:
for _ in source:
pass
@@ -45,6 +45,7 @@ def unpack_tensor(
shapes: List of tensor shapes
dtypes: List of tensor dtypes
tensor_sizes: List of tensor sizes in bytes
"""
unpacked_tensors = packed_tensor.split(tensor_sizes)
@@ -85,6 +86,7 @@ def pack_tensors(
tensor_list: Pre-existing tensor list to append to (for NCCL
multi-buffer reuse). If None, a fresh list is created.
current_size: Byte count already accumulated in tensor_list
"""
if tensor_list is None:
tensor_list = []
@@ -335,6 +337,7 @@ def packed_ipc_producer(
buffer_size_bytes: Exact capacity of the reusable IPC buffer.
Every chunk is guaranteed to fit within this size. A
``ValueError`` is raised if any single tensor exceeds it.
"""
ipc_buffer = torch.empty(buffer_size_bytes, dtype=torch.uint8, device="cuda")
_, ipc_args = reduce_tensor(ipc_buffer)
@@ -463,6 +466,7 @@ def packed_ipc_consumer(
export so the buffer mapping is opened and released exactly once
(see ``PackedBufferImporter``). ``None`` builds an ephemeral one,
which is only safe for single-chunk consumption.
"""
props = torch.cuda.get_device_properties(device_index)
physical_gpu_id = str(props.uuid)
@@ -60,6 +60,7 @@ def check_ray_rdt_version() -> None:
Raises:
ValueError: the installed Ray predates ``RDT_MIN_RAY_VERSION``.
"""
import importlib.metadata
@@ -162,6 +163,7 @@ class RdtRouter:
ValueError: ``workers_per_replica`` does not divide
``num_consumers``, so the deployments are not uniform and two
workers of one deployment would share a block index.
"""
self.num_producers = max(1, num_producers)
self.num_consumers = max(1, num_consumers)
@@ -212,7 +214,8 @@ class RdtRouter:
def class_of(self, name: str) -> int:
"""The name's owner class — the planner's bucketing key, since all names
of a chunk must share one producer."""
of a chunk must share one producer.
"""
return self._class_of.get(name, 0)
def owners(self, name: str) -> list[int]:
@@ -263,6 +266,7 @@ class RdtRouter:
Raises:
ValueError: an owner set is empty or out of range, or a class index
does not resolve.
"""
for c, row in enumerate(self._owner_sets):
if not row:
@@ -282,7 +286,8 @@ class RdtRouter:
def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None:
"""Attach this consumer's producer handles. Rebuilt wholesale per init,
never appended to: every owner index is a position in these lists, so a
rejoining engine that re-inits must not shift them."""
rejoining engine that re-inits must not shift them.
"""
if len(actors) != len(produce_methods):
raise ValueError("actors and produce_methods must be parallel")
self._actors = list(actors)
@@ -345,5 +350,6 @@ def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int:
optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is
allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a
perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers
and the producer's serve rings."""
and the producer's serve rings.
"""
return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20))
@@ -134,7 +134,8 @@ class _PendingPull:
``targets``/``blob`` must stay strongly referenced until it completes:
``set_target_for_ref`` stores WEAKREFS, so dropping them silently reroutes the
transfer into a fallback buffer."""
transfer into a fallback buffer.
"""
ref: "Any"
keys: "list[FetchKey]"
@@ -561,7 +562,8 @@ class ShardedRDTWeightTransferEngine(
def _build_static_plan(self, init_info: ShardedRDTWeightTransferInitInfo) -> None:
"""Build the chunk/free plan once. It never changes across syncs, so
``update_weights`` needs no per-sync names."""
``update_weights`` needs no per-sync names.
"""
if not init_info.group_lens:
raise ValueError(
"Sharded RDT engine requires init_info.group_lens (the gather-"
@@ -585,7 +587,8 @@ class ShardedRDTWeightTransferEngine(
Both sides are sized from the static plan: receive buffers are
``ring_depth`` slots at the largest chunk's ``pack_bytes``, and each bound
producer is asked to pre-register a serve ring at the max bytes this
consumer will pull from it. A no-op when this worker has no chunks."""
consumer will pull from it. A no-op when this worker has no chunks.
"""
plan = self._cached_plan
if plan is None or not plan.chunks:
return
@@ -630,7 +633,8 @@ class ShardedRDTWeightTransferEngine(
``data_parallel_index``, not ``data_parallel_rank``: vLLM resets the
latter to 0 in a dense worker but keeps the former as the distinct global
DP rank. Same formula as the sibling ``nccl_engine``, so dense-via-TP and
MoE-via-DP+EP both yield distinct 0..C-1."""
MoE-via-DP+EP both yield distinct 0..C-1.
"""
pc = self.parallel_config
return pc.data_parallel_index * pc.world_size + pc.rank # world_size = TP*PP
@@ -638,14 +642,16 @@ class ShardedRDTWeightTransferEngine(
"""Total inference-worker count. Prefers the driver-supplied
``init_info.num_consumers`` (authoritative -- the driver knows the whole
fleet); else ``world_size_across_dp``, the same stride
``_global_worker_index`` indexes with, so the two agree at any pp."""
``_global_worker_index`` indexes with, so the two agree at any pp.
"""
if self._num_consumers_override > 0:
return self._num_consumers_override
return self.parallel_config.world_size_across_dp
def start_weight_update(self) -> None:
"""Put the model's params on meta so layerwise reload streams them in
as each layer's slices land. Baked replay uses checkpoint format."""
as each layer's slices land. Baked replay uses checkpoint format.
"""
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
@@ -654,7 +660,8 @@ class ShardedRDTWeightTransferEngine(
def finish_weight_update(self) -> None:
"""Drain the deferred pull/process pipeline (so every layer is fully
loaded) before finalizing the layerwise reload."""
loaded) before finalizing the layerwise reload.
"""
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
@@ -667,7 +674,8 @@ class ShardedRDTWeightTransferEngine(
device sync: post-processing is deferred to background threads and a
sync here would block on them and serialize the pull/process pipeline.
Completion is guaranteed by ``drain_pending`` in
``finish_weight_update``."""
``finish_weight_update``.
"""
self.receive_weights(self.parse_update_info(update_info))
def receive_weights(
@@ -709,7 +717,8 @@ class ShardedRDTWeightTransferEngine(
device: torch.device,
) -> list[tuple[str, torch.Tensor]]:
"""Zero-storage lazies for ``names``, dtype/shape from the init metadata,
all feeding the bake's recording sink."""
all feeding the bake's recording sink.
"""
return [
(
name,
@@ -977,7 +986,8 @@ class ShardedRDTWeightTransferEngine(
def _ensure_proc_worker(self) -> None:
"""Lazily create the per-slot events, the background CUDA stream, the work
queue, and the single processing thread. Idempotent."""
queue, and the single processing thread. Idempotent.
"""
if self._proc_thread is not None:
return
import queue
@@ -1036,7 +1046,8 @@ class ShardedRDTWeightTransferEngine(
def _mark_slot_done(self, slot: int) -> None:
"""Publish that a queued item's read-done event has been recorded (or the
item failed) so a pull waiting to reuse ``slot`` can proceed to its
CUDA-event synchronize."""
CUDA-event synchronize.
"""
with self._slot_cv:
self._slot_done[slot] += 1
self._slot_cv.notify_all()
@@ -1052,7 +1063,8 @@ class ShardedRDTWeightTransferEngine(
"""Block until the background thread has processed every queued item and
its stream work is complete, then re-raise any error it hit. Called from
the worker's ``finish_weight_update`` before ``finalize_layerwise_reload``
so every baked layer is fully loaded (and ``info.reset()``-ed) first."""
so every baked layer is fully loaded (and ``info.reset()``-ed) first.
"""
if self._proc_queue is not None:
self._proc_queue.join() # every put() item task_done()'d
# The scatter thread feeds the quant thread, so join it SECOND (all
@@ -1104,7 +1116,8 @@ class ShardedRDTWeightTransferEngine(
which is exactly what an owner class is. The cut is a pure function of
the bake and the ownership table. Copy order within a chunk is bake
order, and a module's copies may span chunks (materialize/quant fire on
its first/last chunk; see ``_build_call_plan``)."""
its first/last chunk; see ``_build_call_plan``).
"""
assert self._router is not None
by_class: dict[int, list[_Scatter]] = {}
for copies in modules:
@@ -1235,7 +1248,8 @@ class ShardedRDTWeightTransferEngine(
pulls nothing from. Refs are held and drained in ``drain_pending`` so
every signal has EXECUTED before the sync ends: ``begin_sync`` resets
the counters, and a straggler landing in the next sync would credit a
group it does not belong to."""
group it does not belong to.
"""
assert self._router is not None
self._pending_frees.extend(self._router.free_group(group_idx))
@@ -1370,7 +1384,8 @@ class ShardedRDTWeightTransferEngine(
the modules' scatters via ``ready``; touches only the scattered params
(never a receive slot), so it can overlap subsequent chunks' RDMA and
scatters. ``info.reset()`` is what makes finalize skip the layer —
drain_pending joins the quant queue before finalize runs."""
drain_pending joins the quant queue before finalize runs.
"""
from vllm.model_executor.layers.quantization.base_config import (
QuantizeMethodBase,
)
@@ -1403,7 +1418,8 @@ class ShardedRDTWeightTransferEngine(
def _quant_worker_loop(self) -> None:
"""Dedicated quant thread: drains (completed_modules, scatter-done event)
batches. Errors surface via _proc_error like the scatter thread's."""
batches. Errors surface via _proc_error like the scatter thread's.
"""
torch.cuda.set_device(self.device)
q = self._quant_queue
assert q is not None
@@ -206,7 +206,8 @@ class FakeRDTTensor(torch.Tensor):
def _meta(self) -> torch.Tensor:
"""A zero-storage meta tensor of this fake's shape/dtype, so PyTorch
itself computes post-op geometry rather than us reimplementing it."""
itself computes post-op geometry rather than us reimplementing it.
"""
return torch.empty(self.shape, dtype=self.dtype, device="meta")
@classmethod
@@ -250,7 +251,8 @@ class FakeRDTTensor(torch.Tensor):
):
"""Append the op and return a child, or a tuple of children for
multi-return ops. Each child's geometry comes from running the op on a
meta tensor."""
meta tensor.
"""
meta = self_._meta()
with torch._C.DisableTorchFunctionSubclass():
meta_result = func(meta, *args, **kwargs)
@@ -132,7 +132,8 @@ PRODUCE_METHOD_NAME = "rdt_produce_weights_batched"
@dataclass
class _SharedPack:
"""One generation of one slot-sharing group: the rendezvous state for a
single chunk, identified by the consumers' issue index."""
single chunk, identified by the consumers' issue index.
"""
group: int
slot: int
@@ -466,7 +467,8 @@ class _RDTProducerServer:
def end_sync(self) -> list[int]:
"""Block until every published group has been freed by its consumers;
return the remaining freed keys so the engine drops its last refs."""
return the remaining freed keys so the engine drops its last refs.
"""
with self._cache_cond:
self._wait_for(
lambda: bool(self._inflight_groups),
@@ -479,7 +481,8 @@ class _RDTProducerServer:
def set_gather_error(self, message: str) -> None:
"""Record a trainer-side gather failure so blocked serves / publishes
stop waiting and surface it."""
stop waiting and surface it.
"""
with self._cache_cond:
self._gather_error = RuntimeError(message)
self._cache_cond.notify_all()
@@ -508,7 +511,8 @@ class _RDTProducerServer:
def _new_serve_buffer(self, nbytes: int) -> torch.Tensor:
"""Allocate + NIXL-register one serve slot: the single allocation seam,
so registration cannot be skipped on either of the two paths that make
buffers (the init-time reservation and the serve-path backstop)."""
buffers (the init-time reservation and the serve-path backstop).
"""
t = torch.empty(nbytes, dtype=torch.uint8, device=self._serve_device)
with self._reg_lock:
register_nixl_memory(t)
@@ -534,6 +538,7 @@ class _RDTProducerServer:
Raises:
RuntimeError: two consumers of one sharing group disagree on the
chunks they pull from this producer.
"""
sg = self._share_group(consumer_id)
if plan_digest is not None:
@@ -579,13 +584,15 @@ class _RDTProducerServer:
def _share_group(self, consumer_id: int) -> int:
"""The slot-sharing group ``consumer_id`` belongs to: its index within
its own deployment, since that is what fixes the plan. Without a width
the group is the consumer itself and nothing is shared."""
the group is the consumer itself and nothing is shared.
"""
return consumer_id % self._share_width if self._share_width > 0 else consumer_id
def _sharers_of(self, sg: int) -> frozenset:
"""The LIVE consumers of group ``sg``: the rendezvous width. Derived from
``begin_sync``'s live set, so a degraded sync narrows the barrier instead
of waiting forever on a dead deployment. Caller holds ``_cache_cond``."""
of waiting forever on a dead deployment. Caller holds ``_cache_cond``.
"""
cached = self._sharers.get(sg)
if cached is None:
if not self._sharing_active or self._live_ids is None:
@@ -621,7 +628,8 @@ class _RDTProducerServer:
def _await_shared_pack(self, gen: _SharedPack) -> torch.Tensor:
"""Block until this generation's packer published its blob, and return
it. A failed pack is re-raised here, so every sharer of a bad pack fails
instead of reading a half-written slot."""
instead of reading a half-written slot.
"""
with self._cache_cond:
self._wait_for(
lambda: not gen.done and gen.error is None,
@@ -642,7 +650,8 @@ class _RDTProducerServer:
"""Group ``sg``'s ring slot ``idx``, grown if this chunk outgrew the
reservation. Growing registers memory while the fabric is busy, the
hazard ``reserve_serve_buffer`` exists to avoid, so it is a backstop: the
reservation is sized from the same static plan as this pack."""
reservation is sized from the same static plan as this pack.
"""
with self._serve_lock:
buffer = self._serve_rings.setdefault(sg, [None] * self._nring)[idx]
if buffer is not None and buffer.numel() >= need:
@@ -654,7 +663,8 @@ class _RDTProducerServer:
def _fail_shared_pack(self, gen: _SharedPack, exc: BaseException) -> None:
"""Publish a pack failure so waiting sharers raise instead of hanging on
a generation that will never complete."""
a generation that will never complete.
"""
with self._cache_cond:
gen.error = exc
self._cache_cond.notify_all()
@@ -681,6 +691,7 @@ class _RDTProducerServer:
Raises:
ValueError: ``seq`` was not supplied.
"""
needed = sorted({n for n, _ in specs})
if self._served_names is not None:
@@ -855,7 +866,8 @@ class ShardedRDTTrainerWeightTransferEngine(
def _rpc(self, method: str, *args: Any) -> Any:
"""Call one of the server actor's methods and block for the result.
The single seam through which the engine talks to its server, so tests
can inject a local (non-Ray) fake server."""
can inject a local (non-Ray) fake server.
"""
import ray
return ray.get(getattr(self._server, method).remote(*args))
@@ -932,7 +944,8 @@ class ShardedRDTTrainerWeightTransferEngine(
"""Fire publish_group WITHOUT waiting on the RPC (the gather loop
overlaps the publish's server-side rebuild with the next group's
gather) and return a handle that ``_await_publish`` resolves. Ray actor
handle in production; a plain (non-Ray) fake server runs inline."""
handle in production; a plain (non-Ray) fake server runs inline.
"""
method = self._server.publish_group
remote = getattr(method, "remote", None)
if remote is not None:
@@ -943,7 +956,8 @@ class ShardedRDTTrainerWeightTransferEngine(
"""Resolve one async publish so a server-side rebuild error surfaces at
window depth instead of at end_sync. Publishes carry nothing back —
freed groups flow only through wait_freed/end_sync (one channel; see
publish_group)."""
publish_group).
"""
import ray
if isinstance(ref, ray.ObjectRef):
@@ -1023,6 +1037,7 @@ class ShardedRDTTrainerWeightTransferEngine(
Raises:
RuntimeError: called before ``trainer_init`` cached the server names.
"""
if self._server_names is None:
raise RuntimeError(
@@ -1129,7 +1144,8 @@ class ShardedRDTTrainerWeightTransferEngine(
by stall watchdog 300s later: consumers route pulls here, the pull
passes the served-names guard, and the cache wait never completes. With
it, that is an immediate error naming the weight. One set lookup per
name per sync."""
name per sync.
"""
if self._held_names is None:
return
for name, tensor in zip(names, tensors):
@@ -1330,7 +1346,8 @@ class ShardedRDTTrainerWeightTransferEngine(
credit releases when every live consumer has signaled the group). So
the loop self-paces to the consumers' pull rate with at most
`gather_lookahead + 1` groups resident. Runs on every rank; only the
sender has an `update_future` to fail fast on."""
sender has an `update_future` to fail fast on.
"""
assert self.source is not None # guaranteed by trainer_init
# The live IDS ride along with the count: the free barrier needs only
# the count, but the slot-sharing rendezvous needs to know WHICH
@@ -68,7 +68,8 @@ class SparseNCCLTrainerInitInfo(TrainerInitInfo):
Same rendezvous shape as the dense NCCL backend (the sender opens its
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
are never packed. `backend` is the factory dispatch key."""
are never packed. `backend` is the factory dispatch key.
"""
backend: ClassVar[str] = "sparse_nccl"
@@ -113,8 +114,7 @@ class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
class SparseNCCLWeightTransferEngine(
WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo]
):
"""
Sparse weight transfer engine using NCCL.
"""Sparse weight transfer engine using NCCL.
Receives checkpoint-coordinate patches broadcast from the trainer and applies
them through the model's native weight loader. Sparse updates modify initialized
@@ -368,7 +368,8 @@ class SparseNCCLTrainerWeightTransferEngine(
rebuild or free the patch tensors as soon as a send method returns rather
than relying on same-stream ordering. See
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
cross-rank barrier."""
cross-rank barrier.
"""
if torch.cuda.is_available():
torch.cuda.current_stream().synchronize()