Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Weight transfer engines for syncing model weights from trainers
|
||||
"""Weight transfer engines for syncing model weights from trainers
|
||||
to inference workers.
|
||||
"""
|
||||
|
||||
|
||||
@@ -193,6 +193,7 @@ class WeightSource(ABC):
|
||||
|
||||
Returns:
|
||||
The held parameter names, or None to hold every one.
|
||||
|
||||
"""
|
||||
return None
|
||||
|
||||
@@ -329,8 +330,7 @@ class WeightTransferUpdateRequest:
|
||||
|
||||
|
||||
class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
"""
|
||||
Base class for weight transfer engines that handle transport of model weights
|
||||
"""Base class for weight transfer engines that handle transport of model weights
|
||||
from a trainer to inference workers.
|
||||
|
||||
This abstraction separates weight transfer transport logic from the worker
|
||||
@@ -386,14 +386,14 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
device: torch.device,
|
||||
model: torch.nn.Module,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize the weight transfer engine.
|
||||
"""Initialize the weight transfer engine.
|
||||
|
||||
Args:
|
||||
config: The configuration for the weight transfer engine
|
||||
vllm_config: The full vLLM config (provides parallel/model config)
|
||||
device: The device this worker's model lives on
|
||||
model: The local model instance which will receive the weights
|
||||
|
||||
"""
|
||||
self.config = config
|
||||
self.vllm_config = vllm_config
|
||||
@@ -419,8 +419,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
self.model_config = self._default_model_config
|
||||
|
||||
def parse_init_info(self, init_dict: dict[str, Any]) -> TInitInfo:
|
||||
"""
|
||||
Construct typed init info from dict with validation.
|
||||
"""Construct typed init info from dict with validation.
|
||||
|
||||
Args:
|
||||
init_dict: Dictionary containing backend-specific initialization parameters
|
||||
@@ -430,6 +429,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
Raises:
|
||||
ValueError: If init_dict is invalid for this backend
|
||||
|
||||
"""
|
||||
try:
|
||||
return self.init_info_cls(**init_dict)
|
||||
@@ -439,8 +439,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
) from e
|
||||
|
||||
def parse_update_info(self, update_dict: dict[str, Any]) -> TUpdateInfo:
|
||||
"""
|
||||
Construct typed update info from dict with validation.
|
||||
"""Construct typed update info from dict with validation.
|
||||
|
||||
Args:
|
||||
update_dict: Dictionary containing backend-specific update parameters
|
||||
@@ -450,6 +449,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
Raises:
|
||||
ValueError: If update_dict is invalid for this backend
|
||||
|
||||
"""
|
||||
try:
|
||||
return self.update_info_cls(**update_dict)
|
||||
@@ -460,19 +460,18 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
@abstractmethod
|
||||
def init_transfer_engine(self, init_info: TInitInfo) -> None:
|
||||
"""
|
||||
Initialize the weight transfer mechanism.
|
||||
"""Initialize the weight transfer mechanism.
|
||||
This is called once at the beginning of training.
|
||||
|
||||
Args:
|
||||
init_info: Backend-specific initialization info
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def start_weight_update(self) -> None:
|
||||
"""
|
||||
Prepare the engine for a new weight update.
|
||||
"""Prepare the engine for a new weight update.
|
||||
|
||||
Engines that receive weights in checkpoint format initialize layerwise reloading
|
||||
here, else this is typically a no-op.
|
||||
@@ -482,8 +481,7 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
@abstractmethod
|
||||
def finish_weight_update(self) -> None:
|
||||
"""
|
||||
Finalize the current weight update.
|
||||
"""Finalize the current weight update.
|
||||
|
||||
Checkpoint-format engines finalize layerwise reloading here; engines
|
||||
that apply weights in place leave this as a no-op.
|
||||
@@ -491,11 +489,11 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
raise NotImplementedError
|
||||
|
||||
def update_weights(self, update_info: dict[str, Any]) -> None:
|
||||
"""
|
||||
Receive one weight update chunk and load it into the model.
|
||||
"""Receive one weight update chunk and load it into the model.
|
||||
|
||||
Args:
|
||||
update_info: Dictionary containing backend-specific update info
|
||||
|
||||
"""
|
||||
typed_update_info = self.parse_update_info(update_info)
|
||||
self.receive_weights(typed_update_info)
|
||||
@@ -505,19 +503,18 @@ class WeightTransferEngine(ABC, Generic[TInitInfo, TUpdateInfo]):
|
||||
|
||||
@abstractmethod
|
||||
def receive_weights(self, update_info: TUpdateInfo) -> None:
|
||||
"""
|
||||
Receive weights from the trainer and load them into the model.
|
||||
"""Receive weights from the trainer and load them into the model.
|
||||
|
||||
Args:
|
||||
update_info: Backend-specific update info containing parameter metadata
|
||||
and any backend-specific data
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def shutdown(self) -> None:
|
||||
"""
|
||||
Shutdown the weight transfer engine.
|
||||
"""Shutdown the weight transfer engine.
|
||||
This should be called when the worker is shutting down.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -58,6 +58,7 @@ class WeightTransferEngineFactory:
|
||||
|
||||
Raises:
|
||||
ValueError: If an engine with the same name is already registered
|
||||
|
||||
"""
|
||||
if name in cls._registry:
|
||||
raise ValueError(f"Weight transfer engine '{name}' is already registered.")
|
||||
@@ -101,6 +102,7 @@ class WeightTransferEngineFactory:
|
||||
|
||||
Raises:
|
||||
ValueError: If the backend is not registered
|
||||
|
||||
"""
|
||||
backend = config.backend
|
||||
if backend not in cls._registry:
|
||||
@@ -139,7 +141,8 @@ class WeightTransferTrainerFactory:
|
||||
class_name: str | None = None,
|
||||
) -> None:
|
||||
"""Register a trainer engine. Same conventions as
|
||||
`WeightTransferEngineFactory.register_engine`."""
|
||||
`WeightTransferEngineFactory.register_engine`.
|
||||
"""
|
||||
if name in cls._registry:
|
||||
raise ValueError(
|
||||
f"Weight transfer trainer engine '{name}' is already registered."
|
||||
@@ -190,6 +193,7 @@ class WeightTransferTrainerFactory:
|
||||
|
||||
Raises:
|
||||
ValueError: If `init_info.backend` is not registered.
|
||||
|
||||
"""
|
||||
backend = init_info.backend
|
||||
if backend not in cls._registry:
|
||||
|
||||
@@ -40,7 +40,8 @@ class IPCWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
`packed` is a must-agree wire param: the trainer ships it here at the init
|
||||
handshake so the worker decodes with the same setting the trainer encoded
|
||||
with. The consumer rebuilds from the IPC handle + `tensor_sizes`, so it does
|
||||
not need the buffer size (producer-only)."""
|
||||
not need the buffer size (producer-only).
|
||||
"""
|
||||
|
||||
packed: bool = False
|
||||
|
||||
@@ -53,7 +54,8 @@ class IPCTrainerInitInfo(TrainerInitInfo):
|
||||
|
||||
`packed` / `packed_buffer_size_bytes` are the transfer's wire params. The
|
||||
trainer propagates `packed` to the worker at `trainer_init` so the two sides
|
||||
cannot disagree. `backend` is the factory dispatch key."""
|
||||
cannot disagree. `backend` is the factory dispatch key.
|
||||
"""
|
||||
|
||||
backend: ClassVar[str] = "ipc"
|
||||
packed: bool = False
|
||||
@@ -121,8 +123,7 @@ class IPCWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
class IPCWeightTransferEngine(
|
||||
WeightTransferEngine[IPCWeightTransferInitInfo, IPCWeightTransferUpdateInfo]
|
||||
):
|
||||
"""
|
||||
Weight transfer engine using CUDA IPC for communication between trainer and workers.
|
||||
"""Weight transfer engine using CUDA IPC between trainer and workers.
|
||||
|
||||
This implementation uses CUDA IPC to transfer weights from the trainer (rank 0)
|
||||
to all inference workers in a process group. IPC handles are used to share
|
||||
@@ -149,13 +150,13 @@ class IPCWeightTransferEngine(
|
||||
self._packed_importer = PackedBufferImporter()
|
||||
|
||||
def init_transfer_engine(self, init_info: IPCWeightTransferInitInfo) -> None:
|
||||
"""
|
||||
Initialize the weight transfer mechanism. No data-plane rendezvous is
|
||||
"""Initialize the weight transfer mechanism. No data-plane rendezvous is
|
||||
needed for IPC; this just records the trainer-supplied wire params so the
|
||||
worker decodes exactly as the trainer encoded.
|
||||
|
||||
Args:
|
||||
init_info: IPC initialization info (carries `packed`).
|
||||
|
||||
"""
|
||||
self.packed = init_info.packed
|
||||
|
||||
@@ -182,8 +183,7 @@ class IPCWeightTransferEngine(
|
||||
self._packed_importer.close()
|
||||
|
||||
def receive_weights(self, update_info: IPCWeightTransferUpdateInfo) -> None:
|
||||
"""
|
||||
Receive weights from the trainer via CUDA IPC handles and load them.
|
||||
"""Receive weights from the trainer via CUDA IPC handles and load them.
|
||||
|
||||
Whether the transfer is packed is read from `self.packed`, set at the
|
||||
init handshake from the trainer's init info, so it is guaranteed to
|
||||
@@ -193,6 +193,7 @@ class IPCWeightTransferEngine(
|
||||
update_info: IPC update info containing parameter names, dtypes, shapes,
|
||||
and IPC handles. Each IPC handle is a mapping between physical
|
||||
GPU UUID and the rebuild_cuda_tensor args tuple.
|
||||
|
||||
"""
|
||||
# Use the worker's assigned device rather than the ambient current
|
||||
# device: the receive path is no longer wrapped in
|
||||
|
||||
@@ -118,7 +118,8 @@ def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
|
||||
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
|
||||
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
|
||||
only the mode actually in use. Shared by the dense and sparse trainer
|
||||
engines so the two cannot drift."""
|
||||
engines so the two cannot drift.
|
||||
"""
|
||||
return {key: value for key, value in asdict(init_info).items() if value is not None}
|
||||
|
||||
|
||||
@@ -142,8 +143,7 @@ def stateless_init_process_group(
|
||||
world_size: int,
|
||||
device,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""
|
||||
vLLM provides `StatelessProcessGroup` to create a process group
|
||||
"""VLLM provides `StatelessProcessGroup` to create a process group
|
||||
without considering the global process group in torch.distributed.
|
||||
It is recommended to create `StatelessProcessGroup`, and then initialize
|
||||
the data-plane communication (NCCL) between external (train processes)
|
||||
@@ -222,8 +222,7 @@ def worker_init_process_group(
|
||||
def trainer_init(
|
||||
init_info: NCCLRendezvous | dict,
|
||||
) -> "PyNcclCommunicator":
|
||||
"""
|
||||
Initialize NCCL process group for trainer-side weight transfer.
|
||||
"""Initialize NCCL process group for trainer-side weight transfer.
|
||||
|
||||
The trainer is always rank 0 in the process group. Uses the current
|
||||
CUDA device (torch.accelerator.current_device_index()).
|
||||
@@ -237,6 +236,7 @@ def trainer_init(
|
||||
|
||||
Returns:
|
||||
PyNcclCommunicator for weight transfer.
|
||||
|
||||
"""
|
||||
if isinstance(init_info, dict):
|
||||
master_address = init_info["master_address"]
|
||||
|
||||
@@ -69,7 +69,8 @@ class NCCLTrainerInitInfo(TrainerInitInfo):
|
||||
propagates them to the worker at `trainer_init` so the two sides cannot
|
||||
disagree. Note this defaults to packed, unlike the worker-side
|
||||
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
|
||||
ships a value. `backend` is the factory dispatch key."""
|
||||
ships a value. `backend` is the factory dispatch key.
|
||||
"""
|
||||
|
||||
backend: ClassVar[str] = "nccl"
|
||||
|
||||
@@ -113,8 +114,7 @@ class NCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
class NCCLWeightTransferEngine(
|
||||
WeightTransferEngine[NCCLWeightTransferInitInfo, NCCLWeightTransferUpdateInfo]
|
||||
):
|
||||
"""
|
||||
Weight transfer engine using NCCL for communication between trainer and workers.
|
||||
"""Weight transfer engine using NCCL for communication between trainer and workers.
|
||||
|
||||
This implementation uses NCCL broadcast operations to transfer dense
|
||||
checkpoint-format weights from the trainer (rank 0) to all inference workers
|
||||
@@ -142,14 +142,14 @@ class NCCLWeightTransferEngine(
|
||||
self.packed_num_buffers = DEFAULT_PACKED_NUM_BUFFERS
|
||||
|
||||
def init_transfer_engine(self, init_info: NCCLWeightTransferInitInfo) -> None:
|
||||
"""
|
||||
Initialize NCCL process group with the trainer and record the
|
||||
"""Initialize NCCL process group with the trainer and record the
|
||||
trainer-supplied wire params so the worker decodes exactly as the
|
||||
trainer encodes.
|
||||
|
||||
Args:
|
||||
init_info: NCCL initialization info containing master address, port,
|
||||
rank offset, world size, and the packed wire params
|
||||
|
||||
"""
|
||||
self.packed = init_info.packed
|
||||
self.packed_buffer_size_bytes = init_info.packed_buffer_size_bytes
|
||||
@@ -175,8 +175,7 @@ class NCCLWeightTransferEngine(
|
||||
finalize_layerwise_reload(self.model, self.model_config)
|
||||
|
||||
def receive_weights(self, update_info: NCCLWeightTransferUpdateInfo) -> None:
|
||||
"""
|
||||
Receive weights from trainer via NCCL broadcast.
|
||||
"""Receive weights from trainer via NCCL broadcast.
|
||||
|
||||
Whether to use packed broadcasting (and the buffer geometry) is read
|
||||
from `self.packed` / `self.packed_*`, set at the init handshake from the
|
||||
@@ -186,6 +185,7 @@ class NCCLWeightTransferEngine(
|
||||
Args:
|
||||
update_info: NCCL update info containing parameter names, dtypes,
|
||||
and shapes
|
||||
|
||||
"""
|
||||
if self.model_update_group is None:
|
||||
raise RuntimeError(
|
||||
@@ -365,7 +365,8 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
|
||||
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
|
||||
"""Iterate the source (materializing each tensor — a collective on all
|
||||
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
|
||||
Non-sender ranks only replay the iteration to stay in the collective."""
|
||||
Non-sender ranks only replay the iteration to stay in the collective.
|
||||
"""
|
||||
if not self.is_sender:
|
||||
for _ in source:
|
||||
pass
|
||||
|
||||
@@ -45,6 +45,7 @@ def unpack_tensor(
|
||||
shapes: List of tensor shapes
|
||||
dtypes: List of tensor dtypes
|
||||
tensor_sizes: List of tensor sizes in bytes
|
||||
|
||||
"""
|
||||
unpacked_tensors = packed_tensor.split(tensor_sizes)
|
||||
|
||||
@@ -85,6 +86,7 @@ def pack_tensors(
|
||||
tensor_list: Pre-existing tensor list to append to (for NCCL
|
||||
multi-buffer reuse). If None, a fresh list is created.
|
||||
current_size: Byte count already accumulated in tensor_list
|
||||
|
||||
"""
|
||||
if tensor_list is None:
|
||||
tensor_list = []
|
||||
@@ -335,6 +337,7 @@ def packed_ipc_producer(
|
||||
buffer_size_bytes: Exact capacity of the reusable IPC buffer.
|
||||
Every chunk is guaranteed to fit within this size. A
|
||||
``ValueError`` is raised if any single tensor exceeds it.
|
||||
|
||||
"""
|
||||
ipc_buffer = torch.empty(buffer_size_bytes, dtype=torch.uint8, device="cuda")
|
||||
_, ipc_args = reduce_tensor(ipc_buffer)
|
||||
@@ -463,6 +466,7 @@ def packed_ipc_consumer(
|
||||
export so the buffer mapping is opened and released exactly once
|
||||
(see ``PackedBufferImporter``). ``None`` builds an ephemeral one,
|
||||
which is only safe for single-chunk consumption.
|
||||
|
||||
"""
|
||||
props = torch.cuda.get_device_properties(device_index)
|
||||
physical_gpu_id = str(props.uuid)
|
||||
|
||||
@@ -60,6 +60,7 @@ def check_ray_rdt_version() -> None:
|
||||
|
||||
Raises:
|
||||
ValueError: the installed Ray predates ``RDT_MIN_RAY_VERSION``.
|
||||
|
||||
"""
|
||||
import importlib.metadata
|
||||
|
||||
@@ -162,6 +163,7 @@ class RdtRouter:
|
||||
ValueError: ``workers_per_replica`` does not divide
|
||||
``num_consumers``, so the deployments are not uniform and two
|
||||
workers of one deployment would share a block index.
|
||||
|
||||
"""
|
||||
self.num_producers = max(1, num_producers)
|
||||
self.num_consumers = max(1, num_consumers)
|
||||
@@ -212,7 +214,8 @@ class RdtRouter:
|
||||
|
||||
def class_of(self, name: str) -> int:
|
||||
"""The name's owner class — the planner's bucketing key, since all names
|
||||
of a chunk must share one producer."""
|
||||
of a chunk must share one producer.
|
||||
"""
|
||||
return self._class_of.get(name, 0)
|
||||
|
||||
def owners(self, name: str) -> list[int]:
|
||||
@@ -263,6 +266,7 @@ class RdtRouter:
|
||||
Raises:
|
||||
ValueError: an owner set is empty or out of range, or a class index
|
||||
does not resolve.
|
||||
|
||||
"""
|
||||
for c, row in enumerate(self._owner_sets):
|
||||
if not row:
|
||||
@@ -282,7 +286,8 @@ class RdtRouter:
|
||||
def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None:
|
||||
"""Attach this consumer's producer handles. Rebuilt wholesale per init,
|
||||
never appended to: every owner index is a position in these lists, so a
|
||||
rejoining engine that re-inits must not shift them."""
|
||||
rejoining engine that re-inits must not shift them.
|
||||
"""
|
||||
if len(actors) != len(produce_methods):
|
||||
raise ValueError("actors and produce_methods must be parallel")
|
||||
self._actors = list(actors)
|
||||
@@ -345,5 +350,6 @@ def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int:
|
||||
optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is
|
||||
allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a
|
||||
perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers
|
||||
and the producer's serve rings."""
|
||||
and the producer's serve rings.
|
||||
"""
|
||||
return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20))
|
||||
|
||||
@@ -134,7 +134,8 @@ class _PendingPull:
|
||||
|
||||
``targets``/``blob`` must stay strongly referenced until it completes:
|
||||
``set_target_for_ref`` stores WEAKREFS, so dropping them silently reroutes the
|
||||
transfer into a fallback buffer."""
|
||||
transfer into a fallback buffer.
|
||||
"""
|
||||
|
||||
ref: "Any"
|
||||
keys: "list[FetchKey]"
|
||||
@@ -561,7 +562,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
|
||||
def _build_static_plan(self, init_info: ShardedRDTWeightTransferInitInfo) -> None:
|
||||
"""Build the chunk/free plan once. It never changes across syncs, so
|
||||
``update_weights`` needs no per-sync names."""
|
||||
``update_weights`` needs no per-sync names.
|
||||
"""
|
||||
if not init_info.group_lens:
|
||||
raise ValueError(
|
||||
"Sharded RDT engine requires init_info.group_lens (the gather-"
|
||||
@@ -585,7 +587,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
Both sides are sized from the static plan: receive buffers are
|
||||
``ring_depth`` slots at the largest chunk's ``pack_bytes``, and each bound
|
||||
producer is asked to pre-register a serve ring at the max bytes this
|
||||
consumer will pull from it. A no-op when this worker has no chunks."""
|
||||
consumer will pull from it. A no-op when this worker has no chunks.
|
||||
"""
|
||||
plan = self._cached_plan
|
||||
if plan is None or not plan.chunks:
|
||||
return
|
||||
@@ -630,7 +633,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
``data_parallel_index``, not ``data_parallel_rank``: vLLM resets the
|
||||
latter to 0 in a dense worker but keeps the former as the distinct global
|
||||
DP rank. Same formula as the sibling ``nccl_engine``, so dense-via-TP and
|
||||
MoE-via-DP+EP both yield distinct 0..C-1."""
|
||||
MoE-via-DP+EP both yield distinct 0..C-1.
|
||||
"""
|
||||
pc = self.parallel_config
|
||||
return pc.data_parallel_index * pc.world_size + pc.rank # world_size = TP*PP
|
||||
|
||||
@@ -638,14 +642,16 @@ class ShardedRDTWeightTransferEngine(
|
||||
"""Total inference-worker count. Prefers the driver-supplied
|
||||
``init_info.num_consumers`` (authoritative -- the driver knows the whole
|
||||
fleet); else ``world_size_across_dp``, the same stride
|
||||
``_global_worker_index`` indexes with, so the two agree at any pp."""
|
||||
``_global_worker_index`` indexes with, so the two agree at any pp.
|
||||
"""
|
||||
if self._num_consumers_override > 0:
|
||||
return self._num_consumers_override
|
||||
return self.parallel_config.world_size_across_dp
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Put the model's params on meta so layerwise reload streams them in
|
||||
as each layer's slices land. Baked replay uses checkpoint format."""
|
||||
as each layer's slices land. Baked replay uses checkpoint format.
|
||||
"""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
@@ -654,7 +660,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Drain the deferred pull/process pipeline (so every layer is fully
|
||||
loaded) before finalizing the layerwise reload."""
|
||||
loaded) before finalizing the layerwise reload.
|
||||
"""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
)
|
||||
@@ -667,7 +674,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
device sync: post-processing is deferred to background threads and a
|
||||
sync here would block on them and serialize the pull/process pipeline.
|
||||
Completion is guaranteed by ``drain_pending`` in
|
||||
``finish_weight_update``."""
|
||||
``finish_weight_update``.
|
||||
"""
|
||||
self.receive_weights(self.parse_update_info(update_info))
|
||||
|
||||
def receive_weights(
|
||||
@@ -709,7 +717,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
device: torch.device,
|
||||
) -> list[tuple[str, torch.Tensor]]:
|
||||
"""Zero-storage lazies for ``names``, dtype/shape from the init metadata,
|
||||
all feeding the bake's recording sink."""
|
||||
all feeding the bake's recording sink.
|
||||
"""
|
||||
return [
|
||||
(
|
||||
name,
|
||||
@@ -977,7 +986,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
|
||||
def _ensure_proc_worker(self) -> None:
|
||||
"""Lazily create the per-slot events, the background CUDA stream, the work
|
||||
queue, and the single processing thread. Idempotent."""
|
||||
queue, and the single processing thread. Idempotent.
|
||||
"""
|
||||
if self._proc_thread is not None:
|
||||
return
|
||||
import queue
|
||||
@@ -1036,7 +1046,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
def _mark_slot_done(self, slot: int) -> None:
|
||||
"""Publish that a queued item's read-done event has been recorded (or the
|
||||
item failed) so a pull waiting to reuse ``slot`` can proceed to its
|
||||
CUDA-event synchronize."""
|
||||
CUDA-event synchronize.
|
||||
"""
|
||||
with self._slot_cv:
|
||||
self._slot_done[slot] += 1
|
||||
self._slot_cv.notify_all()
|
||||
@@ -1052,7 +1063,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
"""Block until the background thread has processed every queued item and
|
||||
its stream work is complete, then re-raise any error it hit. Called from
|
||||
the worker's ``finish_weight_update`` before ``finalize_layerwise_reload``
|
||||
so every baked layer is fully loaded (and ``info.reset()``-ed) first."""
|
||||
so every baked layer is fully loaded (and ``info.reset()``-ed) first.
|
||||
"""
|
||||
if self._proc_queue is not None:
|
||||
self._proc_queue.join() # every put() item task_done()'d
|
||||
# The scatter thread feeds the quant thread, so join it SECOND (all
|
||||
@@ -1104,7 +1116,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
which is exactly what an owner class is. The cut is a pure function of
|
||||
the bake and the ownership table. Copy order within a chunk is bake
|
||||
order, and a module's copies may span chunks (materialize/quant fire on
|
||||
its first/last chunk; see ``_build_call_plan``)."""
|
||||
its first/last chunk; see ``_build_call_plan``).
|
||||
"""
|
||||
assert self._router is not None
|
||||
by_class: dict[int, list[_Scatter]] = {}
|
||||
for copies in modules:
|
||||
@@ -1235,7 +1248,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
pulls nothing from. Refs are held and drained in ``drain_pending`` so
|
||||
every signal has EXECUTED before the sync ends: ``begin_sync`` resets
|
||||
the counters, and a straggler landing in the next sync would credit a
|
||||
group it does not belong to."""
|
||||
group it does not belong to.
|
||||
"""
|
||||
assert self._router is not None
|
||||
self._pending_frees.extend(self._router.free_group(group_idx))
|
||||
|
||||
@@ -1370,7 +1384,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
the modules' scatters via ``ready``; touches only the scattered params
|
||||
(never a receive slot), so it can overlap subsequent chunks' RDMA and
|
||||
scatters. ``info.reset()`` is what makes finalize skip the layer —
|
||||
drain_pending joins the quant queue before finalize runs."""
|
||||
drain_pending joins the quant queue before finalize runs.
|
||||
"""
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
@@ -1403,7 +1418,8 @@ class ShardedRDTWeightTransferEngine(
|
||||
|
||||
def _quant_worker_loop(self) -> None:
|
||||
"""Dedicated quant thread: drains (completed_modules, scatter-done event)
|
||||
batches. Errors surface via _proc_error like the scatter thread's."""
|
||||
batches. Errors surface via _proc_error like the scatter thread's.
|
||||
"""
|
||||
torch.cuda.set_device(self.device)
|
||||
q = self._quant_queue
|
||||
assert q is not None
|
||||
|
||||
@@ -206,7 +206,8 @@ class FakeRDTTensor(torch.Tensor):
|
||||
|
||||
def _meta(self) -> torch.Tensor:
|
||||
"""A zero-storage meta tensor of this fake's shape/dtype, so PyTorch
|
||||
itself computes post-op geometry rather than us reimplementing it."""
|
||||
itself computes post-op geometry rather than us reimplementing it.
|
||||
"""
|
||||
return torch.empty(self.shape, dtype=self.dtype, device="meta")
|
||||
|
||||
@classmethod
|
||||
@@ -250,7 +251,8 @@ class FakeRDTTensor(torch.Tensor):
|
||||
):
|
||||
"""Append the op and return a child, or a tuple of children for
|
||||
multi-return ops. Each child's geometry comes from running the op on a
|
||||
meta tensor."""
|
||||
meta tensor.
|
||||
"""
|
||||
meta = self_._meta()
|
||||
with torch._C.DisableTorchFunctionSubclass():
|
||||
meta_result = func(meta, *args, **kwargs)
|
||||
|
||||
@@ -132,7 +132,8 @@ PRODUCE_METHOD_NAME = "rdt_produce_weights_batched"
|
||||
@dataclass
|
||||
class _SharedPack:
|
||||
"""One generation of one slot-sharing group: the rendezvous state for a
|
||||
single chunk, identified by the consumers' issue index."""
|
||||
single chunk, identified by the consumers' issue index.
|
||||
"""
|
||||
|
||||
group: int
|
||||
slot: int
|
||||
@@ -466,7 +467,8 @@ class _RDTProducerServer:
|
||||
|
||||
def end_sync(self) -> list[int]:
|
||||
"""Block until every published group has been freed by its consumers;
|
||||
return the remaining freed keys so the engine drops its last refs."""
|
||||
return the remaining freed keys so the engine drops its last refs.
|
||||
"""
|
||||
with self._cache_cond:
|
||||
self._wait_for(
|
||||
lambda: bool(self._inflight_groups),
|
||||
@@ -479,7 +481,8 @@ class _RDTProducerServer:
|
||||
|
||||
def set_gather_error(self, message: str) -> None:
|
||||
"""Record a trainer-side gather failure so blocked serves / publishes
|
||||
stop waiting and surface it."""
|
||||
stop waiting and surface it.
|
||||
"""
|
||||
with self._cache_cond:
|
||||
self._gather_error = RuntimeError(message)
|
||||
self._cache_cond.notify_all()
|
||||
@@ -508,7 +511,8 @@ class _RDTProducerServer:
|
||||
def _new_serve_buffer(self, nbytes: int) -> torch.Tensor:
|
||||
"""Allocate + NIXL-register one serve slot: the single allocation seam,
|
||||
so registration cannot be skipped on either of the two paths that make
|
||||
buffers (the init-time reservation and the serve-path backstop)."""
|
||||
buffers (the init-time reservation and the serve-path backstop).
|
||||
"""
|
||||
t = torch.empty(nbytes, dtype=torch.uint8, device=self._serve_device)
|
||||
with self._reg_lock:
|
||||
register_nixl_memory(t)
|
||||
@@ -534,6 +538,7 @@ class _RDTProducerServer:
|
||||
Raises:
|
||||
RuntimeError: two consumers of one sharing group disagree on the
|
||||
chunks they pull from this producer.
|
||||
|
||||
"""
|
||||
sg = self._share_group(consumer_id)
|
||||
if plan_digest is not None:
|
||||
@@ -579,13 +584,15 @@ class _RDTProducerServer:
|
||||
def _share_group(self, consumer_id: int) -> int:
|
||||
"""The slot-sharing group ``consumer_id`` belongs to: its index within
|
||||
its own deployment, since that is what fixes the plan. Without a width
|
||||
the group is the consumer itself and nothing is shared."""
|
||||
the group is the consumer itself and nothing is shared.
|
||||
"""
|
||||
return consumer_id % self._share_width if self._share_width > 0 else consumer_id
|
||||
|
||||
def _sharers_of(self, sg: int) -> frozenset:
|
||||
"""The LIVE consumers of group ``sg``: the rendezvous width. Derived from
|
||||
``begin_sync``'s live set, so a degraded sync narrows the barrier instead
|
||||
of waiting forever on a dead deployment. Caller holds ``_cache_cond``."""
|
||||
of waiting forever on a dead deployment. Caller holds ``_cache_cond``.
|
||||
"""
|
||||
cached = self._sharers.get(sg)
|
||||
if cached is None:
|
||||
if not self._sharing_active or self._live_ids is None:
|
||||
@@ -621,7 +628,8 @@ class _RDTProducerServer:
|
||||
def _await_shared_pack(self, gen: _SharedPack) -> torch.Tensor:
|
||||
"""Block until this generation's packer published its blob, and return
|
||||
it. A failed pack is re-raised here, so every sharer of a bad pack fails
|
||||
instead of reading a half-written slot."""
|
||||
instead of reading a half-written slot.
|
||||
"""
|
||||
with self._cache_cond:
|
||||
self._wait_for(
|
||||
lambda: not gen.done and gen.error is None,
|
||||
@@ -642,7 +650,8 @@ class _RDTProducerServer:
|
||||
"""Group ``sg``'s ring slot ``idx``, grown if this chunk outgrew the
|
||||
reservation. Growing registers memory while the fabric is busy, the
|
||||
hazard ``reserve_serve_buffer`` exists to avoid, so it is a backstop: the
|
||||
reservation is sized from the same static plan as this pack."""
|
||||
reservation is sized from the same static plan as this pack.
|
||||
"""
|
||||
with self._serve_lock:
|
||||
buffer = self._serve_rings.setdefault(sg, [None] * self._nring)[idx]
|
||||
if buffer is not None and buffer.numel() >= need:
|
||||
@@ -654,7 +663,8 @@ class _RDTProducerServer:
|
||||
|
||||
def _fail_shared_pack(self, gen: _SharedPack, exc: BaseException) -> None:
|
||||
"""Publish a pack failure so waiting sharers raise instead of hanging on
|
||||
a generation that will never complete."""
|
||||
a generation that will never complete.
|
||||
"""
|
||||
with self._cache_cond:
|
||||
gen.error = exc
|
||||
self._cache_cond.notify_all()
|
||||
@@ -681,6 +691,7 @@ class _RDTProducerServer:
|
||||
|
||||
Raises:
|
||||
ValueError: ``seq`` was not supplied.
|
||||
|
||||
"""
|
||||
needed = sorted({n for n, _ in specs})
|
||||
if self._served_names is not None:
|
||||
@@ -855,7 +866,8 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
def _rpc(self, method: str, *args: Any) -> Any:
|
||||
"""Call one of the server actor's methods and block for the result.
|
||||
The single seam through which the engine talks to its server, so tests
|
||||
can inject a local (non-Ray) fake server."""
|
||||
can inject a local (non-Ray) fake server.
|
||||
"""
|
||||
import ray
|
||||
|
||||
return ray.get(getattr(self._server, method).remote(*args))
|
||||
@@ -932,7 +944,8 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
"""Fire publish_group WITHOUT waiting on the RPC (the gather loop
|
||||
overlaps the publish's server-side rebuild with the next group's
|
||||
gather) and return a handle that ``_await_publish`` resolves. Ray actor
|
||||
handle in production; a plain (non-Ray) fake server runs inline."""
|
||||
handle in production; a plain (non-Ray) fake server runs inline.
|
||||
"""
|
||||
method = self._server.publish_group
|
||||
remote = getattr(method, "remote", None)
|
||||
if remote is not None:
|
||||
@@ -943,7 +956,8 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
"""Resolve one async publish so a server-side rebuild error surfaces at
|
||||
window depth instead of at end_sync. Publishes carry nothing back —
|
||||
freed groups flow only through wait_freed/end_sync (one channel; see
|
||||
publish_group)."""
|
||||
publish_group).
|
||||
"""
|
||||
import ray
|
||||
|
||||
if isinstance(ref, ray.ObjectRef):
|
||||
@@ -1023,6 +1037,7 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
|
||||
Raises:
|
||||
RuntimeError: called before ``trainer_init`` cached the server names.
|
||||
|
||||
"""
|
||||
if self._server_names is None:
|
||||
raise RuntimeError(
|
||||
@@ -1129,7 +1144,8 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
by stall watchdog 300s later: consumers route pulls here, the pull
|
||||
passes the served-names guard, and the cache wait never completes. With
|
||||
it, that is an immediate error naming the weight. One set lookup per
|
||||
name per sync."""
|
||||
name per sync.
|
||||
"""
|
||||
if self._held_names is None:
|
||||
return
|
||||
for name, tensor in zip(names, tensors):
|
||||
@@ -1330,7 +1346,8 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
credit releases when every live consumer has signaled the group). So
|
||||
the loop self-paces to the consumers' pull rate with at most
|
||||
`gather_lookahead + 1` groups resident. Runs on every rank; only the
|
||||
sender has an `update_future` to fail fast on."""
|
||||
sender has an `update_future` to fail fast on.
|
||||
"""
|
||||
assert self.source is not None # guaranteed by trainer_init
|
||||
# The live IDS ride along with the count: the free barrier needs only
|
||||
# the count, but the slot-sharing rendezvous needs to know WHICH
|
||||
|
||||
@@ -68,7 +68,8 @@ class SparseNCCLTrainerInitInfo(TrainerInitInfo):
|
||||
|
||||
Same rendezvous shape as the dense NCCL backend (the sender opens its
|
||||
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
|
||||
are never packed. `backend` is the factory dispatch key."""
|
||||
are never packed. `backend` is the factory dispatch key.
|
||||
"""
|
||||
|
||||
backend: ClassVar[str] = "sparse_nccl"
|
||||
|
||||
@@ -113,8 +114,7 @@ class SparseNCCLWeightTransferUpdateInfo(WeightTransferUpdateInfo):
|
||||
class SparseNCCLWeightTransferEngine(
|
||||
WeightTransferEngine[NCCLWeightTransferInitInfo, SparseNCCLWeightTransferUpdateInfo]
|
||||
):
|
||||
"""
|
||||
Sparse weight transfer engine using NCCL.
|
||||
"""Sparse weight transfer engine using NCCL.
|
||||
|
||||
Receives checkpoint-coordinate patches broadcast from the trainer and applies
|
||||
them through the model's native weight loader. Sparse updates modify initialized
|
||||
@@ -368,7 +368,8 @@ class SparseNCCLTrainerWeightTransferEngine(
|
||||
rebuild or free the patch tensors as soon as a send method returns rather
|
||||
than relying on same-stream ordering. See
|
||||
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
|
||||
cross-rank barrier."""
|
||||
cross-rank barrier.
|
||||
"""
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user