[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 10:57:24 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 1085b64425
commit 651a88c09a
693 changed files with 2088 additions and 4171 deletions
@@ -156,8 +156,7 @@ def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor)
def should_nccl_symm_mem_ag_rs() -> bool:
"""Check whether NCCL symmetric memory should be used for
AllGather / ReduceScatter collectives.
"""
AllGather / ReduceScatter collectives."""
from vllm.distributed.device_communicators.pynccl_allocator import (
is_symmetric_memory_enabled,
)
@@ -650,8 +650,7 @@ class MessageQueue:
def shutdown(self):
"""If this is an idle reader, wakes it up so it can clean up and shut
down
"""
down"""
self.shutting_down = True
if self._spin_condition is not None:
self._spin_condition.cancel()
@@ -761,8 +760,7 @@ class MessageQueue:
def should_warn(self) -> bool:
"""Returns true if it's time to log a warning for a timeout that is not
indefinite
"""
indefinite"""
if self._should_warn:
elapsed = time.monotonic() - self.started
if elapsed >= VLLM_RINGBUFFER_WARNING_INTERVAL * self.n_warning:
@@ -15,8 +15,7 @@ from collections import OrderedDict
class CacheEntry:
"""A single cache entry. Read `.block_ids` and `.ready` freely;
mutations only through EmbeddingCache methods.
"""
mutations only through EmbeddingCache methods."""
__slots__ = ("block_ids", "_pin_count")
+2 -4
View File
@@ -84,8 +84,7 @@ class EplbCommunicator(ABC):
@property
def needs_profile_buffer_reservation(self) -> bool:
"""Whether the profile path must run a dummy collective operation to reserve
communication buffers.
"""
communication buffers."""
return True
def set_stream(self, cuda_stream: torch.cuda.Stream | None) -> None:
@@ -443,8 +442,7 @@ class NixlEplbCommunicator(EplbCommunicator):
def _exchange_remote_send_meta(self) -> None:
"""Exchange per-layer per-tensor metadata so receivers can compute
remote RDMA addresses at transfer time.
"""
remote RDMA addresses at transfer time."""
local_meta: dict[tuple[int, int], tuple[int, int, int]] = {}
for layer_idx, layer_tensors in enumerate(self._all_expert_weights):
for t_idx, t in enumerate(layer_tensors):
@@ -51,8 +51,7 @@ def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
class KVOutputAggregator:
"""Utility class to aggregate the output of all workers into a single
output corresponding to Rank 0 for scheduler.
"""
output corresponding to Rank 0 for scheduler."""
def __init__(self, expected_finished_count: int):
# Complete transfer tracker. Used to track finished requests
@@ -278,8 +278,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
def _on_write_done(self, req_id: str, future: Future) -> None:
"""Surface any exception from the disk-write thread and drop the
completed future from the in-flight tracking dict.
"""
completed future from the in-flight tracking dict."""
self._req_futures.pop(req_id, None)
exc = future.exception()
if exc is not None:
@@ -231,8 +231,7 @@ class LMCacheMPRequestTracker:
####
def needs_retrieve(self) -> bool:
"""Check whether the current request needs retrieve, will be used
update_stage_after_alloc
"""
update_stage_after_alloc"""
return (
self.num_lmcache_hit_blocks > self.num_vllm_hit_blocks
and self.state != LMCacheMPRequestState.READY
@@ -240,8 +239,7 @@ class LMCacheMPRequestTracker:
def is_ready_for_retrieving(self) -> bool:
"""Check whether the current request is ready for retrieving,
will be used in process_loading_requests
"""
will be used in process_loading_requests"""
return (
self.state == LMCacheMPRequestState.WAITING_FOR_LOAD
and self.needs_retrieve()
@@ -718,8 +718,7 @@ class MooncakeConnectorScheduler:
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
"""D-side only. Returns N-1 for Mamba models since the decoder
always recomputes the last token and must start from h(N-1).
"""
always recomputes the last token and must start from h(N-1)."""
if self._has_mamba and num_prompt_tokens > 1:
return num_prompt_tokens - 1
return num_prompt_tokens
@@ -730,8 +729,7 @@ class MooncakeConnectorScheduler:
derive h(N) correctly.
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
request is preempted and rescheduled.
"""
request is preempted and rescheduled."""
params = request.kv_transfer_params
if (
params is not None
@@ -1677,8 +1675,7 @@ class MooncakeConnectorWorker:
def _bind_sender_thread_device(self) -> None:
"""ThreadPoolExecutor initializer — binds each pool thread to the
correct CUDA device. CUDA device selection is thread-local, so
without this, NVLink transfers fail for TP ranks > 0.
"""
without this, NVLink transfers fail for TP ranks > 0."""
current_platform.set_device(self.device_id)
def _send_blocks(
@@ -1904,8 +1901,7 @@ class MooncakeConnectorWorker:
def get_kv_connector_stats(self) -> KVConnectorStats | None:
"""Return transfer stats collected since the last call, or None
if nothing has been recorded in this interval.
"""
if nothing has been recorded in this interval."""
if self.xfer_stats.is_empty():
return None
return self.xfer_stats.clone_and_reset()
@@ -73,8 +73,7 @@ class ExternalCachedBlockPool:
class MooncakeStoreCoordinator:
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
``ExternalCachedBlockPool``.
"""
``ExternalCachedBlockPool``."""
def __init__(
self,
@@ -2629,8 +2629,7 @@ class LookupKeyClient:
non_block: bool = False,
) -> MooncakeLookupResult | None:
"""If non_block is True, will return None until the result is ready,
so the caller retries on a later step.
"""
so the caller retries on a later step."""
future = self.futures.get(req_id)
if future is None:
future = self.executor.submit(self._lookup, num_tokens, list(block_hashes))
@@ -548,8 +548,7 @@ class MoRIIOConnectorScheduler:
self, request_id: ReqId, transfer_id: TransferId | None = None
):
"""Unmap request_id/transfer_id. Uses transfer_id for lookup if
exact request_id match fails (handles input_processor mutation).
"""
exact request_id match fails (handles input_processor mutation)."""
if request_id in self.request_id_to_transfer_id:
tid = self.request_id_to_transfer_id[request_id]
del self.request_id_to_transfer_id[request_id]
@@ -400,8 +400,7 @@ class NixlBaseConnectorScheduler:
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
"""D-side only. The number of prompt tokens to load from the prefiller.
Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder
will recompute locally.
"""
will recompute locally."""
backoff = self._prefill_backoff()
if backoff and num_prompt_tokens > backoff:
return num_prompt_tokens - backoff
@@ -414,8 +413,7 @@ class NixlBaseConnectorScheduler:
multi-module MTP it is the drafter's whole lookahead window.
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
request is preempted and rescheduled.
"""
request is preempted and rescheduled."""
backoff = self._prefill_backoff()
params = request.kv_transfer_params
if (
@@ -1873,8 +1873,7 @@ class NixlBaseConnectorWorker:
) -> np.ndarray:
"""Build remote desc regions (conv sub-projections + ssm) per layer.
For hetero-TP, each D rank reads only its sub-projection slice from
the P rank. Returns an Nx3 uint64 array.
"""
the P rank. Returns an Nx3 uint64 array."""
assert nixl_agent_meta.kv_caches_base_addr, (
"Remote KV cache base addresses must not be empty."
)
@@ -2657,8 +2656,7 @@ class NixlBaseConnectorWorker:
@cached_property
def _attention_kv_caches(self) -> list[torch.Tensor]:
"""Device KV caches of attention layers (mamba states excluded),
as consumed by the receive post-process.
"""
as consumed by the receive post-process."""
assert self.device_kv_caches, (
"_attention_kv_caches accessed before register_kv_caches"
)
@@ -315,8 +315,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
executes on the writer; the handshake runs on the background executor
and the request is re-queued onto ``_reg_send_inbox`` once it
completes (at which point ``_ensure_handshake`` returns ``None`` and we
send directly).
"""
send directly)."""
remote_pp_size = reg_data.get("remote_pp_size", 1)
fut = self._ensure_handshake(
reg_data["remote_engine_id"],
@@ -508,8 +507,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
``BlockIds`` is canonically a tuple of per-group lists, but some
registration payloads collapse a single-group case to a flat
list. Re-wrap that case so downstream group-aware helpers see a
consistent shape.
"""
consistent shape."""
if block_ids and not isinstance(block_ids[0], (list, tuple)):
return (list(block_ids),)
return block_ids
@@ -37,8 +37,7 @@ def canonical_format_id(kv_cache_layout: str) -> str:
Canonical pages keep the worker's KV layout family, so the id couples the
format version with that family; consumers must match it exactly. The family
keeps its historical NHD/HND spelling so ids stay stable for KV persisted
before the layout enum existed.
"""
before the layout enum existed."""
layout = KVCacheLayout[kv_cache_layout]
legacy = {KVCacheLayout.LBNHC: "nhd", KVCacheLayout.LBHNC: "hnd"}
family = legacy.get(layout, layout.name.lower())
@@ -122,8 +121,7 @@ def _interleave_cp_tokens(
ctx: _RankContext,
) -> tuple[CopyRun, ...]:
"""Place each region's num_tokens rows at their canonical token positions,
one run per chunk of interleaved tokens.
"""
one run per chunk of interleaved tokens."""
runs: list[CopyRun] = []
for region in regions:
if ctx.cp_size == 1:
@@ -258,8 +256,7 @@ def _attention_byte_regions(
cp_size: int,
) -> list[ByteRegion] | None:
"""Byte regions of an attention page, given this rank's head shard.
None when the physical layout is not recognized (fail closed).
"""
None when the physical layout is not recognized (fail closed)."""
bs, heads, head_size = spec.block_size, spec.num_kv_heads, spec.head_size
if tuple(kv_cache.shape) == (num_blocks, heads, bs, 2 * head_size):
return _packed_kv_regions(kv_cache, spec, head_shard, num_head_shards, cp_size)
@@ -373,8 +370,7 @@ def _is_exact_partition(intervals: list[tuple[int, int]], size: int) -> bool:
def _verify_tiling(layer_name: str, per_rank: list[CanonicalPageMapping]) -> None:
"""Whichever ranks a block elects as writers must tile the canonical page
exactly once, and each rank's runs must cover exactly its local page.
"""
exactly once, and each rank's runs must cover exactly its local page."""
size = per_rank[0].canonical_page_size_bytes
num_writers = per_rank[0].num_writers
for mapping in per_rank:
@@ -79,8 +79,7 @@ def get_offloading_event_group_spec(
class _OffloadEventMetadata:
"""BlockStored payload snapshot for one OffloadKey, captured while the
Request is available and kept until the final matching removal event.
``medium`` and ``ownership`` are forwarded from the OffloadingEvent.
"""
``medium`` and ``ownership`` are forwarded from the OffloadingEvent."""
# The chunk's constituent block hashes; the last one is the OffloadKey.
block_hashes: tuple[BlockHash, ...]
@@ -256,8 +255,7 @@ class OffloadingEventsTracker:
def reset(self) -> None:
"""Drop all tracked state; pending payloads are stale after a
manager cache reset.
"""
manager cache reset."""
self._pending_event_metadata.clear()
def _build_event_metadata(
@@ -268,8 +266,7 @@ class OffloadingEventsTracker:
) -> _OffloadEventMetadata:
"""Build the payload snapshot for one offloaded chunk: its
constituent per-block hashes, the whole chunk's tokens, and the
per-block ``block_size``.
"""
per-block ``block_size``."""
hashes_per_chunk = group_config.hashes_per_chunk
assert hashes_per_chunk > 0
assert chunk_idx >= 0
@@ -676,8 +676,7 @@ class OffloadingConnectorScheduler:
start_chunk_idx: int,
) -> int | None:
"""Return the number of consecutive offloaded chunks from the start,
or None if the backend deferred a lookup.
"""
or None if the backend deferred a lookup."""
hit_count = 0
defer_lookup = False
for local_idx, key in enumerate(keys):
@@ -712,8 +711,7 @@ class OffloadingConnectorScheduler:
"""Return the end index (in `keys`) of the last run of
`sliding_window_size` consecutive hits, scanning from the end.
The first run may need a larger window for a partial rightmost chunk.
Returns 0 on miss, None if the backend deferred a lookup.
"""
Returns 0 on miss, None if the backend deferred a lookup."""
defer_lookup = False
pending_in_window = False
consecutive_hits = 0
+1 -2
View File
@@ -59,8 +59,7 @@ def ensure_divisibility(numerator, denominator):
def divide(numerator, denominator):
"""Ensure that numerator is divisible by the denominator and return
the division value.
"""
the division value."""
ensure_divisibility(numerator, denominator)
return numerator // denominator
+1 -2
View File
@@ -141,8 +141,7 @@ class WeightTransferTrainerFactory:
class_name: str | None = None,
) -> None:
"""Register a trainer engine. Same conventions as
`WeightTransferEngineFactory.register_engine`.
"""
`WeightTransferEngineFactory.register_engine`."""
if name in cls._registry:
raise ValueError(
f"Weight transfer trainer engine '{name}' is already registered."
@@ -40,8 +40,7 @@ class IPCWeightTransferInitInfo(WeightTransferInitInfo):
`packed` is a must-agree wire param: the trainer ships it here at the init
handshake so the worker decodes with the same setting the trainer encoded
with. The consumer rebuilds from the IPC handle + `tensor_sizes`, so it does
not need the buffer size (producer-only).
"""
not need the buffer size (producer-only)."""
packed: bool = False
@@ -54,8 +53,7 @@ class IPCTrainerInitInfo(TrainerInitInfo):
`packed` / `packed_buffer_size_bytes` are the transfer's wire params. The
trainer propagates `packed` to the worker at `trainer_init` so the two sides
cannot disagree. `backend` is the factory dispatch key.
"""
cannot disagree. `backend` is the factory dispatch key."""
backend: ClassVar[str] = "ipc"
packed: bool = False
@@ -118,8 +118,7 @@ def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
only the mode actually in use. Shared by the dense and sparse trainer
engines so the two cannot drift.
"""
engines so the two cannot drift."""
return {key: value for key, value in asdict(init_info).items() if value is not None}
@@ -69,8 +69,7 @@ class NCCLTrainerInitInfo(TrainerInitInfo):
propagates them to the worker at `trainer_init` so the two sides cannot
disagree. Note this defaults to packed, unlike the worker-side
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
ships a value. `backend` is the factory dispatch key.
"""
ships a value. `backend` is the factory dispatch key."""
backend: ClassVar[str] = "nccl"
@@ -365,8 +364,7 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
"""Iterate the source (materializing each tensor — a collective on all
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
Non-sender ranks only replay the iteration to stay in the collective.
"""
Non-sender ranks only replay the iteration to stay in the collective."""
if not self.is_sender:
for _ in source:
pass
@@ -214,8 +214,7 @@ class RdtRouter:
def class_of(self, name: str) -> int:
"""The name's owner class — the planner's bucketing key, since all names
of a chunk must share one producer.
"""
of a chunk must share one producer."""
return self._class_of.get(name, 0)
def owners(self, name: str) -> list[int]:
@@ -286,8 +285,7 @@ class RdtRouter:
def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None:
"""Attach this consumer's producer handles. Rebuilt wholesale per init,
never appended to: every owner index is a position in these lists, so a
rejoining engine that re-inits must not shift them.
"""
rejoining engine that re-inits must not shift them."""
if len(actors) != len(produce_methods):
raise ValueError("actors and produce_methods must be parallel")
self._actors = list(actors)
@@ -350,6 +348,5 @@ def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int:
optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is
allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a
perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers
and the producer's serve rings.
"""
and the producer's serve rings."""
return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20))
@@ -134,8 +134,7 @@ class _PendingPull:
``targets``/``blob`` must stay strongly referenced until it completes:
``set_target_for_ref`` stores WEAKREFS, so dropping them silently reroutes the
transfer into a fallback buffer.
"""
transfer into a fallback buffer."""
ref: "Any"
keys: "list[FetchKey]"
@@ -562,8 +561,7 @@ class ShardedRDTWeightTransferEngine(
def _build_static_plan(self, init_info: ShardedRDTWeightTransferInitInfo) -> None:
"""Build the chunk/free plan once. It never changes across syncs, so
``update_weights`` needs no per-sync names.
"""
``update_weights`` needs no per-sync names."""
if not init_info.group_lens:
raise ValueError(
"Sharded RDT engine requires init_info.group_lens (the gather-"
@@ -587,8 +585,7 @@ class ShardedRDTWeightTransferEngine(
Both sides are sized from the static plan: receive buffers are
``ring_depth`` slots at the largest chunk's ``pack_bytes``, and each bound
producer is asked to pre-register a serve ring at the max bytes this
consumer will pull from it. A no-op when this worker has no chunks.
"""
consumer will pull from it. A no-op when this worker has no chunks."""
plan = self._cached_plan
if plan is None or not plan.chunks:
return
@@ -633,8 +630,7 @@ class ShardedRDTWeightTransferEngine(
``data_parallel_index``, not ``data_parallel_rank``: vLLM resets the
latter to 0 in a dense worker but keeps the former as the distinct global
DP rank. Same formula as the sibling ``nccl_engine``, so dense-via-TP and
MoE-via-DP+EP both yield distinct 0..C-1.
"""
MoE-via-DP+EP both yield distinct 0..C-1."""
pc = self.parallel_config
return pc.data_parallel_index * pc.world_size + pc.rank # world_size = TP*PP
@@ -642,16 +638,14 @@ class ShardedRDTWeightTransferEngine(
"""Total inference-worker count. Prefers the driver-supplied
``init_info.num_consumers`` (authoritative -- the driver knows the whole
fleet); else ``world_size_across_dp``, the same stride
``_global_worker_index`` indexes with, so the two agree at any pp.
"""
``_global_worker_index`` indexes with, so the two agree at any pp."""
if self._num_consumers_override > 0:
return self._num_consumers_override
return self.parallel_config.world_size_across_dp
def start_weight_update(self) -> None:
"""Put the model's params on meta so layerwise reload streams them in
as each layer's slices land. Baked replay uses checkpoint format.
"""
as each layer's slices land. Baked replay uses checkpoint format."""
from vllm.model_executor.model_loader.reload import (
initialize_layerwise_reload,
)
@@ -660,8 +654,7 @@ class ShardedRDTWeightTransferEngine(
def finish_weight_update(self) -> None:
"""Drain the deferred pull/process pipeline (so every layer is fully
loaded) before finalizing the layerwise reload.
"""
loaded) before finalizing the layerwise reload."""
from vllm.model_executor.model_loader.reload import (
finalize_layerwise_reload,
)
@@ -674,8 +667,7 @@ class ShardedRDTWeightTransferEngine(
device sync: post-processing is deferred to background threads and a
sync here would block on them and serialize the pull/process pipeline.
Completion is guaranteed by ``drain_pending`` in
``finish_weight_update``.
"""
``finish_weight_update``."""
self.receive_weights(self.parse_update_info(update_info))
def receive_weights(
@@ -717,8 +709,7 @@ class ShardedRDTWeightTransferEngine(
device: torch.device,
) -> list[tuple[str, torch.Tensor]]:
"""Zero-storage lazies for ``names``, dtype/shape from the init metadata,
all feeding the bake's recording sink.
"""
all feeding the bake's recording sink."""
return [
(
name,
@@ -986,8 +977,7 @@ class ShardedRDTWeightTransferEngine(
def _ensure_proc_worker(self) -> None:
"""Lazily create the per-slot events, the background CUDA stream, the work
queue, and the single processing thread. Idempotent.
"""
queue, and the single processing thread. Idempotent."""
if self._proc_thread is not None:
return
import queue
@@ -1046,8 +1036,7 @@ class ShardedRDTWeightTransferEngine(
def _mark_slot_done(self, slot: int) -> None:
"""Publish that a queued item's read-done event has been recorded (or the
item failed) so a pull waiting to reuse ``slot`` can proceed to its
CUDA-event synchronize.
"""
CUDA-event synchronize."""
with self._slot_cv:
self._slot_done[slot] += 1
self._slot_cv.notify_all()
@@ -1063,8 +1052,7 @@ class ShardedRDTWeightTransferEngine(
"""Block until the background thread has processed every queued item and
its stream work is complete, then re-raise any error it hit. Called from
the worker's ``finish_weight_update`` before ``finalize_layerwise_reload``
so every baked layer is fully loaded (and ``info.reset()``-ed) first.
"""
so every baked layer is fully loaded (and ``info.reset()``-ed) first."""
if self._proc_queue is not None:
self._proc_queue.join() # every put() item task_done()'d
# The scatter thread feeds the quant thread, so join it SECOND (all
@@ -1116,8 +1104,7 @@ class ShardedRDTWeightTransferEngine(
which is exactly what an owner class is. The cut is a pure function of
the bake and the ownership table. Copy order within a chunk is bake
order, and a module's copies may span chunks (materialize/quant fire on
its first/last chunk; see ``_build_call_plan``).
"""
its first/last chunk; see ``_build_call_plan``)."""
assert self._router is not None
by_class: dict[int, list[_Scatter]] = {}
for copies in modules:
@@ -1248,8 +1235,7 @@ class ShardedRDTWeightTransferEngine(
pulls nothing from. Refs are held and drained in ``drain_pending`` so
every signal has EXECUTED before the sync ends: ``begin_sync`` resets
the counters, and a straggler landing in the next sync would credit a
group it does not belong to.
"""
group it does not belong to."""
assert self._router is not None
self._pending_frees.extend(self._router.free_group(group_idx))
@@ -1384,8 +1370,7 @@ class ShardedRDTWeightTransferEngine(
the modules' scatters via ``ready``; touches only the scattered params
(never a receive slot), so it can overlap subsequent chunks' RDMA and
scatters. ``info.reset()`` is what makes finalize skip the layer —
drain_pending joins the quant queue before finalize runs.
"""
drain_pending joins the quant queue before finalize runs."""
from vllm.model_executor.layers.quantization.base_config import (
QuantizeMethodBase,
)
@@ -1418,8 +1403,7 @@ class ShardedRDTWeightTransferEngine(
def _quant_worker_loop(self) -> None:
"""Dedicated quant thread: drains (completed_modules, scatter-done event)
batches. Errors surface via _proc_error like the scatter thread's.
"""
batches. Errors surface via _proc_error like the scatter thread's."""
torch.cuda.set_device(self.device)
q = self._quant_queue
assert q is not None
@@ -206,8 +206,7 @@ class FakeRDTTensor(torch.Tensor):
def _meta(self) -> torch.Tensor:
"""A zero-storage meta tensor of this fake's shape/dtype, so PyTorch
itself computes post-op geometry rather than us reimplementing it.
"""
itself computes post-op geometry rather than us reimplementing it."""
return torch.empty(self.shape, dtype=self.dtype, device="meta")
@classmethod
@@ -251,8 +250,7 @@ class FakeRDTTensor(torch.Tensor):
):
"""Append the op and return a child, or a tuple of children for
multi-return ops. Each child's geometry comes from running the op on a
meta tensor.
"""
meta tensor."""
meta = self_._meta()
with torch._C.DisableTorchFunctionSubclass():
meta_result = func(meta, *args, **kwargs)
@@ -132,8 +132,7 @@ PRODUCE_METHOD_NAME = "rdt_produce_weights_batched"
@dataclass
class _SharedPack:
"""One generation of one slot-sharing group: the rendezvous state for a
single chunk, identified by the consumers' issue index.
"""
single chunk, identified by the consumers' issue index."""
group: int
slot: int
@@ -467,8 +466,7 @@ class _RDTProducerServer:
def end_sync(self) -> list[int]:
"""Block until every published group has been freed by its consumers;
return the remaining freed keys so the engine drops its last refs.
"""
return the remaining freed keys so the engine drops its last refs."""
with self._cache_cond:
self._wait_for(
lambda: bool(self._inflight_groups),
@@ -481,8 +479,7 @@ class _RDTProducerServer:
def set_gather_error(self, message: str) -> None:
"""Record a trainer-side gather failure so blocked serves / publishes
stop waiting and surface it.
"""
stop waiting and surface it."""
with self._cache_cond:
self._gather_error = RuntimeError(message)
self._cache_cond.notify_all()
@@ -511,8 +508,7 @@ class _RDTProducerServer:
def _new_serve_buffer(self, nbytes: int) -> torch.Tensor:
"""Allocate + NIXL-register one serve slot: the single allocation seam,
so registration cannot be skipped on either of the two paths that make
buffers (the init-time reservation and the serve-path backstop).
"""
buffers (the init-time reservation and the serve-path backstop)."""
t = torch.empty(nbytes, dtype=torch.uint8, device=self._serve_device)
with self._reg_lock:
register_nixl_memory(t)
@@ -584,15 +580,13 @@ class _RDTProducerServer:
def _share_group(self, consumer_id: int) -> int:
"""The slot-sharing group ``consumer_id`` belongs to: its index within
its own deployment, since that is what fixes the plan. Without a width
the group is the consumer itself and nothing is shared.
"""
the group is the consumer itself and nothing is shared."""
return consumer_id % self._share_width if self._share_width > 0 else consumer_id
def _sharers_of(self, sg: int) -> frozenset:
"""The LIVE consumers of group ``sg``: the rendezvous width. Derived from
``begin_sync``'s live set, so a degraded sync narrows the barrier instead
of waiting forever on a dead deployment. Caller holds ``_cache_cond``.
"""
of waiting forever on a dead deployment. Caller holds ``_cache_cond``."""
cached = self._sharers.get(sg)
if cached is None:
if not self._sharing_active or self._live_ids is None:
@@ -628,8 +622,7 @@ class _RDTProducerServer:
def _await_shared_pack(self, gen: _SharedPack) -> torch.Tensor:
"""Block until this generation's packer published its blob, and return
it. A failed pack is re-raised here, so every sharer of a bad pack fails
instead of reading a half-written slot.
"""
instead of reading a half-written slot."""
with self._cache_cond:
self._wait_for(
lambda: not gen.done and gen.error is None,
@@ -650,8 +643,7 @@ class _RDTProducerServer:
"""Group ``sg``'s ring slot ``idx``, grown if this chunk outgrew the
reservation. Growing registers memory while the fabric is busy, the
hazard ``reserve_serve_buffer`` exists to avoid, so it is a backstop: the
reservation is sized from the same static plan as this pack.
"""
reservation is sized from the same static plan as this pack."""
with self._serve_lock:
buffer = self._serve_rings.setdefault(sg, [None] * self._nring)[idx]
if buffer is not None and buffer.numel() >= need:
@@ -663,8 +655,7 @@ class _RDTProducerServer:
def _fail_shared_pack(self, gen: _SharedPack, exc: BaseException) -> None:
"""Publish a pack failure so waiting sharers raise instead of hanging on
a generation that will never complete.
"""
a generation that will never complete."""
with self._cache_cond:
gen.error = exc
self._cache_cond.notify_all()
@@ -866,8 +857,7 @@ class ShardedRDTTrainerWeightTransferEngine(
def _rpc(self, method: str, *args: Any) -> Any:
"""Call one of the server actor's methods and block for the result.
The single seam through which the engine talks to its server, so tests
can inject a local (non-Ray) fake server.
"""
can inject a local (non-Ray) fake server."""
import ray
return ray.get(getattr(self._server, method).remote(*args))
@@ -944,8 +934,7 @@ class ShardedRDTTrainerWeightTransferEngine(
"""Fire publish_group WITHOUT waiting on the RPC (the gather loop
overlaps the publish's server-side rebuild with the next group's
gather) and return a handle that ``_await_publish`` resolves. Ray actor
handle in production; a plain (non-Ray) fake server runs inline.
"""
handle in production; a plain (non-Ray) fake server runs inline."""
method = self._server.publish_group
remote = getattr(method, "remote", None)
if remote is not None:
@@ -956,8 +945,7 @@ class ShardedRDTTrainerWeightTransferEngine(
"""Resolve one async publish so a server-side rebuild error surfaces at
window depth instead of at end_sync. Publishes carry nothing back —
freed groups flow only through wait_freed/end_sync (one channel; see
publish_group).
"""
publish_group)."""
import ray
if isinstance(ref, ray.ObjectRef):
@@ -1144,8 +1132,7 @@ class ShardedRDTTrainerWeightTransferEngine(
by stall watchdog 300s later: consumers route pulls here, the pull
passes the served-names guard, and the cache wait never completes. With
it, that is an immediate error naming the weight. One set lookup per
name per sync.
"""
name per sync."""
if self._held_names is None:
return
for name, tensor in zip(names, tensors):
@@ -1346,8 +1333,7 @@ class ShardedRDTTrainerWeightTransferEngine(
credit releases when every live consumer has signaled the group). So
the loop self-paces to the consumers' pull rate with at most
`gather_lookahead + 1` groups resident. Runs on every rank; only the
sender has an `update_future` to fail fast on.
"""
sender has an `update_future` to fail fast on."""
assert self.source is not None # guaranteed by trainer_init
# The live IDS ride along with the count: the free barrier needs only
# the count, but the slot-sharing rendezvous needs to know WHICH
@@ -68,8 +68,7 @@ class SparseNCCLTrainerInitInfo(TrainerInitInfo):
Same rendezvous shape as the dense NCCL backend (the sender opens its
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
are never packed. `backend` is the factory dispatch key.
"""
are never packed. `backend` is the factory dispatch key."""
backend: ClassVar[str] = "sparse_nccl"
@@ -368,8 +367,7 @@ class SparseNCCLTrainerWeightTransferEngine(
rebuild or free the patch tensors as soon as a send method returns rather
than relying on same-stream ordering. See
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
cross-rank barrier.
"""
cross-rank barrier."""
if torch.cuda.is_available():
torch.cuda.current_stream().synchronize()