[CI] Ignore ruff D209, rejoin the docstrings it split, and silence incompatible-rule warnings (#57212)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
1085b64425
commit
651a88c09a
@@ -156,8 +156,7 @@ def should_nccl_symm_mem_allreduce(world_size: int, input_tensor: torch.Tensor)
|
||||
|
||||
def should_nccl_symm_mem_ag_rs() -> bool:
|
||||
"""Check whether NCCL symmetric memory should be used for
|
||||
AllGather / ReduceScatter collectives.
|
||||
"""
|
||||
AllGather / ReduceScatter collectives."""
|
||||
from vllm.distributed.device_communicators.pynccl_allocator import (
|
||||
is_symmetric_memory_enabled,
|
||||
)
|
||||
|
||||
@@ -650,8 +650,7 @@ class MessageQueue:
|
||||
|
||||
def shutdown(self):
|
||||
"""If this is an idle reader, wakes it up so it can clean up and shut
|
||||
down
|
||||
"""
|
||||
down"""
|
||||
self.shutting_down = True
|
||||
if self._spin_condition is not None:
|
||||
self._spin_condition.cancel()
|
||||
@@ -761,8 +760,7 @@ class MessageQueue:
|
||||
|
||||
def should_warn(self) -> bool:
|
||||
"""Returns true if it's time to log a warning for a timeout that is not
|
||||
indefinite
|
||||
"""
|
||||
indefinite"""
|
||||
if self._should_warn:
|
||||
elapsed = time.monotonic() - self.started
|
||||
if elapsed >= VLLM_RINGBUFFER_WARNING_INTERVAL * self.n_warning:
|
||||
|
||||
@@ -15,8 +15,7 @@ from collections import OrderedDict
|
||||
|
||||
class CacheEntry:
|
||||
"""A single cache entry. Read `.block_ids` and `.ready` freely;
|
||||
mutations only through EmbeddingCache methods.
|
||||
"""
|
||||
mutations only through EmbeddingCache methods."""
|
||||
|
||||
__slots__ = ("block_ids", "_pin_count")
|
||||
|
||||
|
||||
@@ -84,8 +84,7 @@ class EplbCommunicator(ABC):
|
||||
@property
|
||||
def needs_profile_buffer_reservation(self) -> bool:
|
||||
"""Whether the profile path must run a dummy collective operation to reserve
|
||||
communication buffers.
|
||||
"""
|
||||
communication buffers."""
|
||||
return True
|
||||
|
||||
def set_stream(self, cuda_stream: torch.cuda.Stream | None) -> None:
|
||||
@@ -443,8 +442,7 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
|
||||
def _exchange_remote_send_meta(self) -> None:
|
||||
"""Exchange per-layer per-tensor metadata so receivers can compute
|
||||
remote RDMA addresses at transfer time.
|
||||
"""
|
||||
remote RDMA addresses at transfer time."""
|
||||
local_meta: dict[tuple[int, int], tuple[int, int, int]] = {}
|
||||
for layer_idx, layer_tensors in enumerate(self._all_expert_weights):
|
||||
for t_idx, t in enumerate(layer_tensors):
|
||||
|
||||
@@ -51,8 +51,7 @@ def get_kv_connector_cache_layout(vllm_config: VllmConfig | None = None):
|
||||
|
||||
class KVOutputAggregator:
|
||||
"""Utility class to aggregate the output of all workers into a single
|
||||
output corresponding to Rank 0 for scheduler.
|
||||
"""
|
||||
output corresponding to Rank 0 for scheduler."""
|
||||
|
||||
def __init__(self, expected_finished_count: int):
|
||||
# Complete transfer tracker. Used to track finished requests
|
||||
|
||||
@@ -278,8 +278,7 @@ class ExampleHiddenStatesConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
def _on_write_done(self, req_id: str, future: Future) -> None:
|
||||
"""Surface any exception from the disk-write thread and drop the
|
||||
completed future from the in-flight tracking dict.
|
||||
"""
|
||||
completed future from the in-flight tracking dict."""
|
||||
self._req_futures.pop(req_id, None)
|
||||
exc = future.exception()
|
||||
if exc is not None:
|
||||
|
||||
@@ -231,8 +231,7 @@ class LMCacheMPRequestTracker:
|
||||
####
|
||||
def needs_retrieve(self) -> bool:
|
||||
"""Check whether the current request needs retrieve, will be used
|
||||
update_stage_after_alloc
|
||||
"""
|
||||
update_stage_after_alloc"""
|
||||
return (
|
||||
self.num_lmcache_hit_blocks > self.num_vllm_hit_blocks
|
||||
and self.state != LMCacheMPRequestState.READY
|
||||
@@ -240,8 +239,7 @@ class LMCacheMPRequestTracker:
|
||||
|
||||
def is_ready_for_retrieving(self) -> bool:
|
||||
"""Check whether the current request is ready for retrieving,
|
||||
will be used in process_loading_requests
|
||||
"""
|
||||
will be used in process_loading_requests"""
|
||||
return (
|
||||
self.state == LMCacheMPRequestState.WAITING_FOR_LOAD
|
||||
and self.needs_retrieve()
|
||||
|
||||
@@ -718,8 +718,7 @@ class MooncakeConnectorScheduler:
|
||||
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
"""D-side only. Returns N-1 for Mamba models since the decoder
|
||||
always recomputes the last token and must start from h(N-1).
|
||||
"""
|
||||
always recomputes the last token and must start from h(N-1)."""
|
||||
if self._has_mamba and num_prompt_tokens > 1:
|
||||
return num_prompt_tokens - 1
|
||||
return num_prompt_tokens
|
||||
@@ -730,8 +729,7 @@ class MooncakeConnectorScheduler:
|
||||
derive h(N) correctly.
|
||||
|
||||
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
|
||||
request is preempted and rescheduled.
|
||||
"""
|
||||
request is preempted and rescheduled."""
|
||||
params = request.kv_transfer_params
|
||||
if (
|
||||
params is not None
|
||||
@@ -1677,8 +1675,7 @@ class MooncakeConnectorWorker:
|
||||
def _bind_sender_thread_device(self) -> None:
|
||||
"""ThreadPoolExecutor initializer — binds each pool thread to the
|
||||
correct CUDA device. CUDA device selection is thread-local, so
|
||||
without this, NVLink transfers fail for TP ranks > 0.
|
||||
"""
|
||||
without this, NVLink transfers fail for TP ranks > 0."""
|
||||
current_platform.set_device(self.device_id)
|
||||
|
||||
def _send_blocks(
|
||||
@@ -1904,8 +1901,7 @@ class MooncakeConnectorWorker:
|
||||
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
"""Return transfer stats collected since the last call, or None
|
||||
if nothing has been recorded in this interval.
|
||||
"""
|
||||
if nothing has been recorded in this interval."""
|
||||
if self.xfer_stats.is_empty():
|
||||
return None
|
||||
return self.xfer_stats.clone_and_reset()
|
||||
|
||||
@@ -73,8 +73,7 @@ class ExternalCachedBlockPool:
|
||||
|
||||
class MooncakeStoreCoordinator:
|
||||
"""Mirror of ``HybridKVCacheCoordinator.find_longest_cache_hit`` over an
|
||||
``ExternalCachedBlockPool``.
|
||||
"""
|
||||
``ExternalCachedBlockPool``."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -2629,8 +2629,7 @@ class LookupKeyClient:
|
||||
non_block: bool = False,
|
||||
) -> MooncakeLookupResult | None:
|
||||
"""If non_block is True, will return None until the result is ready,
|
||||
so the caller retries on a later step.
|
||||
"""
|
||||
so the caller retries on a later step."""
|
||||
future = self.futures.get(req_id)
|
||||
if future is None:
|
||||
future = self.executor.submit(self._lookup, num_tokens, list(block_hashes))
|
||||
|
||||
@@ -548,8 +548,7 @@ class MoRIIOConnectorScheduler:
|
||||
self, request_id: ReqId, transfer_id: TransferId | None = None
|
||||
):
|
||||
"""Unmap request_id/transfer_id. Uses transfer_id for lookup if
|
||||
exact request_id match fails (handles input_processor mutation).
|
||||
"""
|
||||
exact request_id match fails (handles input_processor mutation)."""
|
||||
if request_id in self.request_id_to_transfer_id:
|
||||
tid = self.request_id_to_transfer_id[request_id]
|
||||
del self.request_id_to_transfer_id[request_id]
|
||||
|
||||
@@ -400,8 +400,7 @@ class NixlBaseConnectorScheduler:
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
"""D-side only. The number of prompt tokens to load from the prefiller.
|
||||
Stops short of the trailing ``_prefill_backoff()`` tokens that the decoder
|
||||
will recompute locally.
|
||||
"""
|
||||
will recompute locally."""
|
||||
backoff = self._prefill_backoff()
|
||||
if backoff and num_prompt_tokens > backoff:
|
||||
return num_prompt_tokens - backoff
|
||||
@@ -414,8 +413,7 @@ class NixlBaseConnectorScheduler:
|
||||
multi-module MTP it is the drafter's whole lookahead window.
|
||||
|
||||
Guarded by ``_p_side_truncated`` to avoid repeated truncation if the
|
||||
request is preempted and rescheduled.
|
||||
"""
|
||||
request is preempted and rescheduled."""
|
||||
backoff = self._prefill_backoff()
|
||||
params = request.kv_transfer_params
|
||||
if (
|
||||
|
||||
@@ -1873,8 +1873,7 @@ class NixlBaseConnectorWorker:
|
||||
) -> np.ndarray:
|
||||
"""Build remote desc regions (conv sub-projections + ssm) per layer.
|
||||
For hetero-TP, each D rank reads only its sub-projection slice from
|
||||
the P rank. Returns an Nx3 uint64 array.
|
||||
"""
|
||||
the P rank. Returns an Nx3 uint64 array."""
|
||||
assert nixl_agent_meta.kv_caches_base_addr, (
|
||||
"Remote KV cache base addresses must not be empty."
|
||||
)
|
||||
@@ -2657,8 +2656,7 @@ class NixlBaseConnectorWorker:
|
||||
@cached_property
|
||||
def _attention_kv_caches(self) -> list[torch.Tensor]:
|
||||
"""Device KV caches of attention layers (mamba states excluded),
|
||||
as consumed by the receive post-process.
|
||||
"""
|
||||
as consumed by the receive post-process."""
|
||||
assert self.device_kv_caches, (
|
||||
"_attention_kv_caches accessed before register_kv_caches"
|
||||
)
|
||||
|
||||
@@ -315,8 +315,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
executes on the writer; the handshake runs on the background executor
|
||||
and the request is re-queued onto ``_reg_send_inbox`` once it
|
||||
completes (at which point ``_ensure_handshake`` returns ``None`` and we
|
||||
send directly).
|
||||
"""
|
||||
send directly)."""
|
||||
remote_pp_size = reg_data.get("remote_pp_size", 1)
|
||||
fut = self._ensure_handshake(
|
||||
reg_data["remote_engine_id"],
|
||||
@@ -508,8 +507,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
``BlockIds`` is canonically a tuple of per-group lists, but some
|
||||
registration payloads collapse a single-group case to a flat
|
||||
list. Re-wrap that case so downstream group-aware helpers see a
|
||||
consistent shape.
|
||||
"""
|
||||
consistent shape."""
|
||||
if block_ids and not isinstance(block_ids[0], (list, tuple)):
|
||||
return (list(block_ids),)
|
||||
return block_ids
|
||||
|
||||
@@ -37,8 +37,7 @@ def canonical_format_id(kv_cache_layout: str) -> str:
|
||||
Canonical pages keep the worker's KV layout family, so the id couples the
|
||||
format version with that family; consumers must match it exactly. The family
|
||||
keeps its historical NHD/HND spelling so ids stay stable for KV persisted
|
||||
before the layout enum existed.
|
||||
"""
|
||||
before the layout enum existed."""
|
||||
layout = KVCacheLayout[kv_cache_layout]
|
||||
legacy = {KVCacheLayout.LBNHC: "nhd", KVCacheLayout.LBHNC: "hnd"}
|
||||
family = legacy.get(layout, layout.name.lower())
|
||||
@@ -122,8 +121,7 @@ def _interleave_cp_tokens(
|
||||
ctx: _RankContext,
|
||||
) -> tuple[CopyRun, ...]:
|
||||
"""Place each region's num_tokens rows at their canonical token positions,
|
||||
one run per chunk of interleaved tokens.
|
||||
"""
|
||||
one run per chunk of interleaved tokens."""
|
||||
runs: list[CopyRun] = []
|
||||
for region in regions:
|
||||
if ctx.cp_size == 1:
|
||||
@@ -258,8 +256,7 @@ def _attention_byte_regions(
|
||||
cp_size: int,
|
||||
) -> list[ByteRegion] | None:
|
||||
"""Byte regions of an attention page, given this rank's head shard.
|
||||
None when the physical layout is not recognized (fail closed).
|
||||
"""
|
||||
None when the physical layout is not recognized (fail closed)."""
|
||||
bs, heads, head_size = spec.block_size, spec.num_kv_heads, spec.head_size
|
||||
if tuple(kv_cache.shape) == (num_blocks, heads, bs, 2 * head_size):
|
||||
return _packed_kv_regions(kv_cache, spec, head_shard, num_head_shards, cp_size)
|
||||
@@ -373,8 +370,7 @@ def _is_exact_partition(intervals: list[tuple[int, int]], size: int) -> bool:
|
||||
|
||||
def _verify_tiling(layer_name: str, per_rank: list[CanonicalPageMapping]) -> None:
|
||||
"""Whichever ranks a block elects as writers must tile the canonical page
|
||||
exactly once, and each rank's runs must cover exactly its local page.
|
||||
"""
|
||||
exactly once, and each rank's runs must cover exactly its local page."""
|
||||
size = per_rank[0].canonical_page_size_bytes
|
||||
num_writers = per_rank[0].num_writers
|
||||
for mapping in per_rank:
|
||||
|
||||
@@ -79,8 +79,7 @@ def get_offloading_event_group_spec(
|
||||
class _OffloadEventMetadata:
|
||||
"""BlockStored payload snapshot for one OffloadKey, captured while the
|
||||
Request is available and kept until the final matching removal event.
|
||||
``medium`` and ``ownership`` are forwarded from the OffloadingEvent.
|
||||
"""
|
||||
``medium`` and ``ownership`` are forwarded from the OffloadingEvent."""
|
||||
|
||||
# The chunk's constituent block hashes; the last one is the OffloadKey.
|
||||
block_hashes: tuple[BlockHash, ...]
|
||||
@@ -256,8 +255,7 @@ class OffloadingEventsTracker:
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Drop all tracked state; pending payloads are stale after a
|
||||
manager cache reset.
|
||||
"""
|
||||
manager cache reset."""
|
||||
self._pending_event_metadata.clear()
|
||||
|
||||
def _build_event_metadata(
|
||||
@@ -268,8 +266,7 @@ class OffloadingEventsTracker:
|
||||
) -> _OffloadEventMetadata:
|
||||
"""Build the payload snapshot for one offloaded chunk: its
|
||||
constituent per-block hashes, the whole chunk's tokens, and the
|
||||
per-block ``block_size``.
|
||||
"""
|
||||
per-block ``block_size``."""
|
||||
hashes_per_chunk = group_config.hashes_per_chunk
|
||||
assert hashes_per_chunk > 0
|
||||
assert chunk_idx >= 0
|
||||
|
||||
@@ -676,8 +676,7 @@ class OffloadingConnectorScheduler:
|
||||
start_chunk_idx: int,
|
||||
) -> int | None:
|
||||
"""Return the number of consecutive offloaded chunks from the start,
|
||||
or None if the backend deferred a lookup.
|
||||
"""
|
||||
or None if the backend deferred a lookup."""
|
||||
hit_count = 0
|
||||
defer_lookup = False
|
||||
for local_idx, key in enumerate(keys):
|
||||
@@ -712,8 +711,7 @@ class OffloadingConnectorScheduler:
|
||||
"""Return the end index (in `keys`) of the last run of
|
||||
`sliding_window_size` consecutive hits, scanning from the end.
|
||||
The first run may need a larger window for a partial rightmost chunk.
|
||||
Returns 0 on miss, None if the backend deferred a lookup.
|
||||
"""
|
||||
Returns 0 on miss, None if the backend deferred a lookup."""
|
||||
defer_lookup = False
|
||||
pending_in_window = False
|
||||
consecutive_hits = 0
|
||||
|
||||
@@ -59,8 +59,7 @@ def ensure_divisibility(numerator, denominator):
|
||||
|
||||
def divide(numerator, denominator):
|
||||
"""Ensure that numerator is divisible by the denominator and return
|
||||
the division value.
|
||||
"""
|
||||
the division value."""
|
||||
ensure_divisibility(numerator, denominator)
|
||||
return numerator // denominator
|
||||
|
||||
|
||||
@@ -141,8 +141,7 @@ class WeightTransferTrainerFactory:
|
||||
class_name: str | None = None,
|
||||
) -> None:
|
||||
"""Register a trainer engine. Same conventions as
|
||||
`WeightTransferEngineFactory.register_engine`.
|
||||
"""
|
||||
`WeightTransferEngineFactory.register_engine`."""
|
||||
if name in cls._registry:
|
||||
raise ValueError(
|
||||
f"Weight transfer trainer engine '{name}' is already registered."
|
||||
|
||||
@@ -40,8 +40,7 @@ class IPCWeightTransferInitInfo(WeightTransferInitInfo):
|
||||
`packed` is a must-agree wire param: the trainer ships it here at the init
|
||||
handshake so the worker decodes with the same setting the trainer encoded
|
||||
with. The consumer rebuilds from the IPC handle + `tensor_sizes`, so it does
|
||||
not need the buffer size (producer-only).
|
||||
"""
|
||||
not need the buffer size (producer-only)."""
|
||||
|
||||
packed: bool = False
|
||||
|
||||
@@ -54,8 +53,7 @@ class IPCTrainerInitInfo(TrainerInitInfo):
|
||||
|
||||
`packed` / `packed_buffer_size_bytes` are the transfer's wire params. The
|
||||
trainer propagates `packed` to the worker at `trainer_init` so the two sides
|
||||
cannot disagree. `backend` is the factory dispatch key.
|
||||
"""
|
||||
cannot disagree. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "ipc"
|
||||
packed: bool = False
|
||||
|
||||
@@ -118,8 +118,7 @@ def worker_init_payload(init_info: NCCLWeightTransferInitInfo) -> dict:
|
||||
"""Serialize a worker init info for `init_weight_transfer_engine`, dropping
|
||||
the unset rendezvous field (the UID in TCP mode) so the wire payload carries
|
||||
only the mode actually in use. Shared by the dense and sparse trainer
|
||||
engines so the two cannot drift.
|
||||
"""
|
||||
engines so the two cannot drift."""
|
||||
return {key: value for key, value in asdict(init_info).items() if value is not None}
|
||||
|
||||
|
||||
|
||||
@@ -69,8 +69,7 @@ class NCCLTrainerInitInfo(TrainerInitInfo):
|
||||
propagates them to the worker at `trainer_init` so the two sides cannot
|
||||
disagree. Note this defaults to packed, unlike the worker-side
|
||||
`NCCLWeightTransferInitInfo`, whose default only applies when no trainer
|
||||
ships a value. `backend` is the factory dispatch key.
|
||||
"""
|
||||
ships a value. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "nccl"
|
||||
|
||||
@@ -365,8 +364,7 @@ class NCCLTrainerWeightTransferEngine(TrainerWeightTransferEngine[NCCLTrainerIni
|
||||
def _broadcast(self, source: WeightSource, meta: list[ParamMeta]) -> None:
|
||||
"""Iterate the source (materializing each tensor — a collective on all
|
||||
ranks) and, on the sender, broadcast from rank 0, packed or one-by-one.
|
||||
Non-sender ranks only replay the iteration to stay in the collective.
|
||||
"""
|
||||
Non-sender ranks only replay the iteration to stay in the collective."""
|
||||
if not self.is_sender:
|
||||
for _ in source:
|
||||
pass
|
||||
|
||||
@@ -214,8 +214,7 @@ class RdtRouter:
|
||||
|
||||
def class_of(self, name: str) -> int:
|
||||
"""The name's owner class — the planner's bucketing key, since all names
|
||||
of a chunk must share one producer.
|
||||
"""
|
||||
of a chunk must share one producer."""
|
||||
return self._class_of.get(name, 0)
|
||||
|
||||
def owners(self, name: str) -> list[int]:
|
||||
@@ -286,8 +285,7 @@ class RdtRouter:
|
||||
def bind(self, actors: list, produce_methods: list, consumer_id: int) -> None:
|
||||
"""Attach this consumer's producer handles. Rebuilt wholesale per init,
|
||||
never appended to: every owner index is a position in these lists, so a
|
||||
rejoining engine that re-inits must not shift them.
|
||||
"""
|
||||
rejoining engine that re-inits must not shift them."""
|
||||
if len(actors) != len(produce_methods):
|
||||
raise ValueError("actors and produce_methods must be parallel")
|
||||
self._actors = list(actors)
|
||||
@@ -350,6 +348,5 @@ def buffer_alloc_bytes(nbytes: int, presize: int = 0) -> int:
|
||||
optional ``presize`` floor, and a coarse 256MB round-up, so the buffer is
|
||||
allocated ONCE and never regrows. Regrowth is a correctness hazard, not just a
|
||||
perf one -- see ``buffer_presize_gb``. Shared by the consumer's receive buffers
|
||||
and the producer's serve rings.
|
||||
"""
|
||||
and the producer's serve rings."""
|
||||
return max(nbytes, presize, -(-nbytes // (256 << 20)) * (256 << 20))
|
||||
|
||||
@@ -134,8 +134,7 @@ class _PendingPull:
|
||||
|
||||
``targets``/``blob`` must stay strongly referenced until it completes:
|
||||
``set_target_for_ref`` stores WEAKREFS, so dropping them silently reroutes the
|
||||
transfer into a fallback buffer.
|
||||
"""
|
||||
transfer into a fallback buffer."""
|
||||
|
||||
ref: "Any"
|
||||
keys: "list[FetchKey]"
|
||||
@@ -562,8 +561,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
|
||||
def _build_static_plan(self, init_info: ShardedRDTWeightTransferInitInfo) -> None:
|
||||
"""Build the chunk/free plan once. It never changes across syncs, so
|
||||
``update_weights`` needs no per-sync names.
|
||||
"""
|
||||
``update_weights`` needs no per-sync names."""
|
||||
if not init_info.group_lens:
|
||||
raise ValueError(
|
||||
"Sharded RDT engine requires init_info.group_lens (the gather-"
|
||||
@@ -587,8 +585,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
Both sides are sized from the static plan: receive buffers are
|
||||
``ring_depth`` slots at the largest chunk's ``pack_bytes``, and each bound
|
||||
producer is asked to pre-register a serve ring at the max bytes this
|
||||
consumer will pull from it. A no-op when this worker has no chunks.
|
||||
"""
|
||||
consumer will pull from it. A no-op when this worker has no chunks."""
|
||||
plan = self._cached_plan
|
||||
if plan is None or not plan.chunks:
|
||||
return
|
||||
@@ -633,8 +630,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
``data_parallel_index``, not ``data_parallel_rank``: vLLM resets the
|
||||
latter to 0 in a dense worker but keeps the former as the distinct global
|
||||
DP rank. Same formula as the sibling ``nccl_engine``, so dense-via-TP and
|
||||
MoE-via-DP+EP both yield distinct 0..C-1.
|
||||
"""
|
||||
MoE-via-DP+EP both yield distinct 0..C-1."""
|
||||
pc = self.parallel_config
|
||||
return pc.data_parallel_index * pc.world_size + pc.rank # world_size = TP*PP
|
||||
|
||||
@@ -642,16 +638,14 @@ class ShardedRDTWeightTransferEngine(
|
||||
"""Total inference-worker count. Prefers the driver-supplied
|
||||
``init_info.num_consumers`` (authoritative -- the driver knows the whole
|
||||
fleet); else ``world_size_across_dp``, the same stride
|
||||
``_global_worker_index`` indexes with, so the two agree at any pp.
|
||||
"""
|
||||
``_global_worker_index`` indexes with, so the two agree at any pp."""
|
||||
if self._num_consumers_override > 0:
|
||||
return self._num_consumers_override
|
||||
return self.parallel_config.world_size_across_dp
|
||||
|
||||
def start_weight_update(self) -> None:
|
||||
"""Put the model's params on meta so layerwise reload streams them in
|
||||
as each layer's slices land. Baked replay uses checkpoint format.
|
||||
"""
|
||||
as each layer's slices land. Baked replay uses checkpoint format."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
initialize_layerwise_reload,
|
||||
)
|
||||
@@ -660,8 +654,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
|
||||
def finish_weight_update(self) -> None:
|
||||
"""Drain the deferred pull/process pipeline (so every layer is fully
|
||||
loaded) before finalizing the layerwise reload.
|
||||
"""
|
||||
loaded) before finalizing the layerwise reload."""
|
||||
from vllm.model_executor.model_loader.reload import (
|
||||
finalize_layerwise_reload,
|
||||
)
|
||||
@@ -674,8 +667,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
device sync: post-processing is deferred to background threads and a
|
||||
sync here would block on them and serialize the pull/process pipeline.
|
||||
Completion is guaranteed by ``drain_pending`` in
|
||||
``finish_weight_update``.
|
||||
"""
|
||||
``finish_weight_update``."""
|
||||
self.receive_weights(self.parse_update_info(update_info))
|
||||
|
||||
def receive_weights(
|
||||
@@ -717,8 +709,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
device: torch.device,
|
||||
) -> list[tuple[str, torch.Tensor]]:
|
||||
"""Zero-storage lazies for ``names``, dtype/shape from the init metadata,
|
||||
all feeding the bake's recording sink.
|
||||
"""
|
||||
all feeding the bake's recording sink."""
|
||||
return [
|
||||
(
|
||||
name,
|
||||
@@ -986,8 +977,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
|
||||
def _ensure_proc_worker(self) -> None:
|
||||
"""Lazily create the per-slot events, the background CUDA stream, the work
|
||||
queue, and the single processing thread. Idempotent.
|
||||
"""
|
||||
queue, and the single processing thread. Idempotent."""
|
||||
if self._proc_thread is not None:
|
||||
return
|
||||
import queue
|
||||
@@ -1046,8 +1036,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
def _mark_slot_done(self, slot: int) -> None:
|
||||
"""Publish that a queued item's read-done event has been recorded (or the
|
||||
item failed) so a pull waiting to reuse ``slot`` can proceed to its
|
||||
CUDA-event synchronize.
|
||||
"""
|
||||
CUDA-event synchronize."""
|
||||
with self._slot_cv:
|
||||
self._slot_done[slot] += 1
|
||||
self._slot_cv.notify_all()
|
||||
@@ -1063,8 +1052,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
"""Block until the background thread has processed every queued item and
|
||||
its stream work is complete, then re-raise any error it hit. Called from
|
||||
the worker's ``finish_weight_update`` before ``finalize_layerwise_reload``
|
||||
so every baked layer is fully loaded (and ``info.reset()``-ed) first.
|
||||
"""
|
||||
so every baked layer is fully loaded (and ``info.reset()``-ed) first."""
|
||||
if self._proc_queue is not None:
|
||||
self._proc_queue.join() # every put() item task_done()'d
|
||||
# The scatter thread feeds the quant thread, so join it SECOND (all
|
||||
@@ -1116,8 +1104,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
which is exactly what an owner class is. The cut is a pure function of
|
||||
the bake and the ownership table. Copy order within a chunk is bake
|
||||
order, and a module's copies may span chunks (materialize/quant fire on
|
||||
its first/last chunk; see ``_build_call_plan``).
|
||||
"""
|
||||
its first/last chunk; see ``_build_call_plan``)."""
|
||||
assert self._router is not None
|
||||
by_class: dict[int, list[_Scatter]] = {}
|
||||
for copies in modules:
|
||||
@@ -1248,8 +1235,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
pulls nothing from. Refs are held and drained in ``drain_pending`` so
|
||||
every signal has EXECUTED before the sync ends: ``begin_sync`` resets
|
||||
the counters, and a straggler landing in the next sync would credit a
|
||||
group it does not belong to.
|
||||
"""
|
||||
group it does not belong to."""
|
||||
assert self._router is not None
|
||||
self._pending_frees.extend(self._router.free_group(group_idx))
|
||||
|
||||
@@ -1384,8 +1370,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
the modules' scatters via ``ready``; touches only the scattered params
|
||||
(never a receive slot), so it can overlap subsequent chunks' RDMA and
|
||||
scatters. ``info.reset()`` is what makes finalize skip the layer —
|
||||
drain_pending joins the quant queue before finalize runs.
|
||||
"""
|
||||
drain_pending joins the quant queue before finalize runs."""
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
@@ -1418,8 +1403,7 @@ class ShardedRDTWeightTransferEngine(
|
||||
|
||||
def _quant_worker_loop(self) -> None:
|
||||
"""Dedicated quant thread: drains (completed_modules, scatter-done event)
|
||||
batches. Errors surface via _proc_error like the scatter thread's.
|
||||
"""
|
||||
batches. Errors surface via _proc_error like the scatter thread's."""
|
||||
torch.cuda.set_device(self.device)
|
||||
q = self._quant_queue
|
||||
assert q is not None
|
||||
|
||||
@@ -206,8 +206,7 @@ class FakeRDTTensor(torch.Tensor):
|
||||
|
||||
def _meta(self) -> torch.Tensor:
|
||||
"""A zero-storage meta tensor of this fake's shape/dtype, so PyTorch
|
||||
itself computes post-op geometry rather than us reimplementing it.
|
||||
"""
|
||||
itself computes post-op geometry rather than us reimplementing it."""
|
||||
return torch.empty(self.shape, dtype=self.dtype, device="meta")
|
||||
|
||||
@classmethod
|
||||
@@ -251,8 +250,7 @@ class FakeRDTTensor(torch.Tensor):
|
||||
):
|
||||
"""Append the op and return a child, or a tuple of children for
|
||||
multi-return ops. Each child's geometry comes from running the op on a
|
||||
meta tensor.
|
||||
"""
|
||||
meta tensor."""
|
||||
meta = self_._meta()
|
||||
with torch._C.DisableTorchFunctionSubclass():
|
||||
meta_result = func(meta, *args, **kwargs)
|
||||
|
||||
@@ -132,8 +132,7 @@ PRODUCE_METHOD_NAME = "rdt_produce_weights_batched"
|
||||
@dataclass
|
||||
class _SharedPack:
|
||||
"""One generation of one slot-sharing group: the rendezvous state for a
|
||||
single chunk, identified by the consumers' issue index.
|
||||
"""
|
||||
single chunk, identified by the consumers' issue index."""
|
||||
|
||||
group: int
|
||||
slot: int
|
||||
@@ -467,8 +466,7 @@ class _RDTProducerServer:
|
||||
|
||||
def end_sync(self) -> list[int]:
|
||||
"""Block until every published group has been freed by its consumers;
|
||||
return the remaining freed keys so the engine drops its last refs.
|
||||
"""
|
||||
return the remaining freed keys so the engine drops its last refs."""
|
||||
with self._cache_cond:
|
||||
self._wait_for(
|
||||
lambda: bool(self._inflight_groups),
|
||||
@@ -481,8 +479,7 @@ class _RDTProducerServer:
|
||||
|
||||
def set_gather_error(self, message: str) -> None:
|
||||
"""Record a trainer-side gather failure so blocked serves / publishes
|
||||
stop waiting and surface it.
|
||||
"""
|
||||
stop waiting and surface it."""
|
||||
with self._cache_cond:
|
||||
self._gather_error = RuntimeError(message)
|
||||
self._cache_cond.notify_all()
|
||||
@@ -511,8 +508,7 @@ class _RDTProducerServer:
|
||||
def _new_serve_buffer(self, nbytes: int) -> torch.Tensor:
|
||||
"""Allocate + NIXL-register one serve slot: the single allocation seam,
|
||||
so registration cannot be skipped on either of the two paths that make
|
||||
buffers (the init-time reservation and the serve-path backstop).
|
||||
"""
|
||||
buffers (the init-time reservation and the serve-path backstop)."""
|
||||
t = torch.empty(nbytes, dtype=torch.uint8, device=self._serve_device)
|
||||
with self._reg_lock:
|
||||
register_nixl_memory(t)
|
||||
@@ -584,15 +580,13 @@ class _RDTProducerServer:
|
||||
def _share_group(self, consumer_id: int) -> int:
|
||||
"""The slot-sharing group ``consumer_id`` belongs to: its index within
|
||||
its own deployment, since that is what fixes the plan. Without a width
|
||||
the group is the consumer itself and nothing is shared.
|
||||
"""
|
||||
the group is the consumer itself and nothing is shared."""
|
||||
return consumer_id % self._share_width if self._share_width > 0 else consumer_id
|
||||
|
||||
def _sharers_of(self, sg: int) -> frozenset:
|
||||
"""The LIVE consumers of group ``sg``: the rendezvous width. Derived from
|
||||
``begin_sync``'s live set, so a degraded sync narrows the barrier instead
|
||||
of waiting forever on a dead deployment. Caller holds ``_cache_cond``.
|
||||
"""
|
||||
of waiting forever on a dead deployment. Caller holds ``_cache_cond``."""
|
||||
cached = self._sharers.get(sg)
|
||||
if cached is None:
|
||||
if not self._sharing_active or self._live_ids is None:
|
||||
@@ -628,8 +622,7 @@ class _RDTProducerServer:
|
||||
def _await_shared_pack(self, gen: _SharedPack) -> torch.Tensor:
|
||||
"""Block until this generation's packer published its blob, and return
|
||||
it. A failed pack is re-raised here, so every sharer of a bad pack fails
|
||||
instead of reading a half-written slot.
|
||||
"""
|
||||
instead of reading a half-written slot."""
|
||||
with self._cache_cond:
|
||||
self._wait_for(
|
||||
lambda: not gen.done and gen.error is None,
|
||||
@@ -650,8 +643,7 @@ class _RDTProducerServer:
|
||||
"""Group ``sg``'s ring slot ``idx``, grown if this chunk outgrew the
|
||||
reservation. Growing registers memory while the fabric is busy, the
|
||||
hazard ``reserve_serve_buffer`` exists to avoid, so it is a backstop: the
|
||||
reservation is sized from the same static plan as this pack.
|
||||
"""
|
||||
reservation is sized from the same static plan as this pack."""
|
||||
with self._serve_lock:
|
||||
buffer = self._serve_rings.setdefault(sg, [None] * self._nring)[idx]
|
||||
if buffer is not None and buffer.numel() >= need:
|
||||
@@ -663,8 +655,7 @@ class _RDTProducerServer:
|
||||
|
||||
def _fail_shared_pack(self, gen: _SharedPack, exc: BaseException) -> None:
|
||||
"""Publish a pack failure so waiting sharers raise instead of hanging on
|
||||
a generation that will never complete.
|
||||
"""
|
||||
a generation that will never complete."""
|
||||
with self._cache_cond:
|
||||
gen.error = exc
|
||||
self._cache_cond.notify_all()
|
||||
@@ -866,8 +857,7 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
def _rpc(self, method: str, *args: Any) -> Any:
|
||||
"""Call one of the server actor's methods and block for the result.
|
||||
The single seam through which the engine talks to its server, so tests
|
||||
can inject a local (non-Ray) fake server.
|
||||
"""
|
||||
can inject a local (non-Ray) fake server."""
|
||||
import ray
|
||||
|
||||
return ray.get(getattr(self._server, method).remote(*args))
|
||||
@@ -944,8 +934,7 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
"""Fire publish_group WITHOUT waiting on the RPC (the gather loop
|
||||
overlaps the publish's server-side rebuild with the next group's
|
||||
gather) and return a handle that ``_await_publish`` resolves. Ray actor
|
||||
handle in production; a plain (non-Ray) fake server runs inline.
|
||||
"""
|
||||
handle in production; a plain (non-Ray) fake server runs inline."""
|
||||
method = self._server.publish_group
|
||||
remote = getattr(method, "remote", None)
|
||||
if remote is not None:
|
||||
@@ -956,8 +945,7 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
"""Resolve one async publish so a server-side rebuild error surfaces at
|
||||
window depth instead of at end_sync. Publishes carry nothing back —
|
||||
freed groups flow only through wait_freed/end_sync (one channel; see
|
||||
publish_group).
|
||||
"""
|
||||
publish_group)."""
|
||||
import ray
|
||||
|
||||
if isinstance(ref, ray.ObjectRef):
|
||||
@@ -1144,8 +1132,7 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
by stall watchdog 300s later: consumers route pulls here, the pull
|
||||
passes the served-names guard, and the cache wait never completes. With
|
||||
it, that is an immediate error naming the weight. One set lookup per
|
||||
name per sync.
|
||||
"""
|
||||
name per sync."""
|
||||
if self._held_names is None:
|
||||
return
|
||||
for name, tensor in zip(names, tensors):
|
||||
@@ -1346,8 +1333,7 @@ class ShardedRDTTrainerWeightTransferEngine(
|
||||
credit releases when every live consumer has signaled the group). So
|
||||
the loop self-paces to the consumers' pull rate with at most
|
||||
`gather_lookahead + 1` groups resident. Runs on every rank; only the
|
||||
sender has an `update_future` to fail fast on.
|
||||
"""
|
||||
sender has an `update_future` to fail fast on."""
|
||||
assert self.source is not None # guaranteed by trainer_init
|
||||
# The live IDS ride along with the count: the free barrier needs only
|
||||
# the count, but the slot-sharing rendezvous needs to know WHICH
|
||||
|
||||
@@ -68,8 +68,7 @@ class SparseNCCLTrainerInitInfo(TrainerInitInfo):
|
||||
|
||||
Same rendezvous shape as the dense NCCL backend (the sender opens its
|
||||
endpoint as NCCL rank 0), but with no packed wire params: sparse transfers
|
||||
are never packed. `backend` is the factory dispatch key.
|
||||
"""
|
||||
are never packed. `backend` is the factory dispatch key."""
|
||||
|
||||
backend: ClassVar[str] = "sparse_nccl"
|
||||
|
||||
@@ -368,8 +367,7 @@ class SparseNCCLTrainerWeightTransferEngine(
|
||||
rebuild or free the patch tensors as soon as a send method returns rather
|
||||
than relying on same-stream ordering. See
|
||||
`NCCLTrainerWeightTransferEngine._post_send_sync` for why there is no
|
||||
cross-rank barrier.
|
||||
"""
|
||||
cross-rank barrier."""
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.current_stream().synchronize()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user