Add pydocstyle to the ruff rules (#52136)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Signed-off-by: Harry Mellor <hej.mellor@gmail.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Harry Mellor
2026-09-16 16:44:29 +00:00
committed by GitHub
co-authored by Claude Opus 5
parent 804e5377aa
commit 8c1557a79c
2002 changed files with 15024 additions and 14269 deletions
+125 -64
View File
@@ -53,7 +53,8 @@ META = torch.device("meta")
class _FakeLayer:
"""Stand-in for a leaf module. The planner only stores it and keys on
``id()``; nothing in the plan path touches its attributes."""
``id()``; nothing in the plan path touches its attributes.
"""
def __init__(self, tag: str) -> None:
self.tag = tag
@@ -64,7 +65,8 @@ class _FakeLayer:
def _module(layer, copies):
"""What the bake records for one leaf module: a scatter list whose entries
all carry that module as their ``layer``."""
all carry that module as their ``layer``.
"""
for c in copies:
c.layer = layer
return copies
@@ -81,7 +83,8 @@ def _copy(
dtype=torch.bfloat16,
):
"""A `_Scatter` as the bake would record it: source key, owning layer, and
the meta destination region."""
the meta destination region.
"""
stride = (
(1,)
if len(shape) == 1
@@ -140,7 +143,8 @@ def _planner(baked, *, name_meta, live=None, held_by=None, group_lens=None):
def _one_module_per_layer(n_layers, *, dtype="bfloat16", numel=4):
"""`(baked, name_meta, names, group_lens)` for a pre / N-layer / post model
where every gather group is one fully-baked leaf module."""
where every gather group is one fully-baked leaf module.
"""
names = ["embed.weight"]
names += [f"model.layers.{i}.w" for i in range(n_layers)]
names += ["norm.weight"]
@@ -161,7 +165,8 @@ def _one_module_per_layer(n_layers, *, dtype="bfloat16", numel=4):
class TestFakeOpChains:
"""Every allowlisted op must append itself to the chain and hand back a
child whose shape/dtype PyTorch itself computed. The chain is the wire
format the producer replays, so its exact contents are load-bearing."""
format the producer replays, so its exact contents are load-bearing.
"""
def _fake(self, shape=(4, 6), dtype=torch.bfloat16, sink=None):
return FakeRDTTensor(
@@ -222,7 +227,8 @@ class TestFakeOpChains:
@pytest.mark.parametrize("op,n", [("chunk", 2), ("unbind", 4)])
def test_multi_return_op_emits_one_child_per_output(self, op, n):
"""chunk/unbind hand back a tuple; each child carries the base op plus a
trailing __getitem__(i) so the producer can index the replayed result."""
trailing __getitem__(i) so the producer can index the replayed result.
"""
parts = getattr(self._fake(), op)(*((n,) if op == "chunk" else ()), 0)
assert isinstance(parts, tuple)
assert len(parts) == n
@@ -238,7 +244,8 @@ class TestFakeOpChains:
class TestFakeUnsupportedOps:
"""Anything that needs real data must fail loudly at bake time rather than
silently transferring the wrong bytes."""
silently transferring the wrong bytes.
"""
def _fake(self):
return FakeRDTTensor(
@@ -278,7 +285,8 @@ class TestFakeUnsupportedOps:
class TestBakeRecording:
"""During the dry run the fake's ``copy_`` is the data sink: it records the
source chain plus the meta destination's strided region and moves nothing."""
source chain plus the meta destination's strided region and moves nothing.
"""
def _recorder_and_fake(self, shape=(4, 6)):
rec = BakeSink()
@@ -314,7 +322,8 @@ class TestBakeRecording:
def test_unattributed_copy_is_live_but_unrecorded(self):
"""A copy_ with no loader stamp cannot be attributed to a param, so its
module must fall back to the plain load — but the name still moved data."""
module must fall back to the plain load — but the name still moved data.
"""
rec, fake = self._recorder_and_fake()
rec.current = None
torch.empty((4, 6), dtype=torch.bfloat16, device=META).copy_(fake)
@@ -354,7 +363,8 @@ class TestBakeRecording:
producer packs what the replay yields, so the record must carry the
POST-chain dtype: taking it from the source name's metadata instead sizes
the slice with the wrong itemsize and shifts every later slice in the
chunk, carving the packed blob differently on the two sides."""
chunk, carving the packed blob differently on the two sides.
"""
rec, fake = self._recorder_and_fake(shape=(4,))
layer = _FakeLayer("reinterpreted")
rec.current = (layer, "weight")
@@ -374,7 +384,8 @@ class TestBakeRecording:
the producer packs what the chain yields, and nothing downstream can
catch a mismatch — the consumer's buffer view is exactly prod(dest.shape)
elements, so it reshapes cleanly over bytes laid out at other offsets.
So the bake refuses rather than recording a slice it cannot carve."""
So the bake refuses rather than recording a slice it cannot carve.
"""
rec, fake = self._recorder_and_fake(shape=(1, 6))
rec.current = (_FakeLayer("broadcast"), "weight")
param = torch.empty((4, 6), dtype=torch.bfloat16, device=META)
@@ -390,7 +401,8 @@ class TestBakeRecording:
class TestConsumerIdentity:
"""Every worker in the fleet needs a DISTINCT id in 0..C-1: it selects the
worker's producer block and keys the producer's per-consumer serve ring, so
a collision silently serves two workers out of one ring."""
a collision silently serves two workers out of one ring.
"""
def _engine(self, *, dp_index, rank, world_size):
eng = object.__new__(ShardedRDTWeightTransferEngine)
@@ -427,7 +439,8 @@ class TestConsumerIdentity:
def test_independent_engines_offset_into_distinct_ranges(self):
"""A fleet of separate engines restarts _global_worker_index at 0 in
each, so without the replica offset every engine would claim 0..w-1.
The driver sets replica_rank on the payload; these are the ids it buys."""
The driver sets replica_rank on the payload; these are the ids it buys.
"""
ids = [
self._ids(num_consumers=8, workers=range(4), replica_rank=r, num_replicas=2)
for r in (0, 1)
@@ -453,7 +466,8 @@ def _router(
group_lens=None,
):
"""A router over ``n`` single-name groups unless told otherwise, so a test
can state expectations in group indices."""
can state expectations in group indices.
"""
if names is None:
n = len(name_owner_class or []) or (len(group_lens or []) or 1)
names = [f"g{i}" for i in range(n)]
@@ -482,7 +496,8 @@ class TestRdtRouter:
def test_every_producer_nic_carries_traffic(self):
"""16 producers / 8 consumers: the block rule spreads each consumer's
pulls over its block, alternating by group, so no producer sits idle.
Plain `consumer_id % len(owners)` would idle half of them."""
Plain `consumer_id % len(owners)` would idle half of them.
"""
r = _router(16, 8, group_lens=[1] * 95)
names = [f"g{i}" for i in range(95)]
for c in range(8):
@@ -500,7 +515,8 @@ class TestRdtRouter:
def test_pipeline_stages_route_to_the_owning_stage(self):
"""2 stages x 8 ranks: groups 0-2 on stage 0, 3-5 on stage 1. Two owner
sets, one per stage; each consumer reaches both."""
sets, one per stage; each consumer reaches both.
"""
r = _router(
16,
8,
@@ -516,7 +532,8 @@ class TestRdtRouter:
def test_expert_names_route_to_the_holding_ranks(self):
"""A name held by only some ranks routes to those; a replicated name
keeps the full set."""
keeps the full set.
"""
r = RdtRouter(
8,
4,
@@ -535,7 +552,8 @@ class TestRdtRouter:
def test_consumers_spread_over_an_owner_set(self):
"""Several ranks hold a name (its TP peers); the block rule must spread
consumers across them, not funnel through one."""
consumers across them, not funnel through one.
"""
r = _router(
4, 4, owner_sets=[[0, 1]], name_owner_class=[0] * 6, group_lens=[1] * 6
)
@@ -544,7 +562,8 @@ class TestRdtRouter:
def test_a_group_frees_at_every_owner_of_any_of_its_names(self):
"""The free barrier is per group, so its fan-out is the UNION over the
group's names: a rank holding none of them is not signalled."""
group's names: a rank holding none of them is not signalled.
"""
r = RdtRouter(4, 2, [[0, 1], [2, 3]], [0, 1, 0], ["a", "b", "c"], [2, 1])
assert r.group_owners(0) == [0, 1, 2, 3]
assert r.group_owners(1) == [0, 1]
@@ -614,7 +633,8 @@ class TestReplicaOverlay:
def test_adding_a_deployment_does_not_move_the_first_one(self):
"""The strongest form: an existing consumer's routes are byte-identical
before and after another deployment is provisioned."""
before and after another deployment is provisioned.
"""
one, names = self._fleet(16, 8, 1)
two, _ = self._fleet(16, 8, 2)
for consumer_id in range(8):
@@ -624,7 +644,8 @@ class TestReplicaOverlay:
def test_one_deployment_is_the_historical_carve(self):
"""Default (no width) and width == the whole fleet must agree with the
plain block rule, so a single-deployment fleet is untouched."""
plain block rule, so a single-deployment fleet is untouched.
"""
plain = _router(16, 8, group_lens=[1] * 6)
explicit, names = self._fleet(16, 8, 1)
for consumer_id in range(8):
@@ -638,7 +659,8 @@ class TestReplicaOverlay:
def test_the_overlay_holds_for_partially_owned_names(self):
"""Pipeline stages and experts too: two owner sets, one narrow. The
narrow one already forced agreement (one owner, no block to carve); the
wide one is what the carve decides."""
wide one is what the carve decides.
"""
r, names = self._fleet(
8,
4,
@@ -654,14 +676,16 @@ class TestReplicaOverlay:
def test_a_width_that_does_not_divide_the_fleet_raises(self):
"""A uniform fleet is asserted upstream in ``get_world_size``; if it ever
is not, the overlay would map two workers of one deployment onto the same
block index, so refuse rather than serve the wrong bytes."""
block index, so refuse rather than serve the wrong bytes.
"""
with pytest.raises(ValueError, match="does not divide"):
RdtRouter(4, 6, None, None, ["a"], [1], workers_per_replica=4)
def test_pulls_still_carry_the_global_consumer_id(self):
"""The carve uses the index WITHIN a deployment, but the wire must keep
the fleet-global id: it is what lets the producer tell the sharers of a
slot apart, and count their arrivals separately."""
slot apart, and count their arrivals separately.
"""
class _Method:
def __init__(self):
@@ -704,7 +728,8 @@ def _producer_pack_offsets(slices):
class TestPackedLayout:
def test_consumer_layout_matches_the_producer_rule(self):
"""Mixed dtypes, sizes that do not land on 16B — i.e. the Kimi-style
group (fp8 weights + fp32 scales + bf16 norms) that exposes alignment."""
group (fp8 weights + fp32 scales + bf16 norms) that exposes alignment.
"""
specs = [
("w.fp8", "float8_e4m3fn", [17]),
("w.scale", "float32", [3]),
@@ -779,7 +804,8 @@ class TestPackedLayout:
class TestChunkModuleScatters:
"""The chunk cut: one chunk per distinct owner class present in the copies,
ascending by class index. Derived purely from the bake plus the ownership
table, so vLLM's expert placement needs no cases."""
table, so vLLM's expert placement needs no cases.
"""
def test_unstamped_copies_form_one_chunk(self):
layer = _FakeLayer("l")
@@ -793,7 +819,8 @@ class TestChunkModuleScatters:
"""Classes are numbered by first appearance in metadata order, so the
replicated class (0) leads and the rest follow — the ordering the old
"-1 first, then ascending stamp" rule produced, now falling out of the
table instead of a special case."""
table instead of a special case.
"""
layer = _FakeLayer("moe")
names = ["norm", "e5", "e0", "e2"]
copies = [_copy(n, shape=(4,)) for n in names]
@@ -813,7 +840,8 @@ class TestChunkModuleScatters:
def test_copy_order_within_a_chunk_is_bake_order(self):
"""Scatter order within a chunk must follow the bake, not the stamp
sort: the packed layout and the producer's replay agree on keys order."""
sort: the packed layout and the producer's replay agree on keys order.
"""
layer = _FakeLayer("moe")
names = ["e3", "e1", "e2"] # one stamp, deliberately unsorted names
copies = [_copy(n, shape=(4,)) for n in names]
@@ -827,7 +855,8 @@ class TestChunkModuleScatters:
def test_stamps_interleave_across_modules_without_reordering_within(self):
"""Two modules' copies bucket by stamp; within each bucket the order is
module-then-bake order."""
module-then-bake order.
"""
a, b = _FakeLayer("a"), _FakeLayer("b")
mods = [
_module(a, [_copy("a0", shape=(4,)), _copy("a1", shape=(4,))]),
@@ -845,9 +874,10 @@ class TestChunkModuleScatters:
]
def test_scatters_carry_their_own_dtype(self):
"""dtype rides the record from the bake, where it is the fake's dtype
"""Dtype rides the record from the bake, where it is the fake's dtype
AFTER its op chain — not a plan-time lookup of the source name, which
would be wrong for any chain that reinterprets dtype."""
would be wrong for any chain that reinterprets dtype.
"""
layer = _FakeLayer("l")
eng = _planner({}, name_meta={"w": ("float32", [10])})
((_er, scatters),) = eng._chunk_module_scatters(
@@ -886,7 +916,8 @@ class TestBuildCallPlan:
def test_materialize_fires_on_a_modules_first_chunk_only(self):
"""Empty HF params are allocated once per module, by construction —
including a FusedMoE-like module whose copies span owner-class chunks."""
including a FusedMoE-like module whose copies span owner-class chunks.
"""
layer = _FakeLayer("spanning")
names = [f"w{i}" for i in range(4)]
copies = [_copy(n, shape=(4,)) for n in names]
@@ -921,7 +952,8 @@ class TestBuildCallPlan:
def test_free_signal_waits_for_a_groups_last_chunk_when_it_is_split(self):
"""The load-bearing case: with the group cut across owner-class chunks the
signal must hang off the LAST one. Signaling earlier lets the producers
drop the gather buffers while a later chunk's RDMA is still reading."""
drop the gather buffers while a later chunk's RDMA is still reading.
"""
layer = _FakeLayer("l")
names = [f"w{i}" for i in range(4)]
copies = [_copy(n, shape=(4,)) for n in names]
@@ -937,7 +969,8 @@ class TestBuildCallPlan:
def test_free_signal_timing_across_two_split_groups(self):
"""Two groups x 2 owner-class chunks each: signals land on each group's
last chunk only."""
last chunk only.
"""
names_a = [f"model.layers.0.w{i}" for i in range(4)]
names_b = [f"model.layers.1.w{i}" for i in range(4)]
baked, name_meta, stamps = {}, {}, {}
@@ -955,7 +988,8 @@ class TestBuildCallPlan:
def test_a_group_with_nothing_local_is_signaled_at_sync_start(self):
"""The owners still published it, so this consumer must signal it —
immediately, before the pipeline (signal-before-publish is tolerated),
never hung off another group's chunk."""
never hung off another group's chunk.
"""
baked, name_meta, names, group_lens = _one_module_per_layer(2)
del baked["model.layers.1.w"] # nothing baked, and not live => no pull
eng = _planner(
@@ -976,7 +1010,8 @@ class TestBuildCallPlan:
def test_modules_are_deduped_within_a_group(self):
"""Several names of one fused module map to the same scatter list; the
plan must not chunk it twice."""
plan must not chunk it twice.
"""
layer = _FakeLayer("fused")
names = ["qkv.q", "qkv.k", "qkv.v"]
copies = [_copy(n, shape=(4,)) for n in names]
@@ -992,7 +1027,8 @@ class TestBuildCallPlan:
def test_plan_is_a_pure_function_of_its_inputs(self):
"""Called twice with the same arguments it must produce the same plan —
that is what lets the engine build it once and reuse it every sync."""
that is what lets the engine build it once and reuse it every sync.
"""
baked, name_meta, names, group_lens = _one_module_per_layer(2)
eng = _planner(baked, name_meta=name_meta)
a = eng._build_call_plan(names, group_lens)
@@ -1012,7 +1048,8 @@ class TestBuildCallPlan:
class TestUnbakedNamesGuard:
"""A live name with no baked plan has no way to load — there is no fallback,
and its pull would target groups the pipeline already freed — so the plan
build fails at init, naming the names."""
build fails at init, naming the names.
"""
def test_a_live_unbaked_name_fails_the_plan_build(self):
baked, name_meta, names, group_lens = _one_module_per_layer(2)
@@ -1023,7 +1060,8 @@ class TestUnbakedNamesGuard:
def test_never_copied_names_are_dropped_entirely(self):
"""Experts owned by another EP rank no-op in their loader: not an error,
and not a chunk — their group is pre-freed if nothing else fills it."""
and not a chunk — their group is pre-freed if nothing else fills it.
"""
baked, name_meta, names, group_lens = _one_module_per_layer(2)
del baked["norm.weight"]
eng = _planner(baked, name_meta=name_meta, live=set(names) - {"norm.weight"})
@@ -1035,13 +1073,15 @@ class TestUnbakedNamesGuard:
class TestCallPlanRouting:
"""Under partial ownership each chunk must be pulled from a producer that
actually holds every name in it. Consumers bind EVERY producer (signals fan
out to all owners), so local index == trainer rank."""
out to all owners), so local index == trainer rank.
"""
def _routed_planner(
self, held_by_group, num_consumers, consumer_id, *, held_by=None
):
"""``held_by_group[g]`` = the ranks holding group g's names, so a test
can still speak in groups; ``held_by`` overrides individual names."""
can still speak in groups; ``held_by`` overrides individual names.
"""
baked, name_meta, names, group_lens = _one_module_per_layer(
len(held_by_group) - 2
)
@@ -1068,14 +1108,16 @@ class TestCallPlanRouting:
def test_two_stage_ownership_splits_chunks_between_owners(self):
"""Two PP stages, 4 groups: stage 0 holds the first half, stage 1 the
second. Every chunk goes to a holder of its names."""
second. Every chunk goes to a holder of its names.
"""
eng, names, group_lens = self._routed_planner([[0], [0], [1], [1]], 1, 0)
plan = eng._build_call_plan(names, group_lens)
assert [c.owner for c in plan.chunks] == [0, 0, 1, 1]
def test_a_multi_owner_group_still_resolves_to_one_producer(self):
"""Every chunk is served by exactly ONE producer: splitting a pull only
multiplies produce calls, since the consumer's NIC bounds it."""
multiplies produce calls, since the consumer's NIC bounds it.
"""
eng, names, group_lens = self._routed_planner([[0, 1]] * 4, 1, 0)
plan = eng._build_call_plan(names, group_lens)
assert all(0 <= c.owner < 2 for c in plan.chunks)
@@ -1083,7 +1125,8 @@ class TestCallPlanRouting:
def test_an_expert_name_routes_to_the_rank_holding_it(self):
"""PP ∩ EP: stage 0 = ranks 0-1, stage 1 = ranks 2-3, and one expert
name inside a stage-1 group is held by rank 3 alone. The name's own owner
set is what routes it."""
set is what routes it.
"""
eng, names, group_lens = self._routed_planner(
[[0, 1], [0, 1], [2, 3], [2, 3]],
1,
@@ -1098,7 +1141,8 @@ class TestCallPlanRouting:
def test_an_unheld_name_is_rejected_at_init(self):
"""A name no rank holds can never be served. The trainer raises when it
transposes the holdings; the router's validate() is the consumer-side
backstop, and either way it fails at init rather than at first pull."""
backstop, and either way it fails at init rather than at first pull.
"""
with pytest.raises(ValueError, match="empty"):
RdtRouter(2, 1, [[]], [0], ["w"], [1]).validate()
@@ -1107,14 +1151,16 @@ class TestSignalCompleteness:
"""Every gather group is signaled exactly once — after its last chunk when
the worker pulls from it, at sync start otherwise — for ANY expert
placement. A group signaled twice over-credits the barrier; a group never
signaled parks a producer credit and hangs end_sync."""
signaled parks a producer credit and hangs end_sync.
"""
def _moe_planner(self, worker_experts, *, ep_size=4, n_experts=8, n_layers=2):
"""pre / n_layers MoE layers / post. Each layer: one norm (its own
"""Pre / n_layers MoE layers / post. Each layer: one norm (its own
module, stamp -1) + n_experts expert names stamped ``e // n_local``.
The worker's bake covers only ``worker_experts`` (its placement);
foreign expert names never copied => dropped entirely, as in the real
bake."""
bake.
"""
n_local = n_experts // ep_size
names = ["embed.weight"]
group_lens = [1]
@@ -1173,8 +1219,9 @@ class TestSignalCompleteness:
assert self._signals(plan) == list(range(len(group_lens)))
def test_placement_changes_only_the_chunk_count_never_the_signals(self):
"""linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6
hits all 4. More chunks per group, identical signal set."""
"""Linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6
hits all 4. More chunks per group, identical signal set.
"""
plans = {}
for label, experts in (("linear", [0, 1, 2, 3]), ("round_robin", [0, 2, 4, 6])):
eng, names, group_lens = self._moe_planner(worker_experts=experts)
@@ -1188,7 +1235,8 @@ class TestSignalCompleteness:
def test_the_fused_module_materializes_first_and_quants_last_across_chunks(self):
"""The FusedMoE shape: one module's copies span every owner-class chunk of
its group; materialize on its first, quant on its last."""
its group; materialize on its first, quant on its last.
"""
eng, names, group_lens = self._moe_planner(
worker_experts=[0, 2, 4, 6], n_layers=1
)
@@ -1246,7 +1294,8 @@ class TestLayerwiseGroups:
def test_flattening_a_partition_returns_the_input_order(self):
"""Un-indexed names group by POSITION, not by name class — so one that
looks like a pre-block name still lands after the layers it follows.
Group index therefore means the same thing on every rank."""
Group index therefore means the same thing on every rank.
"""
names = [
"embed.weight",
"model.layers.0.a",
@@ -1264,7 +1313,8 @@ class TestLayerwiseGroups:
def test_stacks_keep_prefix_appearance_order_and_sort_within(self):
"""Two stacks stay in the order they appear — the vision tower before the
text stack — while each sorts by index internally."""
text stack — while each sorts by index internally.
"""
names = [
"visual.blocks.1.w",
"visual.blocks.0.w",
@@ -1281,7 +1331,8 @@ class TestLayerwiseGroups:
def test_the_post_block_lands_last_however_early_it_arrives(self):
"""Megatron-Bridge streams the last pipeline stage's output block before
its layers. Sweeping un-indexed names after the first layer into a
trailing group is what lets the gather loop walk groups ascending."""
trailing group is what lets the gather loop walk groups ascending.
"""
names = ["model.layers.0.w", "model.norm.weight", "model.layers.1.w"]
assert layerwise_groups(names) == [
["model.layers.0.w"],
@@ -1291,7 +1342,8 @@ class TestLayerwiseGroups:
def test_names_sharing_a_layer_coalesce_when_the_source_interleaves(self):
"""A raw-checkpoint source may yield in shard-packing order. Coalescing on
the key keeps that from shattering one layer into many groups."""
the key keeps that from shattering one layer into many groups.
"""
names = [
"model.layers.0.a",
"model.layers.1.a",
@@ -1333,13 +1385,15 @@ class TestLayerwiseGroups:
def test_every_supported_naming_convention_partitions_per_layer(self, prefix):
"""One decoder layer per group under any prefix, with no per-architecture
table. A literal `model.layers.` match silently yields one whole-model
group for most of these."""
group for most of these.
"""
names = [f"{prefix}{i}.self_attn.q_proj.weight" for i in range(3)]
assert layerwise_groups(names) == [[n] for n in names]
def test_a_moe_layer_stays_whole(self):
"""The OUTERMOST index wins, so per-expert names group by layer. Splitting
on the expert would cut FusedMoE — one leaf module — across groups."""
on the expert would cut FusedMoE — one leaf module — across groups.
"""
names = [
"model.layers.0.mlp.experts.0.w1.weight",
"model.layers.0.mlp.experts.1.w1.weight",
@@ -1356,7 +1410,8 @@ class TestLayerwiseGroups:
def test_a_vlm_partitions_the_vision_tower_and_the_text_stack(self):
"""Qwen-VL layout. Today's literal `model.layers.` match puts every one of
these in ONE group — the whole model — because the text stack ships as
`model.language_model.layers.`."""
`model.language_model.layers.`.
"""
names = [
"visual.patch_embed.proj.weight",
"visual.blocks.0.attn.qkv.weight",
@@ -1375,7 +1430,8 @@ class TestLayerwiseGroups:
def test_two_stacks_sharing_an_index_do_not_merge(self):
"""The key carries the prefix, so vision block 0 and text layer 0 are
different groups even though both are index 0."""
different groups even though both are index 0.
"""
names = [
"visual.blocks.0.attn.qkv.weight",
"model.language_model.layers.0.self_attn.q_proj.weight",
@@ -1412,20 +1468,23 @@ class TestOpAllowlistAgreement:
def test_every_bake_recordable_op_is_replayable(self):
"""Anything the bake can record must be serveable, or the failure lands a
whole sync later than the mistake."""
whole sync later than the mistake.
"""
assert not set(SUPPORTED_OPS.values()) - set(ALLOWED_OPS)
def test_the_producer_allows_nothing_the_consumer_cannot_emit(self):
"""The allowlist is also a guard against a misbehaving or spoofed
consumer invoking arbitrary methods on trainer tensors, so it must not be
wider than the ops the bake can actually produce. ``to`` in particular
would let a replay change dtype or device."""
would let a replay change dtype or device.
"""
assert not set(ALLOWED_OPS) - set(SUPPORTED_OPS.values())
assert "to" not in ALLOWED_OPS
def test_transpose_via_t_is_serveable(self):
"""The specific regression: a chain recorded from ``.t()`` must pass the
producer's guard."""
producer's guard.
"""
fake = FakeRDTTensor(
name="w",
shape=torch.Size((4, 6)),
@@ -1470,7 +1529,8 @@ BAKE_MODELS = [
class TestRequiresTheRayExecutor:
"""The engine's data plane is Ray's, so a non-Ray executor cannot work. The
check is at construction because the alternative is an opaque failure during
the first handshake, long after the misconfiguration."""
the first handshake, long after the misconfiguration.
"""
def _construct(self, backend):
cfg = SimpleNamespace(
@@ -1497,7 +1557,8 @@ class TestRequiresTheRayExecutor:
def test_a_custom_executor_class_is_left_alone(self):
"""A `type[Executor]` override is deliberate and unjudgeable here, so it
must not be rejected for merely not being the string "ray"."""
must not be rejected for merely not being the string "ray".
"""
class _CustomExecutor:
pass