Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -18,13 +18,13 @@ DP_RANK = 0
|
||||
|
||||
@pytest.fixture
|
||||
def random_port():
|
||||
"""Generate a random port number for testing"""
|
||||
"""Generate a random port number for testing."""
|
||||
return random.randint(10000, 59900)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def publisher_config(random_port, request):
|
||||
"""Create a publisher config with inproc transport"""
|
||||
"""Create a publisher config with inproc transport."""
|
||||
how = request.param if hasattr(request, "param") else "inproc"
|
||||
|
||||
if how == "inproc":
|
||||
@@ -47,7 +47,7 @@ def publisher_config(random_port, request):
|
||||
|
||||
@pytest.fixture
|
||||
def publisher(publisher_config):
|
||||
"""Create and return a publisher instance"""
|
||||
"""Create and return a publisher instance."""
|
||||
pub = EventPublisherFactory.create(publisher_config, DP_RANK)
|
||||
yield pub
|
||||
pub.shutdown()
|
||||
@@ -55,7 +55,7 @@ def publisher(publisher_config):
|
||||
|
||||
@pytest.fixture
|
||||
def subscriber(publisher_config):
|
||||
"""Create and return a subscriber for testing"""
|
||||
"""Create and return a subscriber for testing."""
|
||||
endpoint = publisher_config.endpoint
|
||||
replay_endpoint = publisher_config.replay_endpoint
|
||||
|
||||
@@ -74,7 +74,7 @@ def subscriber(publisher_config):
|
||||
|
||||
|
||||
class MockSubscriber:
|
||||
"""Helper class to receive and verify published events"""
|
||||
"""Helper class to receive and verify published events."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -113,7 +113,7 @@ class MockSubscriber:
|
||||
self.decoder = msgspec.msgpack.Decoder(type=decode_type)
|
||||
|
||||
def receive_one(self, timeout=1000) -> tuple[int, SampleBatch] | None:
|
||||
"""Receive a single message with timeout"""
|
||||
"""Receive a single message with timeout."""
|
||||
if not self.sub.poll(timeout):
|
||||
return None
|
||||
|
||||
@@ -127,7 +127,7 @@ class MockSubscriber:
|
||||
return seq, data
|
||||
|
||||
def request_replay(self, start_seq: int, socket_idx: int = 0) -> None:
|
||||
"""Request replay of messages starting from start_seq"""
|
||||
"""Request replay of messages starting from start_seq."""
|
||||
if not self.replay_sockets:
|
||||
raise ValueError("Replay sockets not initialized")
|
||||
if socket_idx >= len(self.replay_sockets):
|
||||
@@ -138,7 +138,7 @@ class MockSubscriber:
|
||||
)
|
||||
|
||||
def receive_replay(self, socket_idx: int = 0) -> list[tuple[int, SampleBatch]]:
|
||||
"""Receive replayed messages from a specific replay socket"""
|
||||
"""Receive replayed messages from a specific replay socket."""
|
||||
if not self.replay_sockets:
|
||||
raise ValueError("Replay sockets not initialized")
|
||||
if socket_idx >= len(self.replay_sockets):
|
||||
@@ -170,7 +170,7 @@ class MockSubscriber:
|
||||
return replayed
|
||||
|
||||
def close(self):
|
||||
"""Clean up resources"""
|
||||
"""Clean up resources."""
|
||||
self.sub.close()
|
||||
for replay in self.replay_sockets:
|
||||
replay.close()
|
||||
@@ -179,7 +179,8 @@ class MockSubscriber:
|
||||
@pytest.fixture
|
||||
def enable_ray_v2_backend():
|
||||
"""Set env vars for the Ray V2 executor backend and shut down Ray
|
||||
between tests."""
|
||||
between tests.
|
||||
"""
|
||||
import ray
|
||||
|
||||
saved = {
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
WARNING: This test runs in both single-node (4 GPUs) and multi-node
|
||||
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
|
||||
important to set the distributed backend to "mp" to avoid Ray scheduling
|
||||
all workers in a node other than the head node, which can cause the test
|
||||
to fail.
|
||||
"""WARNING: This test runs in both single-node (4 GPUs) and multi-node
|
||||
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
|
||||
important to set the distributed backend to "mp" to avoid Ray scheduling
|
||||
all workers in a node other than the head node, which can cause the test
|
||||
to fail.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
@@ -29,7 +29,7 @@ def test_eplb_load_stats_reduce_across_ranks():
|
||||
|
||||
|
||||
def test_basic_rebalance():
|
||||
"""Test basic rebalancing functionality"""
|
||||
"""Test basic rebalancing functionality."""
|
||||
# Example from https://github.com/deepseek-ai/eplb
|
||||
weight = torch.tensor(
|
||||
[
|
||||
@@ -92,7 +92,7 @@ def test_basic_rebalance():
|
||||
|
||||
|
||||
def test_single_gpu_case():
|
||||
"""Test single GPU case"""
|
||||
"""Test single GPU case."""
|
||||
weight = torch.tensor([[10, 20, 30, 40]])
|
||||
num_replicas = 4
|
||||
num_groups = 1
|
||||
@@ -115,7 +115,7 @@ def test_single_gpu_case():
|
||||
|
||||
|
||||
def test_equal_weights():
|
||||
"""Test case with equal weights"""
|
||||
"""Test case with equal weights."""
|
||||
weight = torch.tensor([[50, 50, 50, 50, 50, 50, 50, 50]])
|
||||
num_replicas = 8
|
||||
num_groups = 2
|
||||
@@ -139,7 +139,7 @@ def test_equal_weights():
|
||||
|
||||
|
||||
def test_extreme_weight_imbalance():
|
||||
"""Test extreme weight imbalance case"""
|
||||
"""Test extreme weight imbalance case."""
|
||||
weight = torch.tensor([[1000, 1, 1, 1, 1, 1, 1, 1]])
|
||||
num_replicas = 12
|
||||
num_groups = 2
|
||||
@@ -162,7 +162,7 @@ def test_extreme_weight_imbalance():
|
||||
|
||||
|
||||
def test_multiple_layers():
|
||||
"""Test multiple layers case"""
|
||||
"""Test multiple layers case."""
|
||||
weight = torch.tensor(
|
||||
[
|
||||
[10, 20, 30, 40, 50, 60], # First layer
|
||||
@@ -195,7 +195,7 @@ def test_multiple_layers():
|
||||
|
||||
|
||||
def test_parameter_validation():
|
||||
"""Test parameter validation"""
|
||||
"""Test parameter validation."""
|
||||
weight = torch.tensor([[10, 20, 30, 40]])
|
||||
|
||||
# Test non-divisible case - this should handle normally without throwing
|
||||
@@ -213,7 +213,7 @@ def test_parameter_validation():
|
||||
|
||||
|
||||
def test_small_scale_hierarchical():
|
||||
"""Test small-scale hierarchical load balancing"""
|
||||
"""Test small-scale hierarchical load balancing."""
|
||||
weight = torch.tensor(
|
||||
[
|
||||
[100, 50, 200, 75, 150, 25, 300, 80], # 8 experts
|
||||
@@ -243,7 +243,7 @@ def test_small_scale_hierarchical():
|
||||
|
||||
|
||||
def test_global_load_balance_fallback():
|
||||
"""Test global load balancing fallback case"""
|
||||
"""Test global load balancing fallback case."""
|
||||
# When num_groups % num_nodes != 0, should fall back to global load
|
||||
# balancing
|
||||
weight = torch.tensor([[10, 20, 30, 40, 50, 60]])
|
||||
@@ -265,7 +265,7 @@ def test_global_load_balance_fallback():
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
def test_device_compatibility(device):
|
||||
"""Test device compatibility"""
|
||||
"""Test device compatibility."""
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA not available")
|
||||
|
||||
@@ -287,8 +287,7 @@ def test_device_compatibility(device):
|
||||
|
||||
|
||||
def test_additional_cases():
|
||||
"""Test more edge cases and different parameter combinations"""
|
||||
|
||||
"""Test more edge cases and different parameter combinations."""
|
||||
# Test case 1: Large-scale distributed setup
|
||||
weight1 = torch.tensor(
|
||||
[[50, 100, 75, 120, 90, 60, 80, 110, 40, 70, 95, 85, 65, 55, 45, 35]]
|
||||
@@ -320,8 +319,7 @@ def test_additional_cases():
|
||||
|
||||
|
||||
def test_compute_logical_maps_with_negative_indices():
|
||||
"""
|
||||
Test that compute_logical_maps correctly handles physical slots containing
|
||||
"""Test that compute_logical_maps correctly handles physical slots containing
|
||||
-1 (unused slots).
|
||||
"""
|
||||
# 2 layers, 6 physical slots, 4 logical experts.
|
||||
|
||||
@@ -57,9 +57,8 @@ def test_reuse_across_multiple_cycles():
|
||||
|
||||
|
||||
def test_producer_consumer():
|
||||
"""
|
||||
This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared GPU
|
||||
tensor on multiple CPU threads.
|
||||
"""This test uses the CpuGpuEvent to synchronize reads and writes to/from a shared
|
||||
GPU tensor on multiple CPU threads.
|
||||
"""
|
||||
worker_stream = torch.cuda.Stream()
|
||||
# Create a single element counter that will be shared between two threads
|
||||
|
||||
@@ -56,8 +56,7 @@ def create_expert_indices_with_redundancy(
|
||||
total_physical_experts: int,
|
||||
redundancy_config: list[int], # redundancy for each logical expert
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Create expert indices with redundancy.
|
||||
"""Create expert indices with redundancy.
|
||||
|
||||
Args:
|
||||
num_layers: number of layers
|
||||
@@ -67,6 +66,7 @@ def create_expert_indices_with_redundancy(
|
||||
|
||||
Returns:
|
||||
indices: Shape (num_layers, total_physical_experts)
|
||||
|
||||
"""
|
||||
assert sum(redundancy_config) == total_physical_experts
|
||||
assert len(redundancy_config) == num_logical_experts
|
||||
@@ -95,8 +95,7 @@ def create_expert_weights(
|
||||
device: torch.device,
|
||||
physical_to_logical_mapping: torch.Tensor,
|
||||
) -> list[list[torch.Tensor]]:
|
||||
"""
|
||||
Create fake expert weights tensor for testing.
|
||||
"""Create fake expert weights tensor for testing.
|
||||
|
||||
Use `arange` to generate predictable weights values, based on logical
|
||||
expert ID.
|
||||
@@ -105,6 +104,7 @@ def create_expert_weights(
|
||||
Args:
|
||||
physical_to_logical_mapping: Shape (num_layers, num_local_experts)
|
||||
mapping[layer, physical_pos] = logical_expert_id
|
||||
|
||||
"""
|
||||
expert_weights = []
|
||||
|
||||
@@ -209,9 +209,7 @@ def verify_redundant_experts_have_same_weights(
|
||||
world_size: int,
|
||||
num_local_experts: int,
|
||||
) -> bool:
|
||||
"""
|
||||
Verify that all replicas of the same logical expert have the same weights.
|
||||
"""
|
||||
"""Verify that all replicas of the same logical expert have the same weights."""
|
||||
num_layers = len(expert_weights)
|
||||
total_physical_experts = world_size * num_local_experts
|
||||
|
||||
@@ -568,7 +566,6 @@ def test_rearrange_expert_weights_with_redundancy(
|
||||
eplb_communicator,
|
||||
):
|
||||
"""Test the functionality of rearranging expert weights with redundancy."""
|
||||
|
||||
if eplb_communicator == "nixl" and not has_nixl():
|
||||
pytest.skip("NIXL is not available")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
@@ -678,7 +675,6 @@ def test_async_transfer_layer_without_mtp(
|
||||
eplb_communicator: str,
|
||||
):
|
||||
"""Exercise async EPLB transfer path without MTP/spec decode."""
|
||||
|
||||
if eplb_communicator == "nixl" and not has_nixl():
|
||||
pytest.skip("NIXL is not available")
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
@@ -696,11 +692,9 @@ def test_async_transfer_layer_without_mtp(
|
||||
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_rearrange_expert_weights_no_change(world_size):
|
||||
"""
|
||||
Test that when the indices do not change, the weights should remain
|
||||
"""Test that when the indices do not change, the weights should remain
|
||||
unchanged.
|
||||
"""
|
||||
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
@@ -800,7 +794,6 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_rearrange_expert_weights_profile_mode(world_size):
|
||||
"""Test profile mode (should not copy actual weights)"""
|
||||
|
||||
if torch.accelerator.device_count() < world_size:
|
||||
pytest.skip(f"Need at least {world_size} GPUs to run the test")
|
||||
distributed_run(
|
||||
|
||||
@@ -124,7 +124,8 @@ def _build_processed_layer(
|
||||
backend: str, raw: dict[str, torch.Tensor], device: torch.device
|
||||
) -> tuple[CompressedTensorsW4A4Nvfp4MoEMethod, _RoutedExpertsStub]:
|
||||
"""Create the method + layer, load ``raw`` and run the real
|
||||
process_weights_after_loading (which builds the flashinfer kernel)."""
|
||||
process_weights_after_loading (which builds the flashinfer kernel).
|
||||
"""
|
||||
moe_config = _make_moe_config(backend)
|
||||
method = CompressedTensorsW4A4Nvfp4MoEMethod(moe_config, "layer.0", use_a16=False)
|
||||
layer = _RoutedExpertsStub(moe_config).to(device)
|
||||
@@ -157,7 +158,8 @@ def _assert_tensors_equal(
|
||||
|
||||
def _simulate_eplb_rearrangement(layer: torch.nn.Module, perm: torch.Tensor) -> None:
|
||||
"""Permute experts of every registered Parameter in place, the way
|
||||
EPLB's in-place rearrangement moves expert slices along dim 0."""
|
||||
EPLB's in-place rearrangement moves expert slices along dim 0.
|
||||
"""
|
||||
with torch.no_grad():
|
||||
for _, param in layer.named_parameters():
|
||||
param.copy_(param[perm])
|
||||
@@ -165,7 +167,8 @@ def _simulate_eplb_rearrangement(layer: torch.nn.Module, perm: torch.Tensor) ->
|
||||
|
||||
def _ensure_world1_distributed() -> None:
|
||||
"""The enable_eplb config makes weight processing all-reduce activation
|
||||
scale amaxes over the EP group, so a (single-rank) EP group must exist."""
|
||||
scale amaxes over the EP group, so a (single-rank) EP group must exist.
|
||||
"""
|
||||
from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
init_distributed_environment,
|
||||
@@ -253,7 +256,8 @@ def test_nvfp4_eplb_rearrangement_matches_reload(backend: str) -> None:
|
||||
def test_fp8_per_tensor_alphas_registered_and_aliased() -> None:
|
||||
"""CPU-only: the fp8 per-tensor oracle must register the fused
|
||||
(w_scale * a_scale) products as layer Parameters aliased by the quant
|
||||
config, so EPLB rearrangement keeps kernels consistent."""
|
||||
config, so EPLB rearrangement keeps kernels consistent.
|
||||
"""
|
||||
from vllm.model_executor.layers.fused_moe.oracle.fp8 import (
|
||||
Fp8MoeBackend,
|
||||
make_fp8_moe_quant_config,
|
||||
|
||||
@@ -75,8 +75,7 @@ def test_eplb_spec_decode(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
model_setup: tuple[str, str, str, int, float],
|
||||
):
|
||||
"""
|
||||
Test the correctness of EPLB speculative decoding with GSM8K dataset.
|
||||
"""Test the correctness of EPLB speculative decoding with GSM8K dataset.
|
||||
Applicable to MoE models with mtp or eagle spec decode.
|
||||
"""
|
||||
method, model_name, spec_model_name, tp_size, expected_gsm8k_value = model_setup
|
||||
@@ -109,10 +108,7 @@ def test_eplb_spec_decode(
|
||||
|
||||
@large_gpu_mark(min_gb=80)
|
||||
def test_eplb_spec_decode_qwen3_next_mtp_async() -> None:
|
||||
"""
|
||||
Ensure async EPLB works with MTP speculative decoding for Qwen3-Next.
|
||||
"""
|
||||
|
||||
"""Ensure async EPLB works with MTP speculative decoding for Qwen3-Next."""
|
||||
TASK = "gsm8k"
|
||||
FILTER = "exact_match,strict-match"
|
||||
RTOL = 0.03
|
||||
|
||||
@@ -30,8 +30,7 @@ def _make_model_state(
|
||||
|
||||
|
||||
def test_commit_eplb_maps_shape_change():
|
||||
"""
|
||||
When the number of physical experts changes, resize the active map within its
|
||||
"""When the number of physical experts changes, resize the active map within its
|
||||
preallocated storage.
|
||||
"""
|
||||
num_layers, num_logical, num_physical = 2, 4, 6
|
||||
@@ -65,8 +64,7 @@ def test_commit_eplb_maps_shape_change():
|
||||
|
||||
|
||||
def test_commit_eplb_maps_for_layer_logical_padding():
|
||||
"""
|
||||
Test that logical_to_physical_map is padded with -1 to fill the
|
||||
"""Test that logical_to_physical_map is padded with -1 to fill the
|
||||
pre-allocated slots when the new map has fewer replicas than the max.
|
||||
"""
|
||||
num_layers, num_logical, num_physical = 2, 4, 6
|
||||
|
||||
@@ -20,27 +20,26 @@ class EventSample(
|
||||
tag=True, # type: ignore
|
||||
array_like=True, # type: ignore
|
||||
):
|
||||
"""Test event for publisher testing"""
|
||||
"""Test event for publisher testing."""
|
||||
|
||||
id: int
|
||||
value: str
|
||||
|
||||
|
||||
class SampleBatch(EventBatch):
|
||||
"""Test event batch for publisher testing"""
|
||||
"""Test event batch for publisher testing."""
|
||||
|
||||
events: list[EventSample]
|
||||
|
||||
|
||||
def create_test_events(count: int) -> SampleBatch:
|
||||
"""Create a batch of test events"""
|
||||
"""Create a batch of test events."""
|
||||
events = [EventSample(id=i, value=f"test-{i}") for i in range(count)]
|
||||
return SampleBatch(ts=time.time(), events=events)
|
||||
|
||||
|
||||
def test_basic_publishing(publisher, subscriber):
|
||||
"""Test basic event publishing works"""
|
||||
|
||||
"""Test basic event publishing works."""
|
||||
test_batch = create_test_events(5)
|
||||
publisher.publish(test_batch)
|
||||
|
||||
@@ -58,7 +57,7 @@ def test_basic_publishing(publisher, subscriber):
|
||||
|
||||
|
||||
def test_multiple_events(publisher, subscriber):
|
||||
"""Test publishing and receiving multiple event batches"""
|
||||
"""Test publishing and receiving multiple event batches."""
|
||||
for _ in range(10):
|
||||
batch = create_test_events(2)
|
||||
publisher.publish(batch)
|
||||
@@ -75,7 +74,7 @@ def test_multiple_events(publisher, subscriber):
|
||||
|
||||
|
||||
def test_replay_mechanism(publisher, subscriber):
|
||||
"""Test the replay mechanism works correctly"""
|
||||
"""Test the replay mechanism works correctly."""
|
||||
for _ in range(19):
|
||||
batch = create_test_events(1)
|
||||
publisher.publish(batch)
|
||||
@@ -96,7 +95,7 @@ def test_replay_mechanism(publisher, subscriber):
|
||||
|
||||
|
||||
def test_replay_includes_topic(publisher, subscriber, publisher_config):
|
||||
"""Test that replay responses include the topic, matching PUB format"""
|
||||
"""Test that replay responses include the topic, matching PUB format."""
|
||||
for _ in range(5):
|
||||
publisher.publish(create_test_events(1))
|
||||
|
||||
@@ -115,7 +114,7 @@ def test_replay_includes_topic(publisher, subscriber, publisher_config):
|
||||
|
||||
|
||||
def test_buffer_limit(publisher, subscriber, publisher_config):
|
||||
"""Test buffer limit behavior"""
|
||||
"""Test buffer limit behavior."""
|
||||
buffer_size = publisher_config.buffer_steps
|
||||
|
||||
# Publish more events than the buffer can hold
|
||||
@@ -139,9 +138,7 @@ def test_buffer_limit(publisher, subscriber, publisher_config):
|
||||
|
||||
|
||||
def test_topic_filtering(publisher_config):
|
||||
"""
|
||||
Test that a subscriber only receives messages matching its topic filter
|
||||
"""
|
||||
"""Test that a subscriber only receives messages matching its topic filter."""
|
||||
publisher_config.replay_endpoint = None
|
||||
|
||||
publisher_config.topic = "foo"
|
||||
@@ -174,7 +171,7 @@ def test_topic_filtering(publisher_config):
|
||||
|
||||
|
||||
def test_high_volume(publisher, subscriber):
|
||||
"""Test publishing and receiving a high volume of events"""
|
||||
"""Test publishing and receiving a high volume of events."""
|
||||
num_batches = 10_000
|
||||
events_per_batch = 100
|
||||
|
||||
@@ -210,7 +207,7 @@ def test_high_volume(publisher, subscriber):
|
||||
|
||||
|
||||
def test_null_publisher():
|
||||
"""Test that NullEventPublisher can be used without errors"""
|
||||
"""Test that NullEventPublisher can be used without errors."""
|
||||
publisher = NullEventPublisher(DP_RANK)
|
||||
|
||||
# This should not raise any errors
|
||||
@@ -220,8 +217,7 @@ def test_null_publisher():
|
||||
|
||||
|
||||
def test_data_parallel_rank_tagging(publisher_config):
|
||||
"""Test that events are properly tagged with their data parallel rank"""
|
||||
|
||||
"""Test that events are properly tagged with their data parallel rank."""
|
||||
publisher_config.topic = "foo"
|
||||
pub_0 = EventPublisherFactory.create(publisher_config, DP_RANK)
|
||||
pub_1 = EventPublisherFactory.create(publisher_config, DP_RANK + 1)
|
||||
@@ -285,7 +281,7 @@ def test_data_parallel_rank_tagging(publisher_config):
|
||||
|
||||
|
||||
def test_event_publisher_factory(random_port):
|
||||
"""Test event publisher factory creation behavior under different configurations"""
|
||||
"""Test event publisher factory creation behavior under different configurations."""
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed.kv_events import ZmqEventPublisher
|
||||
|
||||
|
||||
@@ -49,7 +49,6 @@ def verify_round_robin_pattern(expert_map, ep_rank, ep_size, global_num_experts)
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_expert_placement_various_sizes(expert_placement_strategy, world_size):
|
||||
"""Test round_robin expert placement with various expert counts."""
|
||||
|
||||
# Test with different global_num_experts values
|
||||
# Include both divisible and non-divisible cases
|
||||
if world_size == 2:
|
||||
@@ -116,7 +115,6 @@ def test_expert_placement_various_sizes(expert_placement_strategy, world_size):
|
||||
@pytest.mark.parametrize("world_size", [2, 4])
|
||||
def test_expert_placement_edge_cases(expert_placement_strategy, world_size):
|
||||
"""Test edge cases for round_robin expert placement."""
|
||||
|
||||
# Test case 1: ep_size = 1 (should return None for expert_map)
|
||||
local_num_experts, expert_map, _ = determine_expert_map(
|
||||
ep_size=1,
|
||||
@@ -139,7 +137,6 @@ def test_expert_placement_edge_cases(expert_placement_strategy, world_size):
|
||||
|
||||
def test_determine_expert_map_comprehensive():
|
||||
"""Test of determine_expert_map function with various configurations."""
|
||||
|
||||
# Test cases: (ep_size, ep_rank, global_num_experts,
|
||||
# expert_placement_strategy, expected_local, expected_map_pattern)
|
||||
test_cases = [
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Tests for MNNVL AllToAll operations.
|
||||
"""Tests for MNNVL AllToAll operations.
|
||||
|
||||
Requires: docker run ... --cap-add=SYS_PTRACE ...
|
||||
Run: pytest tests/distributed/test_mnnvl_alltoall.py -v
|
||||
@@ -87,6 +86,7 @@ def _run_worker(rank, world_size, port, worker_fn, dp_size, dp_port, err_queue):
|
||||
Otherwise use tp=world_size (default for EP-based tests).
|
||||
dp_port: Separate port for the DP master (only used when dp_size is set).
|
||||
err_queue: Queue for propagating tracebacks to the parent process.
|
||||
|
||||
"""
|
||||
try:
|
||||
os.environ.pop("CUDA_VISIBLE_DEVICES", None)
|
||||
@@ -115,6 +115,7 @@ def _init_dp_environment(world_size, rank, port, dp_size, dp_port):
|
||||
Args:
|
||||
port: Port for torch.distributed init.
|
||||
dp_port: Separate port for the DP master group init.
|
||||
|
||||
"""
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
@@ -159,7 +160,8 @@ def _make_forward_context(rank, world_size, num_tokens_per_rank):
|
||||
class _AttnMeta:
|
||||
"""Minimal placeholder so set_forward_context's
|
||||
``attn_metadata is not None`` guard (forward_context.py:334)
|
||||
is satisfied. The real DPMetadata is built from num_tokens_across_dp."""
|
||||
is satisfied. The real DPMetadata is built from num_tokens_across_dp.
|
||||
"""
|
||||
|
||||
dp_metadata = None
|
||||
|
||||
@@ -458,7 +460,8 @@ def _one_sided_workspace_grow_worker(rank, world_size):
|
||||
def test_one_sided_manager_workspace_grow(world_size):
|
||||
"""A later initialize() with a larger per-token payload must grow the
|
||||
workspace and rebuild MoeAlltoAll; a later initialize() with a smaller
|
||||
payload must no-op."""
|
||||
payload must no-op.
|
||||
"""
|
||||
_spawn_workers(
|
||||
_one_sided_workspace_grow_worker,
|
||||
world_size,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Test that MessageQueue uses the local node's IP for binding,
|
||||
"""Test that MessageQueue uses the local node's IP for binding,
|
||||
not a remote master_addr. This validates the fix for cross-node
|
||||
data-parallel where each DP group leader must bind to its own IP.
|
||||
|
||||
@@ -22,7 +21,8 @@ from vllm.utils.network_utils import get_ip
|
||||
|
||||
def test_mq_bind_with_local_ip():
|
||||
"""MessageQueue with remote readers should successfully bind
|
||||
when connect_ip is the local node's IP."""
|
||||
when connect_ip is the local node's IP.
|
||||
"""
|
||||
# n_reader=2, n_local_reader=1 means 1 remote reader,
|
||||
# which triggers the remote ZMQ socket bind.
|
||||
mq = MessageQueue(
|
||||
@@ -44,7 +44,8 @@ def test_mq_bind_with_local_ip():
|
||||
def test_mq_bind_with_non_local_ip_fails():
|
||||
"""MessageQueue should fail to bind when connect_ip is a
|
||||
non-local IP address (simulating the bug where master_addr
|
||||
from a different node was used)."""
|
||||
from a different node was used).
|
||||
"""
|
||||
# Use a non-local IP that we definitely can't bind to.
|
||||
# 198.51.100.1 is from TEST-NET-2 (RFC 5737), never locally assigned.
|
||||
non_local_ip = "198.51.100.1"
|
||||
@@ -58,7 +59,8 @@ def test_mq_bind_with_non_local_ip_fails():
|
||||
|
||||
def test_mq_bind_defaults_to_local_ip():
|
||||
"""When connect_ip is None, MessageQueue should auto-detect
|
||||
the local IP and bind successfully."""
|
||||
the local IP and bind successfully.
|
||||
"""
|
||||
mq = MessageQueue(
|
||||
n_reader=2,
|
||||
n_local_reader=1,
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Integration tests for MultiprocExecutor at the executor level.
|
||||
"""Integration tests for MultiprocExecutor at the executor level.
|
||||
This test directly tests the executor without going through the LLM interface,
|
||||
focusing on executor initialization, RPC calls, and distributed execution.
|
||||
"""
|
||||
@@ -377,8 +376,7 @@ def test_multiproc_executor_properties():
|
||||
|
||||
@multi_gpu_test(num_gpus=4)
|
||||
def test_multiproc_executor_multi_node():
|
||||
"""
|
||||
Test MultiprocExecutor with multi-node configuration.
|
||||
"""Test MultiprocExecutor with multi-node configuration.
|
||||
This simulates 2 nodes with TP=4:
|
||||
- Node 0 (rank 0): Uses GPUs 0,1 (CUDA_VISIBLE_DEVICES=0,1) with TP=2
|
||||
- Node 1 (rank 1): Uses GPUs 2,3 (CUDA_VISIBLE_DEVICES=2,3) with TP=2
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
WARNING: This test runs in both single-node (4 GPUs) and multi-node
|
||||
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
|
||||
important to set the distributed backend to "mp" to avoid Ray scheduling
|
||||
all workers in a node other than the head node, which can cause the test
|
||||
to fail.
|
||||
"""WARNING: This test runs in both single-node (4 GPUs) and multi-node
|
||||
(2 node with 2 GPUs each) modes. If the test only uses 2 GPUs, it is
|
||||
important to set the distributed backend to "mp" to avoid Ray scheduling
|
||||
all workers in a node other than the head node, which can cause the test
|
||||
to fail.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
@@ -495,7 +495,8 @@ def test_pynccl_suspend_resume_idempotent():
|
||||
|
||||
def test_pynccl_suspend_noop_without_symbol():
|
||||
"""RCCL / NCCL < 2.29.7 lack ncclCommSuspend; suspend() must no-op, not
|
||||
crash, and resume() must then no-op too since nothing was suspended."""
|
||||
crash, and resume() must then no-op too since nothing was suspended.
|
||||
"""
|
||||
from unittest.mock import Mock
|
||||
|
||||
comm = object.__new__(PyNcclCommunicator)
|
||||
|
||||
@@ -396,8 +396,7 @@ def test_custom_quick_allreduce_bf16_cast(monkeypatch: pytest.MonkeyPatch):
|
||||
|
||||
|
||||
def qr_variable_input(rank, world_size):
|
||||
"""
|
||||
When the tensor parallelism is set to 4 or 8, frequent changes
|
||||
"""When the tensor parallelism is set to 4 or 8, frequent changes
|
||||
in the input shape can cause QuickReduce to hang (this issue
|
||||
has been observed with the gpt_oss model).
|
||||
"""
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Integration tests for RayExecutorV2 at the executor level.
|
||||
"""Integration tests for RayExecutorV2 at the executor level.
|
||||
Validates executor initialization, placement group support, RPC calls,
|
||||
and distributed execution with various TP/PP configurations.
|
||||
"""
|
||||
@@ -299,7 +298,6 @@ def test_ray_v2_run_refs_stored_for_monitoring(executor):
|
||||
@pytest.mark.parametrize("tp_size, pp_size", [(2, 1), (2, 2)])
|
||||
def test_ray_v2_single_node_generation(tp_size, pp_size):
|
||||
"""End-to-end LLM generation with RayExecutorV2."""
|
||||
|
||||
llm = LLM(
|
||||
model=MODEL,
|
||||
tensor_parallel_size=tp_size,
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Orchestration-level integration tests for RayExecutorV2.
|
||||
"""
|
||||
"""Orchestration-level integration tests for RayExecutorV2."""
|
||||
|
||||
import gc
|
||||
import os
|
||||
@@ -25,7 +23,8 @@ def _ray_init():
|
||||
|
||||
Without this, workers cannot unpickle actor classes defined in the
|
||||
``tests`` package, causing FunctionActorManager to fall back to
|
||||
TemporaryActor which drops async method signatures."""
|
||||
TemporaryActor which drops async method signatures.
|
||||
"""
|
||||
project_root = str(pathlib.Path(__file__).resolve().parents[2])
|
||||
ray.init(
|
||||
ignore_reinit_error=True,
|
||||
@@ -158,8 +157,7 @@ def test_multi_replicas_with_bundle_indices(ray_init):
|
||||
|
||||
|
||||
def test_env_var_and_runtime_env_propagation():
|
||||
"""
|
||||
Verify env vars (NCCL_, HF_) and parallel_config.ray_runtime_env
|
||||
"""Verify env vars (NCCL_, HF_) and parallel_config.ray_runtime_env
|
||||
propagate to RayWorkerProc actors.
|
||||
"""
|
||||
sentinel_vars = {
|
||||
|
||||
@@ -53,7 +53,8 @@ META = torch.device("meta")
|
||||
|
||||
class _FakeLayer:
|
||||
"""Stand-in for a leaf module. The planner only stores it and keys on
|
||||
``id()``; nothing in the plan path touches its attributes."""
|
||||
``id()``; nothing in the plan path touches its attributes.
|
||||
"""
|
||||
|
||||
def __init__(self, tag: str) -> None:
|
||||
self.tag = tag
|
||||
@@ -64,7 +65,8 @@ class _FakeLayer:
|
||||
|
||||
def _module(layer, copies):
|
||||
"""What the bake records for one leaf module: a scatter list whose entries
|
||||
all carry that module as their ``layer``."""
|
||||
all carry that module as their ``layer``.
|
||||
"""
|
||||
for c in copies:
|
||||
c.layer = layer
|
||||
return copies
|
||||
@@ -81,7 +83,8 @@ def _copy(
|
||||
dtype=torch.bfloat16,
|
||||
):
|
||||
"""A `_Scatter` as the bake would record it: source key, owning layer, and
|
||||
the meta destination region."""
|
||||
the meta destination region.
|
||||
"""
|
||||
stride = (
|
||||
(1,)
|
||||
if len(shape) == 1
|
||||
@@ -140,7 +143,8 @@ def _planner(baked, *, name_meta, live=None, held_by=None, group_lens=None):
|
||||
|
||||
def _one_module_per_layer(n_layers, *, dtype="bfloat16", numel=4):
|
||||
"""`(baked, name_meta, names, group_lens)` for a pre / N-layer / post model
|
||||
where every gather group is one fully-baked leaf module."""
|
||||
where every gather group is one fully-baked leaf module.
|
||||
"""
|
||||
names = ["embed.weight"]
|
||||
names += [f"model.layers.{i}.w" for i in range(n_layers)]
|
||||
names += ["norm.weight"]
|
||||
@@ -161,7 +165,8 @@ def _one_module_per_layer(n_layers, *, dtype="bfloat16", numel=4):
|
||||
class TestFakeOpChains:
|
||||
"""Every allowlisted op must append itself to the chain and hand back a
|
||||
child whose shape/dtype PyTorch itself computed. The chain is the wire
|
||||
format the producer replays, so its exact contents are load-bearing."""
|
||||
format the producer replays, so its exact contents are load-bearing.
|
||||
"""
|
||||
|
||||
def _fake(self, shape=(4, 6), dtype=torch.bfloat16, sink=None):
|
||||
return FakeRDTTensor(
|
||||
@@ -222,7 +227,8 @@ class TestFakeOpChains:
|
||||
@pytest.mark.parametrize("op,n", [("chunk", 2), ("unbind", 4)])
|
||||
def test_multi_return_op_emits_one_child_per_output(self, op, n):
|
||||
"""chunk/unbind hand back a tuple; each child carries the base op plus a
|
||||
trailing __getitem__(i) so the producer can index the replayed result."""
|
||||
trailing __getitem__(i) so the producer can index the replayed result.
|
||||
"""
|
||||
parts = getattr(self._fake(), op)(*((n,) if op == "chunk" else ()), 0)
|
||||
assert isinstance(parts, tuple)
|
||||
assert len(parts) == n
|
||||
@@ -238,7 +244,8 @@ class TestFakeOpChains:
|
||||
|
||||
class TestFakeUnsupportedOps:
|
||||
"""Anything that needs real data must fail loudly at bake time rather than
|
||||
silently transferring the wrong bytes."""
|
||||
silently transferring the wrong bytes.
|
||||
"""
|
||||
|
||||
def _fake(self):
|
||||
return FakeRDTTensor(
|
||||
@@ -278,7 +285,8 @@ class TestFakeUnsupportedOps:
|
||||
|
||||
class TestBakeRecording:
|
||||
"""During the dry run the fake's ``copy_`` is the data sink: it records the
|
||||
source chain plus the meta destination's strided region and moves nothing."""
|
||||
source chain plus the meta destination's strided region and moves nothing.
|
||||
"""
|
||||
|
||||
def _recorder_and_fake(self, shape=(4, 6)):
|
||||
rec = BakeSink()
|
||||
@@ -314,7 +322,8 @@ class TestBakeRecording:
|
||||
|
||||
def test_unattributed_copy_is_live_but_unrecorded(self):
|
||||
"""A copy_ with no loader stamp cannot be attributed to a param, so its
|
||||
module must fall back to the plain load — but the name still moved data."""
|
||||
module must fall back to the plain load — but the name still moved data.
|
||||
"""
|
||||
rec, fake = self._recorder_and_fake()
|
||||
rec.current = None
|
||||
torch.empty((4, 6), dtype=torch.bfloat16, device=META).copy_(fake)
|
||||
@@ -354,7 +363,8 @@ class TestBakeRecording:
|
||||
producer packs what the replay yields, so the record must carry the
|
||||
POST-chain dtype: taking it from the source name's metadata instead sizes
|
||||
the slice with the wrong itemsize and shifts every later slice in the
|
||||
chunk, carving the packed blob differently on the two sides."""
|
||||
chunk, carving the packed blob differently on the two sides.
|
||||
"""
|
||||
rec, fake = self._recorder_and_fake(shape=(4,))
|
||||
layer = _FakeLayer("reinterpreted")
|
||||
rec.current = (layer, "weight")
|
||||
@@ -374,7 +384,8 @@ class TestBakeRecording:
|
||||
the producer packs what the chain yields, and nothing downstream can
|
||||
catch a mismatch — the consumer's buffer view is exactly prod(dest.shape)
|
||||
elements, so it reshapes cleanly over bytes laid out at other offsets.
|
||||
So the bake refuses rather than recording a slice it cannot carve."""
|
||||
So the bake refuses rather than recording a slice it cannot carve.
|
||||
"""
|
||||
rec, fake = self._recorder_and_fake(shape=(1, 6))
|
||||
rec.current = (_FakeLayer("broadcast"), "weight")
|
||||
param = torch.empty((4, 6), dtype=torch.bfloat16, device=META)
|
||||
@@ -390,7 +401,8 @@ class TestBakeRecording:
|
||||
class TestConsumerIdentity:
|
||||
"""Every worker in the fleet needs a DISTINCT id in 0..C-1: it selects the
|
||||
worker's producer block and keys the producer's per-consumer serve ring, so
|
||||
a collision silently serves two workers out of one ring."""
|
||||
a collision silently serves two workers out of one ring.
|
||||
"""
|
||||
|
||||
def _engine(self, *, dp_index, rank, world_size):
|
||||
eng = object.__new__(ShardedRDTWeightTransferEngine)
|
||||
@@ -427,7 +439,8 @@ class TestConsumerIdentity:
|
||||
def test_independent_engines_offset_into_distinct_ranges(self):
|
||||
"""A fleet of separate engines restarts _global_worker_index at 0 in
|
||||
each, so without the replica offset every engine would claim 0..w-1.
|
||||
The driver sets replica_rank on the payload; these are the ids it buys."""
|
||||
The driver sets replica_rank on the payload; these are the ids it buys.
|
||||
"""
|
||||
ids = [
|
||||
self._ids(num_consumers=8, workers=range(4), replica_rank=r, num_replicas=2)
|
||||
for r in (0, 1)
|
||||
@@ -453,7 +466,8 @@ def _router(
|
||||
group_lens=None,
|
||||
):
|
||||
"""A router over ``n`` single-name groups unless told otherwise, so a test
|
||||
can state expectations in group indices."""
|
||||
can state expectations in group indices.
|
||||
"""
|
||||
if names is None:
|
||||
n = len(name_owner_class or []) or (len(group_lens or []) or 1)
|
||||
names = [f"g{i}" for i in range(n)]
|
||||
@@ -482,7 +496,8 @@ class TestRdtRouter:
|
||||
def test_every_producer_nic_carries_traffic(self):
|
||||
"""16 producers / 8 consumers: the block rule spreads each consumer's
|
||||
pulls over its block, alternating by group, so no producer sits idle.
|
||||
Plain `consumer_id % len(owners)` would idle half of them."""
|
||||
Plain `consumer_id % len(owners)` would idle half of them.
|
||||
"""
|
||||
r = _router(16, 8, group_lens=[1] * 95)
|
||||
names = [f"g{i}" for i in range(95)]
|
||||
for c in range(8):
|
||||
@@ -500,7 +515,8 @@ class TestRdtRouter:
|
||||
|
||||
def test_pipeline_stages_route_to_the_owning_stage(self):
|
||||
"""2 stages x 8 ranks: groups 0-2 on stage 0, 3-5 on stage 1. Two owner
|
||||
sets, one per stage; each consumer reaches both."""
|
||||
sets, one per stage; each consumer reaches both.
|
||||
"""
|
||||
r = _router(
|
||||
16,
|
||||
8,
|
||||
@@ -516,7 +532,8 @@ class TestRdtRouter:
|
||||
|
||||
def test_expert_names_route_to_the_holding_ranks(self):
|
||||
"""A name held by only some ranks routes to those; a replicated name
|
||||
keeps the full set."""
|
||||
keeps the full set.
|
||||
"""
|
||||
r = RdtRouter(
|
||||
8,
|
||||
4,
|
||||
@@ -535,7 +552,8 @@ class TestRdtRouter:
|
||||
|
||||
def test_consumers_spread_over_an_owner_set(self):
|
||||
"""Several ranks hold a name (its TP peers); the block rule must spread
|
||||
consumers across them, not funnel through one."""
|
||||
consumers across them, not funnel through one.
|
||||
"""
|
||||
r = _router(
|
||||
4, 4, owner_sets=[[0, 1]], name_owner_class=[0] * 6, group_lens=[1] * 6
|
||||
)
|
||||
@@ -544,7 +562,8 @@ class TestRdtRouter:
|
||||
|
||||
def test_a_group_frees_at_every_owner_of_any_of_its_names(self):
|
||||
"""The free barrier is per group, so its fan-out is the UNION over the
|
||||
group's names: a rank holding none of them is not signalled."""
|
||||
group's names: a rank holding none of them is not signalled.
|
||||
"""
|
||||
r = RdtRouter(4, 2, [[0, 1], [2, 3]], [0, 1, 0], ["a", "b", "c"], [2, 1])
|
||||
assert r.group_owners(0) == [0, 1, 2, 3]
|
||||
assert r.group_owners(1) == [0, 1]
|
||||
@@ -614,7 +633,8 @@ class TestReplicaOverlay:
|
||||
|
||||
def test_adding_a_deployment_does_not_move_the_first_one(self):
|
||||
"""The strongest form: an existing consumer's routes are byte-identical
|
||||
before and after another deployment is provisioned."""
|
||||
before and after another deployment is provisioned.
|
||||
"""
|
||||
one, names = self._fleet(16, 8, 1)
|
||||
two, _ = self._fleet(16, 8, 2)
|
||||
for consumer_id in range(8):
|
||||
@@ -624,7 +644,8 @@ class TestReplicaOverlay:
|
||||
|
||||
def test_one_deployment_is_the_historical_carve(self):
|
||||
"""Default (no width) and width == the whole fleet must agree with the
|
||||
plain block rule, so a single-deployment fleet is untouched."""
|
||||
plain block rule, so a single-deployment fleet is untouched.
|
||||
"""
|
||||
plain = _router(16, 8, group_lens=[1] * 6)
|
||||
explicit, names = self._fleet(16, 8, 1)
|
||||
for consumer_id in range(8):
|
||||
@@ -638,7 +659,8 @@ class TestReplicaOverlay:
|
||||
def test_the_overlay_holds_for_partially_owned_names(self):
|
||||
"""Pipeline stages and experts too: two owner sets, one narrow. The
|
||||
narrow one already forced agreement (one owner, no block to carve); the
|
||||
wide one is what the carve decides."""
|
||||
wide one is what the carve decides.
|
||||
"""
|
||||
r, names = self._fleet(
|
||||
8,
|
||||
4,
|
||||
@@ -654,14 +676,16 @@ class TestReplicaOverlay:
|
||||
def test_a_width_that_does_not_divide_the_fleet_raises(self):
|
||||
"""A uniform fleet is asserted upstream in ``get_world_size``; if it ever
|
||||
is not, the overlay would map two workers of one deployment onto the same
|
||||
block index, so refuse rather than serve the wrong bytes."""
|
||||
block index, so refuse rather than serve the wrong bytes.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="does not divide"):
|
||||
RdtRouter(4, 6, None, None, ["a"], [1], workers_per_replica=4)
|
||||
|
||||
def test_pulls_still_carry_the_global_consumer_id(self):
|
||||
"""The carve uses the index WITHIN a deployment, but the wire must keep
|
||||
the fleet-global id: it is what lets the producer tell the sharers of a
|
||||
slot apart, and count their arrivals separately."""
|
||||
slot apart, and count their arrivals separately.
|
||||
"""
|
||||
|
||||
class _Method:
|
||||
def __init__(self):
|
||||
@@ -704,7 +728,8 @@ def _producer_pack_offsets(slices):
|
||||
class TestPackedLayout:
|
||||
def test_consumer_layout_matches_the_producer_rule(self):
|
||||
"""Mixed dtypes, sizes that do not land on 16B — i.e. the Kimi-style
|
||||
group (fp8 weights + fp32 scales + bf16 norms) that exposes alignment."""
|
||||
group (fp8 weights + fp32 scales + bf16 norms) that exposes alignment.
|
||||
"""
|
||||
specs = [
|
||||
("w.fp8", "float8_e4m3fn", [17]),
|
||||
("w.scale", "float32", [3]),
|
||||
@@ -779,7 +804,8 @@ class TestPackedLayout:
|
||||
class TestChunkModuleScatters:
|
||||
"""The chunk cut: one chunk per distinct owner class present in the copies,
|
||||
ascending by class index. Derived purely from the bake plus the ownership
|
||||
table, so vLLM's expert placement needs no cases."""
|
||||
table, so vLLM's expert placement needs no cases.
|
||||
"""
|
||||
|
||||
def test_unstamped_copies_form_one_chunk(self):
|
||||
layer = _FakeLayer("l")
|
||||
@@ -793,7 +819,8 @@ class TestChunkModuleScatters:
|
||||
"""Classes are numbered by first appearance in metadata order, so the
|
||||
replicated class (0) leads and the rest follow — the ordering the old
|
||||
"-1 first, then ascending stamp" rule produced, now falling out of the
|
||||
table instead of a special case."""
|
||||
table instead of a special case.
|
||||
"""
|
||||
layer = _FakeLayer("moe")
|
||||
names = ["norm", "e5", "e0", "e2"]
|
||||
copies = [_copy(n, shape=(4,)) for n in names]
|
||||
@@ -813,7 +840,8 @@ class TestChunkModuleScatters:
|
||||
|
||||
def test_copy_order_within_a_chunk_is_bake_order(self):
|
||||
"""Scatter order within a chunk must follow the bake, not the stamp
|
||||
sort: the packed layout and the producer's replay agree on keys order."""
|
||||
sort: the packed layout and the producer's replay agree on keys order.
|
||||
"""
|
||||
layer = _FakeLayer("moe")
|
||||
names = ["e3", "e1", "e2"] # one stamp, deliberately unsorted names
|
||||
copies = [_copy(n, shape=(4,)) for n in names]
|
||||
@@ -827,7 +855,8 @@ class TestChunkModuleScatters:
|
||||
|
||||
def test_stamps_interleave_across_modules_without_reordering_within(self):
|
||||
"""Two modules' copies bucket by stamp; within each bucket the order is
|
||||
module-then-bake order."""
|
||||
module-then-bake order.
|
||||
"""
|
||||
a, b = _FakeLayer("a"), _FakeLayer("b")
|
||||
mods = [
|
||||
_module(a, [_copy("a0", shape=(4,)), _copy("a1", shape=(4,))]),
|
||||
@@ -845,9 +874,10 @@ class TestChunkModuleScatters:
|
||||
]
|
||||
|
||||
def test_scatters_carry_their_own_dtype(self):
|
||||
"""dtype rides the record from the bake, where it is the fake's dtype
|
||||
"""Dtype rides the record from the bake, where it is the fake's dtype
|
||||
AFTER its op chain — not a plan-time lookup of the source name, which
|
||||
would be wrong for any chain that reinterprets dtype."""
|
||||
would be wrong for any chain that reinterprets dtype.
|
||||
"""
|
||||
layer = _FakeLayer("l")
|
||||
eng = _planner({}, name_meta={"w": ("float32", [10])})
|
||||
((_er, scatters),) = eng._chunk_module_scatters(
|
||||
@@ -886,7 +916,8 @@ class TestBuildCallPlan:
|
||||
|
||||
def test_materialize_fires_on_a_modules_first_chunk_only(self):
|
||||
"""Empty HF params are allocated once per module, by construction —
|
||||
including a FusedMoE-like module whose copies span owner-class chunks."""
|
||||
including a FusedMoE-like module whose copies span owner-class chunks.
|
||||
"""
|
||||
layer = _FakeLayer("spanning")
|
||||
names = [f"w{i}" for i in range(4)]
|
||||
copies = [_copy(n, shape=(4,)) for n in names]
|
||||
@@ -921,7 +952,8 @@ class TestBuildCallPlan:
|
||||
def test_free_signal_waits_for_a_groups_last_chunk_when_it_is_split(self):
|
||||
"""The load-bearing case: with the group cut across owner-class chunks the
|
||||
signal must hang off the LAST one. Signaling earlier lets the producers
|
||||
drop the gather buffers while a later chunk's RDMA is still reading."""
|
||||
drop the gather buffers while a later chunk's RDMA is still reading.
|
||||
"""
|
||||
layer = _FakeLayer("l")
|
||||
names = [f"w{i}" for i in range(4)]
|
||||
copies = [_copy(n, shape=(4,)) for n in names]
|
||||
@@ -937,7 +969,8 @@ class TestBuildCallPlan:
|
||||
|
||||
def test_free_signal_timing_across_two_split_groups(self):
|
||||
"""Two groups x 2 owner-class chunks each: signals land on each group's
|
||||
last chunk only."""
|
||||
last chunk only.
|
||||
"""
|
||||
names_a = [f"model.layers.0.w{i}" for i in range(4)]
|
||||
names_b = [f"model.layers.1.w{i}" for i in range(4)]
|
||||
baked, name_meta, stamps = {}, {}, {}
|
||||
@@ -955,7 +988,8 @@ class TestBuildCallPlan:
|
||||
def test_a_group_with_nothing_local_is_signaled_at_sync_start(self):
|
||||
"""The owners still published it, so this consumer must signal it —
|
||||
immediately, before the pipeline (signal-before-publish is tolerated),
|
||||
never hung off another group's chunk."""
|
||||
never hung off another group's chunk.
|
||||
"""
|
||||
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
||||
del baked["model.layers.1.w"] # nothing baked, and not live => no pull
|
||||
eng = _planner(
|
||||
@@ -976,7 +1010,8 @@ class TestBuildCallPlan:
|
||||
|
||||
def test_modules_are_deduped_within_a_group(self):
|
||||
"""Several names of one fused module map to the same scatter list; the
|
||||
plan must not chunk it twice."""
|
||||
plan must not chunk it twice.
|
||||
"""
|
||||
layer = _FakeLayer("fused")
|
||||
names = ["qkv.q", "qkv.k", "qkv.v"]
|
||||
copies = [_copy(n, shape=(4,)) for n in names]
|
||||
@@ -992,7 +1027,8 @@ class TestBuildCallPlan:
|
||||
|
||||
def test_plan_is_a_pure_function_of_its_inputs(self):
|
||||
"""Called twice with the same arguments it must produce the same plan —
|
||||
that is what lets the engine build it once and reuse it every sync."""
|
||||
that is what lets the engine build it once and reuse it every sync.
|
||||
"""
|
||||
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
||||
eng = _planner(baked, name_meta=name_meta)
|
||||
a = eng._build_call_plan(names, group_lens)
|
||||
@@ -1012,7 +1048,8 @@ class TestBuildCallPlan:
|
||||
class TestUnbakedNamesGuard:
|
||||
"""A live name with no baked plan has no way to load — there is no fallback,
|
||||
and its pull would target groups the pipeline already freed — so the plan
|
||||
build fails at init, naming the names."""
|
||||
build fails at init, naming the names.
|
||||
"""
|
||||
|
||||
def test_a_live_unbaked_name_fails_the_plan_build(self):
|
||||
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
||||
@@ -1023,7 +1060,8 @@ class TestUnbakedNamesGuard:
|
||||
|
||||
def test_never_copied_names_are_dropped_entirely(self):
|
||||
"""Experts owned by another EP rank no-op in their loader: not an error,
|
||||
and not a chunk — their group is pre-freed if nothing else fills it."""
|
||||
and not a chunk — their group is pre-freed if nothing else fills it.
|
||||
"""
|
||||
baked, name_meta, names, group_lens = _one_module_per_layer(2)
|
||||
del baked["norm.weight"]
|
||||
eng = _planner(baked, name_meta=name_meta, live=set(names) - {"norm.weight"})
|
||||
@@ -1035,13 +1073,15 @@ class TestUnbakedNamesGuard:
|
||||
class TestCallPlanRouting:
|
||||
"""Under partial ownership each chunk must be pulled from a producer that
|
||||
actually holds every name in it. Consumers bind EVERY producer (signals fan
|
||||
out to all owners), so local index == trainer rank."""
|
||||
out to all owners), so local index == trainer rank.
|
||||
"""
|
||||
|
||||
def _routed_planner(
|
||||
self, held_by_group, num_consumers, consumer_id, *, held_by=None
|
||||
):
|
||||
"""``held_by_group[g]`` = the ranks holding group g's names, so a test
|
||||
can still speak in groups; ``held_by`` overrides individual names."""
|
||||
can still speak in groups; ``held_by`` overrides individual names.
|
||||
"""
|
||||
baked, name_meta, names, group_lens = _one_module_per_layer(
|
||||
len(held_by_group) - 2
|
||||
)
|
||||
@@ -1068,14 +1108,16 @@ class TestCallPlanRouting:
|
||||
|
||||
def test_two_stage_ownership_splits_chunks_between_owners(self):
|
||||
"""Two PP stages, 4 groups: stage 0 holds the first half, stage 1 the
|
||||
second. Every chunk goes to a holder of its names."""
|
||||
second. Every chunk goes to a holder of its names.
|
||||
"""
|
||||
eng, names, group_lens = self._routed_planner([[0], [0], [1], [1]], 1, 0)
|
||||
plan = eng._build_call_plan(names, group_lens)
|
||||
assert [c.owner for c in plan.chunks] == [0, 0, 1, 1]
|
||||
|
||||
def test_a_multi_owner_group_still_resolves_to_one_producer(self):
|
||||
"""Every chunk is served by exactly ONE producer: splitting a pull only
|
||||
multiplies produce calls, since the consumer's NIC bounds it."""
|
||||
multiplies produce calls, since the consumer's NIC bounds it.
|
||||
"""
|
||||
eng, names, group_lens = self._routed_planner([[0, 1]] * 4, 1, 0)
|
||||
plan = eng._build_call_plan(names, group_lens)
|
||||
assert all(0 <= c.owner < 2 for c in plan.chunks)
|
||||
@@ -1083,7 +1125,8 @@ class TestCallPlanRouting:
|
||||
def test_an_expert_name_routes_to_the_rank_holding_it(self):
|
||||
"""PP ∩ EP: stage 0 = ranks 0-1, stage 1 = ranks 2-3, and one expert
|
||||
name inside a stage-1 group is held by rank 3 alone. The name's own owner
|
||||
set is what routes it."""
|
||||
set is what routes it.
|
||||
"""
|
||||
eng, names, group_lens = self._routed_planner(
|
||||
[[0, 1], [0, 1], [2, 3], [2, 3]],
|
||||
1,
|
||||
@@ -1098,7 +1141,8 @@ class TestCallPlanRouting:
|
||||
def test_an_unheld_name_is_rejected_at_init(self):
|
||||
"""A name no rank holds can never be served. The trainer raises when it
|
||||
transposes the holdings; the router's validate() is the consumer-side
|
||||
backstop, and either way it fails at init rather than at first pull."""
|
||||
backstop, and either way it fails at init rather than at first pull.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="empty"):
|
||||
RdtRouter(2, 1, [[]], [0], ["w"], [1]).validate()
|
||||
|
||||
@@ -1107,14 +1151,16 @@ class TestSignalCompleteness:
|
||||
"""Every gather group is signaled exactly once — after its last chunk when
|
||||
the worker pulls from it, at sync start otherwise — for ANY expert
|
||||
placement. A group signaled twice over-credits the barrier; a group never
|
||||
signaled parks a producer credit and hangs end_sync."""
|
||||
signaled parks a producer credit and hangs end_sync.
|
||||
"""
|
||||
|
||||
def _moe_planner(self, worker_experts, *, ep_size=4, n_experts=8, n_layers=2):
|
||||
"""pre / n_layers MoE layers / post. Each layer: one norm (its own
|
||||
"""Pre / n_layers MoE layers / post. Each layer: one norm (its own
|
||||
module, stamp -1) + n_experts expert names stamped ``e // n_local``.
|
||||
The worker's bake covers only ``worker_experts`` (its placement);
|
||||
foreign expert names never copied => dropped entirely, as in the real
|
||||
bake."""
|
||||
bake.
|
||||
"""
|
||||
n_local = n_experts // ep_size
|
||||
names = ["embed.weight"]
|
||||
group_lens = [1]
|
||||
@@ -1173,8 +1219,9 @@ class TestSignalCompleteness:
|
||||
assert self._signals(plan) == list(range(len(group_lens)))
|
||||
|
||||
def test_placement_changes_only_the_chunk_count_never_the_signals(self):
|
||||
"""linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6
|
||||
hits all 4. More chunks per group, identical signal set."""
|
||||
"""Linear experts 0-3 hit 2 owner classes; round_robin 0,2,4,6
|
||||
hits all 4. More chunks per group, identical signal set.
|
||||
"""
|
||||
plans = {}
|
||||
for label, experts in (("linear", [0, 1, 2, 3]), ("round_robin", [0, 2, 4, 6])):
|
||||
eng, names, group_lens = self._moe_planner(worker_experts=experts)
|
||||
@@ -1188,7 +1235,8 @@ class TestSignalCompleteness:
|
||||
|
||||
def test_the_fused_module_materializes_first_and_quants_last_across_chunks(self):
|
||||
"""The FusedMoE shape: one module's copies span every owner-class chunk of
|
||||
its group; materialize on its first, quant on its last."""
|
||||
its group; materialize on its first, quant on its last.
|
||||
"""
|
||||
eng, names, group_lens = self._moe_planner(
|
||||
worker_experts=[0, 2, 4, 6], n_layers=1
|
||||
)
|
||||
@@ -1246,7 +1294,8 @@ class TestLayerwiseGroups:
|
||||
def test_flattening_a_partition_returns_the_input_order(self):
|
||||
"""Un-indexed names group by POSITION, not by name class — so one that
|
||||
looks like a pre-block name still lands after the layers it follows.
|
||||
Group index therefore means the same thing on every rank."""
|
||||
Group index therefore means the same thing on every rank.
|
||||
"""
|
||||
names = [
|
||||
"embed.weight",
|
||||
"model.layers.0.a",
|
||||
@@ -1264,7 +1313,8 @@ class TestLayerwiseGroups:
|
||||
|
||||
def test_stacks_keep_prefix_appearance_order_and_sort_within(self):
|
||||
"""Two stacks stay in the order they appear — the vision tower before the
|
||||
text stack — while each sorts by index internally."""
|
||||
text stack — while each sorts by index internally.
|
||||
"""
|
||||
names = [
|
||||
"visual.blocks.1.w",
|
||||
"visual.blocks.0.w",
|
||||
@@ -1281,7 +1331,8 @@ class TestLayerwiseGroups:
|
||||
def test_the_post_block_lands_last_however_early_it_arrives(self):
|
||||
"""Megatron-Bridge streams the last pipeline stage's output block before
|
||||
its layers. Sweeping un-indexed names after the first layer into a
|
||||
trailing group is what lets the gather loop walk groups ascending."""
|
||||
trailing group is what lets the gather loop walk groups ascending.
|
||||
"""
|
||||
names = ["model.layers.0.w", "model.norm.weight", "model.layers.1.w"]
|
||||
assert layerwise_groups(names) == [
|
||||
["model.layers.0.w"],
|
||||
@@ -1291,7 +1342,8 @@ class TestLayerwiseGroups:
|
||||
|
||||
def test_names_sharing_a_layer_coalesce_when_the_source_interleaves(self):
|
||||
"""A raw-checkpoint source may yield in shard-packing order. Coalescing on
|
||||
the key keeps that from shattering one layer into many groups."""
|
||||
the key keeps that from shattering one layer into many groups.
|
||||
"""
|
||||
names = [
|
||||
"model.layers.0.a",
|
||||
"model.layers.1.a",
|
||||
@@ -1333,13 +1385,15 @@ class TestLayerwiseGroups:
|
||||
def test_every_supported_naming_convention_partitions_per_layer(self, prefix):
|
||||
"""One decoder layer per group under any prefix, with no per-architecture
|
||||
table. A literal `model.layers.` match silently yields one whole-model
|
||||
group for most of these."""
|
||||
group for most of these.
|
||||
"""
|
||||
names = [f"{prefix}{i}.self_attn.q_proj.weight" for i in range(3)]
|
||||
assert layerwise_groups(names) == [[n] for n in names]
|
||||
|
||||
def test_a_moe_layer_stays_whole(self):
|
||||
"""The OUTERMOST index wins, so per-expert names group by layer. Splitting
|
||||
on the expert would cut FusedMoE — one leaf module — across groups."""
|
||||
on the expert would cut FusedMoE — one leaf module — across groups.
|
||||
"""
|
||||
names = [
|
||||
"model.layers.0.mlp.experts.0.w1.weight",
|
||||
"model.layers.0.mlp.experts.1.w1.weight",
|
||||
@@ -1356,7 +1410,8 @@ class TestLayerwiseGroups:
|
||||
def test_a_vlm_partitions_the_vision_tower_and_the_text_stack(self):
|
||||
"""Qwen-VL layout. Today's literal `model.layers.` match puts every one of
|
||||
these in ONE group — the whole model — because the text stack ships as
|
||||
`model.language_model.layers.`."""
|
||||
`model.language_model.layers.`.
|
||||
"""
|
||||
names = [
|
||||
"visual.patch_embed.proj.weight",
|
||||
"visual.blocks.0.attn.qkv.weight",
|
||||
@@ -1375,7 +1430,8 @@ class TestLayerwiseGroups:
|
||||
|
||||
def test_two_stacks_sharing_an_index_do_not_merge(self):
|
||||
"""The key carries the prefix, so vision block 0 and text layer 0 are
|
||||
different groups even though both are index 0."""
|
||||
different groups even though both are index 0.
|
||||
"""
|
||||
names = [
|
||||
"visual.blocks.0.attn.qkv.weight",
|
||||
"model.language_model.layers.0.self_attn.q_proj.weight",
|
||||
@@ -1412,20 +1468,23 @@ class TestOpAllowlistAgreement:
|
||||
|
||||
def test_every_bake_recordable_op_is_replayable(self):
|
||||
"""Anything the bake can record must be serveable, or the failure lands a
|
||||
whole sync later than the mistake."""
|
||||
whole sync later than the mistake.
|
||||
"""
|
||||
assert not set(SUPPORTED_OPS.values()) - set(ALLOWED_OPS)
|
||||
|
||||
def test_the_producer_allows_nothing_the_consumer_cannot_emit(self):
|
||||
"""The allowlist is also a guard against a misbehaving or spoofed
|
||||
consumer invoking arbitrary methods on trainer tensors, so it must not be
|
||||
wider than the ops the bake can actually produce. ``to`` in particular
|
||||
would let a replay change dtype or device."""
|
||||
would let a replay change dtype or device.
|
||||
"""
|
||||
assert not set(ALLOWED_OPS) - set(SUPPORTED_OPS.values())
|
||||
assert "to" not in ALLOWED_OPS
|
||||
|
||||
def test_transpose_via_t_is_serveable(self):
|
||||
"""The specific regression: a chain recorded from ``.t()`` must pass the
|
||||
producer's guard."""
|
||||
producer's guard.
|
||||
"""
|
||||
fake = FakeRDTTensor(
|
||||
name="w",
|
||||
shape=torch.Size((4, 6)),
|
||||
@@ -1470,7 +1529,8 @@ BAKE_MODELS = [
|
||||
class TestRequiresTheRayExecutor:
|
||||
"""The engine's data plane is Ray's, so a non-Ray executor cannot work. The
|
||||
check is at construction because the alternative is an opaque failure during
|
||||
the first handshake, long after the misconfiguration."""
|
||||
the first handshake, long after the misconfiguration.
|
||||
"""
|
||||
|
||||
def _construct(self, backend):
|
||||
cfg = SimpleNamespace(
|
||||
@@ -1497,7 +1557,8 @@ class TestRequiresTheRayExecutor:
|
||||
|
||||
def test_a_custom_executor_class_is_left_alone(self):
|
||||
"""A `type[Executor]` override is deliberate and unjudgeable here, so it
|
||||
must not be rejected for merely not being the string "ray"."""
|
||||
must not be rejected for merely not being the string "ray".
|
||||
"""
|
||||
|
||||
class _CustomExecutor:
|
||||
pass
|
||||
|
||||
@@ -236,7 +236,8 @@ class TestPublishAndRebuild:
|
||||
def test_views_of_one_storage_do_not_overlap(self, server_factory):
|
||||
"""One IPC export per storage, one as_strided view per name — the whole
|
||||
point of the storage/view split. Overlapping views would serve the same
|
||||
bytes for two different weights."""
|
||||
bytes for two different weights.
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(1)
|
||||
_publish(server, GI_A, GROUP_A, nbytes=64)
|
||||
@@ -259,7 +260,8 @@ class TestPublishAndRebuild:
|
||||
def test_the_default_lookahead_is_one(self):
|
||||
"""1 is the sweet spot: group N+1 is gathered AND pullable while N is
|
||||
being pulled, with resident memory at its floor of 2 groups — the bound
|
||||
the larger-model runs size against."""
|
||||
the larger-model runs size against.
|
||||
"""
|
||||
assert DEFAULT_GATHER_LOOKAHEAD == 1
|
||||
|
||||
|
||||
@@ -274,7 +276,8 @@ class TestFreeBarrier:
|
||||
|
||||
def test_group_is_held_until_the_last_live_consumer_signals(self, server_factory):
|
||||
"""Every live consumer signals every owner; releasing on the first
|
||||
signal would drop storage other consumers are still reading."""
|
||||
signal would drop storage other consumers are still reading.
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(3)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -301,7 +304,8 @@ class TestFreeBarrier:
|
||||
):
|
||||
"""A consumer with nothing to pull from a group signals it at sync
|
||||
start, which can precede the publish. The publish must then release the
|
||||
group rather than wait for a signal that will never come again."""
|
||||
group rather than wait for a signal that will never come again.
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(1)
|
||||
server.free_group(GI_A)
|
||||
@@ -324,7 +328,8 @@ class TestFreeBarrier:
|
||||
the server says the group is gone — and it hears that ONLY through
|
||||
``wait_freed`` / ``end_sync``, never a publish return (a freed notice
|
||||
riding an unharvested async publish while the engine blocks in
|
||||
``wait_freed`` would wedge the loop)."""
|
||||
``wait_freed`` would wedge the loop).
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(1)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -353,7 +358,8 @@ class TestFreeBarrier:
|
||||
|
||||
class _CountingSource:
|
||||
"""`iter_groups` yielding one CPU-tensor group per call, built lazily so a
|
||||
group's tensors exist only once the engine's credit gate lets it gather."""
|
||||
group's tensors exist only once the engine's credit gate lets it gather.
|
||||
"""
|
||||
|
||||
def __init__(self, groups):
|
||||
self._groups = groups
|
||||
@@ -366,7 +372,8 @@ class _CountingSource:
|
||||
class _ResidencyDict(dict):
|
||||
"""The engine's `_inflight`, instrumented: `max_len` is the high-water mark
|
||||
of groups whose CUDA-IPC export refs were alive at once — the trainer's
|
||||
resident-memory bound in groups."""
|
||||
resident-memory bound in groups.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
@@ -380,7 +387,8 @@ class _ResidencyDict(dict):
|
||||
def _loop_engine(server, n_groups, *, lookahead):
|
||||
"""A real engine wired straight to a real (non-Ray) server, with only the
|
||||
state `_run_gather_loop` reads. The `_rpc` seam dispatches inline;
|
||||
`_publish_async` sees no `.remote` and runs inline too."""
|
||||
`_publish_async` sees no `.remote` and runs inline too.
|
||||
"""
|
||||
groups = [(f"model.layers.{gi}.w",) for gi in range(n_groups)]
|
||||
e = ShardedRDTTrainerWeightTransferEngine.__new__(
|
||||
ShardedRDTTrainerWeightTransferEngine
|
||||
@@ -403,14 +411,16 @@ class TestGatherCredit:
|
||||
serveable immediately, and the ENGINE's loop stops gathering while more than
|
||||
`gather_lookahead` groups are unfreed. So at most `lookahead + 1` groups are
|
||||
resident — two at the default — while group N+1 is pulled the instant N's
|
||||
pulls finish."""
|
||||
pulls finish.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def gather_engine(self, monkeypatch):
|
||||
"""Neutralize the two CUDA touches in the export path so the REAL
|
||||
gather loop runs on CPU: `.cuda()` becomes identity, and the storage
|
||||
export ships the `(nbytes, ..., device, ...)` tuple the fixture's
|
||||
stubbed `rebuild_cuda_tensor` expects."""
|
||||
stubbed `rebuild_cuda_tensor` expects.
|
||||
"""
|
||||
monkeypatch.setattr(torch.Tensor, "cuda", lambda self: self)
|
||||
monkeypatch.setattr(
|
||||
trainer_mod,
|
||||
@@ -422,7 +432,8 @@ class TestGatherCredit:
|
||||
def test_publish_never_blocks(self, server_factory):
|
||||
"""Publishing must not park on a credit: the whole point is that a
|
||||
gathered group becomes serveable immediately. Three publishes with no
|
||||
frees, inline on this thread — a block here hangs the test."""
|
||||
frees, inline on this thread — a block here hangs the test.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=1)
|
||||
server.begin_sync(1)
|
||||
for gi in range(3):
|
||||
@@ -436,7 +447,8 @@ class TestGatherCredit:
|
||||
than 2 groups of gathered weights, no matter how the consumer paces —
|
||||
and the pipeline still overlaps, because the consumer here refuses to
|
||||
free group N until N+1 is already published (pullable). A gate an
|
||||
off-by-one too tight deadlocks this test; too loose fails max_len."""
|
||||
off-by-one too tight deadlocks this test; too loose fails max_len.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=1)
|
||||
n_groups = 8
|
||||
engine = gather_engine(server, n_groups, lookahead=1)
|
||||
@@ -478,7 +490,8 @@ class TestGatherCredit:
|
||||
):
|
||||
"""0 is the serialized baseline: the loop waits for group N to be freed
|
||||
before gathering N+1, so exactly ONE group is ever resident and nothing
|
||||
overlaps. A floor of 1 on the bound would silently pipeline instead."""
|
||||
overlaps. A floor of 1 on the bound would silently pipeline instead.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=0)
|
||||
engine = gather_engine(server, 4, lookahead=0)
|
||||
|
||||
@@ -503,8 +516,9 @@ class TestGatherCredit:
|
||||
def test_the_residency_bound_scales_with_the_lookahead(
|
||||
self, server_factory, gather_engine
|
||||
):
|
||||
"""lookahead + 1, not a hardcoded 2: at lookahead=2 a free-nothing
|
||||
consumer sees exactly 3 groups gathered before the loop parks."""
|
||||
"""Lookahead + 1, not a hardcoded 2: at lookahead=2 a free-nothing
|
||||
consumer sees exactly 3 groups gathered before the loop parks.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=2)
|
||||
engine = gather_engine(server, 8, lookahead=2)
|
||||
|
||||
@@ -540,7 +554,8 @@ class TestGatherCredit:
|
||||
def test_a_gather_error_releases_a_blocked_wait_freed(self, server_factory):
|
||||
"""Otherwise a trainer-side failure on another rank deadlocks this one
|
||||
inside its credit gate. It must RAISE, not return empty — an empty
|
||||
return would spin the engine straight back into the wait."""
|
||||
return would spin the engine straight back into the wait.
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(1)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -621,7 +636,8 @@ class TestBeginSync:
|
||||
|
||||
def test_the_target_is_floored_at_one(self, server_factory):
|
||||
"""A bare/zero call must not make every group free instantly (or divide
|
||||
the barrier by zero)."""
|
||||
the barrier by zero).
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(0)
|
||||
assert server._live_count == 1
|
||||
@@ -629,7 +645,8 @@ class TestBeginSync:
|
||||
def test_a_degraded_sync_lowers_the_target(self, server_factory):
|
||||
"""FT degraded sync: the live count is the WHOLE degraded-sync
|
||||
mechanism on this side — the group releases after the live consumers
|
||||
alone."""
|
||||
alone.
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(2) # 4 provisioned, 2 alive
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -653,7 +670,8 @@ class TestBeginSync:
|
||||
def test_a_straggler_signal_cannot_credit_the_next_sync(self, server_factory):
|
||||
"""begin_sync resets the counts, so a signal that leaked past the sync
|
||||
boundary would over-credit — the reason the consumer drains its fired
|
||||
signals before finishing. Pin the reset side of that contract."""
|
||||
signals before finishing. Pin the reset side of that contract.
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(2)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -675,7 +693,8 @@ class TestBeginSync:
|
||||
self, server_factory
|
||||
):
|
||||
"""The packed layout repeats every sync — that is what makes caching the
|
||||
destination views worth ~5ms per 384-spec group."""
|
||||
destination views worth ~5ms per 384-spec group.
|
||||
"""
|
||||
server = server_factory()
|
||||
server._pack_dsts[("sentinel",)] = (0, [])
|
||||
server.begin_sync(1)
|
||||
@@ -686,7 +705,8 @@ class TestServedNamesGuard:
|
||||
"""A producer serves only the names it holds: its stage's groups, and
|
||||
within them its own EP coordinate's experts plus the replicated names.
|
||||
Without this guard a misrouted pull would block forever in the cache wait
|
||||
for a name this rank never publishes."""
|
||||
for a name this rank never publishes.
|
||||
"""
|
||||
|
||||
def test_a_pull_for_an_unserved_name_fails_loudly(self, server_factory):
|
||||
server = server_factory(served_names=list(GROUP_A))
|
||||
@@ -702,7 +722,8 @@ class TestServedNamesGuard:
|
||||
|
||||
def test_served_names_none_accepts_anything(self, server_factory):
|
||||
"""Gather-to-all: every producer holds every group, so there is nothing
|
||||
to guard. The call proceeds to the cache wait (not exercised here)."""
|
||||
to guard. The call proceeds to the cache wait (not exercised here).
|
||||
"""
|
||||
server = server_factory(served_names=None)
|
||||
assert server._served_names is None
|
||||
|
||||
@@ -749,7 +770,8 @@ class TestStallWatchdog:
|
||||
|
||||
def test_end_sync_fails_when_a_consumer_never_signals(self, server_factory):
|
||||
"""The exact mid-sync death signature: the group is published, one of
|
||||
the two live consumers that owed a signal is gone."""
|
||||
the two live consumers that owed a signal is gone.
|
||||
"""
|
||||
server = server_factory(stall_timeout_s=0.3)
|
||||
server.begin_sync(2)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -763,7 +785,8 @@ class TestStallWatchdog:
|
||||
"""The watchdog fires on the existing `set_gather_error` channel, so one
|
||||
stall unwinds the whole rank through one path rather than each waiter
|
||||
timing out separately — here both a consumer parked in the serve cache
|
||||
wait and the engine parked in its credit gate."""
|
||||
wait and the engine parked in its credit gate.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
|
||||
server.begin_sync(1)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -796,7 +819,8 @@ class TestStallWatchdog:
|
||||
"""A consumer that is slow but signaling must never trip it: the stamp
|
||||
is global to the producer, so a steady trickle of signals holds the
|
||||
whole rank open. Nine sequential publishes with a 0.3s timeout each take
|
||||
longer in total than the timeout, and none may fire."""
|
||||
longer in total than the timeout, and none may fire.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
|
||||
server.begin_sync(1)
|
||||
for gi in range(9):
|
||||
@@ -809,7 +833,8 @@ class TestStallWatchdog:
|
||||
|
||||
def test_begin_sync_resets_the_progress_stamp(self, server_factory):
|
||||
"""Syncs are minutes apart. Without the reset the first publish of sync N+1
|
||||
would measure its stall from somewhere inside sync N and fire immediately."""
|
||||
would measure its stall from somewhere inside sync N and fire immediately.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=1, stall_timeout_s=0.3)
|
||||
server.begin_sync(1)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -831,7 +856,8 @@ class TestConcurrentPublishAndFree:
|
||||
self, server_factory
|
||||
):
|
||||
"""Every group published must end up freed and out of the cache, with
|
||||
end_sync returning cleanly — the property whose violation is a hang."""
|
||||
end_sync returning cleanly — the property whose violation is a hang.
|
||||
"""
|
||||
server = server_factory(gather_lookahead=2)
|
||||
server.begin_sync(1)
|
||||
groups = [(gi, (f"model.layers.{gi}.w",)) for gi in range(12)]
|
||||
@@ -866,7 +892,8 @@ class TestConcurrentPublishAndFree:
|
||||
):
|
||||
"""All live consumers signaling at once: the release must happen on the
|
||||
last signal and only once, or the engine drops IPC refs twice (or
|
||||
never)."""
|
||||
never).
|
||||
"""
|
||||
server = server_factory()
|
||||
server.begin_sync(8)
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -893,7 +920,8 @@ class TestFakeServerAgreesWithTheRealOne:
|
||||
"""`test_sharded_rdt_trainer.py`'s `_FakeProducerServer` is a second,
|
||||
independent model of this protocol that the engine-side tests assert on. Pin
|
||||
the two against each other so the fake cannot silently drift from the
|
||||
semantics the real server enforces."""
|
||||
semantics the real server enforces.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _fake():
|
||||
@@ -924,7 +952,8 @@ class TestExportRing:
|
||||
"""The packed export path. Its one hard invariant is that the trainer packs
|
||||
a group exactly the way ``publish_group`` rebuilds it — a layout
|
||||
disagreement would serve the wrong bytes with nothing downstream to catch
|
||||
it."""
|
||||
it.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _engine():
|
||||
@@ -1016,7 +1045,8 @@ class TestExportRingSlotSafety:
|
||||
):
|
||||
"""Free the later group of each pair first and hold the earlier one live
|
||||
until the NEXT group is gathered -- the window a modular slot counter got
|
||||
wrong (at lookahead=1 there are 2 slots, so group 2 took group 0's)."""
|
||||
wrong (at lookahead=1 there are 2 slots, so group 2 took group 0's).
|
||||
"""
|
||||
server = server_factory(gather_lookahead=1)
|
||||
n_groups = 8
|
||||
engine = gather_engine(server, n_groups, lookahead=1)
|
||||
@@ -1140,7 +1170,8 @@ class TestSharedServeSlots:
|
||||
|
||||
def test_one_deployment_never_shares(self, sharing_server):
|
||||
"""With the width equal to the live count every group is a singleton, so
|
||||
each call packs for itself out of its own ring."""
|
||||
each call packs for itself out of its own ring.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
server.begin_sync(2, [0, 1])
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -1220,7 +1251,8 @@ class TestSharedServeSlots:
|
||||
|
||||
def test_a_seqless_call_is_refused(self, sharing_server):
|
||||
"""The slot comes from ``seq``, so a caller that sends none cannot be
|
||||
served safely."""
|
||||
served safely.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
server.begin_sync(1, [0])
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -1230,7 +1262,8 @@ class TestSharedServeSlots:
|
||||
|
||||
def test_a_failed_pack_reaches_every_sharer(self, sharing_server):
|
||||
"""Only the packer touches the buffer, so its failure has to be
|
||||
republished: a waiter must raise rather than read a half-written slot."""
|
||||
republished: a waiter must raise rather than read a half-written slot.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
server.begin_sync(4, [0, 1, 2, 3])
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -1254,7 +1287,8 @@ class TestSharedServeSlots:
|
||||
|
||||
def test_a_degraded_sync_narrows_the_rendezvous(self, sharing_server):
|
||||
"""A dead deployment must not be waited for: a survivor left alone in
|
||||
its group packs for itself instead of blocking on the corpse."""
|
||||
its group packs for itself instead of blocking on the corpse.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2, stall_timeout_s=30.0)
|
||||
server.begin_sync(3, [0, 1, 3]) # consumer 2 died
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -1268,7 +1302,8 @@ class TestSharedServeSlots:
|
||||
|
||||
def test_a_sharer_that_never_arrives_trips_the_watchdog(self, sharing_server):
|
||||
"""A consumer that dies INSIDE the sync window has no detector, exactly
|
||||
as for the free barrier; the rendezvous waits on the same channel."""
|
||||
as for the free barrier; the rendezvous waits on the same channel.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2, stall_timeout_s=0.3)
|
||||
server.begin_sync(4, [0, 1, 2, 3])
|
||||
_publish(server, GI_A, GROUP_A)
|
||||
@@ -1280,7 +1315,8 @@ class TestSharedServeSlots:
|
||||
|
||||
class TestSharedReservation:
|
||||
"""``reserve_serve_buffer`` under sharing: one ring per GROUP, and the
|
||||
init-time check that the group's members pull the same chunks."""
|
||||
init-time check that the group's members pull the same chunks.
|
||||
"""
|
||||
|
||||
def test_matching_plans_reserve_one_ring_between_them(self, sharing_server):
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
@@ -1302,7 +1338,8 @@ class TestSharedReservation:
|
||||
def test_mismatched_plans_fail_at_reservation(self, sharing_server):
|
||||
"""Sharing rests on the sharers pulling the same chunks. Checked here,
|
||||
where it names both consumers, rather than at serve time, where it is a
|
||||
rendezvous nobody completes."""
|
||||
rendezvous nobody completes.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
server.reserve_serve_buffer(0, 1024, "plan-a")
|
||||
|
||||
@@ -1311,7 +1348,8 @@ class TestSharedReservation:
|
||||
|
||||
def test_different_groups_may_differ(self, sharing_server):
|
||||
"""Only the members of ONE group have to agree; worker 0 and worker 1 of
|
||||
the same deployment hold different slices and never share a slot."""
|
||||
the same deployment hold different slices and never share a slot.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
server.reserve_serve_buffer(0, 1024, "plan-a")
|
||||
server.reserve_serve_buffer(1, 1024, "plan-b")
|
||||
@@ -1320,7 +1358,8 @@ class TestSharedReservation:
|
||||
|
||||
def test_a_missing_digest_skips_the_check(self, sharing_server):
|
||||
"""The digest is optional on the wire, so an engine that does not send
|
||||
one still reserves."""
|
||||
one still reserves.
|
||||
"""
|
||||
server = sharing_server(workers_per_replica=2)
|
||||
server.reserve_serve_buffer(0, 1024, None)
|
||||
server.reserve_serve_buffer(2, 1024, None)
|
||||
|
||||
@@ -34,7 +34,8 @@ from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
|
||||
class _ListSource(WeightSource):
|
||||
"""A WeightSource over an explicit ordered (name, cpu-tensor) list, so the
|
||||
sharded-RDT group/order logic can be tested without a real model."""
|
||||
sharded-RDT group/order logic can be tested without a real model.
|
||||
"""
|
||||
|
||||
def __init__(self, pairs):
|
||||
self._pairs = list(pairs)
|
||||
@@ -54,7 +55,8 @@ class _FakeProducerServer:
|
||||
per-group barrier: publish/free are keyed by GROUP INDEX, signals count
|
||||
to the ``begin_sync`` live total, publish_group returns nothing, and freed
|
||||
groups flow back only through wait_freed / end_sync (see
|
||||
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne)."""
|
||||
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne).
|
||||
"""
|
||||
|
||||
def __init__(self, auto_free=True):
|
||||
self.order: list[str] = []
|
||||
@@ -98,7 +100,8 @@ class _FakeProducerServer:
|
||||
must already have a banked credit when the gate asks (auto_free, or a
|
||||
test's own free_group) — anything else is the deadlock the real
|
||||
watchdog would kill, so fail loudly. Not appended to ``order``: it is
|
||||
pacing, not a lifecycle milestone."""
|
||||
pacing, not a lifecycle milestone.
|
||||
"""
|
||||
assert self._pending_freed, (
|
||||
"wait_freed with nothing freed: the gather loop would deadlock"
|
||||
)
|
||||
@@ -121,7 +124,8 @@ def _rdt_engine_with_fake_server(
|
||||
):
|
||||
"""Build a ShardedRDTTrainerWeightTransferEngine wired to an in-process fake
|
||||
server (no Ray, no CUDA IPC): bypass trainer_init's spawn, set the
|
||||
group-major metadata, and route _rpc to the fake."""
|
||||
group-major metadata, and route _rpc to the fake.
|
||||
"""
|
||||
import vllm.distributed.weight_transfer.sharded_rdt_trainer as mod
|
||||
|
||||
# reduce_tensor needs CUDA; the fake server never rebuilds, so stub it.
|
||||
@@ -321,7 +325,8 @@ def test_sharded_rdt_send_weights_surfaces_update_error(monkeypatch):
|
||||
|
||||
class _OwnedSource(_ListSource):
|
||||
"""A source holding only some groups' names, like a pipeline-parallel rank.
|
||||
Takes group indices for convenience and declares the names inside them."""
|
||||
Takes group indices for convenience and declares the names inside them.
|
||||
"""
|
||||
|
||||
def __init__(self, pairs, owned_group_idx):
|
||||
super().__init__(pairs)
|
||||
@@ -387,7 +392,8 @@ def test_sharded_rdt_owned_group_order_mismatch_raises(monkeypatch):
|
||||
)
|
||||
def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
|
||||
"""The free barrier's target is one integer per sync: the live consumer
|
||||
count, defaulting to the whole provisioned fleet."""
|
||||
count, defaulting to the whole provisioned fleet.
|
||||
"""
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
@@ -405,7 +411,8 @@ def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
|
||||
class TestLiveCountPlumbing:
|
||||
"""``send_weights(live_consumer_ids)`` -> the barrier target. The
|
||||
provisioned geometry is frozen; a degraded sync only lowers the target and
|
||||
narrows each slot-sharing group's rendezvous."""
|
||||
narrows each slot-sharing group's rendezvous.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _engine(num_consumers, world=2, rank=0):
|
||||
@@ -438,7 +445,8 @@ class TestLiveCountPlumbing:
|
||||
|
||||
def test_which_consumers_died_does_not_matter_only_how_many(self):
|
||||
"""The whole point of the barrier: no routed per-producer targets, so
|
||||
the identity of the dead consumer is irrelevant to the producers."""
|
||||
the identity of the dead consumer is irrelevant to the producers.
|
||||
"""
|
||||
counts = []
|
||||
for live in ([0, 1, 2, 3], [4, 5, 6, 7], [0, 2, 4, 6]):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
@@ -449,7 +457,8 @@ class TestLiveCountPlumbing:
|
||||
def test_the_live_ids_travel_with_the_count(self):
|
||||
"""The count sizes the free barrier, the ids size the slot-sharing
|
||||
rendezvous, so a producer that shares slots can tell WHICH consumers it
|
||||
is still waiting for. They must describe the same set."""
|
||||
is still waiting for. They must describe the same set.
|
||||
"""
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights([6, 0, 2, 2])
|
||||
((count, ids),) = got
|
||||
@@ -517,7 +526,8 @@ def test_sharded_rdt_rejects_a_rank_holding_nothing(monkeypatch):
|
||||
|
||||
def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
|
||||
"""Every name must be held somewhere or it can never be served — caught
|
||||
when the holdings are transposed, naming the orphan."""
|
||||
when the holdings are transposed, naming the orphan.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="no trainer rank holds"):
|
||||
_rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
|
||||
@@ -531,7 +541,8 @@ def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
|
||||
|
||||
def test_sharded_rdt_rejects_metadata_disagreement_across_ranks(monkeypatch):
|
||||
"""Only the sender's metadata reaches the consumers, so a rank describing
|
||||
just its own share must fail loudly rather than silently drop the rest."""
|
||||
just its own share must fail loudly rather than silently drop the rest.
|
||||
"""
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
|
||||
is_sender=False,
|
||||
@@ -554,7 +565,8 @@ def _serve_ring_server(src_name, src):
|
||||
"""A producer server with one cached tensor and a pre-seeded serve ring, so a
|
||||
pull needs no Ray and no NIXL registration. Returns (server, serve) where
|
||||
``serve(chain)`` packs one spec into the SAME ring slot every time — which is
|
||||
what puts the destination-view cache, and only it, under test."""
|
||||
what puts the destination-view cache, and only it, under test.
|
||||
"""
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
_RDTProducerServer,
|
||||
)
|
||||
|
||||
@@ -43,6 +43,7 @@ def distributed_run(fn, world_size, timeout=60):
|
||||
fn: Function to run in each process
|
||||
world_size: Number of processes to spawn
|
||||
timeout: Maximum time in seconds to wait for processes (default: 60)
|
||||
|
||||
"""
|
||||
number_of_processes = world_size
|
||||
processes = []
|
||||
@@ -418,7 +419,8 @@ def test_tensor_broadcast():
|
||||
|
||||
def _dumps_oob(obj) -> tuple[bytes, list]:
|
||||
"""Pickle `obj` the same way `MessageQueue.enqueue` does: tensor
|
||||
dispatch table + out-of-band buffers >= 1MiB."""
|
||||
dispatch table + out-of-band buffers >= 1MiB.
|
||||
"""
|
||||
buffers = []
|
||||
|
||||
def callback(buf: pickle.PickleBuffer) -> bool:
|
||||
@@ -489,7 +491,8 @@ def test_tensor_pickle_roundtrip(case: str):
|
||||
@pytest.mark.parametrize("case", ["cuda", "requires_grad", "conj"])
|
||||
def test_reduce_tensor_fallback(case: str):
|
||||
"""Tensors the zero-copy reducer can't safely alias must fall back to
|
||||
torch's default reduction."""
|
||||
torch's default reduction.
|
||||
"""
|
||||
if case == "cuda":
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("requires CUDA")
|
||||
@@ -634,11 +637,9 @@ def test_acquire_read_releases_slot_when_reader_raises():
|
||||
|
||||
|
||||
def test_warning_logs(caplog_vllm):
|
||||
"""
|
||||
Test that warning logs are emitted at VLLM_RINGBUFFER_WARNING_INTERVAL intervals
|
||||
"""Test that warning logs are emitted at VLLM_RINGBUFFER_WARNING_INTERVAL intervals
|
||||
when indefinite=False, and are not emitted when indefinite=True.
|
||||
"""
|
||||
|
||||
# Patch the warning log interval to every 1 ms during reads
|
||||
with mock.patch(
|
||||
"vllm.distributed.device_communicators.shm_broadcast.VLLM_RINGBUFFER_WARNING_INTERVAL",
|
||||
@@ -754,7 +755,8 @@ def test_remote_subscribe_addr_unique_concurrent_writers(
|
||||
Pre-fix, the writer probed a port with get_open_port() and bound it
|
||||
afterwards; pinning the probe to one free port makes every writer
|
||||
bind the same port and fail deterministically on that code path,
|
||||
while the late-binding implementation never consults the probe."""
|
||||
while the late-binding implementation never consults the probe.
|
||||
"""
|
||||
from vllm.distributed.device_communicators import shm_broadcast
|
||||
|
||||
colliding_port = get_open_port()
|
||||
|
||||
@@ -12,20 +12,20 @@ from vllm.distributed.device_communicators.shm_object_storage import (
|
||||
|
||||
|
||||
class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
"""Test suite for the ring buffer implementation"""
|
||||
"""Test suite for the ring buffer implementation."""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test fixtures"""
|
||||
"""Set up test fixtures."""
|
||||
self.buffer_size = 4096
|
||||
self.ring_buffer = None
|
||||
|
||||
def tearDown(self):
|
||||
"""Clean up after tests"""
|
||||
"""Clean up after tests."""
|
||||
if self.ring_buffer:
|
||||
self.ring_buffer.close()
|
||||
|
||||
def test_buffer_opening(self):
|
||||
"""Test opening an existing buffer"""
|
||||
"""Test opening an existing buffer."""
|
||||
# First create a buffer
|
||||
self.ring_buffer = SingleWriterShmRingBuffer(
|
||||
data_buffer_size=self.buffer_size, create=True
|
||||
@@ -39,7 +39,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
)
|
||||
|
||||
def test_buffer_access(self):
|
||||
"""Test accessing allocated buffers"""
|
||||
"""Test accessing allocated buffers."""
|
||||
self.ring_buffer = SingleWriterShmRingBuffer(
|
||||
data_buffer_size=self.buffer_size, create=True
|
||||
)
|
||||
@@ -61,7 +61,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
self.assertEqual(read_id, monotonic_id)
|
||||
|
||||
def test_memory_error_on_full_buffer(self):
|
||||
"""Test that MemoryError is raised when buffer is full"""
|
||||
"""Test that MemoryError is raised when buffer is full."""
|
||||
small_buffer_size = 200
|
||||
self.ring_buffer = SingleWriterShmRingBuffer(
|
||||
data_buffer_size=small_buffer_size, create=True
|
||||
@@ -76,7 +76,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
self.ring_buffer.allocate_buf(1) # Would exceed buffer capacity
|
||||
|
||||
def test_allocation_and_free(self):
|
||||
"""Test allocation and freeing of buffers"""
|
||||
"""Test allocation and freeing of buffers."""
|
||||
small_buffer_size = 200
|
||||
self.ring_buffer = SingleWriterShmRingBuffer(
|
||||
data_buffer_size=small_buffer_size, create=True
|
||||
@@ -96,7 +96,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
self.assertEqual(freed_ids[0], i)
|
||||
|
||||
def test_clear_buffer(self):
|
||||
"""Test clearing the buffer"""
|
||||
"""Test clearing the buffer."""
|
||||
self.ring_buffer = SingleWriterShmRingBuffer(
|
||||
data_buffer_size=self.buffer_size, create=True
|
||||
)
|
||||
@@ -180,7 +180,7 @@ class TestSingleWriterShmRingBuffer(unittest.TestCase):
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function demonstrating usage and running tests"""
|
||||
"""Main function demonstrating usage and running tests."""
|
||||
print("=== SingleWriterShmRingBuffer Test Suite ===\n")
|
||||
|
||||
# Run unit tests
|
||||
|
||||
@@ -31,7 +31,8 @@ class _CUDADeviceCountStatelessTestActor:
|
||||
|
||||
def test_cuda_device_count_stateless():
|
||||
"""Test that cuda_device_count_stateless changes return value if
|
||||
CUDA_VISIBLE_DEVICES is changed."""
|
||||
CUDA_VISIBLE_DEVICES is changed.
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip("Skip for ROCm because Ray uses HIP_VISIBLE_DEVICES.")
|
||||
actor = _CUDADeviceCountStatelessTestActor.options( # type: ignore
|
||||
|
||||
@@ -1166,7 +1166,8 @@ class RecordingClient:
|
||||
|
||||
def _module_with(*pairs):
|
||||
"""A tiny nn.Module exposing the given (name, tensor) pairs as parameters,
|
||||
so trainer tests can build a ModuleSource without a real model."""
|
||||
so trainer tests can build a ModuleSource without a real model.
|
||||
"""
|
||||
module = torch.nn.Module()
|
||||
for name, tensor in pairs:
|
||||
module.register_parameter(name, torch.nn.Parameter(tensor, requires_grad=False))
|
||||
@@ -1287,7 +1288,8 @@ class TestModuleSource:
|
||||
|
||||
def test_metadata_agrees_with_iteration(self):
|
||||
"""The two channels must line up element-for-element: engines declare
|
||||
the round from `metadata()` and then send what iteration yields."""
|
||||
the round from `metadata()` and then send what iteration yields.
|
||||
"""
|
||||
source = ModuleSource(
|
||||
_module_with(("w", torch.zeros(2, 3)), ("b", torch.zeros(3)))
|
||||
)
|
||||
@@ -1301,11 +1303,13 @@ class TestModuleSource:
|
||||
class TestWeightSourceGroupContract:
|
||||
"""`groups()` / `iter_groups()` on the WeightSource ABC. Groups are what
|
||||
backends gather and free by, so the default must agree with
|
||||
`layerwise_groups` over `metadata()`, restricted to what this rank holds."""
|
||||
`layerwise_groups` over `metadata()`, restricted to what this rank holds.
|
||||
"""
|
||||
|
||||
class _Source(WeightSource):
|
||||
"""Minimal source over an ordered (name, tensor) list, optionally holding
|
||||
only some names (in which case it iterates only their groups)."""
|
||||
only some names (in which case it iterates only their groups).
|
||||
"""
|
||||
|
||||
def __init__(self, names, held=None, reverse=False):
|
||||
self._pairs = [(n, torch.full((2,), float(i))) for i, n in enumerate(names)]
|
||||
@@ -1360,7 +1364,8 @@ class TestWeightSourceGroupContract:
|
||||
def test_groups_order_follows_metadata_not_the_declaration(self):
|
||||
"""The declaration is a SET of names; the groups it selects still come
|
||||
out in metadata order, so each pairs with the right ``iter_groups()``
|
||||
batch however the source listed them."""
|
||||
batch however the source listed them.
|
||||
"""
|
||||
names = ["embed.w", "model.layers.0.a", "model.layers.1.a", "norm.w"]
|
||||
source = self._source(
|
||||
names, held=["model.layers.1.a", "model.layers.0.a", "model.layers.1.a"]
|
||||
@@ -1394,14 +1399,16 @@ class TestWeightSourceGroupContract:
|
||||
|
||||
def test_out_of_order_iteration_raises(self):
|
||||
"""Materializing is usually a collective, so a rank that iterates out of
|
||||
order deadlocks its peers -- fail loudly instead."""
|
||||
order deadlocks its peers -- fail loudly instead.
|
||||
"""
|
||||
source = self._source(["model.layers.0.a", "model.layers.0.b"], reverse=True)
|
||||
with pytest.raises(RuntimeError, match="iteration order must match"):
|
||||
list(source.iter_groups())
|
||||
|
||||
def test_a_source_may_override_iter_groups(self):
|
||||
"""The extension point: materialize a whole group in one step instead of
|
||||
one generator resume per tensor."""
|
||||
one generator resume per tensor.
|
||||
"""
|
||||
calls = []
|
||||
|
||||
class _Batched(TestWeightSourceGroupContract._Source):
|
||||
@@ -1422,7 +1429,8 @@ class TestDeferredProcessingContract:
|
||||
caller that takes over the update tail (running its own
|
||||
`finalize_layerwise_reload` instead of going through `finish_weight_update`)
|
||||
reads the flag and calls the method. Both must be answerable on any engine, or
|
||||
that caller ends up reaching through a getattr."""
|
||||
that caller ends up reaching through a getattr.
|
||||
"""
|
||||
|
||||
def _engines(self):
|
||||
return {
|
||||
@@ -1436,7 +1444,8 @@ class TestDeferredProcessingContract:
|
||||
|
||||
def test_every_engine_can_be_drained(self):
|
||||
"""The default is a no-op, so a caller never has to check whether the
|
||||
method exists before calling it."""
|
||||
method exists before calling it.
|
||||
"""
|
||||
for name, cls in self._engines().items():
|
||||
assert callable(cls.drain_pending), name
|
||||
|
||||
@@ -1530,7 +1539,8 @@ class TestTrainerEngineBase:
|
||||
)
|
||||
def test_ipc_trainer_send_weights_drives_client_in_order():
|
||||
"""send_weights issues start -> update -> finish and ships per-round metadata;
|
||||
the packed wire param rides the init info, not the per-round update_info."""
|
||||
the packed wire param rides the init info, not the per-round update_info.
|
||||
"""
|
||||
client = RecordingClient()
|
||||
engine = IPCTrainerWeightTransferEngine(
|
||||
client=client,
|
||||
@@ -1549,7 +1559,8 @@ def test_ipc_trainer_send_weights_drives_client_in_order():
|
||||
|
||||
def test_ipc_trainer_init_ships_packed_to_worker():
|
||||
"""trainer_init drives the inference-side init handshake and propagates the
|
||||
must-agree `packed` flag to the worker."""
|
||||
must-agree `packed` flag to the worker.
|
||||
"""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU (CUDA IPC handles).")
|
||||
|
||||
@@ -1570,7 +1581,8 @@ def test_ipc_trainer_init_ships_packed_to_worker():
|
||||
def test_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
||||
"""The sender's trainer_init drives the inference-side init handshake with
|
||||
the worker-shaped init info (rank_offset=1) while opening its own endpoint,
|
||||
and propagates the must-agree wire params to the worker."""
|
||||
and propagates the must-agree wire params to the worker.
|
||||
"""
|
||||
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
|
||||
|
||||
# Bypass the real NCCL rendezvous.
|
||||
@@ -1611,7 +1623,8 @@ def test_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
||||
def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch):
|
||||
"""The worker engine reads packed + buffer geometry from the
|
||||
trainer-supplied init info at the handshake, not from the config or the
|
||||
per-round update info."""
|
||||
per-round update info.
|
||||
"""
|
||||
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
@@ -1644,7 +1657,8 @@ def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch):
|
||||
|
||||
def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client():
|
||||
"""Non-sender trainer ranks build an engine without opening an endpoint or
|
||||
touching the client; they only join the collectives in send_weights."""
|
||||
touching the client; they only join the collectives in send_weights.
|
||||
"""
|
||||
client = RecordingClient()
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=NCCLTrainerInitInfo(
|
||||
@@ -1674,7 +1688,8 @@ def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client():
|
||||
def test_nccl_trainer_send_weights_drives_client_in_order():
|
||||
"""send_weights issues start -> update -> finish and ships per-round
|
||||
metadata; the packed wire params ride the init handshake, not the
|
||||
per-round update_info."""
|
||||
per-round update_info.
|
||||
"""
|
||||
client = RecordingClient()
|
||||
engine = NCCLTrainerWeightTransferEngine(
|
||||
client=client,
|
||||
@@ -1695,7 +1710,8 @@ def test_nccl_trainer_send_weights_drives_client_in_order():
|
||||
|
||||
class _ScriptedSource(WeightSource):
|
||||
"""Declares `meta` but yields whatever `pairs` says — used to drive the
|
||||
metadata/iteration agreement checks."""
|
||||
metadata/iteration agreement checks.
|
||||
"""
|
||||
|
||||
def __init__(self, meta, pairs):
|
||||
self._meta = meta
|
||||
@@ -1731,7 +1747,8 @@ def test_nccl_trainer_init_requires_source():
|
||||
|
||||
def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch):
|
||||
"""The worker sizes its buffers (and cuts packed chunks) from metadata(), so
|
||||
iteration disagreeing with it must raise rather than corrupt the stream."""
|
||||
iteration disagreeing with it must raise rather than corrupt the stream.
|
||||
"""
|
||||
meta = [
|
||||
ParamMeta("w", torch.float32, (4,)),
|
||||
ParamMeta("b", torch.float32, (2,)),
|
||||
@@ -1745,7 +1762,8 @@ def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch):
|
||||
|
||||
def test_nccl_trainer_send_weights_rejects_dtype_disagreement(monkeypatch):
|
||||
"""A source that declares one wire dtype and materializes another would make
|
||||
the two sides disagree on every byte offset."""
|
||||
the two sides disagree on every byte offset.
|
||||
"""
|
||||
meta = [ParamMeta("w", torch.float32, (4,))]
|
||||
engine = _mock_group_engine(
|
||||
_ScriptedSource(meta, [("w", torch.zeros(4, dtype=torch.bfloat16))]),
|
||||
@@ -1772,7 +1790,8 @@ def test_nccl_trainer_send_weights_rejects_truncated_source(monkeypatch):
|
||||
|
||||
def test_nccl_trainer_send_weights_broadcasts_contiguous(monkeypatch):
|
||||
"""NCCL sends numel elements from data_ptr(), so a non-contiguous view must
|
||||
be linearized first or the worker receives unrelated memory."""
|
||||
be linearized first or the worker receives unrelated memory.
|
||||
"""
|
||||
base = torch.arange(6, dtype=torch.float32).reshape(2, 3)
|
||||
view = base.t() # non-contiguous
|
||||
meta = [ParamMeta("w", torch.float32, tuple(view.shape))]
|
||||
@@ -1842,7 +1861,8 @@ def _sparse_patch(device: str = "cpu") -> SparseWeightPatch:
|
||||
def test_sparse_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
||||
"""The sender's trainer_init drives the init handshake with the
|
||||
worker-shaped init info; sparse ships no packed wire params, so the worker
|
||||
keeps its unpacked defaults. Sparse takes no `source`."""
|
||||
keeps its unpacked defaults. Sparse takes no `source`.
|
||||
"""
|
||||
import vllm.distributed.weight_transfer.sparse_nccl_engine as sparse_mod
|
||||
|
||||
monkeypatch.setattr(sparse_mod, "open_trainer_endpoint", lambda info: MagicMock())
|
||||
@@ -1895,7 +1915,8 @@ def test_sparse_nccl_trainer_send_weights_drives_client_in_order(monkeypatch):
|
||||
|
||||
def test_sparse_nccl_trainer_send_weights_empty_round_is_noop():
|
||||
"""A round with no patches must not touch the client (an empty sparse
|
||||
update info is invalid by construction)."""
|
||||
update info is invalid by construction).
|
||||
"""
|
||||
client = RecordingClient()
|
||||
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
||||
engine.model_update_group = MagicMock()
|
||||
@@ -1931,7 +1952,8 @@ def test_sparse_nccl_trainer_rejects_source():
|
||||
def test_sparse_nccl_trainer_validates_patch_before_any_rpc():
|
||||
"""Malformed patches must fail on the trainer, before start_weight_update:
|
||||
the worker's own checks only run once the broadcasts are already in flight,
|
||||
where a size mismatch wedges both sides instead of raising."""
|
||||
where a size mismatch wedges both sides instead of raising.
|
||||
"""
|
||||
client = RecordingClient()
|
||||
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
||||
engine.model_update_group = MagicMock()
|
||||
|
||||
@@ -403,7 +403,8 @@ def trainer_broadcast_tensor_torch_free(
|
||||
def _run_uid_transfer(trainer_task) -> None:
|
||||
"""Mint a unique id, run the worker and the given trainer peer concurrently,
|
||||
and assert the tensor round-trips. Both ranks must enter init together --
|
||||
there is no store barrier on the UID path."""
|
||||
there is no store barrier on the UID path.
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
|
||||
|
||||
_init_ray_for_weight_transfer()
|
||||
@@ -450,7 +451,8 @@ def _run_uid_transfer(trainer_task) -> None:
|
||||
)
|
||||
def test_nccl_weight_transfer_between_processes_uid():
|
||||
"""Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both
|
||||
ranks on vLLM's ``PyNcclCommunicator``."""
|
||||
ranks on vLLM's ``PyNcclCommunicator``.
|
||||
"""
|
||||
_run_uid_transfer(trainer_broadcast_tensor_uid)
|
||||
|
||||
|
||||
@@ -460,6 +462,7 @@ def test_nccl_weight_transfer_between_processes_uid():
|
||||
)
|
||||
def test_nccl_weight_transfer_torch_free_trainer():
|
||||
"""UID weight transfer against a foreign peer that uses no torch and no vLLM
|
||||
communicator, only a cupy buffer and raw ``ncclBroadcast``."""
|
||||
communicator, only a cupy buffer and raw ``ncclBroadcast``.
|
||||
"""
|
||||
pytest.importorskip("cupy")
|
||||
_run_uid_transfer(trainer_broadcast_tensor_torch_free)
|
||||
|
||||
Reference in New Issue
Block a user