[EPLB] Add nixl-based eplb communicator (#36276)
Signed-off-by: ilmarkov <markovilya197@gmail.com> Signed-off-by: Markov Ilya <markovilya19@gmail.com>
This commit is contained in:
@@ -541,23 +541,29 @@ def rearrange_expert_weights_inplace(
|
||||
assert num_physical_experts == ep_size * num_local_physical_experts
|
||||
|
||||
first_layer_weights = list(expert_weights[0])
|
||||
|
||||
if is_profile:
|
||||
if communicator.needs_profile_buffer_reservation:
|
||||
# Reserve NCCL communication buffers via a dummy all_gather.
|
||||
# Backends that pre-allocate their own transfer buffers
|
||||
# skip this to avoid the extra memory spike during profiling.
|
||||
weights_buffer: list[torch.Tensor] = [
|
||||
torch.empty_like(w) for w in first_layer_weights
|
||||
]
|
||||
for weight, buffer in zip(expert_weights[0], weights_buffer):
|
||||
dummy_recv_buffer = [buffer for _ in range(ep_size)]
|
||||
torch.distributed.barrier()
|
||||
all_gather(
|
||||
dummy_recv_buffer,
|
||||
weight,
|
||||
group=ep_group,
|
||||
)
|
||||
return
|
||||
|
||||
# Buffers to hold the expert weights during the exchange.
|
||||
# NOTE: Currently we assume the same weights across different layers
|
||||
# have the same shape.
|
||||
weights_buffer: list[torch.Tensor] = [
|
||||
torch.empty_like(w) for w in first_layer_weights
|
||||
]
|
||||
if is_profile:
|
||||
# Reserve communication buffers via a minimal dummy all_gather on first layer
|
||||
for weight, buffer in zip(expert_weights[0], weights_buffer):
|
||||
dummy_recv_buffer = [buffer for _ in range(ep_size)]
|
||||
torch.distributed.barrier()
|
||||
all_gather(
|
||||
dummy_recv_buffer,
|
||||
weight,
|
||||
group=ep_group,
|
||||
)
|
||||
return
|
||||
weights_buffer = [torch.empty_like(w) for w in first_layer_weights]
|
||||
|
||||
# NOTE(bowen): We need this synchronize to run, but I don't know why.
|
||||
# If you figure out the reason, please let me know -- thank you!
|
||||
|
||||
Reference in New Issue
Block a user