[EPLB] Nixl communicator optimization. Zero-copy transfers (#41633)
Signed-off-by: ilmarkov <markovilya197@gmail.com> Signed-off-by: Markov Ilya <markovilya19@gmail.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Markov Ilya <markovilya19@gmail.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
Markov Ilya
Nicolò Lucchesi
Tyler Michael Smith
Tyler Michael Smith
parent
f0cd590d62
commit
4f423bd5bc
@@ -1287,10 +1287,12 @@ def _test_body_eplb(
|
||||
|
||||
expert_weights = [list(eplb_moe_layer.get_expert_weights())]
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
communicator = create_eplb_communicator(
|
||||
group_coordinator=get_eplb_group(),
|
||||
backend=vllm_config.parallel_config.eplb_config.communicator,
|
||||
expert_weights=expert_weights[0],
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
|
||||
# Rearrange expert weights across EP ranks
|
||||
@@ -1298,6 +1300,7 @@ def _test_body_eplb(
|
||||
old_global_expert_indices=initial_indices.unsqueeze(0),
|
||||
new_global_expert_indices=shuffled_indices.unsqueeze(0),
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
ep_group=cpu_group,
|
||||
communicator=communicator,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user