[EPLB] Nixl communicator optimization. Zero-copy transfers (#41633)
Signed-off-by: ilmarkov <markovilya197@gmail.com> Signed-off-by: Markov Ilya <markovilya19@gmail.com> Signed-off-by: Tyler Michael Smith <tlrmchlsmth@gmail.com> Co-authored-by: Markov Ilya <markovilya19@gmail.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com> Co-authored-by: Tyler Michael Smith <tyler@neuralmagic.com> Co-authored-by: Tyler Michael Smith <tlrmchlsmth@gmail.com>
This commit is contained in:
co-authored by
Markov Ilya
Nicolò Lucchesi
Tyler Michael Smith
Tyler Michael Smith
parent
f0cd590d62
commit
4f423bd5bc
@@ -277,12 +277,15 @@ def assert_verification_synced(local_ok: bool, msg: str) -> None:
|
||||
assert bool(ok_tensor.item()), msg
|
||||
|
||||
|
||||
def create_eplb_communicator_or_raise(*, group_coordinator, backend, expert_weights):
|
||||
def create_eplb_communicator_or_raise(
|
||||
*, group_coordinator, backend, expert_weights, expert_buffer
|
||||
):
|
||||
try:
|
||||
return create_eplb_communicator(
|
||||
group_coordinator=group_coordinator,
|
||||
backend=backend,
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
@@ -355,7 +358,8 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend=eplb_communicator,
|
||||
expert_weights=expert_weights[0],
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
communicator.set_stream(cuda_stream)
|
||||
|
||||
@@ -368,6 +372,7 @@ def _test_async_transfer_layer_without_mtp_worker(
|
||||
ep_group=ep_group,
|
||||
communicator=communicator,
|
||||
cuda_stream=cuda_stream,
|
||||
layer_idx=layer_idx,
|
||||
)
|
||||
cuda_stream.synchronize()
|
||||
move_from_buffer(
|
||||
@@ -460,10 +465,12 @@ def _test_rearrange_expert_weights_with_redundancy(
|
||||
num_layers, num_local_experts, hidden_sizes, ep_rank, device, old_indices
|
||||
)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend=eplb_communicator,
|
||||
expert_weights=expert_weights[0],
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
|
||||
# Execute weight rearrangement
|
||||
@@ -471,9 +478,9 @@ def _test_rearrange_expert_weights_with_redundancy(
|
||||
old_indices,
|
||||
new_indices,
|
||||
expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
is_profile=False,
|
||||
communicator=communicator,
|
||||
communicator,
|
||||
)
|
||||
|
||||
# Verify the rearrangement result
|
||||
@@ -593,10 +600,12 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
layer_copy.append(weight.clone())
|
||||
original_weights.append(layer_copy)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend="torch_nccl",
|
||||
expert_weights=expert_weights[0],
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
|
||||
# Execute rearrangement (should be no change)
|
||||
@@ -604,9 +613,9 @@ def _test_rearrange_expert_weights_no_change(env, world_size) -> None:
|
||||
indices,
|
||||
indices, # Same indices
|
||||
expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
communicator,
|
||||
is_profile=False,
|
||||
)
|
||||
|
||||
# Verify that the weights have not changed
|
||||
@@ -726,10 +735,12 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
layer_copy.append(weight.clone())
|
||||
original_weights.append(layer_copy)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in expert_weights[0]]
|
||||
communicator = create_eplb_communicator_or_raise(
|
||||
group_coordinator=ep_group_coordinator,
|
||||
backend="torch_nccl",
|
||||
expert_weights=expert_weights[0],
|
||||
expert_weights=expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
|
||||
# Execute profile mode rearrangement
|
||||
@@ -737,9 +748,10 @@ def _test_rearrange_expert_weights_profile_mode(env, world_size) -> None:
|
||||
old_indices,
|
||||
new_indices,
|
||||
expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
communicator,
|
||||
is_profile=True, # Profile mode
|
||||
is_profile=True,
|
||||
)
|
||||
|
||||
# In profile mode, the weights should remain unchanged
|
||||
|
||||
@@ -9,9 +9,11 @@ import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
|
||||
from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace
|
||||
from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
get_eplb_group,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
@@ -213,12 +215,20 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
|
||||
for lidx in range(test_config.num_layers):
|
||||
shuffled_indices[lidx] = torch.randperm(test_config.num_experts)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in rank_expert_weights[0]]
|
||||
communicator = create_eplb_communicator(
|
||||
group_coordinator=get_eplb_group(),
|
||||
backend="torch_nccl",
|
||||
expert_weights=rank_expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
rearrange_expert_weights_inplace(
|
||||
indices,
|
||||
shuffled_indices,
|
||||
rank_expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
is_profile=False,
|
||||
communicator,
|
||||
)
|
||||
|
||||
num_local_experts = test_config.num_local_experts
|
||||
|
||||
@@ -10,11 +10,13 @@ import torch
|
||||
|
||||
from tests.kernels.moe.utils import make_test_quant_config
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed.eplb.eplb_communicator import create_eplb_communicator
|
||||
from vllm.distributed.eplb.eplb_state import EplbLayerState
|
||||
from vllm.distributed.eplb.rebalance_execute import rearrange_expert_weights_inplace
|
||||
from vllm.distributed.parallel_state import (
|
||||
ensure_model_parallel_initialized,
|
||||
get_dp_group,
|
||||
get_eplb_group,
|
||||
)
|
||||
from vllm.forward_context import set_forward_context
|
||||
from vllm.model_executor.layers.fused_moe.layer import FusedMoE
|
||||
@@ -171,12 +173,20 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
|
||||
for lidx in range(test_config.num_layers):
|
||||
shuffled_indices[lidx] = torch.randperm(test_config.num_experts)
|
||||
|
||||
expert_buffer = [torch.empty_like(w) for w in rank_expert_weights[0]]
|
||||
communicator = create_eplb_communicator(
|
||||
group_coordinator=get_eplb_group(),
|
||||
backend="torch_nccl",
|
||||
expert_weights=rank_expert_weights,
|
||||
expert_buffer=expert_buffer,
|
||||
)
|
||||
rearrange_expert_weights_inplace(
|
||||
indices,
|
||||
shuffled_indices,
|
||||
rank_expert_weights,
|
||||
expert_buffer,
|
||||
ep_group,
|
||||
is_profile=False,
|
||||
communicator,
|
||||
)
|
||||
|
||||
num_global_experts = test_config.num_experts
|
||||
|
||||
Reference in New Issue
Block a user