Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
The async worker that transfers experts in the background.
|
||||
"""
|
||||
"""The async worker that transfers experts in the background."""
|
||||
|
||||
import threading
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
EPLB communicator implementations and factory.
|
||||
"""
|
||||
"""EPLB communicator implementations and factory."""
|
||||
|
||||
import contextlib
|
||||
import time
|
||||
@@ -86,7 +84,8 @@ class EplbCommunicator(ABC):
|
||||
@property
|
||||
def needs_profile_buffer_reservation(self) -> bool:
|
||||
"""Whether the profile path must run a dummy collective operation to reserve
|
||||
communication buffers."""
|
||||
communication buffers.
|
||||
"""
|
||||
return True
|
||||
|
||||
def set_stream(self, cuda_stream: torch.cuda.Stream | None) -> None:
|
||||
@@ -258,6 +257,7 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
cpu_group: CPU process group for metadata exchange.
|
||||
all_expert_weights: Expert weight tensors for all MoE layers.
|
||||
expert_buffer: Pre-allocated receive buffer tensors.
|
||||
|
||||
"""
|
||||
assert all_expert_weights, (
|
||||
"NixlEplbCommunicator requires non-empty all_expert_weights."
|
||||
@@ -443,7 +443,8 @@ class NixlEplbCommunicator(EplbCommunicator):
|
||||
|
||||
def _exchange_remote_send_meta(self) -> None:
|
||||
"""Exchange per-layer per-tensor metadata so receivers can compute
|
||||
remote RDMA addresses at transfer time."""
|
||||
remote RDMA addresses at transfer time.
|
||||
"""
|
||||
local_meta: dict[tuple[int, int], tuple[int, int, int]] = {}
|
||||
for layer_idx, layer_tensors in enumerate(self._all_expert_weights):
|
||||
for t_idx, t in enumerate(layer_tensors):
|
||||
@@ -666,6 +667,7 @@ def create_eplb_communicator(
|
||||
zero-copy RDMA reads.
|
||||
expert_buffer: Pre-allocated receive buffer tensors (one per
|
||||
weight tensor in a single layer).
|
||||
|
||||
"""
|
||||
first_layer = expert_weights[0] if expert_weights else []
|
||||
tensor_device_type = first_layer[0].device.type if first_layer else "cpu"
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Expert parallelism load balancer (EPLB) metrics and states.
|
||||
"""Expert parallelism load balancer (EPLB) metrics and states.
|
||||
|
||||
# Glossary
|
||||
|
||||
@@ -74,9 +73,7 @@ def _compute_eplb_load_stats(
|
||||
|
||||
@dataclass
|
||||
class EplbStats:
|
||||
"""
|
||||
Model stats used in EPLB rebalancing algorithm.
|
||||
"""
|
||||
"""Model stats used in EPLB rebalancing algorithm."""
|
||||
|
||||
global_expert_load_window: torch.Tensor
|
||||
"""
|
||||
@@ -233,9 +230,7 @@ class EplbModelState:
|
||||
|
||||
|
||||
class EplbState:
|
||||
"""
|
||||
EplbState of each expert parallel model. Key is the model config hash.
|
||||
"""
|
||||
"""EplbState of each expert parallel model. Key is the model config hash."""
|
||||
|
||||
def __init__(self, parallel_config: ParallelConfig, device: torch.device):
|
||||
self.parallel_config = parallel_config
|
||||
@@ -305,14 +300,14 @@ class EplbState:
|
||||
num_routed_experts: int,
|
||||
num_redundant_experts: int,
|
||||
) -> Sequence[int]:
|
||||
"""
|
||||
Build an initial expert arrangement using the following structure:
|
||||
"""Build an initial expert arrangement using the following structure:
|
||||
[original routed experts, redundant experts]
|
||||
|
||||
Returns:
|
||||
physical_to_logical_map (Sequence[int]): A list of integers,
|
||||
where each integer is the index of the logical expert
|
||||
that the corresponding physical expert maps to.
|
||||
|
||||
"""
|
||||
global_physical_to_logical_map = list(range(num_routed_experts))
|
||||
global_physical_to_logical_map += [
|
||||
@@ -321,8 +316,7 @@ class EplbState:
|
||||
return global_physical_to_logical_map
|
||||
|
||||
def validate_ep_configuration(self, new_model: MixtureOfExperts):
|
||||
"""
|
||||
Validate that the expert parallel configuration of
|
||||
"""Validate that the expert parallel configuration of
|
||||
the new model is the same as the existing models.
|
||||
"""
|
||||
if len(self.model_states) > 0:
|
||||
@@ -361,9 +355,7 @@ class EplbState:
|
||||
model: MixtureOfExperts,
|
||||
model_config: ModelConfig,
|
||||
):
|
||||
"""
|
||||
Build the initial EPLB state.
|
||||
"""
|
||||
"""Build the initial EPLB state."""
|
||||
self.validate_ep_configuration(model)
|
||||
self.is_async = self.parallel_config.eplb_config.use_async
|
||||
|
||||
@@ -534,6 +526,7 @@ class EplbState:
|
||||
ubatch_slices: When DBO is active, a list of
|
||||
``UBatchSlice`` objects describing each micro-batch's
|
||||
token range. When ``None``, only ``tensors[0]`` is filled.
|
||||
|
||||
"""
|
||||
model_state = self.model_states.get(compute_hash_cached(model_config))
|
||||
if model_state is None or model_state.num_unpadded_tokens_tensors is None:
|
||||
@@ -556,8 +549,7 @@ class EplbState:
|
||||
is_profile: bool = False,
|
||||
log_stats: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Step the EPLB state.
|
||||
"""Step the EPLB state.
|
||||
|
||||
Args:
|
||||
is_dummy (bool): If `True`, this is a dummy step and the load
|
||||
@@ -574,6 +566,7 @@ class EplbState:
|
||||
- `avg_tokens`: The average load across ranks.
|
||||
- `max_tokens`: The maximum load across ranks.
|
||||
- `balancedness`: The ratio of average load to maximum load.
|
||||
|
||||
"""
|
||||
ep_group = get_ep_group().device_group
|
||||
if is_profile:
|
||||
@@ -752,8 +745,7 @@ class EplbState:
|
||||
is_profile: bool = False,
|
||||
rank_mapping: dict[int, int] | None = None,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
Rearrange the experts according to the current load.
|
||||
"""Rearrange the experts according to the current load.
|
||||
|
||||
Args:
|
||||
is_profile (bool): If `True`, perform a dummy rearrangement.
|
||||
@@ -761,8 +753,8 @@ class EplbState:
|
||||
no memory movement will be performed. Default is False.
|
||||
rank_mapping (dict[int, int] | None): The rank mapping
|
||||
when scaling is done in EEP.
|
||||
"""
|
||||
|
||||
"""
|
||||
ep_group = get_ep_group().device_group
|
||||
ep_rank = ep_group.rank()
|
||||
|
||||
@@ -1033,9 +1025,7 @@ class EplbState:
|
||||
return int(flag.item()) == device_group.size()
|
||||
|
||||
def _allreduce_list(self, tensor_list: list[torch.Tensor]) -> list[torch.Tensor]:
|
||||
"""
|
||||
All-reduce a list of tensors.
|
||||
"""
|
||||
"""All-reduce a list of tensors."""
|
||||
ep_group = get_ep_group().device_group
|
||||
if len(tensor_list) == 1:
|
||||
all_reduce(tensor_list[0], group=ep_group)
|
||||
@@ -1060,8 +1050,7 @@ class EplbState:
|
||||
return all_reduce_list
|
||||
|
||||
def _sync_load_pass(self) -> list[torch.Tensor]:
|
||||
"""
|
||||
Sync the expert load pass across all ranks for log stats.
|
||||
"""Sync the expert load pass across all ranks for log stats.
|
||||
Doesn't update the expert load pass in eplb_model_state.
|
||||
"""
|
||||
load_pass_list = []
|
||||
@@ -1250,8 +1239,7 @@ def compute_logical_maps(
|
||||
physical_to_logical_map: torch.Tensor,
|
||||
num_logical_experts: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Derive logical_to_physical_map and logical_replica_count from
|
||||
"""Derive logical_to_physical_map and logical_replica_count from
|
||||
physical_to_logical_map.
|
||||
|
||||
Args:
|
||||
@@ -1264,6 +1252,7 @@ def compute_logical_maps(
|
||||
physical slots per logical expert; -1 where unused
|
||||
logical_replica_count: [num_layers, num_logical_experts], number of
|
||||
physical replicas per logical expert
|
||||
|
||||
"""
|
||||
device = physical_to_logical_map.device
|
||||
assert physical_to_logical_map.device.type == "cpu"
|
||||
@@ -1339,13 +1328,11 @@ def _commit_eplb_maps_for_layer(
|
||||
new_physical_to_logical_map: torch.Tensor,
|
||||
layer: int,
|
||||
) -> None:
|
||||
"""
|
||||
Per-layer version of _commit_eplb_maps that's used by the sync portion of EPLB
|
||||
"""Per-layer version of _commit_eplb_maps that's used by the sync portion of EPLB
|
||||
when running async EPLB. Copies all of the new_* maps into model_state. After this
|
||||
function completes, the new mappings will become the current mappings and will be
|
||||
visible to the model.
|
||||
"""
|
||||
|
||||
# Commit physical_to_logical_map
|
||||
src = new_physical_to_logical_map
|
||||
dst = model_state.physical_to_logical_map[layer]
|
||||
@@ -1376,12 +1363,10 @@ def _commit_eplb_maps(
|
||||
model_state: EplbModelState,
|
||||
new_physical_to_logical_map: torch.Tensor,
|
||||
) -> None:
|
||||
"""
|
||||
Copies all of the new_* maps into model_state. After this function completes,
|
||||
"""Copies all of the new_* maps into model_state. After this function completes,
|
||||
the new mappings will become the current mappings and will be visible to the
|
||||
model.
|
||||
"""
|
||||
|
||||
# Commit physical_to_logical_map
|
||||
src = new_physical_to_logical_map
|
||||
dst = model_state.physical_to_logical_map
|
||||
|
||||
@@ -14,8 +14,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CpuGpuEvent:
|
||||
"""
|
||||
Combines a CUDA event with a CPU threading event to enforce record->wait
|
||||
"""Combines a CUDA event with a CPU threading event to enforce record->wait
|
||||
ordering across two threads.
|
||||
|
||||
This class is designed for exactly two threads: one producer that calls
|
||||
@@ -35,8 +34,7 @@ class CpuGpuEvent:
|
||||
self._recorded = threading.Event()
|
||||
|
||||
def wait(self, stream: torch.cuda.Stream | None = None):
|
||||
"""
|
||||
Blocks the calling thread until record finishes. Used to guarantee that the
|
||||
"""Blocks the calling thread until record finishes. Used to guarantee that the
|
||||
record kernel is called before wait.
|
||||
|
||||
Should only be called by the Async Eplb thread.
|
||||
@@ -46,8 +44,7 @@ class CpuGpuEvent:
|
||||
self._recorded.clear()
|
||||
|
||||
def record(self, stream: torch.cuda.Stream | None = None):
|
||||
"""
|
||||
Unblocks the waiting thread after calling event.record().
|
||||
"""Unblocks the waiting thread after calling event.record().
|
||||
|
||||
Should only be called by the main thread.
|
||||
"""
|
||||
@@ -65,12 +62,12 @@ def override_envs_for_eplb(
|
||||
parallel_config: ParallelConfig,
|
||||
moe_backend: str | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Override environment variables for EPLB when specific conditions are met.
|
||||
"""Override environment variables for EPLB when specific conditions are met.
|
||||
|
||||
Args:
|
||||
parallel_config: The parallel configuration object.
|
||||
moe_backend: The configured MoE backend (e.g. ``deep_gemm_mega_moe``).
|
||||
|
||||
"""
|
||||
is_data_parallel = parallel_config.data_parallel_size > 1
|
||||
is_eplb_enabled = parallel_config.enable_eplb
|
||||
|
||||
@@ -18,10 +18,9 @@ class AbstractEplbPolicy(ABC):
|
||||
num_ranks: int,
|
||||
old_global_expert_indices: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Entry point for expert-parallelism load balancer.
|
||||
"""Entry point for expert-parallelism load balancer.
|
||||
|
||||
Parameters:
|
||||
Args:
|
||||
weight: [layers, num_logical_experts], the load statistics
|
||||
for all logical experts
|
||||
num_replicas: number of physical experts, must be a multiple of
|
||||
@@ -32,8 +31,10 @@ class AbstractEplbPolicy(ABC):
|
||||
old_global_expert_indices: [layers, num_logical_experts], the old global
|
||||
expert indices. Used to avoid unnecessary weight copying
|
||||
for experts moving within one rank.
|
||||
|
||||
Returns:
|
||||
physical_to_logical_map: [layers, num_replicas], the expert
|
||||
index of each replica
|
||||
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Expert parallelism load balancer (EPLB) for vLLM.
|
||||
"""Expert parallelism load balancer (EPLB) for vLLM.
|
||||
|
||||
This module implements the core rearrangement algorithm.
|
||||
|
||||
@@ -23,17 +22,17 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
def balanced_packing(
|
||||
cls, weight: np.ndarray, num_packs: int
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Pack n weighted objects to m packs, such that each bin contains exactly
|
||||
"""Pack n weighted objects to m packs, such that each bin contains exactly
|
||||
n/m objects and the weights of all packs are as balanced as possible.
|
||||
|
||||
Parameters:
|
||||
Args:
|
||||
weight: [X, n], the weight of each item
|
||||
num_packs: number of packs
|
||||
|
||||
Returns:
|
||||
pack_index: [X, n], the pack index of each item
|
||||
rank_in_pack: [X, n], the rank of the item in the pack
|
||||
|
||||
"""
|
||||
num_layers, num_groups = weight.shape
|
||||
assert num_groups % num_packs == 0
|
||||
@@ -76,17 +75,17 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
def replicate_experts(
|
||||
cls, weight: np.ndarray, num_phy: int
|
||||
) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""
|
||||
Replicate `num_log` experts to `num_phy` replicas, such that the maximum
|
||||
"""Replicate `num_log` experts to `num_phy` replicas, such that the maximum
|
||||
load of all replicas is minimized.
|
||||
|
||||
Parameters:
|
||||
Args:
|
||||
weight: [X, num_log]
|
||||
num_phy: total number of experts after replication
|
||||
|
||||
Returns:
|
||||
phy2log: [X, num_phy], logical expert id of each physical expert
|
||||
logcnt: [X, num_log], number of replicas for each logical expert
|
||||
|
||||
"""
|
||||
n, num_log = weight.shape
|
||||
num_redundant = num_phy - num_log
|
||||
@@ -109,8 +108,7 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
num_nodes: int,
|
||||
num_gpus: int,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Parameters:
|
||||
"""Parameters
|
||||
weight: [num_moe_layers, num_logical_experts]
|
||||
num_physical_experts: number of physical experts after replication
|
||||
num_groups: number of expert groups
|
||||
@@ -121,6 +119,7 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
Returns:
|
||||
phy2log: [layers, num_replicas], the expert
|
||||
index of each replica
|
||||
|
||||
"""
|
||||
num_layers, num_logical_experts = weight.shape
|
||||
assert num_logical_experts % num_groups == 0
|
||||
@@ -195,8 +194,7 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
num_ranks: int,
|
||||
old_phy2log: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
"""
|
||||
Reorder the new mapping per GPU so that experts that remain on the same GPU
|
||||
"""Reorder the new mapping per GPU so that experts that remain on the same GPU
|
||||
keep their previous slot positions when possible. Incoming experts to that GPU
|
||||
fill any remaining available slots. This is applied only when the number of GPUs
|
||||
is unchanged and the slots per GPU remain the same between
|
||||
@@ -281,10 +279,9 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
num_ranks: int,
|
||||
old_global_expert_indices: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Entry point for expert-parallelism load balancer.
|
||||
"""Entry point for expert-parallelism load balancer.
|
||||
|
||||
Parameters:
|
||||
Args:
|
||||
weight: [layers, num_logical_experts], the load statistics for all
|
||||
logical experts
|
||||
num_replicas: number of physical experts, must be a multiple of
|
||||
@@ -296,9 +293,12 @@ class DefaultEplbPolicy(AbstractEplbPolicy):
|
||||
old_global_expert_indices: [layers, num_logical_experts], the old global
|
||||
expert indices. Used to avoid unnecessary weight copying
|
||||
for experts moving within one rank.
|
||||
|
||||
Returns:
|
||||
-------
|
||||
phy2log: [layers, num_replicas], the expert
|
||||
index of each replica
|
||||
|
||||
"""
|
||||
weight_np = weight.float().cpu().numpy()
|
||||
old_phy2log_np = (
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
The actual execution of the rearrangement.
|
||||
"""The actual execution of the rearrangement.
|
||||
|
||||
This involves the exchange of expert weights between GPUs.
|
||||
"""
|
||||
@@ -41,9 +40,7 @@ class TransferMetadata:
|
||||
|
||||
@dataclass
|
||||
class AsyncEplbLayerResult:
|
||||
"""
|
||||
The result of one completed async EPLB layer transfer.
|
||||
"""
|
||||
"""The result of one completed async EPLB layer transfer."""
|
||||
|
||||
layer_idx: int
|
||||
"""Index of the MoE layer that was transferred."""
|
||||
@@ -69,8 +66,7 @@ def get_ep_ranks_with_experts_batch(
|
||||
old_indices: np.ndarray,
|
||||
new_indices: np.ndarray,
|
||||
) -> tuple[dict[int, list[int]], dict[int, list[int]]]:
|
||||
"""
|
||||
Get the ranks of the experts that need to be exchanged.
|
||||
"""Get the ranks of the experts that need to be exchanged.
|
||||
|
||||
Args:
|
||||
expert_ids: 1D array of expert indices to query.
|
||||
@@ -82,6 +78,7 @@ def get_ep_ranks_with_experts_batch(
|
||||
A tuple of two dictionaries mapping expert_id to:
|
||||
- ranks_to_send: The ranks that have this expert and need to send.
|
||||
- ranks_to_recv: The ranks that need to receive this expert.
|
||||
|
||||
"""
|
||||
ranks_to_send_map: dict[int, list[int]] = {}
|
||||
ranks_to_recv_map: dict[int, list[int]] = {}
|
||||
@@ -181,8 +178,7 @@ def move_to_buffer(
|
||||
communicator: EplbCommunicator,
|
||||
layer_idx: int = 0,
|
||||
) -> TransferMetadata:
|
||||
"""
|
||||
Rearranges expert weights during EPLB rebalancing.
|
||||
"""Rearranges expert weights during EPLB rebalancing.
|
||||
|
||||
Args:
|
||||
num_local_experts: Number of local experts.
|
||||
@@ -199,6 +195,7 @@ def move_to_buffer(
|
||||
|
||||
Returns:
|
||||
TransferMetadata: Metadata needed for completing remote weight transfers.
|
||||
|
||||
"""
|
||||
assert old_indices.shape == new_indices.shape
|
||||
recv_primary_mask = np.zeros((num_local_experts,), dtype=np.bool_)
|
||||
@@ -355,9 +352,8 @@ def move_from_buffer(
|
||||
new_indices: np.ndarray,
|
||||
ep_rank: int,
|
||||
) -> None:
|
||||
"""
|
||||
Copies expert weights from communication buffers back to the target weight tensors
|
||||
after EPLB rebalancing.
|
||||
"""Copies expert weights from communication buffers back to the target weight
|
||||
tensors after EPLB rebalancing.
|
||||
|
||||
Args:
|
||||
expert_weights: List of the actual MoE layer weights used in the execution.
|
||||
@@ -367,6 +363,7 @@ def move_from_buffer(
|
||||
new_indices: (num_experts_total,) mapping from local rows to desired
|
||||
(possibly global) expert id, after rebalance.
|
||||
ep_rank: Rank of the process in the expert parallel group.
|
||||
|
||||
"""
|
||||
is_unchanged = transfer_metadata.is_unchanged
|
||||
is_received_locally = transfer_metadata.is_received_locally
|
||||
@@ -437,8 +434,7 @@ def transfer_layer(
|
||||
rank_mapping: dict[int, int] | None = None,
|
||||
layer_idx: int = 0,
|
||||
) -> TransferMetadata:
|
||||
"""
|
||||
Rearranges the expert weights in place according to the new expert indices.
|
||||
"""Rearranges the expert weights in place according to the new expert indices.
|
||||
|
||||
The value of the indices arguments are logical indices of the experts,
|
||||
while keys are physical.
|
||||
@@ -462,6 +458,7 @@ def transfer_layer(
|
||||
Returns:
|
||||
TransferMetadata: Metadata needed for completing remote weight transfers,
|
||||
including is_unchanged and is_received_locally masks.
|
||||
|
||||
"""
|
||||
ep_size = ep_group.size()
|
||||
if rank_mapping is not None:
|
||||
@@ -519,8 +516,7 @@ def rearrange_expert_weights_inplace(
|
||||
is_profile: bool = False,
|
||||
rank_mapping: dict[int, int] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Rearranges the expert weights in place according to the new expert indices.
|
||||
"""Rearranges the expert weights in place according to the new expert indices.
|
||||
|
||||
The value of the indices arguments are logical indices of the experts,
|
||||
while keys are physical.
|
||||
@@ -540,6 +536,7 @@ def rearrange_expert_weights_inplace(
|
||||
This is used during profile run, where we only perform dummy
|
||||
communications to reserve enough memory for the buffers.
|
||||
rank_mapping: A dictionary mapping old rank to new rank.
|
||||
|
||||
"""
|
||||
if rank_mapping is not None:
|
||||
if len(rank_mapping) == ep_group.size():
|
||||
@@ -624,8 +621,7 @@ def _map_old_expert_indices_with_rank_mapping(
|
||||
rank_mapping: dict[int, int],
|
||||
new_ep_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Map the old global expert indices to the new global expert indices.
|
||||
"""Map the old global expert indices to the new global expert indices.
|
||||
|
||||
Args:
|
||||
old_global_expert_indices:
|
||||
@@ -636,6 +632,7 @@ def _map_old_expert_indices_with_rank_mapping(
|
||||
Returns:
|
||||
Mapped expert indices with shape
|
||||
(num_layers, new_ep_size * num_local_physical_experts).
|
||||
|
||||
"""
|
||||
num_layers, old_num_physical_experts = old_global_expert_indices.shape
|
||||
assert rank_mapping, "Rank mapping is required"
|
||||
|
||||
Reference in New Issue
Block a user