[BUG] Exclude SKIP_TENSORS from get_layer_size() + new weight sync example for dpep (#37334)
Signed-off-by: ahao-anyscale <ahao@anyscale.com>
This commit is contained in:
@@ -27,5 +27,15 @@ def get_layer_params_buffers(layer: torch.nn.Module) -> LayerTensors:
|
||||
|
||||
|
||||
def get_layer_size(layer: torch.nn.Module) -> int:
|
||||
"""Calculate total number of elements across all tensors in a layer."""
|
||||
return sum(tensor.numel() for tensor in get_layer_tensors(layer).values())
|
||||
"""Calculate total number of elements across loadable tensors in a layer.
|
||||
|
||||
Excludes SKIP_TENSORS (e.g. _expert_map) which are never moved to meta
|
||||
device and never loaded via weight_loader during layerwise reload.
|
||||
"""
|
||||
from .meta import SKIP_TENSORS
|
||||
|
||||
return sum(
|
||||
tensor.numel()
|
||||
for name, tensor in get_layer_tensors(layer).items()
|
||||
if name not in SKIP_TENSORS
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user