[BUG] Exclude SKIP_TENSORS from get_layer_size() + new weight sync example for dpep (#37334)

Signed-off-by: ahao-anyscale <ahao@anyscale.com>
This commit is contained in:
Aaron Hao
2026-03-19 00:45:10 +00:00
committed by GitHub
parent c32a58cc2a
commit 5f82706a21
2 changed files with 351 additions and 2 deletions
@@ -27,5 +27,15 @@ def get_layer_params_buffers(layer: torch.nn.Module) -> LayerTensors:
def get_layer_size(layer: torch.nn.Module) -> int:
"""Calculate total number of elements across all tensors in a layer."""
return sum(tensor.numel() for tensor in get_layer_tensors(layer).values())
"""Calculate total number of elements across loadable tensors in a layer.
Excludes SKIP_TENSORS (e.g. _expert_map) which are never moved to meta
device and never loaded via weight_loader during layerwise reload.
"""
from .meta import SKIP_TENSORS
return sum(
tensor.numel()
for name, tensor in get_layer_tensors(layer).items()
if name not in SKIP_TENSORS
)