[Bugfix] Wait for the linear bias before layerwise online processing (#49805)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
26d725c334
commit
9321aff536
@@ -33,15 +33,15 @@ def get_layer_params_buffers(layer: torch.nn.Module) -> LayerTensors:
|
||||
def get_layer_size(layer: torch.nn.Module) -> int:
|
||||
"""Calculate total number of elements across loadable tensors in a layer.
|
||||
|
||||
Excludes SKIP_TENSORS (e.g. _expert_map) which are never moved to meta
|
||||
device and never loaded via weight_loader during layerwise reload.
|
||||
Excludes SKIP_LOAD_TENSORS (e.g. _expert_map) which are never loaded via
|
||||
weight_loader during layerwise reload.
|
||||
"""
|
||||
from .meta import SKIP_TENSORS
|
||||
from .meta import SKIP_LOAD_TENSORS
|
||||
|
||||
return sum(
|
||||
tensor.numel()
|
||||
for name, tensor in get_layer_tensors(layer).items()
|
||||
if name not in SKIP_TENSORS
|
||||
if name not in SKIP_LOAD_TENSORS
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user