[Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699)
Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com> Signed-off-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Dakai An <dakaian108@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
Dakai An
OpenAI Codex
aoshen02
parent
117cf43fb6
commit
78fdf4efb8
@@ -9,7 +9,7 @@ from torch.utils._python_dispatch import TorchDispatchMode
|
||||
|
||||
from .sanitize import restore_layer_refs, sanitize_layer_refs
|
||||
from .types import LayerReloadingInfo, LayerTensors
|
||||
from .utils import get_layer_params_buffers, get_layer_tensors
|
||||
from .utils import get_layer_params_buffers, get_layer_tensors, get_tensor_load_numel
|
||||
|
||||
__all__ = [
|
||||
"to_meta_tensor",
|
||||
@@ -206,5 +206,5 @@ def get_numel_loaded(
|
||||
numel = counter.copied_numel
|
||||
param = args.arguments.get("param", None)
|
||||
if isinstance(param, torch.Tensor):
|
||||
numel = min(numel, param.numel())
|
||||
numel = min(numel, get_tensor_load_numel(param))
|
||||
return numel, return_value
|
||||
|
||||
@@ -11,6 +11,7 @@ __all__ = [
|
||||
"get_layer_tensors",
|
||||
"get_layer_params_buffers",
|
||||
"get_layer_size",
|
||||
"get_tensor_load_numel",
|
||||
"has_device_tensors",
|
||||
"get_info_size",
|
||||
]
|
||||
@@ -30,6 +31,11 @@ def get_layer_params_buffers(layer: torch.nn.Module) -> LayerTensors:
|
||||
)
|
||||
|
||||
|
||||
def get_tensor_load_numel(tensor: torch.Tensor) -> int:
|
||||
"""Count checkpoint elements, excluding padding declared by the weight creator."""
|
||||
return getattr(tensor, "weight_loader_numel", tensor.numel())
|
||||
|
||||
|
||||
def get_layer_size(layer: torch.nn.Module) -> int:
|
||||
"""Calculate total number of elements across loadable tensors in a layer.
|
||||
|
||||
@@ -39,7 +45,7 @@ def get_layer_size(layer: torch.nn.Module) -> int:
|
||||
from .meta import SKIP_LOAD_TENSORS
|
||||
|
||||
return sum(
|
||||
tensor.numel()
|
||||
get_tensor_load_numel(tensor)
|
||||
for name, tensor in get_layer_tensors(layer).items()
|
||||
if name not in SKIP_LOAD_TENSORS
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user