[Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699)

Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Yuchen Wang
2026-09-17 22:00:04 +08:00
committed by GitHub
co-authored by Dakai An OpenAI Codex aoshen02
parent 117cf43fb6
commit 78fdf4efb8
9 changed files with 667 additions and 68 deletions
@@ -9,7 +9,7 @@ from torch.utils._python_dispatch import TorchDispatchMode
from .sanitize import restore_layer_refs, sanitize_layer_refs
from .types import LayerReloadingInfo, LayerTensors
from .utils import get_layer_params_buffers, get_layer_tensors
from .utils import get_layer_params_buffers, get_layer_tensors, get_tensor_load_numel
__all__ = [
"to_meta_tensor",
@@ -206,5 +206,5 @@ def get_numel_loaded(
numel = counter.copied_numel
param = args.arguments.get("param", None)
if isinstance(param, torch.Tensor):
numel = min(numel, param.numel())
numel = min(numel, get_tensor_load_numel(param))
return numel, return_value
@@ -11,6 +11,7 @@ __all__ = [
"get_layer_tensors",
"get_layer_params_buffers",
"get_layer_size",
"get_tensor_load_numel",
"has_device_tensors",
"get_info_size",
]
@@ -30,6 +31,11 @@ def get_layer_params_buffers(layer: torch.nn.Module) -> LayerTensors:
)
def get_tensor_load_numel(tensor: torch.Tensor) -> int:
"""Count checkpoint elements, excluding padding declared by the weight creator."""
return getattr(tensor, "weight_loader_numel", tensor.numel())
def get_layer_size(layer: torch.nn.Module) -> int:
"""Calculate total number of elements across loadable tensors in a layer.
@@ -39,7 +45,7 @@ def get_layer_size(layer: torch.nn.Module) -> int:
from .meta import SKIP_LOAD_TENSORS
return sum(
tensor.numel()
get_tensor_load_numel(tensor)
for name, tensor in get_layer_tensors(layer).items()
if name not in SKIP_LOAD_TENSORS
)