[Bugfix][MoE] Convert FlashInfer BF16 weights in place (#54699)
Signed-off-by: Yuchen Wang <93700456+yuchenwang3@users.noreply.github.com> Signed-off-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Dakai An <dakaian108@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
Dakai An
OpenAI Codex
aoshen02
parent
117cf43fb6
commit
78fdf4efb8
@@ -9,7 +9,7 @@ from torch.utils._python_dispatch import TorchDispatchMode
|
||||
|
||||
from .sanitize import restore_layer_refs, sanitize_layer_refs
|
||||
from .types import LayerReloadingInfo, LayerTensors
|
||||
from .utils import get_layer_params_buffers, get_layer_tensors
|
||||
from .utils import get_layer_params_buffers, get_layer_tensors, get_tensor_load_numel
|
||||
|
||||
__all__ = [
|
||||
"to_meta_tensor",
|
||||
@@ -206,5 +206,5 @@ def get_numel_loaded(
|
||||
numel = counter.copied_numel
|
||||
param = args.arguments.get("param", None)
|
||||
if isinstance(param, torch.Tensor):
|
||||
numel = min(numel, param.numel())
|
||||
numel = min(numel, get_tensor_load_numel(param))
|
||||
return numel, return_value
|
||||
|
||||
Reference in New Issue
Block a user