[BugFix] weights processing peak memory reduction for nvfp4 MoE layers (#46276)

Signed-off-by: Jimmy Lee <hirejimmylee@gmail.com>
This commit is contained in:
Jimmy Lee
2026-07-11 02:05:35 +00:00
committed by GitHub
parent 1bf3997eae
commit 092387963c
2 changed files with 36 additions and 29 deletions
@@ -23,7 +23,6 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
__all__ = [
"reorder_w1w3_to_w3w1",
]
@@ -32,18 +31,35 @@ __all__ = [
def reorder_w1w3_to_w3w1(
weight: torch.Tensor, scale: torch.Tensor, dim: int = -2
) -> tuple[torch.Tensor, torch.Tensor]:
"""Re-order the concatenated `[w1, w3]` tensors to `[w3, w1]`"""
"""Re-order concatenated `[w1, w3]` tensors to `[w3, w1]` in-place.
`weight` and `scale` must be contiguous; they remain contiguous on return.
"""
assert weight.is_contiguous(), "weight must be contiguous"
assert scale.is_contiguous(), "scale must be contiguous"
size = weight.size(dim)
assert size % 2 == 0, f"Expected even size in dim {dim}, got {size}"
half = size // 2
d = dim % weight.dim()
w1, w3 = weight.split(half, dim=dim)
s1, s3 = scale.split(half, dim=dim)
return (
torch.cat([w3, w1], dim=dim).contiguous(),
torch.cat([s3, s1], dim=dim).contiguous(),
# 64 MB transient cap
bytes_per_row = max(
weight.numel() // size * weight.element_size(),
scale.numel() // size * scale.element_size(),
)
chunk = max(1, min(half, (64 << 20) // max(bytes_per_row, 1)))
fa, fb = [slice(None)] * weight.dim(), [slice(None)] * weight.dim()
for off in range(0, half, chunk):
end = min(off + chunk, half)
fa[d], fb[d] = slice(off, end), slice(half + off, half + end)
a, b = tuple(fa), tuple(fb)
for t in (weight, scale):
tmp = t[b].clone()
t[b] = t[a]
t[a] = tmp
return weight, scale
def interleave_linear_and_gate(