[BugFix] weights processing peak memory reduction for nvfp4 MoE layers (#46276)
Signed-off-by: Jimmy Lee <hirejimmylee@gmail.com>
This commit is contained in:
@@ -23,7 +23,6 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"reorder_w1w3_to_w3w1",
|
||||
]
|
||||
@@ -32,18 +31,35 @@ __all__ = [
|
||||
def reorder_w1w3_to_w3w1(
|
||||
weight: torch.Tensor, scale: torch.Tensor, dim: int = -2
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Re-order the concatenated `[w1, w3]` tensors to `[w3, w1]`"""
|
||||
"""Re-order concatenated `[w1, w3]` tensors to `[w3, w1]` in-place.
|
||||
|
||||
`weight` and `scale` must be contiguous; they remain contiguous on return.
|
||||
"""
|
||||
assert weight.is_contiguous(), "weight must be contiguous"
|
||||
assert scale.is_contiguous(), "scale must be contiguous"
|
||||
size = weight.size(dim)
|
||||
assert size % 2 == 0, f"Expected even size in dim {dim}, got {size}"
|
||||
half = size // 2
|
||||
d = dim % weight.dim()
|
||||
|
||||
w1, w3 = weight.split(half, dim=dim)
|
||||
s1, s3 = scale.split(half, dim=dim)
|
||||
|
||||
return (
|
||||
torch.cat([w3, w1], dim=dim).contiguous(),
|
||||
torch.cat([s3, s1], dim=dim).contiguous(),
|
||||
# 64 MB transient cap
|
||||
bytes_per_row = max(
|
||||
weight.numel() // size * weight.element_size(),
|
||||
scale.numel() // size * scale.element_size(),
|
||||
)
|
||||
chunk = max(1, min(half, (64 << 20) // max(bytes_per_row, 1)))
|
||||
|
||||
fa, fb = [slice(None)] * weight.dim(), [slice(None)] * weight.dim()
|
||||
for off in range(0, half, chunk):
|
||||
end = min(off + chunk, half)
|
||||
fa[d], fb[d] = slice(off, end), slice(half + off, half + end)
|
||||
a, b = tuple(fa), tuple(fb)
|
||||
for t in (weight, scale):
|
||||
tmp = t[b].clone()
|
||||
t[b] = t[a]
|
||||
t[a] = tmp
|
||||
|
||||
return weight, scale
|
||||
|
||||
|
||||
def interleave_linear_and_gate(
|
||||
|
||||
Reference in New Issue
Block a user