[Bugfix] Preserve Marlin runtime tensor storage across weight reload (#48438)

Signed-off-by: Ryan Clark <ryanclark2k@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Ryan Clark
2026-07-30 14:50:03 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent e9096fcb12
commit 68fb303a41
6 changed files with 330 additions and 17 deletions
@@ -397,14 +397,31 @@ def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tenso
def marlin_make_workspace_new(
device: torch.device, max_blocks_per_sm: int = 1
device: torch.device,
max_blocks_per_sm: int = 1,
existing: torch.Tensor | None = None,
) -> torch.Tensor:
# In the new marlin kernel, we use the num of threadblocks as workspace
# size. The num of threadblocks is sms_count * max_blocks_per_sm.
sms = num_compute_units(device.index)
return torch.zeros(
sms * max_blocks_per_sm, dtype=torch.int, device=device, requires_grad=False
)
size = sms * max_blocks_per_sm
# On weight reload, reuse the existing storage so the workspace address
# captured by CUDA graphs stays valid.
if existing is not None:
if (
existing.device != device
or existing.dtype != torch.int
or existing.numel() != size
):
raise ValueError(
f"Existing Marlin workspace is incompatible "
f"(device={existing.device}, dtype={existing.dtype}, "
f"numel={existing.numel()}; expected device={device}, "
f"dtype={torch.int}, numel={size}). Reload must reuse the "
f"workspace storage captured by CUDA graphs."
)
return existing.zero_()
return torch.zeros(size, dtype=torch.int, device=device, requires_grad=False)
def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
@@ -240,7 +240,9 @@ def prepare_fp4_layer_for_marlin(
device = layer.weight.device
# WORKSPACE
layer.workspace = marlin_make_workspace_new(device)
layer.workspace = marlin_make_workspace_new(
device, existing=getattr(layer, "workspace", None)
)
# WEIGHT
# Repack weights to marlin format
@@ -396,7 +398,9 @@ def prepare_nvfp4_moe_layer_for_marlin(
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
# WORKSPACE
layer.workspace = marlin_make_workspace_new(device, 4)
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
perm = torch.empty(0, dtype=torch.int, device=device)
# WEIGHT
@@ -485,7 +489,9 @@ def prepare_moe_fp4_layer_for_marlin(
# WORKSPACE
device = layer.w13_weight.device
param_dtype = layer.params_dtype
layer.workspace = marlin_make_workspace_new(device, 4)
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
perm = torch.empty(0, dtype=torch.int, device=device)
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
@@ -132,7 +132,9 @@ def prepare_fp8_layer_for_marlin(
device = layer.weight.device
# WORKSPACE
layer.workspace = marlin_make_workspace_new(device)
layer.workspace = marlin_make_workspace_new(
device, existing=getattr(layer, "workspace", None)
)
# WEIGHT
# Repack weights to marlin format
@@ -278,9 +280,9 @@ def prepare_fp8_moe_layer_for_marlin(
# WORKSPACE
device = layer.w13_weight.device
# NOTE(rob): we do not need to register the workspace as a param
# because it is not used as part of the weight reloading process.
layer.workspace = marlin_make_workspace_new(device, 4)
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
perm = torch.empty(0, dtype=torch.int, device=device)
# WEIGHT
@@ -463,7 +465,9 @@ def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None:
device = layer.weight.device
# WORKSPACE
layer.workspace = marlin_make_workspace_new(device)
layer.workspace = marlin_make_workspace_new(
device, existing=getattr(layer, "workspace", None)
)
# WEIGHT - repack FP8 weights to Marlin format
perm = torch.empty(0, dtype=torch.int, device=device)
@@ -549,7 +553,9 @@ def prepare_mxfp8_moe_layer_for_marlin(
param_dtype = torch.get_default_dtype()
perm = torch.empty(0, dtype=torch.int, device=device)
layer.workspace = marlin_make_workspace_new(device, 4)
layer.workspace = marlin_make_workspace_new(
device, 4, existing=getattr(layer, "workspace", None)
)
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
if "w13" in name: