[Bugfix] Preserve Marlin runtime tensor storage across weight reload (#48438)
Signed-off-by: Ryan Clark <ryanclark2k@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
e9096fcb12
commit
68fb303a41
@@ -397,14 +397,31 @@ def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tenso
|
||||
|
||||
|
||||
def marlin_make_workspace_new(
|
||||
device: torch.device, max_blocks_per_sm: int = 1
|
||||
device: torch.device,
|
||||
max_blocks_per_sm: int = 1,
|
||||
existing: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# In the new marlin kernel, we use the num of threadblocks as workspace
|
||||
# size. The num of threadblocks is sms_count * max_blocks_per_sm.
|
||||
sms = num_compute_units(device.index)
|
||||
return torch.zeros(
|
||||
sms * max_blocks_per_sm, dtype=torch.int, device=device, requires_grad=False
|
||||
)
|
||||
size = sms * max_blocks_per_sm
|
||||
# On weight reload, reuse the existing storage so the workspace address
|
||||
# captured by CUDA graphs stays valid.
|
||||
if existing is not None:
|
||||
if (
|
||||
existing.device != device
|
||||
or existing.dtype != torch.int
|
||||
or existing.numel() != size
|
||||
):
|
||||
raise ValueError(
|
||||
f"Existing Marlin workspace is incompatible "
|
||||
f"(device={existing.device}, dtype={existing.dtype}, "
|
||||
f"numel={existing.numel()}; expected device={device}, "
|
||||
f"dtype={torch.int}, numel={size}). Reload must reuse the "
|
||||
f"workspace storage captured by CUDA graphs."
|
||||
)
|
||||
return existing.zero_()
|
||||
return torch.zeros(size, dtype=torch.int, device=device, requires_grad=False)
|
||||
|
||||
|
||||
def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
|
||||
|
||||
@@ -240,7 +240,9 @@ def prepare_fp4_layer_for_marlin(
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(device)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
@@ -396,7 +398,9 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
# WEIGHT
|
||||
@@ -485,7 +489,9 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
# WORKSPACE
|
||||
device = layer.w13_weight.device
|
||||
param_dtype = layer.params_dtype
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
|
||||
@@ -132,7 +132,9 @@ def prepare_fp8_layer_for_marlin(
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(device)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
@@ -278,9 +280,9 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
|
||||
# WORKSPACE
|
||||
device = layer.w13_weight.device
|
||||
# NOTE(rob): we do not need to register the workspace as a param
|
||||
# because it is not used as part of the weight reloading process.
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
# WEIGHT
|
||||
@@ -463,7 +465,9 @@ def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(device)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT - repack FP8 weights to Marlin format
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
@@ -549,7 +553,9 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
param_dtype = torch.get_default_dtype()
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
|
||||
if "w13" in name:
|
||||
|
||||
Reference in New Issue
Block a user