[Bugfix] Preserve Marlin runtime tensor storage across weight reload (#48438)

Signed-off-by: Ryan Clark <ryanclark2k@gmail.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Ryan Clark
2026-07-30 14:50:03 -07:00
committed by GitHub
co-authored by Claude Fable 5
parent e9096fcb12
commit 68fb303a41
6 changed files with 330 additions and 17 deletions
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
unpack_cols,
)
from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.scalar_type import scalar_types
@@ -111,8 +112,10 @@ class MarlinLinearKernel(MPLinearKernel):
else:
padded_n, padded_k = marlin_padded_nk(size_n, size_k, c.group_size)
# Allocate marlin workspace.
self.workspace = marlin_make_workspace_new(device)
# Allocate marlin workspace, reusing existing storage on reload.
self.workspace = marlin_make_workspace_new(
device, existing=getattr(self, "workspace", None)
)
# Default names since marlin requires empty parameters for these,
# TODO: remove this requirement from marlin (allow optional tensors)
@@ -174,7 +177,9 @@ class MarlinLinearKernel(MPLinearKernel):
getattr(layer, self.w_gidx_name)
)
self._transform_param(layer, self.w_gidx_name, lambda _: g_idx)
layer.g_idx_sort_indices = g_idx_sort_indices
replace_parameter(
layer, "g_idx_sort_indices", g_idx_sort_indices, prefer_copy=True
)
else:
setattr(layer, self.w_gidx_name, marlin_make_empty_g_idx(device))
layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)