[Bugfix] Preserve Marlin runtime tensor storage across weight reload (#48438)
Signed-off-by: Ryan Clark <ryanclark2k@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
e9096fcb12
commit
68fb303a41
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
unpack_cols,
|
||||
)
|
||||
from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
@@ -111,8 +112,10 @@ class MarlinLinearKernel(MPLinearKernel):
|
||||
else:
|
||||
padded_n, padded_k = marlin_padded_nk(size_n, size_k, c.group_size)
|
||||
|
||||
# Allocate marlin workspace.
|
||||
self.workspace = marlin_make_workspace_new(device)
|
||||
# Allocate marlin workspace, reusing existing storage on reload.
|
||||
self.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(self, "workspace", None)
|
||||
)
|
||||
|
||||
# Default names since marlin requires empty parameters for these,
|
||||
# TODO: remove this requirement from marlin (allow optional tensors)
|
||||
@@ -174,7 +177,9 @@ class MarlinLinearKernel(MPLinearKernel):
|
||||
getattr(layer, self.w_gidx_name)
|
||||
)
|
||||
self._transform_param(layer, self.w_gidx_name, lambda _: g_idx)
|
||||
layer.g_idx_sort_indices = g_idx_sort_indices
|
||||
replace_parameter(
|
||||
layer, "g_idx_sort_indices", g_idx_sort_indices, prefer_copy=True
|
||||
)
|
||||
else:
|
||||
setattr(layer, self.w_gidx_name, marlin_make_empty_g_idx(device))
|
||||
layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
|
||||
|
||||
Reference in New Issue
Block a user