[Bugfix] Preserve Marlin runtime tensor storage across weight reload (#48438)
Signed-off-by: Ryan Clark <ryanclark2k@gmail.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
e9096fcb12
commit
68fb303a41
@@ -167,6 +167,283 @@ def test_materialize_layer_preserves_non_meta_tensors():
|
||||
assert torch.equal(layer.bias.data, bias_values)
|
||||
|
||||
|
||||
_MARLIN_SIZE_K, _MARLIN_SIZE_N, _MARLIN_GROUP_SIZE = 128, 64, 64
|
||||
|
||||
|
||||
def _stub_marlin_ops(monkeypatch):
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils import marlin_utils
|
||||
|
||||
monkeypatch.setattr(marlin_utils, "num_compute_units", lambda _: 4)
|
||||
monkeypatch.setattr(
|
||||
ops,
|
||||
"gptq_marlin_repack",
|
||||
lambda w, perm, size_k, size_n, num_bits, is_a_8bit=False: torch.zeros(
|
||||
size_k // 16, size_n * 2, dtype=torch.int32
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _make_act_order_marlin_kernel():
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.marlin import (
|
||||
MarlinLinearKernel,
|
||||
)
|
||||
from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import (
|
||||
MPLinearLayerConfig,
|
||||
)
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
kernel = object.__new__(MarlinLinearKernel)
|
||||
kernel.config = MPLinearLayerConfig(
|
||||
full_weight_shape=(_MARLIN_SIZE_K, _MARLIN_SIZE_N),
|
||||
partition_weight_shape=(_MARLIN_SIZE_K, _MARLIN_SIZE_N),
|
||||
weight_type=scalar_types.uint4b8,
|
||||
act_type=torch.float16,
|
||||
group_size=_MARLIN_GROUP_SIZE,
|
||||
zero_points=False,
|
||||
has_g_idx=True,
|
||||
)
|
||||
kernel.w_q_name = "qweight"
|
||||
kernel.w_s_name = "scales"
|
||||
kernel.w_zp_name = None
|
||||
kernel.w_gidx_name = "g_idx"
|
||||
return kernel
|
||||
|
||||
|
||||
def _load_marlin_checkpoint_format_weights(layer, g_idx):
|
||||
from vllm.model_executor.parameter import (
|
||||
GroupQuantScaleParameter,
|
||||
PackedvLLMParameter,
|
||||
RowvLLMParameter,
|
||||
)
|
||||
|
||||
layer.qweight = PackedvLLMParameter(
|
||||
data=torch.zeros(_MARLIN_SIZE_K // 8, _MARLIN_SIZE_N, dtype=torch.int32),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
packed_dim=0,
|
||||
packed_factor=8,
|
||||
weight_loader=default_weight_loader,
|
||||
)
|
||||
layer.scales = GroupQuantScaleParameter(
|
||||
data=torch.ones(
|
||||
_MARLIN_SIZE_K // _MARLIN_GROUP_SIZE, _MARLIN_SIZE_N, dtype=torch.float16
|
||||
),
|
||||
input_dim=0,
|
||||
output_dim=1,
|
||||
weight_loader=default_weight_loader,
|
||||
)
|
||||
layer.g_idx = RowvLLMParameter(
|
||||
data=g_idx.clone(),
|
||||
input_dim=0,
|
||||
weight_loader=default_weight_loader,
|
||||
)
|
||||
|
||||
|
||||
def _random_g_idx(generator):
|
||||
return torch.randint(
|
||||
0,
|
||||
_MARLIN_SIZE_K // _MARLIN_GROUP_SIZE,
|
||||
(_MARLIN_SIZE_K,),
|
||||
dtype=torch.int32,
|
||||
generator=generator,
|
||||
)
|
||||
|
||||
|
||||
def test_marlin_post_load_preserves_runtime_tensor_addresses(monkeypatch, dist_init):
|
||||
"""Marlin workspace and act-order sort indices must be recomputed into
|
||||
the same storage when weights are reloaded (RL weight sync), so device
|
||||
addresses captured by CUDA graphs remain valid."""
|
||||
from vllm.model_executor.layers.quantization.utils import marlin_utils
|
||||
|
||||
_stub_marlin_ops(monkeypatch)
|
||||
kernel = _make_act_order_marlin_kernel()
|
||||
|
||||
generator = torch.Generator().manual_seed(0)
|
||||
first_g_idx = _random_g_idx(generator)
|
||||
second_g_idx = _random_g_idx(generator)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
_load_marlin_checkpoint_format_weights(layer, first_g_idx)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
workspace_ptr = kernel.workspace.data_ptr()
|
||||
sort_indices_ptr = layer.g_idx_sort_indices.data_ptr()
|
||||
|
||||
# Reload: fresh checkpoint-format tensors with a different act-order
|
||||
_load_marlin_checkpoint_format_weights(layer, second_g_idx)
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
assert kernel.workspace.data_ptr() == workspace_ptr
|
||||
assert torch.all(kernel.workspace == 0)
|
||||
assert layer.g_idx_sort_indices.data_ptr() == sort_indices_ptr
|
||||
expected_sort_indices = marlin_utils.marlin_sort_g_idx(second_g_idx)[1]
|
||||
assert torch.equal(layer.g_idx_sort_indices.data, expected_sort_indices)
|
||||
# registered as a Parameter so layerwise reload copy-back preserves it
|
||||
assert isinstance(layer.g_idx_sort_indices, torch.nn.Parameter)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("variant", ["fp8", "mxfp8", "nvfp4"])
|
||||
def test_marlin_prepare_layer_preserves_workspace_address(monkeypatch, variant):
|
||||
"""The Marlin fallback prepare_* functions rerun on weight reload and must
|
||||
reuse the workspace storage whose address captured CUDA graphs hold."""
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils import (
|
||||
marlin_utils,
|
||||
marlin_utils_fp4,
|
||||
marlin_utils_fp8,
|
||||
)
|
||||
|
||||
size_k, size_n = 128, 64
|
||||
|
||||
monkeypatch.setattr(marlin_utils, "num_compute_units", lambda _: 4)
|
||||
monkeypatch.setattr(
|
||||
ops,
|
||||
"gptq_marlin_repack",
|
||||
lambda b_q_weight, perm, size_k, size_n, num_bits, is_a_8bit=False: torch.zeros(
|
||||
size_k // 16, size_n * 2, dtype=torch.int32
|
||||
),
|
||||
)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.output_size_per_partition = size_n
|
||||
layer.input_size_per_partition = size_k
|
||||
layer.orig_dtype = torch.float16
|
||||
layer.params_dtype = torch.float16
|
||||
|
||||
if variant == "fp8":
|
||||
prepare = marlin_utils_fp8.prepare_fp8_layer_for_marlin
|
||||
|
||||
def load_checkpoint_format_weights():
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.zeros(size_k, size_n, dtype=torch.float8_e4m3fn),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
torch.ones(1, dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
elif variant == "mxfp8":
|
||||
prepare = marlin_utils_fp8.prepare_mxfp8_layer_for_marlin
|
||||
|
||||
def load_checkpoint_format_weights():
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.zeros(size_n, size_k, dtype=torch.float8_e4m3fn),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
torch.full((size_n, size_k // 32), 127, dtype=torch.uint8),
|
||||
requires_grad=False,
|
||||
)
|
||||
else:
|
||||
prepare = marlin_utils_fp4.prepare_fp4_layer_for_marlin
|
||||
|
||||
def load_checkpoint_format_weights():
|
||||
layer.weight = torch.nn.Parameter(
|
||||
torch.zeros(size_n, size_k // 2, dtype=torch.uint8),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.weight_scale = torch.nn.Parameter(
|
||||
torch.ones(size_n, size_k // 16, dtype=torch.float8_e4m3fn),
|
||||
requires_grad=False,
|
||||
)
|
||||
layer.weight_global_scale = torch.nn.Parameter(
|
||||
torch.ones(1, dtype=torch.float32), requires_grad=False
|
||||
)
|
||||
|
||||
load_checkpoint_format_weights()
|
||||
prepare(layer)
|
||||
workspace_ptr = layer.workspace.data_ptr()
|
||||
|
||||
# Reload: fresh checkpoint-format tensors, prepare runs again
|
||||
load_checkpoint_format_weights()
|
||||
prepare(layer)
|
||||
|
||||
assert layer.workspace.data_ptr() == workspace_ptr
|
||||
assert torch.all(layer.workspace == 0)
|
||||
|
||||
|
||||
def test_marlin_make_workspace_new_rejects_incompatible_existing(monkeypatch):
|
||||
"""An incompatible existing workspace means the address captured by CUDA
|
||||
graphs is already unusable; allocating a replacement would hide that."""
|
||||
from vllm.model_executor.layers.quantization.utils import marlin_utils
|
||||
|
||||
monkeypatch.setattr(marlin_utils, "num_compute_units", lambda _: 4)
|
||||
device = torch.device("cpu")
|
||||
|
||||
workspace = marlin_utils.marlin_make_workspace_new(device)
|
||||
reused = marlin_utils.marlin_make_workspace_new(device, existing=workspace)
|
||||
assert reused is workspace
|
||||
|
||||
with pytest.raises(ValueError, match="incompatible"):
|
||||
marlin_utils.marlin_make_workspace_new(device, 4, existing=workspace)
|
||||
with pytest.raises(ValueError, match="incompatible"):
|
||||
marlin_utils.marlin_make_workspace_new(
|
||||
device, existing=workspace.to(torch.int64)
|
||||
)
|
||||
|
||||
|
||||
def test_marlin_act_order_layerwise_reload_accounting(monkeypatch, dist_init):
|
||||
"""`g_idx_sort_indices` is generated during weight processing and never
|
||||
loaded from checkpoints. Registering it as a Parameter must not count it
|
||||
toward `load_numel_total`: reload restores the construction-time tensor
|
||||
set before sizing, so act-order layers still process during streaming
|
||||
instead of deferring (and buffering weights) until finalization."""
|
||||
from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils import marlin_utils
|
||||
from vllm.model_executor.model_loader.reload.layerwise import get_layerwise_info
|
||||
|
||||
_stub_marlin_ops(monkeypatch)
|
||||
kernel = _make_act_order_marlin_kernel()
|
||||
|
||||
class _KernelQuantMethod(QuantizeMethodBase):
|
||||
def create_weights(self, layer, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def apply(self, layer, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
kernel.process_weights_after_loading(layer)
|
||||
|
||||
generator = torch.Generator().manual_seed(0)
|
||||
layer = torch.nn.Module()
|
||||
layer.quant_method = _KernelQuantMethod()
|
||||
_load_marlin_checkpoint_format_weights(layer, _random_g_idx(generator))
|
||||
|
||||
# Metadata is recorded at model construction, before any processing
|
||||
record_metadata_for_reloading(layer)
|
||||
checkpoint_numel = sum(t.numel() for t in get_layer_tensors(layer).values())
|
||||
|
||||
kernel.process_weights_after_loading(layer)
|
||||
sort_indices = layer.g_idx_sort_indices
|
||||
|
||||
initialize_layerwise_reload(layer)
|
||||
info = get_layerwise_info(layer)
|
||||
assert info.load_numel_total == checkpoint_numel
|
||||
|
||||
# Stream a new checkpoint; the layer must process as soon as its last
|
||||
# tensor arrives
|
||||
new_g_idx = _random_g_idx(generator)
|
||||
checkpoint = {
|
||||
"qweight": torch.zeros(_MARLIN_SIZE_K // 8, _MARLIN_SIZE_N, dtype=torch.int32),
|
||||
"scales": torch.ones(
|
||||
_MARLIN_SIZE_K // _MARLIN_GROUP_SIZE, _MARLIN_SIZE_N, dtype=torch.float16
|
||||
),
|
||||
"g_idx": new_g_idx,
|
||||
}
|
||||
for name, weight in checkpoint.items():
|
||||
param = getattr(layer, name)
|
||||
param.weight_loader(param, weight)
|
||||
|
||||
assert not info.can_load()
|
||||
assert not info.loaded_weights
|
||||
assert layer.g_idx_sort_indices is sort_indices
|
||||
expected_sort_indices = marlin_utils.marlin_sort_g_idx(new_g_idx)[1]
|
||||
assert torch.equal(layer.g_idx_sort_indices.data, expected_sort_indices)
|
||||
|
||||
|
||||
def test_model_cleanup(dist_init, default_vllm_config):
|
||||
layer = QKVParallelLinear(2, 3, 4)
|
||||
assert layer.weight.weight_loader.__self__ is layer
|
||||
|
||||
@@ -25,6 +25,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import (
|
||||
unpack_cols,
|
||||
)
|
||||
from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_
|
||||
from vllm.model_executor.utils import replace_parameter
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.scalar_type import scalar_types
|
||||
|
||||
@@ -111,8 +112,10 @@ class MarlinLinearKernel(MPLinearKernel):
|
||||
else:
|
||||
padded_n, padded_k = marlin_padded_nk(size_n, size_k, c.group_size)
|
||||
|
||||
# Allocate marlin workspace.
|
||||
self.workspace = marlin_make_workspace_new(device)
|
||||
# Allocate marlin workspace, reusing existing storage on reload.
|
||||
self.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(self, "workspace", None)
|
||||
)
|
||||
|
||||
# Default names since marlin requires empty parameters for these,
|
||||
# TODO: remove this requirement from marlin (allow optional tensors)
|
||||
@@ -174,7 +177,9 @@ class MarlinLinearKernel(MPLinearKernel):
|
||||
getattr(layer, self.w_gidx_name)
|
||||
)
|
||||
self._transform_param(layer, self.w_gidx_name, lambda _: g_idx)
|
||||
layer.g_idx_sort_indices = g_idx_sort_indices
|
||||
replace_parameter(
|
||||
layer, "g_idx_sort_indices", g_idx_sort_indices, prefer_copy=True
|
||||
)
|
||||
else:
|
||||
setattr(layer, self.w_gidx_name, marlin_make_empty_g_idx(device))
|
||||
layer.g_idx_sort_indices = marlin_make_empty_g_idx(device)
|
||||
|
||||
+3
-1
@@ -504,7 +504,9 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod):
|
||||
and self.wna16_backend != WNA16MoEBackend.EMULATION
|
||||
):
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
layer.w13_weight_g_idx.device, 4
|
||||
layer.w13_weight_g_idx.device,
|
||||
4,
|
||||
existing=getattr(layer, "workspace", None),
|
||||
)
|
||||
|
||||
# Alias packed weights to w13_weight/w2_weight for the modular kernel interface
|
||||
|
||||
@@ -397,14 +397,31 @@ def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tenso
|
||||
|
||||
|
||||
def marlin_make_workspace_new(
|
||||
device: torch.device, max_blocks_per_sm: int = 1
|
||||
device: torch.device,
|
||||
max_blocks_per_sm: int = 1,
|
||||
existing: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
# In the new marlin kernel, we use the num of threadblocks as workspace
|
||||
# size. The num of threadblocks is sms_count * max_blocks_per_sm.
|
||||
sms = num_compute_units(device.index)
|
||||
return torch.zeros(
|
||||
sms * max_blocks_per_sm, dtype=torch.int, device=device, requires_grad=False
|
||||
)
|
||||
size = sms * max_blocks_per_sm
|
||||
# On weight reload, reuse the existing storage so the workspace address
|
||||
# captured by CUDA graphs stays valid.
|
||||
if existing is not None:
|
||||
if (
|
||||
existing.device != device
|
||||
or existing.dtype != torch.int
|
||||
or existing.numel() != size
|
||||
):
|
||||
raise ValueError(
|
||||
f"Existing Marlin workspace is incompatible "
|
||||
f"(device={existing.device}, dtype={existing.dtype}, "
|
||||
f"numel={existing.numel()}; expected device={device}, "
|
||||
f"dtype={torch.int}, numel={size}). Reload must reuse the "
|
||||
f"workspace storage captured by CUDA graphs."
|
||||
)
|
||||
return existing.zero_()
|
||||
return torch.zeros(size, dtype=torch.int, device=device, requires_grad=False)
|
||||
|
||||
|
||||
def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool:
|
||||
|
||||
@@ -240,7 +240,9 @@ def prepare_fp4_layer_for_marlin(
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(device)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
@@ -396,7 +398,9 @@ def prepare_nvfp4_moe_layer_for_marlin(
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
# WEIGHT
|
||||
@@ -485,7 +489,9 @@ def prepare_moe_fp4_layer_for_marlin(
|
||||
# WORKSPACE
|
||||
device = layer.w13_weight.device
|
||||
param_dtype = layer.params_dtype
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1
|
||||
|
||||
|
||||
@@ -132,7 +132,9 @@ def prepare_fp8_layer_for_marlin(
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(device)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT
|
||||
# Repack weights to marlin format
|
||||
@@ -278,9 +280,9 @@ def prepare_fp8_moe_layer_for_marlin(
|
||||
|
||||
# WORKSPACE
|
||||
device = layer.w13_weight.device
|
||||
# NOTE(rob): we do not need to register the workspace as a param
|
||||
# because it is not used as part of the weight reloading process.
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
# WEIGHT
|
||||
@@ -463,7 +465,9 @@ def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None:
|
||||
device = layer.weight.device
|
||||
|
||||
# WORKSPACE
|
||||
layer.workspace = marlin_make_workspace_new(device)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
# WEIGHT - repack FP8 weights to Marlin format
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
@@ -549,7 +553,9 @@ def prepare_mxfp8_moe_layer_for_marlin(
|
||||
param_dtype = torch.get_default_dtype()
|
||||
perm = torch.empty(0, dtype=torch.int, device=device)
|
||||
|
||||
layer.workspace = marlin_make_workspace_new(device, 4)
|
||||
layer.workspace = marlin_make_workspace_new(
|
||||
device, 4, existing=getattr(layer, "workspace", None)
|
||||
)
|
||||
|
||||
def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor:
|
||||
if "w13" in name:
|
||||
|
||||
Reference in New Issue
Block a user