diff --git a/tests/model_executor/model_loader/test_reload.py b/tests/model_executor/model_loader/test_reload.py index e390f8d25a..c21d9b1e02 100644 --- a/tests/model_executor/model_loader/test_reload.py +++ b/tests/model_executor/model_loader/test_reload.py @@ -167,6 +167,283 @@ def test_materialize_layer_preserves_non_meta_tensors(): assert torch.equal(layer.bias.data, bias_values) +_MARLIN_SIZE_K, _MARLIN_SIZE_N, _MARLIN_GROUP_SIZE = 128, 64, 64 + + +def _stub_marlin_ops(monkeypatch): + from vllm import _custom_ops as ops + from vllm.model_executor.layers.quantization.utils import marlin_utils + + monkeypatch.setattr(marlin_utils, "num_compute_units", lambda _: 4) + monkeypatch.setattr( + ops, + "gptq_marlin_repack", + lambda w, perm, size_k, size_n, num_bits, is_a_8bit=False: torch.zeros( + size_k // 16, size_n * 2, dtype=torch.int32 + ), + ) + + +def _make_act_order_marlin_kernel(): + from vllm.model_executor.kernels.linear.mixed_precision.marlin import ( + MarlinLinearKernel, + ) + from vllm.model_executor.kernels.linear.mixed_precision.MPLinearKernel import ( + MPLinearLayerConfig, + ) + from vllm.scalar_type import scalar_types + + kernel = object.__new__(MarlinLinearKernel) + kernel.config = MPLinearLayerConfig( + full_weight_shape=(_MARLIN_SIZE_K, _MARLIN_SIZE_N), + partition_weight_shape=(_MARLIN_SIZE_K, _MARLIN_SIZE_N), + weight_type=scalar_types.uint4b8, + act_type=torch.float16, + group_size=_MARLIN_GROUP_SIZE, + zero_points=False, + has_g_idx=True, + ) + kernel.w_q_name = "qweight" + kernel.w_s_name = "scales" + kernel.w_zp_name = None + kernel.w_gidx_name = "g_idx" + return kernel + + +def _load_marlin_checkpoint_format_weights(layer, g_idx): + from vllm.model_executor.parameter import ( + GroupQuantScaleParameter, + PackedvLLMParameter, + RowvLLMParameter, + ) + + layer.qweight = PackedvLLMParameter( + data=torch.zeros(_MARLIN_SIZE_K // 8, _MARLIN_SIZE_N, dtype=torch.int32), + input_dim=0, + output_dim=1, + packed_dim=0, + packed_factor=8, + weight_loader=default_weight_loader, + ) + layer.scales = GroupQuantScaleParameter( + data=torch.ones( + _MARLIN_SIZE_K // _MARLIN_GROUP_SIZE, _MARLIN_SIZE_N, dtype=torch.float16 + ), + input_dim=0, + output_dim=1, + weight_loader=default_weight_loader, + ) + layer.g_idx = RowvLLMParameter( + data=g_idx.clone(), + input_dim=0, + weight_loader=default_weight_loader, + ) + + +def _random_g_idx(generator): + return torch.randint( + 0, + _MARLIN_SIZE_K // _MARLIN_GROUP_SIZE, + (_MARLIN_SIZE_K,), + dtype=torch.int32, + generator=generator, + ) + + +def test_marlin_post_load_preserves_runtime_tensor_addresses(monkeypatch, dist_init): + """Marlin workspace and act-order sort indices must be recomputed into + the same storage when weights are reloaded (RL weight sync), so device + addresses captured by CUDA graphs remain valid.""" + from vllm.model_executor.layers.quantization.utils import marlin_utils + + _stub_marlin_ops(monkeypatch) + kernel = _make_act_order_marlin_kernel() + + generator = torch.Generator().manual_seed(0) + first_g_idx = _random_g_idx(generator) + second_g_idx = _random_g_idx(generator) + + layer = torch.nn.Module() + _load_marlin_checkpoint_format_weights(layer, first_g_idx) + kernel.process_weights_after_loading(layer) + + workspace_ptr = kernel.workspace.data_ptr() + sort_indices_ptr = layer.g_idx_sort_indices.data_ptr() + + # Reload: fresh checkpoint-format tensors with a different act-order + _load_marlin_checkpoint_format_weights(layer, second_g_idx) + kernel.process_weights_after_loading(layer) + + assert kernel.workspace.data_ptr() == workspace_ptr + assert torch.all(kernel.workspace == 0) + assert layer.g_idx_sort_indices.data_ptr() == sort_indices_ptr + expected_sort_indices = marlin_utils.marlin_sort_g_idx(second_g_idx)[1] + assert torch.equal(layer.g_idx_sort_indices.data, expected_sort_indices) + # registered as a Parameter so layerwise reload copy-back preserves it + assert isinstance(layer.g_idx_sort_indices, torch.nn.Parameter) + + +@pytest.mark.parametrize("variant", ["fp8", "mxfp8", "nvfp4"]) +def test_marlin_prepare_layer_preserves_workspace_address(monkeypatch, variant): + """The Marlin fallback prepare_* functions rerun on weight reload and must + reuse the workspace storage whose address captured CUDA graphs hold.""" + from vllm import _custom_ops as ops + from vllm.model_executor.layers.quantization.utils import ( + marlin_utils, + marlin_utils_fp4, + marlin_utils_fp8, + ) + + size_k, size_n = 128, 64 + + monkeypatch.setattr(marlin_utils, "num_compute_units", lambda _: 4) + monkeypatch.setattr( + ops, + "gptq_marlin_repack", + lambda b_q_weight, perm, size_k, size_n, num_bits, is_a_8bit=False: torch.zeros( + size_k // 16, size_n * 2, dtype=torch.int32 + ), + ) + + layer = torch.nn.Module() + layer.output_size_per_partition = size_n + layer.input_size_per_partition = size_k + layer.orig_dtype = torch.float16 + layer.params_dtype = torch.float16 + + if variant == "fp8": + prepare = marlin_utils_fp8.prepare_fp8_layer_for_marlin + + def load_checkpoint_format_weights(): + layer.weight = torch.nn.Parameter( + torch.zeros(size_k, size_n, dtype=torch.float8_e4m3fn), + requires_grad=False, + ) + layer.weight_scale = torch.nn.Parameter( + torch.ones(1, dtype=torch.float32), requires_grad=False + ) + elif variant == "mxfp8": + prepare = marlin_utils_fp8.prepare_mxfp8_layer_for_marlin + + def load_checkpoint_format_weights(): + layer.weight = torch.nn.Parameter( + torch.zeros(size_n, size_k, dtype=torch.float8_e4m3fn), + requires_grad=False, + ) + layer.weight_scale = torch.nn.Parameter( + torch.full((size_n, size_k // 32), 127, dtype=torch.uint8), + requires_grad=False, + ) + else: + prepare = marlin_utils_fp4.prepare_fp4_layer_for_marlin + + def load_checkpoint_format_weights(): + layer.weight = torch.nn.Parameter( + torch.zeros(size_n, size_k // 2, dtype=torch.uint8), + requires_grad=False, + ) + layer.weight_scale = torch.nn.Parameter( + torch.ones(size_n, size_k // 16, dtype=torch.float8_e4m3fn), + requires_grad=False, + ) + layer.weight_global_scale = torch.nn.Parameter( + torch.ones(1, dtype=torch.float32), requires_grad=False + ) + + load_checkpoint_format_weights() + prepare(layer) + workspace_ptr = layer.workspace.data_ptr() + + # Reload: fresh checkpoint-format tensors, prepare runs again + load_checkpoint_format_weights() + prepare(layer) + + assert layer.workspace.data_ptr() == workspace_ptr + assert torch.all(layer.workspace == 0) + + +def test_marlin_make_workspace_new_rejects_incompatible_existing(monkeypatch): + """An incompatible existing workspace means the address captured by CUDA + graphs is already unusable; allocating a replacement would hide that.""" + from vllm.model_executor.layers.quantization.utils import marlin_utils + + monkeypatch.setattr(marlin_utils, "num_compute_units", lambda _: 4) + device = torch.device("cpu") + + workspace = marlin_utils.marlin_make_workspace_new(device) + reused = marlin_utils.marlin_make_workspace_new(device, existing=workspace) + assert reused is workspace + + with pytest.raises(ValueError, match="incompatible"): + marlin_utils.marlin_make_workspace_new(device, 4, existing=workspace) + with pytest.raises(ValueError, match="incompatible"): + marlin_utils.marlin_make_workspace_new( + device, existing=workspace.to(torch.int64) + ) + + +def test_marlin_act_order_layerwise_reload_accounting(monkeypatch, dist_init): + """`g_idx_sort_indices` is generated during weight processing and never + loaded from checkpoints. Registering it as a Parameter must not count it + toward `load_numel_total`: reload restores the construction-time tensor + set before sizing, so act-order layers still process during streaming + instead of deferring (and buffering weights) until finalization.""" + from vllm.model_executor.layers.quantization.base_config import ( + QuantizeMethodBase, + ) + from vllm.model_executor.layers.quantization.utils import marlin_utils + from vllm.model_executor.model_loader.reload.layerwise import get_layerwise_info + + _stub_marlin_ops(monkeypatch) + kernel = _make_act_order_marlin_kernel() + + class _KernelQuantMethod(QuantizeMethodBase): + def create_weights(self, layer, *args, **kwargs): + raise NotImplementedError + + def apply(self, layer, *args, **kwargs): + raise NotImplementedError + + def process_weights_after_loading(self, layer): + kernel.process_weights_after_loading(layer) + + generator = torch.Generator().manual_seed(0) + layer = torch.nn.Module() + layer.quant_method = _KernelQuantMethod() + _load_marlin_checkpoint_format_weights(layer, _random_g_idx(generator)) + + # Metadata is recorded at model construction, before any processing + record_metadata_for_reloading(layer) + checkpoint_numel = sum(t.numel() for t in get_layer_tensors(layer).values()) + + kernel.process_weights_after_loading(layer) + sort_indices = layer.g_idx_sort_indices + + initialize_layerwise_reload(layer) + info = get_layerwise_info(layer) + assert info.load_numel_total == checkpoint_numel + + # Stream a new checkpoint; the layer must process as soon as its last + # tensor arrives + new_g_idx = _random_g_idx(generator) + checkpoint = { + "qweight": torch.zeros(_MARLIN_SIZE_K // 8, _MARLIN_SIZE_N, dtype=torch.int32), + "scales": torch.ones( + _MARLIN_SIZE_K // _MARLIN_GROUP_SIZE, _MARLIN_SIZE_N, dtype=torch.float16 + ), + "g_idx": new_g_idx, + } + for name, weight in checkpoint.items(): + param = getattr(layer, name) + param.weight_loader(param, weight) + + assert not info.can_load() + assert not info.loaded_weights + assert layer.g_idx_sort_indices is sort_indices + expected_sort_indices = marlin_utils.marlin_sort_g_idx(new_g_idx)[1] + assert torch.equal(layer.g_idx_sort_indices.data, expected_sort_indices) + + def test_model_cleanup(dist_init, default_vllm_config): layer = QKVParallelLinear(2, 3, 4) assert layer.weight.weight_loader.__self__ is layer diff --git a/vllm/model_executor/kernels/linear/mixed_precision/marlin.py b/vllm/model_executor/kernels/linear/mixed_precision/marlin.py index 87ed8d1b58..5a4f652971 100644 --- a/vllm/model_executor/kernels/linear/mixed_precision/marlin.py +++ b/vllm/model_executor/kernels/linear/mixed_precision/marlin.py @@ -25,6 +25,7 @@ from vllm.model_executor.layers.quantization.utils.marlin_utils import ( unpack_cols, ) from vllm.model_executor.parameter import BasevLLMParameter, permute_param_layout_ +from vllm.model_executor.utils import replace_parameter from vllm.platforms import current_platform from vllm.scalar_type import scalar_types @@ -111,8 +112,10 @@ class MarlinLinearKernel(MPLinearKernel): else: padded_n, padded_k = marlin_padded_nk(size_n, size_k, c.group_size) - # Allocate marlin workspace. - self.workspace = marlin_make_workspace_new(device) + # Allocate marlin workspace, reusing existing storage on reload. + self.workspace = marlin_make_workspace_new( + device, existing=getattr(self, "workspace", None) + ) # Default names since marlin requires empty parameters for these, # TODO: remove this requirement from marlin (allow optional tensors) @@ -174,7 +177,9 @@ class MarlinLinearKernel(MPLinearKernel): getattr(layer, self.w_gidx_name) ) self._transform_param(layer, self.w_gidx_name, lambda _: g_idx) - layer.g_idx_sort_indices = g_idx_sort_indices + replace_parameter( + layer, "g_idx_sort_indices", g_idx_sort_indices, prefer_copy=True + ) else: setattr(layer, self.w_gidx_name, marlin_make_empty_g_idx(device)) layer.g_idx_sort_indices = marlin_make_empty_g_idx(device) diff --git a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py index 516d94fae6..f29b229249 100644 --- a/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py +++ b/vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors_moe/compressed_tensors_moe_wna16_marlin.py @@ -504,7 +504,9 @@ class CompressedTensorsWNA16MarlinMoEMethod(CompressedTensorsMoEMethod): and self.wna16_backend != WNA16MoEBackend.EMULATION ): layer.workspace = marlin_make_workspace_new( - layer.w13_weight_g_idx.device, 4 + layer.w13_weight_g_idx.device, + 4, + existing=getattr(layer, "workspace", None), ) # Alias packed weights to w13_weight/w2_weight for the modular kernel interface diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils.py b/vllm/model_executor/layers/quantization/utils/marlin_utils.py index f6d96f574d..742e791faf 100644 --- a/vllm/model_executor/layers/quantization/utils/marlin_utils.py +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils.py @@ -397,14 +397,31 @@ def marlin_moe_intermediate_size(w1_packed: torch.Tensor, w2_packed: torch.Tenso def marlin_make_workspace_new( - device: torch.device, max_blocks_per_sm: int = 1 + device: torch.device, + max_blocks_per_sm: int = 1, + existing: torch.Tensor | None = None, ) -> torch.Tensor: # In the new marlin kernel, we use the num of threadblocks as workspace # size. The num of threadblocks is sms_count * max_blocks_per_sm. sms = num_compute_units(device.index) - return torch.zeros( - sms * max_blocks_per_sm, dtype=torch.int, device=device, requires_grad=False - ) + size = sms * max_blocks_per_sm + # On weight reload, reuse the existing storage so the workspace address + # captured by CUDA graphs stays valid. + if existing is not None: + if ( + existing.device != device + or existing.dtype != torch.int + or existing.numel() != size + ): + raise ValueError( + f"Existing Marlin workspace is incompatible " + f"(device={existing.device}, dtype={existing.dtype}, " + f"numel={existing.numel()}; expected device={device}, " + f"dtype={torch.int}, numel={size}). Reload must reuse the " + f"workspace storage captured by CUDA graphs." + ) + return existing.zero_() + return torch.zeros(size, dtype=torch.int, device=device, requires_grad=False) def marlin_is_k_full(act_order: bool, is_row_parallel: bool) -> bool: diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils_fp4.py b/vllm/model_executor/layers/quantization/utils/marlin_utils_fp4.py index cc8538e5ca..01d28e5e91 100644 --- a/vllm/model_executor/layers/quantization/utils/marlin_utils_fp4.py +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils_fp4.py @@ -240,7 +240,9 @@ def prepare_fp4_layer_for_marlin( device = layer.weight.device # WORKSPACE - layer.workspace = marlin_make_workspace_new(device) + layer.workspace = marlin_make_workspace_new( + device, existing=getattr(layer, "workspace", None) + ) # WEIGHT # Repack weights to marlin format @@ -396,7 +398,9 @@ def prepare_nvfp4_moe_layer_for_marlin( is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1 # WORKSPACE - layer.workspace = marlin_make_workspace_new(device, 4) + layer.workspace = marlin_make_workspace_new( + device, 4, existing=getattr(layer, "workspace", None) + ) perm = torch.empty(0, dtype=torch.int, device=device) # WEIGHT @@ -485,7 +489,9 @@ def prepare_moe_fp4_layer_for_marlin( # WORKSPACE device = layer.w13_weight.device param_dtype = layer.params_dtype - layer.workspace = marlin_make_workspace_new(device, 4) + layer.workspace = marlin_make_workspace_new( + device, 4, existing=getattr(layer, "workspace", None) + ) perm = torch.empty(0, dtype=torch.int, device=device) is_a_8bit = input_dtype is not None and input_dtype.itemsize == 1 diff --git a/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py b/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py index 739f76659c..aaa1d82090 100644 --- a/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py +++ b/vllm/model_executor/layers/quantization/utils/marlin_utils_fp8.py @@ -132,7 +132,9 @@ def prepare_fp8_layer_for_marlin( device = layer.weight.device # WORKSPACE - layer.workspace = marlin_make_workspace_new(device) + layer.workspace = marlin_make_workspace_new( + device, existing=getattr(layer, "workspace", None) + ) # WEIGHT # Repack weights to marlin format @@ -278,9 +280,9 @@ def prepare_fp8_moe_layer_for_marlin( # WORKSPACE device = layer.w13_weight.device - # NOTE(rob): we do not need to register the workspace as a param - # because it is not used as part of the weight reloading process. - layer.workspace = marlin_make_workspace_new(device, 4) + layer.workspace = marlin_make_workspace_new( + device, 4, existing=getattr(layer, "workspace", None) + ) perm = torch.empty(0, dtype=torch.int, device=device) # WEIGHT @@ -463,7 +465,9 @@ def prepare_mxfp8_layer_for_marlin(layer: torch.nn.Module) -> None: device = layer.weight.device # WORKSPACE - layer.workspace = marlin_make_workspace_new(device) + layer.workspace = marlin_make_workspace_new( + device, existing=getattr(layer, "workspace", None) + ) # WEIGHT - repack FP8 weights to Marlin format perm = torch.empty(0, dtype=torch.int, device=device) @@ -549,7 +553,9 @@ def prepare_mxfp8_moe_layer_for_marlin( param_dtype = torch.get_default_dtype() perm = torch.empty(0, dtype=torch.int, device=device) - layer.workspace = marlin_make_workspace_new(device, 4) + layer.workspace = marlin_make_workspace_new( + device, 4, existing=getattr(layer, "workspace", None) + ) def repack_weight(weight: torch.Tensor, name: str) -> torch.Tensor: if "w13" in name: