diff --git a/tests/model_executor/model_loader/test_reload.py b/tests/model_executor/model_loader/test_reload.py index 1480191083..e390f8d25a 100644 --- a/tests/model_executor/model_loader/test_reload.py +++ b/tests/model_executor/model_loader/test_reload.py @@ -10,9 +10,11 @@ from torch.nn.parameter import UninitializedParameter import vllm.model_executor.model_loader.reload.meta as reload_meta from vllm.model_executor.layers.linear import QKVParallelLinear +from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase from vllm.model_executor.model_loader.reload.layerwise import ( finalize_layerwise_reload, initialize_layerwise_reload, + initialize_online_processing, record_metadata_for_reloading, ) from vllm.model_executor.model_loader.reload.meta import ( @@ -278,6 +280,58 @@ def test_layerwise_reload_composed_loader_does_not_drop_params(monkeypatch): assert torch.equal(layer.D, loaded["D"]) +class _RecordingQuantMethod(QuantizeMethodBase): + """Records the layer's bias at the moment processing runs.""" + + uses_meta_device = True + + def __init__(self): + self.bias_at_process = None + + def create_weights(self, layer, *weight_args, **extra_weight_attrs): + pass + + def apply(self, layer, *args, **kwargs): + raise NotImplementedError + + def process_weights_after_loading(self, layer): + self.bias_at_process = layer.bias.detach().clone() + + +class _LateBiasLayer(torch.nn.Module): + """Mimics an online-quantized linear: `weight` is created on meta by + `create_weights()`, which wraps the loaders, and the linear base registers + `bias` afterwards.""" + + def __init__(self, quant_method): + super().__init__() + self.quant_method = quant_method + weight = torch.nn.Parameter(torch.empty(4, 2, device="meta")) + weight.weight_loader = default_weight_loader + self.register_parameter("weight", weight) + initialize_online_processing(self) + bias = torch.nn.Parameter(torch.zeros(4)) + bias.weight_loader = default_weight_loader + self.register_parameter("bias", bias) + + +def test_online_processing_waits_for_late_registered_bias(): + # Regression test: `bias` is skipped by the meta device paths, but it is + # still loaded by a weight loader. Excluding it from the processing trigger + # finalized the layer one load early, so the trailing bias was written into + # an already-processed layer (e.g. over FP8 Marlin's permuted bias). + quant_method = _RecordingQuantMethod() + layer = _LateBiasLayer(quant_method) + loaded_bias = torch.full((4,), 3.0) + + layer.weight.weight_loader(layer.weight, torch.full((4, 2), 2.0)) + assert quant_method.bias_at_process is None + + layer.bias.weight_loader(layer.bias, loaded_bias) + assert quant_method.bias_at_process is not None + assert torch.equal(quant_method.bias_at_process, loaded_bias) + + def test_layerwise_reload_skips_non_persistent_parameter_alias_buffers(monkeypatch): layer = _AliasedBufferLayer() model = torch.nn.Sequential(layer) diff --git a/vllm/model_executor/model_loader/reload/layerwise.py b/vllm/model_executor/model_loader/reload/layerwise.py index 92f454f9a5..d6d4eb4b93 100644 --- a/vllm/model_executor/model_loader/reload/layerwise.py +++ b/vllm/model_executor/model_loader/reload/layerwise.py @@ -14,7 +14,7 @@ from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBa from vllm.model_executor.model_loader.weight_utils import default_weight_loader from .meta import ( - SKIP_TENSORS, + SKIP_LOAD_TENSORS, capture_layer_to_meta, get_numel_loaded, materialize_layer, @@ -140,7 +140,7 @@ def _wrap_parameters_weight_loader(layer: torch.nn.Module) -> None: """Wrap each parameter's weight loader.""" # Note that nested wrapping will occur for shared tensors for name, tensor in get_layer_tensors(layer).items(): - if name in SKIP_TENSORS: + if name in SKIP_LOAD_TENSORS: continue if _get_weight_loader(tensor).__name__ != "online_process_loader": tensor.weight_loader = make_online_process_loader(layer, name) diff --git a/vllm/model_executor/model_loader/reload/meta.py b/vllm/model_executor/model_loader/reload/meta.py index 1ccf2a34c6..213a9aaeaf 100644 --- a/vllm/model_executor/model_loader/reload/meta.py +++ b/vllm/model_executor/model_loader/reload/meta.py @@ -20,20 +20,23 @@ __all__ = [ "get_numel_loaded", ] +# Modules whose tensors are never moved to, or materialized from, the meta device. SKIP_MODULES: set[str] = {"HadamardTransform"} -SKIP_TENSORS: set[str] = { +# Tensors never loaded by a weight loader, so the layerwise trigger ignores them. +SKIP_LOAD_TENSORS: set[str] = { "_expert_map", "expert_mask", "expert_global_to_physical", "expert_physical_to_global", "expert_local_to_global", "e_score_correction_bias", - # Built after create_weights(), so it is not tracked by the layerwise-reload - # trigger and would be re-materialized into uninitialized memory. Skip it. - "bias", } +# Tensors which are never moved to, or materialized from, the meta device. +# `bias` is built after create_weights(), so it is never on meta to begin with. +SKIP_TENSORS: set[str] = SKIP_LOAD_TENSORS | {"bias"} + def to_meta_tensor(tensor: torch.Tensor) -> torch.Tensor: """Convert a tensor to a meta tensor while preserving class and attributes.""" diff --git a/vllm/model_executor/model_loader/reload/utils.py b/vllm/model_executor/model_loader/reload/utils.py index f0078d0f9d..b6d129a794 100644 --- a/vllm/model_executor/model_loader/reload/utils.py +++ b/vllm/model_executor/model_loader/reload/utils.py @@ -33,15 +33,15 @@ def get_layer_params_buffers(layer: torch.nn.Module) -> LayerTensors: def get_layer_size(layer: torch.nn.Module) -> int: """Calculate total number of elements across loadable tensors in a layer. - Excludes SKIP_TENSORS (e.g. _expert_map) which are never moved to meta - device and never loaded via weight_loader during layerwise reload. + Excludes SKIP_LOAD_TENSORS (e.g. _expert_map) which are never loaded via + weight_loader during layerwise reload. """ - from .meta import SKIP_TENSORS + from .meta import SKIP_LOAD_TENSORS return sum( tensor.numel() for name, tensor in get_layer_tensors(layer).items() - if name not in SKIP_TENSORS + if name not in SKIP_LOAD_TENSORS )