[Bugfix] Wait for the linear bias before layerwise online processing (#49805)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
26d725c334
commit
9321aff536
@@ -10,9 +10,11 @@ from torch.nn.parameter import UninitializedParameter
|
||||
|
||||
import vllm.model_executor.model_loader.reload.meta as reload_meta
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.model_loader.reload.layerwise import (
|
||||
finalize_layerwise_reload,
|
||||
initialize_layerwise_reload,
|
||||
initialize_online_processing,
|
||||
record_metadata_for_reloading,
|
||||
)
|
||||
from vllm.model_executor.model_loader.reload.meta import (
|
||||
@@ -278,6 +280,58 @@ def test_layerwise_reload_composed_loader_does_not_drop_params(monkeypatch):
|
||||
assert torch.equal(layer.D, loaded["D"])
|
||||
|
||||
|
||||
class _RecordingQuantMethod(QuantizeMethodBase):
|
||||
"""Records the layer's bias at the moment processing runs."""
|
||||
|
||||
uses_meta_device = True
|
||||
|
||||
def __init__(self):
|
||||
self.bias_at_process = None
|
||||
|
||||
def create_weights(self, layer, *weight_args, **extra_weight_attrs):
|
||||
pass
|
||||
|
||||
def apply(self, layer, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def process_weights_after_loading(self, layer):
|
||||
self.bias_at_process = layer.bias.detach().clone()
|
||||
|
||||
|
||||
class _LateBiasLayer(torch.nn.Module):
|
||||
"""Mimics an online-quantized linear: `weight` is created on meta by
|
||||
`create_weights()`, which wraps the loaders, and the linear base registers
|
||||
`bias` afterwards."""
|
||||
|
||||
def __init__(self, quant_method):
|
||||
super().__init__()
|
||||
self.quant_method = quant_method
|
||||
weight = torch.nn.Parameter(torch.empty(4, 2, device="meta"))
|
||||
weight.weight_loader = default_weight_loader
|
||||
self.register_parameter("weight", weight)
|
||||
initialize_online_processing(self)
|
||||
bias = torch.nn.Parameter(torch.zeros(4))
|
||||
bias.weight_loader = default_weight_loader
|
||||
self.register_parameter("bias", bias)
|
||||
|
||||
|
||||
def test_online_processing_waits_for_late_registered_bias():
|
||||
# Regression test: `bias` is skipped by the meta device paths, but it is
|
||||
# still loaded by a weight loader. Excluding it from the processing trigger
|
||||
# finalized the layer one load early, so the trailing bias was written into
|
||||
# an already-processed layer (e.g. over FP8 Marlin's permuted bias).
|
||||
quant_method = _RecordingQuantMethod()
|
||||
layer = _LateBiasLayer(quant_method)
|
||||
loaded_bias = torch.full((4,), 3.0)
|
||||
|
||||
layer.weight.weight_loader(layer.weight, torch.full((4, 2), 2.0))
|
||||
assert quant_method.bias_at_process is None
|
||||
|
||||
layer.bias.weight_loader(layer.bias, loaded_bias)
|
||||
assert quant_method.bias_at_process is not None
|
||||
assert torch.equal(quant_method.bias_at_process, loaded_bias)
|
||||
|
||||
|
||||
def test_layerwise_reload_skips_non_persistent_parameter_alias_buffers(monkeypatch):
|
||||
layer = _AliasedBufferLayer()
|
||||
model = torch.nn.Sequential(layer)
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBa
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from .meta import (
|
||||
SKIP_TENSORS,
|
||||
SKIP_LOAD_TENSORS,
|
||||
capture_layer_to_meta,
|
||||
get_numel_loaded,
|
||||
materialize_layer,
|
||||
@@ -140,7 +140,7 @@ def _wrap_parameters_weight_loader(layer: torch.nn.Module) -> None:
|
||||
"""Wrap each parameter's weight loader."""
|
||||
# Note that nested wrapping will occur for shared tensors
|
||||
for name, tensor in get_layer_tensors(layer).items():
|
||||
if name in SKIP_TENSORS:
|
||||
if name in SKIP_LOAD_TENSORS:
|
||||
continue
|
||||
if _get_weight_loader(tensor).__name__ != "online_process_loader":
|
||||
tensor.weight_loader = make_online_process_loader(layer, name)
|
||||
|
||||
@@ -20,20 +20,23 @@ __all__ = [
|
||||
"get_numel_loaded",
|
||||
]
|
||||
|
||||
# Modules whose tensors are never moved to, or materialized from, the meta device.
|
||||
SKIP_MODULES: set[str] = {"HadamardTransform"}
|
||||
|
||||
SKIP_TENSORS: set[str] = {
|
||||
# Tensors never loaded by a weight loader, so the layerwise trigger ignores them.
|
||||
SKIP_LOAD_TENSORS: set[str] = {
|
||||
"_expert_map",
|
||||
"expert_mask",
|
||||
"expert_global_to_physical",
|
||||
"expert_physical_to_global",
|
||||
"expert_local_to_global",
|
||||
"e_score_correction_bias",
|
||||
# Built after create_weights(), so it is not tracked by the layerwise-reload
|
||||
# trigger and would be re-materialized into uninitialized memory. Skip it.
|
||||
"bias",
|
||||
}
|
||||
|
||||
# Tensors which are never moved to, or materialized from, the meta device.
|
||||
# `bias` is built after create_weights(), so it is never on meta to begin with.
|
||||
SKIP_TENSORS: set[str] = SKIP_LOAD_TENSORS | {"bias"}
|
||||
|
||||
|
||||
def to_meta_tensor(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Convert a tensor to a meta tensor while preserving class and attributes."""
|
||||
|
||||
@@ -33,15 +33,15 @@ def get_layer_params_buffers(layer: torch.nn.Module) -> LayerTensors:
|
||||
def get_layer_size(layer: torch.nn.Module) -> int:
|
||||
"""Calculate total number of elements across loadable tensors in a layer.
|
||||
|
||||
Excludes SKIP_TENSORS (e.g. _expert_map) which are never moved to meta
|
||||
device and never loaded via weight_loader during layerwise reload.
|
||||
Excludes SKIP_LOAD_TENSORS (e.g. _expert_map) which are never loaded via
|
||||
weight_loader during layerwise reload.
|
||||
"""
|
||||
from .meta import SKIP_TENSORS
|
||||
from .meta import SKIP_LOAD_TENSORS
|
||||
|
||||
return sum(
|
||||
tensor.numel()
|
||||
for name, tensor in get_layer_tensors(layer).items()
|
||||
if name not in SKIP_TENSORS
|
||||
if name not in SKIP_LOAD_TENSORS
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user