[Bugfix] Fix level-2 sleep/wake/reload with enable_lora=True (#39935)

Signed-off-by: Silen Naihin <silen.naihin@gmail.com>
Co-authored-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
Silen Naihin
2026-08-06 09:13:10 +08:00
committed by GitHub
co-authored by Dakai An Jiangyun Zhu
parent 8cfa01cdd2
commit b50fdebce0
11 changed files with 252 additions and 4 deletions
+13
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import TYPE_CHECKING, overload
import torch
@@ -14,6 +15,18 @@ if TYPE_CHECKING:
class BaseLayerWithLoRA(nn.Module):
def load_weights(
self, weights: Iterable[tuple[str, torch.Tensor]]
) -> Iterable[str]:
"""Load checkpoint weights into the wrapped base layer."""
base_load_weights = getattr(self.base_layer, "load_weights", None)
if callable(base_load_weights):
return base_load_weights(weights)
from vllm.model_executor.models.utils import AutoWeightsLoader
return AutoWeightsLoader(self.base_layer).load_weights(weights)
@overload
def slice_lora_a(
self, lora_a: list[torch.Tensor | None]
+12
View File
@@ -118,6 +118,18 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
else:
self.sharded_to_full_mapping_gpu = None
def reset_sharded_to_full_mapping(self) -> None:
"""Restore the TP logits mapping after its GPU memory is reused."""
mapping_gpu = self.sharded_to_full_mapping_gpu
if mapping_gpu is not None:
mapping_gpu.copy_(
torch.tensor(
self.sharded_to_full_mapping,
device=mapping_gpu.device,
dtype=mapping_gpu.dtype,
)
)
def reset_lora(self, index: int):
self.lora_a_stacked[index] = 0
self.lora_b_stacked[index] = 0
+2
View File
@@ -387,6 +387,8 @@ class LoRAModelManager:
self._registered_adapters.clear()
self.lora_index_to_id = [None] * self.lora_slots
self._active_adapters.clear()
self._last_mapping = None
self._last_slot_layout = None
def _create_lora_modules(self):
def _parent_module(module_name: str) -> str:
+18 -3
View File
@@ -56,7 +56,7 @@ from vllm.forward_context import (
set_forward_context,
)
from vllm.logger import init_logger
from vllm.lora.layers import LoRAMapping, LoRAMappingType
from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping, LoRAMappingType
from vllm.model_executor.layers.attention import Attention, MLAAttention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
@@ -250,6 +250,16 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
def _get_parameter_for_reload(model: nn.Module, name: str) -> nn.Parameter:
"""Resolve checkpoint names without changing the model's module tree."""
module_name, _, parameter_name = name.rpartition(".")
module = model.get_submodule(module_name)
if isinstance(module, BaseLayerWithLoRA):
module = module.base_layer
return module.get_parameter(parameter_name)
AttnMetadataDict: TypeAlias = dict[str, AttentionMetadata]
# list when ubatching is enabled
PerLayerAttnMetadata: TypeAlias = list[AttnMetadataDict] | AttnMetadataDict
@@ -5535,7 +5545,10 @@ class GPUModelRunner(
)
model = self.get_model()
weights_to_load = {name for name, _ in model.named_parameters()}
weights_to_load = {
name.replace(".base_layer.", ".") if self.lora_config else name
for name, _ in model.named_parameters()
}
counter_before_reloading = time.perf_counter()
# load weights from disk if none are provided
@@ -5572,10 +5585,12 @@ class GPUModelRunner(
)
loaded_weights = set()
for name, loaded_weight in weights_iterator:
param = model.get_parameter(name) # TODO: buffers?
param = _get_parameter_for_reload(model, name) # TODO: buffers?
param.copy_(loaded_weight)
loaded_weights.add(name)
self.reset_lora_state()
# logging and validation
counter_after_reloading = time.perf_counter()
diff_seconds = counter_after_reloading - counter_before_reloading
+6
View File
@@ -160,6 +160,7 @@ class Worker(WorkerBase):
# is available, since the engine needs a reference to the model.
self.weight_transfer_engine: WeightTransferEngine | None = None
self._weight_update_active = False
self._weight_update_is_draft = False
# Torch/CUDA profiler. Enabled and configured through profiler_config.
# Profiler wrapper is created lazily in profile() when start is called,
@@ -1272,6 +1273,7 @@ class Worker(WorkerBase):
self.weight_transfer_engine.reset_weight_update_target()
raise
self._weight_update_active = True
self._weight_update_is_draft = is_draft
def update_weights(self, update_info: dict) -> None:
"""
@@ -1316,6 +1318,10 @@ class Worker(WorkerBase):
self.weight_transfer_engine.reset_weight_update_target()
self._weight_update_active = False
# Weight transfer bypasses GPUModelRunner.reload_weights().
if not self._weight_update_is_draft:
self.model_runner.reset_lora_state()
def shutdown(self) -> None:
gc.unfreeze()
+16
View File
@@ -4,6 +4,7 @@
Define LoRA functionality mixin for model runners.
"""
from collections.abc import Callable
from contextlib import contextmanager
from typing import TypeAlias
@@ -28,6 +29,21 @@ logger = init_logger(__name__)
# Defined as a mixin for GPUModelRunner
class LoRAModelRunnerMixin:
lora_config: LoRAConfig | None
get_model: Callable[[], nn.Module]
def reset_lora_state(self) -> None:
"""Invalidate LoRA state after base weights are replaced."""
if not self.lora_config:
return
from vllm.lora.layers.logits_processor import LogitsProcessorWithLoRA
self.lora_manager.remove_all_adapters()
for module in self.get_model().modules():
if isinstance(module, LogitsProcessorWithLoRA):
module.reset_sharded_to_full_mapping()
def load_lora_model(
self,
model: nn.Module,