[Bugfix] Fix level-2 sleep/wake/reload with enable_lora=True (#39935)
Signed-off-by: Silen Naihin <silen.naihin@gmail.com> Co-authored-by: Dakai An <dakaian108@gmail.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
co-authored by
Dakai An
Jiangyun Zhu
parent
8cfa01cdd2
commit
b50fdebce0
@@ -56,7 +56,7 @@ from vllm.forward_context import (
|
||||
set_forward_context,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.layers import LoRAMapping, LoRAMappingType
|
||||
from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping, LoRAMappingType
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager
|
||||
@@ -250,6 +250,16 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_parameter_for_reload(model: nn.Module, name: str) -> nn.Parameter:
|
||||
"""Resolve checkpoint names without changing the model's module tree."""
|
||||
module_name, _, parameter_name = name.rpartition(".")
|
||||
module = model.get_submodule(module_name)
|
||||
if isinstance(module, BaseLayerWithLoRA):
|
||||
module = module.base_layer
|
||||
return module.get_parameter(parameter_name)
|
||||
|
||||
|
||||
AttnMetadataDict: TypeAlias = dict[str, AttentionMetadata]
|
||||
# list when ubatching is enabled
|
||||
PerLayerAttnMetadata: TypeAlias = list[AttnMetadataDict] | AttnMetadataDict
|
||||
@@ -5535,7 +5545,10 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
model = self.get_model()
|
||||
weights_to_load = {name for name, _ in model.named_parameters()}
|
||||
weights_to_load = {
|
||||
name.replace(".base_layer.", ".") if self.lora_config else name
|
||||
for name, _ in model.named_parameters()
|
||||
}
|
||||
counter_before_reloading = time.perf_counter()
|
||||
|
||||
# load weights from disk if none are provided
|
||||
@@ -5572,10 +5585,12 @@ class GPUModelRunner(
|
||||
)
|
||||
loaded_weights = set()
|
||||
for name, loaded_weight in weights_iterator:
|
||||
param = model.get_parameter(name) # TODO: buffers?
|
||||
param = _get_parameter_for_reload(model, name) # TODO: buffers?
|
||||
param.copy_(loaded_weight)
|
||||
loaded_weights.add(name)
|
||||
|
||||
self.reset_lora_state()
|
||||
|
||||
# logging and validation
|
||||
counter_after_reloading = time.perf_counter()
|
||||
diff_seconds = counter_after_reloading - counter_before_reloading
|
||||
|
||||
Reference in New Issue
Block a user