[Bugfix] Fix level-2 sleep/wake/reload with enable_lora=True (#39935)

Signed-off-by: Silen Naihin <silen.naihin@gmail.com>
Co-authored-by: Dakai An <dakaian108@gmail.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
This commit is contained in:
Silen Naihin
2026-08-06 09:13:10 +08:00
committed by GitHub
co-authored by Dakai An Jiangyun Zhu
parent 8cfa01cdd2
commit b50fdebce0
11 changed files with 252 additions and 4 deletions
+93
View File
@@ -209,6 +209,99 @@ def test_deep_sleep():
assert output[0].outputs[0].text == output2[0].outputs[0].text
@create_new_process_for_each_test()
def test_deep_sleep_lora():
"""Level-2 sleep/wake/reload with enable_lora=True.
LoRA wrapping moves parameters under base_layer and adds LoRA
stacked tensors that are plain attributes, not restored by the
reload machinery — reload must forward checkpoint weights through
the wrappers and reset the LoRA state afterwards.
"""
model = "hmellor/tiny-random-LlamaForCausalLM"
llm = LLM(
model,
enable_sleep_mode=True,
enable_lora=True,
max_lora_rank=8,
enforce_eager=True,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
output = llm.generate(prompt, sampling_params)
# Level-2 sleep discards all GPU memory
llm.sleep(level=2)
# Reload weights from checkpoint
llm.wake_up(tags=["weights"])
llm.collective_rpc("reload_weights")
llm.wake_up(tags=["kv_cache"])
output2 = llm.generate(prompt, sampling_params)
assert output[0].outputs[0].text == output2[0].outputs[0].text
# Multiple cycles should not accumulate corruption
for _ in range(3):
llm.sleep(level=2)
llm.wake_up(tags=["weights"])
llm.collective_rpc("reload_weights")
llm.wake_up(tags=["kv_cache"])
output3 = llm.generate(prompt, sampling_params)
assert output[0].outputs[0].text == output3[0].outputs[0].text
def _lora_logits_mapping_present(model) -> bool:
from vllm.lora.layers.logits_processor import LogitsProcessorWithLoRA
return any(
isinstance(m, LogitsProcessorWithLoRA)
and m.sharded_to_full_mapping_gpu is not None
for m in model.modules()
)
@create_new_process_for_each_test()
def test_deep_sleep_lora_tp2(num_gpus_available, monkeypatch):
"""Level-2 sleep/wake/reload with enable_lora=True and TP=2.
With TP > 1 the LoRA logits processor carries
``sharded_to_full_mapping_gpu``, a permanent index mapping used to
reorder gathered logits. Like the LoRA stacked tensors it is a plain
attribute allocated in the sleep-mode pool, so level-2 sleep destroys
its contents — it must be restored after reload.
"""
if num_gpus_available < 2:
pytest.skip("Requires at least 2 GPUs")
# Needed for apply_model to reach the multiproc TP workers below.
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
model = "hmellor/tiny-random-LlamaForCausalLM"
llm = LLM(
model,
enable_sleep_mode=True,
enable_lora=True,
max_lora_rank=8,
tensor_parallel_size=2,
enforce_eager=True,
)
# Guard against this test silently not exercising the TP>1 reindex
# path (e.g. if lm_head wrapping conditions change).
assert all(llm.apply_model(_lora_logits_mapping_present))
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
output = llm.generate(prompt, sampling_params)
llm.sleep(level=2)
llm.wake_up(tags=["weights"])
llm.collective_rpc("reload_weights")
llm.wake_up(tags=["kv_cache"])
output2 = llm.generate(prompt, sampling_params)
assert output[0].outputs[0].text == output2[0].outputs[0].text
@create_new_process_for_each_test()
def test_deep_sleep_async():
async def test():
+47
View File
@@ -83,6 +83,21 @@ NUM_RANDOM_SEEDS = 2
VOCAB_PARALLEL_EMBEDDING_TEST_NUM_RANDOM_SEEDS = 2
def test_base_layer_with_lora_delegates_load_weights():
class BaseLayer(torch.nn.Module):
def load_weights(self, weights):
self.loaded_weights = list(weights)
return {"weight"}
base_layer = BaseLayer()
layer = BaseLayerWithLoRA()
layer.base_layer = base_layer
weights = [("weight", torch.ones(1))]
assert layer.load_weights(weights) == {"weight"}
assert base_layer.loaded_weights == weights
@pytest.fixture(autouse=True)
def clean_cache_reset_device(reset_default_device):
# Release any memory we might be holding on to. CI runs OOMs otherwise.
@@ -509,6 +524,38 @@ def test_lm_head_logits_processor_invalid_vocab_size(
lora_logits_processor.create_lora_weights(max_loras, lora_config)
@torch.inference_mode()
@pytest.mark.parametrize("device", DEVICES)
def test_lm_head_reset_sharded_to_full_mapping(
default_vllm_config, dist_init, device
) -> None:
if current_platform.is_cuda_alike() or current_platform.is_xpu():
torch.accelerator.set_device_index(device)
torch.set_default_device(device)
max_loras = 8
vocab_size = 1024
lora_config = LoRAConfig(
max_loras=max_loras, max_lora_rank=8, lora_dtype=torch.float16
)
logits_processor = LogitsProcessor(vocab_size)
sharded_to_full_mapping = list(reversed(range(vocab_size)))
lora_logits_processor = LogitsProcessorWithLoRA(
logits_processor, 1024, torch.float16, device, sharded_to_full_mapping
)
lora_logits_processor.create_lora_weights(max_loras, lora_config)
lora_logits_processor.sharded_to_full_mapping_gpu.fill_(-1)
lora_logits_processor.reset_sharded_to_full_mapping()
torch.testing.assert_close(
lora_logits_processor.sharded_to_full_mapping_gpu,
torch.tensor(sharded_to_full_mapping, dtype=torch.long),
)
@torch.inference_mode()
@pytest.mark.parametrize("num_loras", [1, 2, 4])
@pytest.mark.parametrize("device", DEVICES)
+4
View File
@@ -699,6 +699,10 @@ def test_set_adapter_mapping_refreshes_after_slot_reassignment(
manager.set_adapter_mapping(LoRAMapping((1, 2), (1, 2)))
assert punica_wrapper.token_lora_indices.tolist() == [1, 0]
manager.remove_all_adapters()
assert manager._last_mapping is None
assert manager._last_slot_layout is None
@pytest.mark.parametrize("device", DEVICES)
def test_lru_cache_worker_adapter_manager(dist_init, dummy_model, device, tmp_path):
@@ -8,8 +8,12 @@ session is active. These tests verify that delegation and the session guard.
"""
import pytest
import torch
import torch.nn as nn
from vllm.config import VllmConfig, get_current_vllm_config
from vllm.lora.layers import BaseLayerWithLoRA
from vllm.v1.worker.gpu_model_runner import _get_parameter_for_reload
from vllm.v1.worker.gpu_worker import Worker
@@ -21,6 +25,7 @@ class _RecordingEngine:
self.started = False
self.finished = False
self.reset_count = 0
self.supports_draft_weight_update = False
self.update_calls: list[dict] = []
self.seen_configs: list[VllmConfig] = []
@@ -48,16 +53,22 @@ class _RecordingEngine:
class _RecordingModelRunner:
def __init__(self) -> None:
self.seen_config: VllmConfig | None = None
self.reset_lora_calls = 0
def reload_weights(self) -> None:
self.seen_config = get_current_vllm_config()
def reset_lora_state(self) -> None:
self.reset_lora_calls += 1
def _make_worker(engine: _RecordingEngine | None) -> Worker:
worker = object.__new__(Worker)
worker.vllm_config = VllmConfig()
worker.weight_transfer_engine = engine
worker._weight_update_active = False
worker._weight_update_is_draft = False
worker.model_runner = _RecordingModelRunner()
return worker
@@ -71,6 +82,22 @@ def test_reload_weights_sets_current_config():
assert model_runner.seen_config is worker.vllm_config
def test_reload_parameter_lookup_preserves_lora_module_names():
base_layer = nn.Module()
qweight = nn.Parameter(torch.ones(1))
base_layer.register_parameter("qweight", qweight)
wrapper = BaseLayerWithLoRA()
wrapper.base_layer = base_layer
model = nn.Module()
model.proj = wrapper
named_parameters = dict(model.named_parameters())
assert set(named_parameters) == {"proj.base_layer.qweight"}
assert named_parameters["proj.base_layer.qweight"] is qweight
assert model.get_parameter("proj.base_layer.qweight") is qweight
assert _get_parameter_for_reload(model, "proj.qweight") is qweight
def test_start_update_finish_delegates_to_engine():
engine = _RecordingEngine()
worker = _make_worker(engine)
@@ -88,6 +115,19 @@ def test_start_update_finish_delegates_to_engine():
assert engine.reset_count == 1
assert worker._weight_update_active is False
assert engine.seen_configs == [worker.vllm_config] * 3
assert worker.model_runner.reset_lora_calls == 1
def test_finish_draft_session_keeps_lora_state():
engine = _RecordingEngine()
engine.supports_draft_weight_update = True
worker = _make_worker(engine)
worker._set_draft_weight_update_target = lambda: None
Worker.start_draft_weight_update(worker)
Worker.finish_weight_update(worker)
assert worker.model_runner.reset_lora_calls == 0
def test_double_start_raises():