[BugFix][LoRA] Refresh punica metadata when LoRA slots are reassigned under an unchanged mapping (#47725)

Signed-off-by: AmeenP <ameenp360@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ameen Patel
2026-07-07 08:53:55 -07:00
committed by GitHub
co-authored by Claude mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 21b396abe1
commit 392d1b4d2e
2 changed files with 56 additions and 1 deletions
+49
View File
@@ -651,6 +651,55 @@ def test_lru_lora_model_manager(default_vllm_config, dist_init, dummy_model, dev
assert manager.device == device
@pytest.mark.parametrize("device", DEVICES)
def test_set_adapter_mapping_refreshes_after_slot_reassignment(
default_vllm_config, dist_init, dummy_model, device
):
# An out-of-band add_lora() can LRU-evict and reassign GPU slots while the
# running batch -- and therefore its LoRAMapping -- is unchanged. The
# punica metadata must still be re-derived, otherwise in-flight requests
# are routed to the evicted layout and decode with the wrong adapter.
model = dummy_model
model_lora1 = create_lora(1, model, ["dense1", "dense2", "lm_head"], device=device)
model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"], device=device)
model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"], device=device)
manager = LRUCacheLoRAModelManager(
model,
2,
2,
2,
LoRAConfig(
max_lora_rank=8, max_cpu_loras=3, max_loras=2, lora_dtype=DEFAULT_DTYPE
),
device=device,
vllm_config=default_vllm_config,
)
punica_wrapper = manager.punica_wrapper_mapping[DEFAULT_LANGUAGE_WRAPPER_KEY]
assert manager.add_adapter(model_lora1)
assert manager.activate_adapter(1)
assert manager.add_adapter(model_lora2)
assert manager.activate_adapter(2)
assert manager.lora_index_to_id == [1, 2]
# Two in-flight requests, one token each, on adapters 1 and 2.
manager.set_adapter_mapping(LoRAMapping((1, 2), (1, 2)))
assert punica_wrapper.token_lora_indices.tolist() == [0, 1]
# Out-of-band add_lora() with both slots held by the running batch:
# activating 3 evicts 1; re-activating the batch's adapters lands them
# in swapped slots while the batch itself is unchanged.
assert manager.add_adapter(model_lora3)
assert manager.activate_adapter(3)
assert manager.activate_adapter(1)
assert manager.activate_adapter(2)
assert manager.lora_index_to_id == [2, 1]
# Identical mapping, but the metadata must follow the new slot layout.
manager.set_adapter_mapping(LoRAMapping((1, 2), (1, 2)))
assert punica_wrapper.token_lora_indices.tolist() == [1, 0]
@pytest.mark.parametrize("device", DEVICES)
def test_lru_cache_worker_adapter_manager(dist_init, dummy_model, device, tmp_path):
lora_config = LoRAConfig(
+7 -1
View File
@@ -117,6 +117,7 @@ class LoRAModelManager:
self.packed_modules: dict[str, list[str]] = {}
self.modules: dict[str, BaseLayerWithLoRA] = {}
self._last_mapping: LoRAMapping | None = None
self._last_slot_layout: tuple[int | None, ...] | None = None
is_moe = is_moe_model(self.model)
self._is_moe = is_moe
@@ -1147,9 +1148,14 @@ class LoRAModelManager:
return True
def set_adapter_mapping(self, mapping: LoRAMapping) -> None:
if self._last_mapping != mapping:
# The punica metadata derives from the slot layout as well as the
# mapping: an out-of-band add_lora() can LRU-evict and reassign slots
# while the running batch, and thus the mapping, is unchanged.
slot_layout = tuple(self.lora_index_to_id)
if self._last_mapping != mapping or self._last_slot_layout != slot_layout:
self._set_adapter_mapping(mapping)
self._last_mapping = mapping
self._last_slot_layout = slot_layout
def remove_adapter(self, adapter_id: int) -> bool:
self.deactivate_adapter(adapter_id)