[BugFix][LoRA] Refresh punica metadata when LoRA slots are reassigned under an unchanged mapping (#47725)
Signed-off-by: AmeenP <ameenp360@gmail.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Claude
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
21b396abe1
commit
392d1b4d2e
@@ -651,6 +651,55 @@ def test_lru_lora_model_manager(default_vllm_config, dist_init, dummy_model, dev
|
||||
assert manager.device == device
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_set_adapter_mapping_refreshes_after_slot_reassignment(
|
||||
default_vllm_config, dist_init, dummy_model, device
|
||||
):
|
||||
# An out-of-band add_lora() can LRU-evict and reassign GPU slots while the
|
||||
# running batch -- and therefore its LoRAMapping -- is unchanged. The
|
||||
# punica metadata must still be re-derived, otherwise in-flight requests
|
||||
# are routed to the evicted layout and decode with the wrong adapter.
|
||||
model = dummy_model
|
||||
model_lora1 = create_lora(1, model, ["dense1", "dense2", "lm_head"], device=device)
|
||||
model_lora2 = create_lora(2, model, ["dense1", "dense2", "lm_head"], device=device)
|
||||
model_lora3 = create_lora(3, model, ["dense1", "dense2", "lm_head"], device=device)
|
||||
manager = LRUCacheLoRAModelManager(
|
||||
model,
|
||||
2,
|
||||
2,
|
||||
2,
|
||||
LoRAConfig(
|
||||
max_lora_rank=8, max_cpu_loras=3, max_loras=2, lora_dtype=DEFAULT_DTYPE
|
||||
),
|
||||
device=device,
|
||||
vllm_config=default_vllm_config,
|
||||
)
|
||||
punica_wrapper = manager.punica_wrapper_mapping[DEFAULT_LANGUAGE_WRAPPER_KEY]
|
||||
|
||||
assert manager.add_adapter(model_lora1)
|
||||
assert manager.activate_adapter(1)
|
||||
assert manager.add_adapter(model_lora2)
|
||||
assert manager.activate_adapter(2)
|
||||
assert manager.lora_index_to_id == [1, 2]
|
||||
|
||||
# Two in-flight requests, one token each, on adapters 1 and 2.
|
||||
manager.set_adapter_mapping(LoRAMapping((1, 2), (1, 2)))
|
||||
assert punica_wrapper.token_lora_indices.tolist() == [0, 1]
|
||||
|
||||
# Out-of-band add_lora() with both slots held by the running batch:
|
||||
# activating 3 evicts 1; re-activating the batch's adapters lands them
|
||||
# in swapped slots while the batch itself is unchanged.
|
||||
assert manager.add_adapter(model_lora3)
|
||||
assert manager.activate_adapter(3)
|
||||
assert manager.activate_adapter(1)
|
||||
assert manager.activate_adapter(2)
|
||||
assert manager.lora_index_to_id == [2, 1]
|
||||
|
||||
# Identical mapping, but the metadata must follow the new slot layout.
|
||||
manager.set_adapter_mapping(LoRAMapping((1, 2), (1, 2)))
|
||||
assert punica_wrapper.token_lora_indices.tolist() == [1, 0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", DEVICES)
|
||||
def test_lru_cache_worker_adapter_manager(dist_init, dummy_model, device, tmp_path):
|
||||
lora_config = LoRAConfig(
|
||||
|
||||
@@ -117,6 +117,7 @@ class LoRAModelManager:
|
||||
self.packed_modules: dict[str, list[str]] = {}
|
||||
self.modules: dict[str, BaseLayerWithLoRA] = {}
|
||||
self._last_mapping: LoRAMapping | None = None
|
||||
self._last_slot_layout: tuple[int | None, ...] | None = None
|
||||
is_moe = is_moe_model(self.model)
|
||||
self._is_moe = is_moe
|
||||
|
||||
@@ -1147,9 +1148,14 @@ class LoRAModelManager:
|
||||
return True
|
||||
|
||||
def set_adapter_mapping(self, mapping: LoRAMapping) -> None:
|
||||
if self._last_mapping != mapping:
|
||||
# The punica metadata derives from the slot layout as well as the
|
||||
# mapping: an out-of-band add_lora() can LRU-evict and reassign slots
|
||||
# while the running batch, and thus the mapping, is unchanged.
|
||||
slot_layout = tuple(self.lora_index_to_id)
|
||||
if self._last_mapping != mapping or self._last_slot_layout != slot_layout:
|
||||
self._set_adapter_mapping(mapping)
|
||||
self._last_mapping = mapping
|
||||
self._last_slot_layout = slot_layout
|
||||
|
||||
def remove_adapter(self, adapter_id: int) -> bool:
|
||||
self.deactivate_adapter(adapter_id)
|
||||
|
||||
Reference in New Issue
Block a user