[Bugfix][LoRA] Reject PEFT adapter features vLLM does not implement (#60225)

Signed-off-by: suraj <surajsharan85@gmail.com>
Co-authored-by: linitra24 <renshuang.zhou@daocloud.io>

tests/entrypoints/serve/lora/test_lora_adapters.py

@@ -23,6 +23,7 @@ BADREQUEST_CASES = [
         "is greater than max_lora_rank",
     ),
     ("test_dora", {"use_dora": True}, "does not yet support DoRA"),
+    ("test_pissa", {"init_lora_weights": "pissa"}, "init_lora_weights='pissa'"),
     (
         "test_modules_to_save",
         {"modules_to_save": ["lm_head"]},

tests/lora/test_peft_helper.py

@@ -24,6 +24,25 @@ ERROR_CASES = [
     ),
     ("test_rank_zero", {"r": 0}, "must be a positive integer"),
     ("test_rank_negative", {"r": -8}, "must be a positive integer"),
+    ("test_lora_bias", {"lora_bias": True}, "does not support LoRA bias"),
+    ("test_pissa", {"init_lora_weights": "pissa"}, "init_lora_weights='pissa'"),
+    (
+        "test_pissa_niter",
+        {"init_lora_weights": "pissa_niter_4"},
+        "init_lora_weights='pissa_niter_4'",
+    ),
+    ("test_olora", {"init_lora_weights": "olora"}, "init_lora_weights='olora'"),
+    ("test_corda", {"init_lora_weights": "corda"}, "init_lora_weights='corda'"),
+    ("test_loftq", {"init_lora_weights": "loftq"}, "init_lora_weights='loftq'"),
+    (
+        "test_unknown_init",
+        {"init_lora_weights": "future_init"},
+        "init_lora_weights='future_init'",
+    ),
+    ("test_alora", {"alora_invocation_tokens": [1, 2]}, "Activated LoRA"),
+    ("test_layer_replication", {"layer_replication": [[0, 2]]}, "layer_replication"),
+    ("test_bdlora", {"use_bdlora": {"nblocks": 2}}, "BD-LoRA"),
+    ("test_qalora", {"use_qalora": True}, "QALoRA"),
 ]
 
 
@@ -114,3 +133,23 @@ def test_peft_helper_invalid_rank_direct(bad_rank: int):
     """
     with pytest.raises(ValueError, match="must be a positive integer"):
         PEFTHelper(r=bad_rank, lora_alpha=16, target_modules=["q_proj"])
+
+
+@pytest.mark.parametrize(
+    "init_lora_weights",
+    [True, False, "gaussian", "eva", "orthogonal", "mica", "lora_ga"],
+)
+def test_peft_helper_init_lora_weights_supported(init_lora_weights):
+    """A saved adapter with these inits loads in PEFT as a plain LoRA."""
+    lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
+    PEFTHelper.from_dict(
+        {
+            "r": 8,
+            "lora_alpha": 16,
+            "target_modules": ["q_proj"],
+            "init_lora_weights": init_lora_weights,
+            "alora_invocation_tokens": None,
+            "layer_replication": None,
+            "use_bdlora": None,
+        }
+    ).validate_legal(lora_config)

tests/lora/test_utils.py

@@ -106,6 +106,8 @@ def test_parse_fine_tuned_lora_name_invalid():
     fixture = {
         "base_model.weight",
         "base_model.model.weight",
+        "base_model.model.model.layers.9.mlp.down_proj.lora_A.bias",
+        "base_model.model.model.layers.9.mlp.down_proj.lora_B.bias",
     }
     for name in fixture:
         with pytest.raises(ValueError, match="unsupported LoRA weight"):

vllm/lora/peft_helper.py

@@ -33,6 +33,14 @@ class PEFTHelper:
     use_rslora: bool = field(default=False)
     # True to use Weight-Decomposed Low-Rank Adaptation (DoRA, see: https://arxiv.org/abs/2402.09353)
     use_dora: bool = field(default=False)
+    # PEFT features that change the adapter's semantics and that vLLM does not
+    # implement. They are read only to reject such adapters.
+    lora_bias: bool = field(default=False)
+    init_lora_weights: bool | str = field(default=True)
+    alora_invocation_tokens: list[int] | None = field(default=None)
+    layer_replication: list[list[int]] | None = field(default=None)
+    use_bdlora: dict | None = field(default=None)
+    use_qalora: bool = field(default=False)
     # Extra vllm field, start with 'vllm_' to avoid conflict
     vllm_lora_scaling_factor: float = field(default=1.0)
     vllm_max_position_embeddings: int | None = field(default=False)
@@ -52,8 +60,40 @@ class PEFTHelper:
                 )
         if self.use_dora:
             error_msg.append("vLLM does not yet support DoRA.")
+        if self.lora_bias:
+            error_msg.append("vLLM does not support LoRA bias (lora_bias).")
+        if not self._init_is_supported():
+            error_msg.append(
+                f"vLLM does not support init_lora_weights={self.init_lora_weights!r}."
+                " Inits such as PiSSA, OLoRA, CorDA and LoftQ modify the base model"
+                " weights when PEFT loads the adapter. Convert it to a regular LoRA"
+                " adapter with PEFT's path_initial_model_for_weight_conversion, or"
+                " set init_lora_weights to true if the served base model already"
+                " contains the modified weights."
+            )
+        if self.alora_invocation_tokens:
+            error_msg.append("vLLM does not support Activated LoRA (aLoRA).")
+        if self.layer_replication:
+            error_msg.append("vLLM does not support layer_replication.")
+        if self.use_bdlora:
+            error_msg.append("vLLM does not support block-diagonal LoRA (BD-LoRA).")
+        if self.use_qalora:
+            error_msg.append("vLLM does not support QALoRA.")
         return error_msg
 
+    def _init_is_supported(self) -> bool:
+        # These inits only set the initial adapter weights, so a saved adapter
+        # loads in PEFT as a plain LoRA. lora_ga modifies the base weights only
+        # during training, when its gradients are attached.
+        init = self.init_lora_weights
+        return not isinstance(init, str) or init.lower() in (
+            "gaussian",
+            "eva",
+            "orthogonal",
+            "mica",
+            "lora_ga",
+        )
+
     def __post_init__(self):
         if self.r <= 0:
             raise ValueError(f"LoRA rank `r` must be a positive integer, got {self.r}.")

vllm/lora/utils.py

@@ -205,7 +205,7 @@ def parse_fine_tuned_lora_name(
 
     parts = name.split(".")
     if (parts[-1] == "weight" or parts[-1] == "bias") and len(parts) >= 2:
-        if parts[-2] in ["lora_A", "lora_B"]:
+        if parts[-2] in ["lora_A", "lora_B"] and parts[-1] == "weight":
             new_name = ".".join(parts[start_index:-2])
             return new_name, parts[-2] == "lora_A"
         # For modules_to_save in classification.