[Bugfix][LoRA] Validate incompatible MoE adapter formats (#59924)

Signed-off-by: Brian Weston <brianweston@gmail.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
Co-authored-by: linitra24 <renshuang.zhou@daocloud.io>

docs/features/lora.md

@@ -280,10 +280,10 @@ curl -X POST http://localhost:8000/v1/load_lora_adapter \
 
 !!! warning "You must know your adapter's layout"
     Under `--enable-mixed-moe-lora-format`, vLLM trusts whatever
-    `is_3d_lora_weight` the caller declares — it does **not** inspect the
-    checkpoint to verify. A wrong declaration will load weights into the
-    wrong stacked buffers and silently produce garbage outputs, with no
-    error at load time. Confirm the layout before serving:
+    `is_3d_lora_weight` the caller declares. Loading checks reject fused 3D
+    expert weights routed to a 2D wrapper without the required flags, but
+    do not validate every layout mismatch. A wrong declaration can still
+    cause loading failures or incorrect outputs. Confirm the layout before serving:
 
     - **2D (per-expert, megatron-style)** → set `is_3d_lora_weight: false`.
       Adapter keys look like `...experts.{idx}.gate_proj.lora_A.weight`,
@@ -299,6 +299,11 @@ is ignored: vLLM picks the wrapper from the base model's
 `is_3d_moe_weight` and the adapter is required to match. The field is
 also ignored for non-MoE models.
 
+For a model using a 2D MoE wrapper (such as Qwen3-MoE), loading fused 3D
+expert adapter weights requires both `enable_mixed_moe_lora_format=True`
+on the engine and `is_3d_lora_weight=True` on the request. Otherwise, adapter
+loading raises a `ValueError` naming these settings before activation.
+
 ## LoRA model lineage in model card
 
 The new format of `--lora-modules` is mainly to support the display of parent model information in the model card. Here's an explanation of how your current response supports this:

tests/lora/test_lora_manager.py

@@ -16,10 +16,12 @@ from vllm.lora.layers import (
     ReplicatedLinearWithLoRA,
     RowParallelLinearWithLoRA,
 )
+from vllm.lora.layers.fused_moe import FusedMoE3DWithLoRA, FusedMoEWithLoRA
 from vllm.lora.lora_model import LoRAModel
 from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
 from vllm.lora.model_manager import (
     DEFAULT_LANGUAGE_WRAPPER_KEY,
+    AdapterLRUCache,
     LoRAMapping,
     LoRAModelManager,
     LRUCacheLoRAModelManager,
@@ -47,6 +49,68 @@ DEVICES = (
 DEFAULT_DTYPE = torch.get_default_dtype()
 
 
+@pytest.mark.parametrize(
+    "mixed,declared_3d", [(False, False), (False, True), (True, False)]
+)
+def test_reject_fused_moe_lora_before_registration(mixed, declared_3d):
+    """Missing format flags should produce guidance without caching the adapter."""
+    module_name = "model.layers.0.mlp.experts"
+    manager = LoRAModelManager.__new__(LoRAModelManager)
+    manager.lora_config = LoRAConfig(max_cpu_loras=1)
+    manager._registered_adapters = AdapterLRUCache(1, manager.deactivate_adapter)
+    manager._enable_mixed_moe_lora_format = mixed
+    manager.is_pooling_model = False
+    manager.modules = {module_name: FusedMoEWithLoRA.__new__(FusedMoEWithLoRA)}
+    adapter = LoRAModel(
+        1,
+        8,
+        {
+            module_name: LoRALayerWeights(
+                module_name, 8, 16, torch.zeros(16, 4), torch.zeros(4, 16)
+            )
+        },
+    )
+    adapter.is_3d_lora_weight = declared_3d
+
+    with pytest.raises(ValueError, match="enable_mixed_moe_lora_format=True") as exc:
+        manager.add_adapter(adapter)
+    assert "is_3d_lora_weight=True" in str(exc.value)
+    assert module_name in str(exc.value)
+    assert manager.list_adapters() == {}
+
+
+@pytest.mark.parametrize(
+    "wrapper,mixed,declared_3d,weight_name",
+    [
+        (FusedMoE3DWithLoRA, False, False, "experts"),
+        (FusedMoEWithLoRA, True, True, "experts"),
+        (FusedMoEWithLoRA, False, False, "experts.0.down_proj"),
+        (FusedMoEWithLoRA, True, False, "experts.0.down_proj"),
+        (FusedMoEWithLoRA, False, False, "q_proj"),
+    ],
+)
+def test_moe_format_validation_allows_supported_layouts(
+    wrapper, mixed, declared_3d, weight_name
+):
+    """Preserve native 3D, mixed-format, per-expert and attention-only adapters."""
+    manager = LoRAModelManager.__new__(LoRAModelManager)
+    manager._enable_mixed_moe_lora_format = mixed
+    manager.is_pooling_model = False
+    manager.modules = {"experts": wrapper.__new__(wrapper)}
+    adapter = LoRAModel(
+        1,
+        8,
+        {
+            weight_name: LoRALayerWeights(
+                weight_name, 8, 16, torch.zeros(8, 4), torch.zeros(4, 8)
+            )
+        },
+    )
+    adapter.is_3d_lora_weight = declared_3d
+
+    manager._validate_moe_lora_format(adapter)
+
+
 @pytest.mark.parametrize("device", DEVICES)
 def test_from_lora_tensors(qwen3_lora_files, device):
     tensors = load_file(os.path.join(qwen3_lora_files, "adapter_model.safetensors"))

vllm/lora/model_manager.py

@@ -414,6 +414,7 @@ class LoRAModelManager:
             pass
 
     def _add_adapter(self, lora: LoRAModel):
+        self._validate_moe_lora_format(lora)
         self._create_merged_loras_inplace(lora)
         self._registered_adapters[lora.id] = lora
 
@@ -1252,6 +1253,27 @@ class LoRAModelManager:
             )
         return weights
 
+    def _validate_moe_lora_format(self, lora_model: LoRAModel) -> None:
+        if self._enable_mixed_moe_lora_format and getattr(
+            lora_model, "is_3d_lora_weight", False
+        ):
+            return
+        for module_name, module in self.modules.items():
+            if not isinstance(module, FusedMoEWithLoRA) or isinstance(
+                module, FusedMoE3DWithLoRA
+            ):
+                continue
+            module_lora = self._get_lora_layer_weights(lora_model, module_name)
+            if module_lora is not None and torch.is_tensor(module_lora.lora_a):
+                raise ValueError(
+                    f"LoRA adapter {lora_model.id} contains fused 3D MoE weights "
+                    f"for {module_name!r}, but the model uses a 2D MoE LoRA "
+                    "wrapper. Start the engine with "
+                    "enable_mixed_moe_lora_format=True "
+                    "(--enable-mixed-moe-lora-format) and set "
+                    "is_3d_lora_weight=True on the LoRA request."
+                )
+
     def _validate_modules_to_save(self, lora_model: LoRAModel) -> None:
         if not lora_model.modules_to_save:
             return

vllm/lora/worker_manager.py

@@ -153,6 +153,7 @@ class WorkerLoRAManager:
             # adapter manager can route 3D-format checkpoints through the
             # 3D->2D conversion when running under the universal 2D wrapper.
             lora.is_3d_lora_weight = lora_request.is_3d_lora_weight
+            self._adapter_manager._validate_moe_lora_format(lora)
 
             # Validate classification-head weights.
             self._adapter_manager._validate_modules_to_save(lora)