diff --git a/docs/features/lora.md b/docs/features/lora.md index 4c95bde8cd..c05bc8c0fa 100644 --- a/docs/features/lora.md +++ b/docs/features/lora.md @@ -280,10 +280,10 @@ curl -X POST http://localhost:8000/v1/load_lora_adapter \ !!! warning "You must know your adapter's layout" Under `--enable-mixed-moe-lora-format`, vLLM trusts whatever - `is_3d_lora_weight` the caller declares — it does **not** inspect the - checkpoint to verify. A wrong declaration will load weights into the - wrong stacked buffers and silently produce garbage outputs, with no - error at load time. Confirm the layout before serving: + `is_3d_lora_weight` the caller declares. Loading checks reject fused 3D + expert weights routed to a 2D wrapper without the required flags, but + do not validate every layout mismatch. A wrong declaration can still + cause loading failures or incorrect outputs. Confirm the layout before serving: - **2D (per-expert, megatron-style)** → set `is_3d_lora_weight: false`. Adapter keys look like `...experts.{idx}.gate_proj.lora_A.weight`, @@ -299,6 +299,11 @@ is ignored: vLLM picks the wrapper from the base model's `is_3d_moe_weight` and the adapter is required to match. The field is also ignored for non-MoE models. +For a model using a 2D MoE wrapper (such as Qwen3-MoE), loading fused 3D +expert adapter weights requires both `enable_mixed_moe_lora_format=True` +on the engine and `is_3d_lora_weight=True` on the request. Otherwise, adapter +loading raises a `ValueError` naming these settings before activation. + ## LoRA model lineage in model card The new format of `--lora-modules` is mainly to support the display of parent model information in the model card. Here's an explanation of how your current response supports this: diff --git a/tests/lora/test_lora_manager.py b/tests/lora/test_lora_manager.py index 43fa1975bd..ffb064ebfb 100644 --- a/tests/lora/test_lora_manager.py +++ b/tests/lora/test_lora_manager.py @@ -16,10 +16,12 @@ from vllm.lora.layers import ( ReplicatedLinearWithLoRA, RowParallelLinearWithLoRA, ) +from vllm.lora.layers.fused_moe import FusedMoE3DWithLoRA, FusedMoEWithLoRA from vllm.lora.lora_model import LoRAModel from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights from vllm.lora.model_manager import ( DEFAULT_LANGUAGE_WRAPPER_KEY, + AdapterLRUCache, LoRAMapping, LoRAModelManager, LRUCacheLoRAModelManager, @@ -47,6 +49,68 @@ DEVICES = ( DEFAULT_DTYPE = torch.get_default_dtype() +@pytest.mark.parametrize( + "mixed,declared_3d", [(False, False), (False, True), (True, False)] +) +def test_reject_fused_moe_lora_before_registration(mixed, declared_3d): + """Missing format flags should produce guidance without caching the adapter.""" + module_name = "model.layers.0.mlp.experts" + manager = LoRAModelManager.__new__(LoRAModelManager) + manager.lora_config = LoRAConfig(max_cpu_loras=1) + manager._registered_adapters = AdapterLRUCache(1, manager.deactivate_adapter) + manager._enable_mixed_moe_lora_format = mixed + manager.is_pooling_model = False + manager.modules = {module_name: FusedMoEWithLoRA.__new__(FusedMoEWithLoRA)} + adapter = LoRAModel( + 1, + 8, + { + module_name: LoRALayerWeights( + module_name, 8, 16, torch.zeros(16, 4), torch.zeros(4, 16) + ) + }, + ) + adapter.is_3d_lora_weight = declared_3d + + with pytest.raises(ValueError, match="enable_mixed_moe_lora_format=True") as exc: + manager.add_adapter(adapter) + assert "is_3d_lora_weight=True" in str(exc.value) + assert module_name in str(exc.value) + assert manager.list_adapters() == {} + + +@pytest.mark.parametrize( + "wrapper,mixed,declared_3d,weight_name", + [ + (FusedMoE3DWithLoRA, False, False, "experts"), + (FusedMoEWithLoRA, True, True, "experts"), + (FusedMoEWithLoRA, False, False, "experts.0.down_proj"), + (FusedMoEWithLoRA, True, False, "experts.0.down_proj"), + (FusedMoEWithLoRA, False, False, "q_proj"), + ], +) +def test_moe_format_validation_allows_supported_layouts( + wrapper, mixed, declared_3d, weight_name +): + """Preserve native 3D, mixed-format, per-expert and attention-only adapters.""" + manager = LoRAModelManager.__new__(LoRAModelManager) + manager._enable_mixed_moe_lora_format = mixed + manager.is_pooling_model = False + manager.modules = {"experts": wrapper.__new__(wrapper)} + adapter = LoRAModel( + 1, + 8, + { + weight_name: LoRALayerWeights( + weight_name, 8, 16, torch.zeros(8, 4), torch.zeros(4, 8) + ) + }, + ) + adapter.is_3d_lora_weight = declared_3d + + manager._validate_moe_lora_format(adapter) + + @pytest.mark.parametrize("device", DEVICES) def test_from_lora_tensors(qwen3_lora_files, device): tensors = load_file(os.path.join(qwen3_lora_files, "adapter_model.safetensors")) diff --git a/vllm/lora/model_manager.py b/vllm/lora/model_manager.py index b97a0a412a..8dc7b56056 100644 --- a/vllm/lora/model_manager.py +++ b/vllm/lora/model_manager.py @@ -414,6 +414,7 @@ class LoRAModelManager: pass def _add_adapter(self, lora: LoRAModel): + self._validate_moe_lora_format(lora) self._create_merged_loras_inplace(lora) self._registered_adapters[lora.id] = lora @@ -1252,6 +1253,27 @@ class LoRAModelManager: ) return weights + def _validate_moe_lora_format(self, lora_model: LoRAModel) -> None: + if self._enable_mixed_moe_lora_format and getattr( + lora_model, "is_3d_lora_weight", False + ): + return + for module_name, module in self.modules.items(): + if not isinstance(module, FusedMoEWithLoRA) or isinstance( + module, FusedMoE3DWithLoRA + ): + continue + module_lora = self._get_lora_layer_weights(lora_model, module_name) + if module_lora is not None and torch.is_tensor(module_lora.lora_a): + raise ValueError( + f"LoRA adapter {lora_model.id} contains fused 3D MoE weights " + f"for {module_name!r}, but the model uses a 2D MoE LoRA " + "wrapper. Start the engine with " + "enable_mixed_moe_lora_format=True " + "(--enable-mixed-moe-lora-format) and set " + "is_3d_lora_weight=True on the LoRA request." + ) + def _validate_modules_to_save(self, lora_model: LoRAModel) -> None: if not lora_model.modules_to_save: return diff --git a/vllm/lora/worker_manager.py b/vllm/lora/worker_manager.py index fe266879a8..9427b3e134 100644 --- a/vllm/lora/worker_manager.py +++ b/vllm/lora/worker_manager.py @@ -153,6 +153,7 @@ class WorkerLoRAManager: # adapter manager can route 3D-format checkpoints through the # 3D->2D conversion when running under the universal 2D wrapper. lora.is_3d_lora_weight = lora_request.is_3d_lora_weight + self._adapter_manager._validate_moe_lora_format(lora) # Validate classification-head weights. self._adapter_manager._validate_modules_to_save(lora)