diff --git a/tests/lora/test_moe_lora_ep_load.py b/tests/lora/test_moe_lora_ep_load.py new file mode 100644 index 0000000000..ead40b2552 --- /dev/null +++ b/tests/lora/test_moe_lora_ep_load.py @@ -0,0 +1,90 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +""" +End-to-end correctness test for 2D MoE LoRA expert-parallel +load-time slicing +""" + +import pytest +import torch + +from vllm.lora.lora_model import LoRAModel, MoEEPLoadSpec +from vllm.lora.peft_helper import PEFTHelper + +NUM_LAYERS = 48 +GLOBAL_NUM_EXPERTS = 128 +LOCAL_NUM_EXPERTS = 64 # ep_size = 2 +EXPERT_PROJECTIONS = ("down_proj", "gate_proj", "up_proj") +NON_EXPERT_MODULES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate") + + +def _expected_lora_modules() -> set[str]: + """Replicate the set ``WorkerLoRAManager._load_adapter`` would build + from this model's ``packed_modules_mapping``.""" + expected: set[str] = set(NON_EXPERT_MODULES) + for expert in range(GLOBAL_NUM_EXPERTS): + for proj in EXPERT_PROJECTIONS: + expected.add(f"experts.{expert}.{proj}") + return expected + + +def _load(lora_dir, peft_helper, *, moe_ep_spec, lora_id): + return LoRAModel.from_local_checkpoint( + lora_dir, + _expected_lora_modules(), + peft_helper=peft_helper, + lora_model_id=lora_id, + device="cpu", + moe_ep_spec=moe_ep_spec, + ) + + +@pytest.mark.parametrize("ep_rank", [0, 1]) +def test_moe_lora_ep2_real_qwen3moe(qwen3moe_lora_files, ep_rank): + """ep_size=2 against the real Qwen3-MoE adapter: each rank's loaded + LoRA has the right size, the right expert membership, and the + right tensor values.""" + peft_helper = PEFTHelper.from_local_dir( + qwen3moe_lora_files, max_position_embeddings=4096 + ) + + # Baseline: no spec → loads every expert × projection × layer plus + # all non-expert LoRA modules. + ground_truth = _load(qwen3moe_lora_files, peft_helper, moe_ep_spec=None, lora_id=1) + expected_baseline = ( + GLOBAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS + + len(NON_EXPERT_MODULES) * NUM_LAYERS + ) + assert len(ground_truth.loras) == expected_baseline + + # Sliced load: only this rank's experts; non-expert LoRA is untouched. + spec = MoEEPLoadSpec( + ep_rank=ep_rank, + local_num_experts=LOCAL_NUM_EXPERTS, + global_num_experts=GLOBAL_NUM_EXPERTS, + ) + sliced = _load( + qwen3moe_lora_files, + peft_helper, + moe_ep_spec=spec, + lora_id=100 + ep_rank, + ) + + expected_sliced = ( + LOCAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS + + len(NON_EXPERT_MODULES) * NUM_LAYERS + ) + assert len(sliced.loras) == expected_sliced + + expert_start = ep_rank * LOCAL_NUM_EXPERTS + expert_end = expert_start + LOCAL_NUM_EXPERTS + + for name, lora in sliced.loras.items(): + gt = ground_truth.loras[name] + torch.testing.assert_close(lora.lora_a, gt.lora_a) + torch.testing.assert_close(lora.lora_b, gt.lora_b) + if ".experts." in name: + expert_idx = int(name.split(".experts.")[-1].split(".")[0]) + assert expert_start <= expert_idx < expert_end, ( + f"non-local expert {expert_idx} leaked: {name}" + ) diff --git a/vllm/lora/lora_model.py b/vllm/lora/lora_model.py index 357c1b5818..e3cb82e356 100644 --- a/vllm/lora/lora_model.py +++ b/vllm/lora/lora_model.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import os +from dataclasses import dataclass import safetensors import torch @@ -21,6 +22,41 @@ from vllm.utils.platform_utils import is_pin_memory_available logger = init_logger(__name__) +@dataclass(frozen=True) +class MoEEPLoadSpec: + """Per-expert-parallel slicing metadata for one FusedMoE LoRA module. + + Threaded into the LoRA loader so per-expert weights from EP ranks + other than this one can be skipped before they ever hit CPU memory. + """ + + ep_rank: int + local_num_experts: int + global_num_experts: int + + +_EXPERTS_SEPARATOR = ".experts." + + +def _is_remote_expert_key(raw_name: str, spec: "MoEEPLoadSpec") -> bool: + """ + Decide whether a checkpoint key belongs to a non-local expert. + """ + pos = raw_name.find(_EXPERTS_SEPARATOR) + if pos < 0: + return False + idx_start = pos + len(_EXPERTS_SEPARATOR) + idx_end = raw_name.find(".", idx_start) + if idx_end < 0: + return False + idx_str = raw_name[idx_start:idx_end] + if not idx_str.isdigit(): + return False + expert_idx = int(idx_str) + local_start = spec.ep_rank * spec.local_num_experts + return not (local_start <= expert_idx < local_start + spec.local_num_experts) + + class LoRAModel: """A LoRA fine-tuned model.""" @@ -141,6 +177,7 @@ class LoRAModel: weights_mapper: WeightsMapper | None = None, tensorizer_config_dict: dict | None = None, skip_prefixes: list[str] | None = None, + moe_ep_spec: MoEEPLoadSpec | None = None, ) -> "LoRAModel": """Create a LoRAModel from a local checkpoint. @@ -156,6 +193,11 @@ class LoRAModel: skip_prefixes: List of module name prefixes to skip during loading. Models can define this to skip modules not used in inference (e.g., MTP layers). Format: ["mtp."] + moe_ep_spec: When 2D FusedMoE LoRA modules are present with + expert parallelism enabled, the (ep_rank, local, global) + slicing metadata shared across all MoE layers. Non-local + expert weights are skipped at read time instead of being + loaded and discarded later. Returns: Loaded LoRA Model. @@ -227,6 +269,10 @@ class LoRAModel: # Load tensors if there are only expected modules. check_unexpected_modules(f) for module in f.keys(): # noqa + if moe_ep_spec is not None and _is_remote_expert_key( + module, moe_ep_spec + ): + continue tensors[module] = f.get_tensor(module) elif os.path.isfile(lora_bin_file_path) or os.path.isfile(lora_pt_file_path): lora_file_path = ( @@ -236,6 +282,15 @@ class LoRAModel: ) tensors = torch.load(lora_file_path, map_location=device, weights_only=True) check_unexpected_modules(tensors) + if moe_ep_spec is not None: + # `.bin`/`.pt` adapters can't be lazy-loaded, but pruning + # the dict here still frees the non-local expert tensors + # before the dtype cast / pin_memory work that follows. + tensors = { + k: v + for k, v in tensors.items() + if not _is_remote_expert_key(k, moe_ep_spec) + } else: raise ValueError(f"{lora_dir} doesn't contain tensors") diff --git a/vllm/lora/model_manager.py b/vllm/lora/model_manager.py index bb471daf5d..07df1b53da 100644 --- a/vllm/lora/model_manager.py +++ b/vllm/lora/model_manager.py @@ -18,7 +18,7 @@ from vllm.lora.layers import ( LoRAMapping, LoRAMappingType, ) -from vllm.lora.lora_model import LoRAModel +from vllm.lora.lora_model import LoRAModel, MoEEPLoadSpec from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights from vllm.lora.punica_wrapper import PunicaWrapperBase, get_punica_wrapper from vllm.lora.utils import ( @@ -109,8 +109,8 @@ class LoRAModelManager: # Dict instead of a set for compatibility with LRUCache. self._last_mapping: LoRAMapping | None = None is_moe = is_moe_model(self.model) - # Whether the underlying model class declares 3D fused MoE weights. - self._model_is_3d_moe = is_moe and self.model.is_3d_moe_weight + self._is_moe = is_moe + # When the engine is started with enable_mixed_moe_lora_format=True # we force the universal 2D wrapper (FusedMoEWithLoRA) regardless of # the model's 3D flag, so 2D and 3D adapters can coexist. @@ -118,15 +118,22 @@ class LoRAModelManager: is_moe and lora_config.enable_mixed_moe_lora_format ) self._is_3d_moe_model = ( - self._model_is_3d_moe and not self._enable_mixed_moe_lora_format + self._is_moe + and self.model.is_3d_moe_weight + and not self._enable_mixed_moe_lora_format ) self.packed_modules_mapping = process_packed_modules_mapping( self.model, force_2d_moe=self._enable_mixed_moe_lora_format ) self._is_non_gated_moe = is_moe and self.model.is_non_gated_moe + self._use_ep = bool( + vllm_config and vllm_config.parallel_config.enable_expert_parallel + ) self._init_punica_wrapper(max_num_batched_tokens, vllm_config) self._create_lora_modules() + self.moe_ep_load_spec: MoEEPLoadSpec | None = self._build_moe_ep_load_spec() + self.model.lora_manager = self def _init_punica_wrapper( @@ -715,15 +722,21 @@ class LoRAModelManager: def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None: for module_name, new_module_names in self.packed_modules.items(): + # For 2D FusedMoE modules with EP, narrow the per-expert + # sub-module list to this rank's owned experts so pack_moe + # produces a tensor sized to local_num_experts directly. + packed_module_names = new_module_names + if module_name.endswith(".experts"): + new_module_names = self._restrict_to_local_experts( + module_name, new_module_names + ) replacement_loras: list[LoRALayerWeights | None] = [] - replaced_module: set[str] = set() has_replacement = False for r in new_module_names: lora = self._get_lora_layer_weights(lora_model, r) replacement_loras.append(lora) if lora: has_replacement = True - replaced_module.add(r) if not has_replacement: continue for i in range(len(replacement_loras)): @@ -749,8 +762,11 @@ class LoRAModelManager: lora_model.loras[module_name] = PackedLoRALayerWeights.pack( replacement_loras ) - # Remove the modules that have been replaced. - for module in replaced_module: + # Drop every candidate sub-module, including non-local expert + # entries that were loaded but did not contribute to the + # packed result. Without this they would keep extra CPU + # memory alive after pack_moe. + for module in packed_module_names: lora_model.loras.pop(module, None) for lora in lora_model.loras.values(): @@ -998,6 +1014,13 @@ class LoRAModelManager: is active each rank only owns local_num_experts; without this slice the CPU LoRAModel keeps the full global weight and set_lora has to re-slice on every activation. + + With the load-time / pack-time slicing in + ``_restrict_to_local_experts``, the stacked tensors already match + ``local_num_experts`` and the inner branch becomes a no-op. The + guard remains so checkpoints that bypassed the pre-slicing (e.g. + ``.bin``/``.pt`` adapters with weights mappers we don't recognize) + still get sliced here. """ if not module.base_layer.use_ep: return @@ -1022,6 +1045,65 @@ class LoRAModelManager: module_lora.lora_a = new_lora_a module_lora.lora_b = new_lora_b + def _restrict_to_local_experts( + self, module_name: str, new_module_names: list[str] + ) -> list[str]: + """Narrow a flat expert-major sub-module list to this rank's experts. + + ``new_module_names`` is produced by + ``FusedMoE.make_expert_params_mapping`` and is ordered + ``[e=0,w1, e=0,w2, e=0,w3, e=1,w1, ...]`` (non-gated MoE has 2 + entries per expert instead of 3). When the module is a 2D + ``FusedMoEWithLoRA`` with EP enabled, we slice the list to the + contiguous block of experts owned by this rank so the downstream + ``pack_moe`` call only consumes local weights and produces a + tensor sized to ``local_num_experts`` directly. + + Returns the original list unchanged for non-MoE modules, the 3D + MoE path (handled separately by ``_stack_moe_lora_weights``), + modules without EP, or layouts we cannot cleanly partition. + """ + module = self.modules.get(module_name) + if not isinstance(module, FusedMoEWithLoRA): + return new_module_names + if isinstance(module, FusedMoE3DWithLoRA): + return new_module_names + if not getattr(module.base_layer, "use_ep", False): + return new_module_names + global_num_experts = module.base_layer.global_num_experts + local_num_experts = module.base_layer.local_num_experts + ep_rank = module.base_layer.ep_rank + if global_num_experts <= 0 or len(new_module_names) % global_num_experts != 0: + return new_module_names + per_expert = len(new_module_names) // global_num_experts + start = ep_rank * local_num_experts * per_expert + end = start + local_num_experts * per_expert + return new_module_names[start:end] + + def _build_moe_ep_load_spec(self) -> MoEEPLoadSpec | None: + """ + Per-rank slicing metadata for 2D FusedMoE LoRA modules. + """ + if not self._use_ep or not self._is_moe: + return None + module = next( + ( + m + for m in self.modules.values() + if isinstance(m, FusedMoEWithLoRA) + and not isinstance(m, FusedMoE3DWithLoRA) + ), + None, + ) + if module is None: + return None + base = module.base_layer + return MoEEPLoadSpec( + ep_rank=base.ep_rank, + local_num_experts=base.local_num_experts, + global_num_experts=base.global_num_experts, + ) + def _get_lora_layer_weights( self, lora_model: LoRAModel, module_name: str ) -> LoRALayerWeights | None: diff --git a/vllm/lora/worker_manager.py b/vllm/lora/worker_manager.py index 32d26fa6dc..166d5c36ba 100644 --- a/vllm/lora/worker_manager.py +++ b/vllm/lora/worker_manager.py @@ -140,6 +140,7 @@ class WorkerLoRAManager: tensorizer_config_dict=lora_request.tensorizer_config_dict, weights_mapper=hf_to_vllm_mapper, skip_prefixes=lora_skip_prefixes, + moe_ep_spec=self._adapter_manager.moe_ep_load_spec, ) # Stamp the on-disk MoE layout onto the loaded model so the # adapter manager can route 3D-format checkpoints through the