[LoRA] Reduce memory of 2D weights when EP is set (#42737)

Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-05-22 06:41:56 -07:00
committed by GitHub
parent 79ff0ffa98
commit 15f7cd33dc
4 changed files with 236 additions and 8 deletions
+90
View File
@@ -0,0 +1,90 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
End-to-end correctness test for 2D MoE LoRA expert-parallel
load-time slicing
"""
import pytest
import torch
from vllm.lora.lora_model import LoRAModel, MoEEPLoadSpec
from vllm.lora.peft_helper import PEFTHelper
NUM_LAYERS = 48
GLOBAL_NUM_EXPERTS = 128
LOCAL_NUM_EXPERTS = 64 # ep_size = 2
EXPERT_PROJECTIONS = ("down_proj", "gate_proj", "up_proj")
NON_EXPERT_MODULES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate")
def _expected_lora_modules() -> set[str]:
"""Replicate the set ``WorkerLoRAManager._load_adapter`` would build
from this model's ``packed_modules_mapping``."""
expected: set[str] = set(NON_EXPERT_MODULES)
for expert in range(GLOBAL_NUM_EXPERTS):
for proj in EXPERT_PROJECTIONS:
expected.add(f"experts.{expert}.{proj}")
return expected
def _load(lora_dir, peft_helper, *, moe_ep_spec, lora_id):
return LoRAModel.from_local_checkpoint(
lora_dir,
_expected_lora_modules(),
peft_helper=peft_helper,
lora_model_id=lora_id,
device="cpu",
moe_ep_spec=moe_ep_spec,
)
@pytest.mark.parametrize("ep_rank", [0, 1])
def test_moe_lora_ep2_real_qwen3moe(qwen3moe_lora_files, ep_rank):
"""ep_size=2 against the real Qwen3-MoE adapter: each rank's loaded
LoRA has the right size, the right expert membership, and the
right tensor values."""
peft_helper = PEFTHelper.from_local_dir(
qwen3moe_lora_files, max_position_embeddings=4096
)
# Baseline: no spec → loads every expert × projection × layer plus
# all non-expert LoRA modules.
ground_truth = _load(qwen3moe_lora_files, peft_helper, moe_ep_spec=None, lora_id=1)
expected_baseline = (
GLOBAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
+ len(NON_EXPERT_MODULES) * NUM_LAYERS
)
assert len(ground_truth.loras) == expected_baseline
# Sliced load: only this rank's experts; non-expert LoRA is untouched.
spec = MoEEPLoadSpec(
ep_rank=ep_rank,
local_num_experts=LOCAL_NUM_EXPERTS,
global_num_experts=GLOBAL_NUM_EXPERTS,
)
sliced = _load(
qwen3moe_lora_files,
peft_helper,
moe_ep_spec=spec,
lora_id=100 + ep_rank,
)
expected_sliced = (
LOCAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
+ len(NON_EXPERT_MODULES) * NUM_LAYERS
)
assert len(sliced.loras) == expected_sliced
expert_start = ep_rank * LOCAL_NUM_EXPERTS
expert_end = expert_start + LOCAL_NUM_EXPERTS
for name, lora in sliced.loras.items():
gt = ground_truth.loras[name]
torch.testing.assert_close(lora.lora_a, gt.lora_a)
torch.testing.assert_close(lora.lora_b, gt.lora_b)
if ".experts." in name:
expert_idx = int(name.split(".experts.")[-1].split(".")[0])
assert expert_start <= expert_idx < expert_end, (
f"non-local expert {expert_idx} leaked: {name}"
)
+55
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
from dataclasses import dataclass
import safetensors
import torch
@@ -21,6 +22,41 @@ from vllm.utils.platform_utils import is_pin_memory_available
logger = init_logger(__name__)
@dataclass(frozen=True)
class MoEEPLoadSpec:
"""Per-expert-parallel slicing metadata for one FusedMoE LoRA module.
Threaded into the LoRA loader so per-expert weights from EP ranks
other than this one can be skipped before they ever hit CPU memory.
"""
ep_rank: int
local_num_experts: int
global_num_experts: int
_EXPERTS_SEPARATOR = ".experts."
def _is_remote_expert_key(raw_name: str, spec: "MoEEPLoadSpec") -> bool:
"""
Decide whether a checkpoint key belongs to a non-local expert.
"""
pos = raw_name.find(_EXPERTS_SEPARATOR)
if pos < 0:
return False
idx_start = pos + len(_EXPERTS_SEPARATOR)
idx_end = raw_name.find(".", idx_start)
if idx_end < 0:
return False
idx_str = raw_name[idx_start:idx_end]
if not idx_str.isdigit():
return False
expert_idx = int(idx_str)
local_start = spec.ep_rank * spec.local_num_experts
return not (local_start <= expert_idx < local_start + spec.local_num_experts)
class LoRAModel:
"""A LoRA fine-tuned model."""
@@ -141,6 +177,7 @@ class LoRAModel:
weights_mapper: WeightsMapper | None = None,
tensorizer_config_dict: dict | None = None,
skip_prefixes: list[str] | None = None,
moe_ep_spec: MoEEPLoadSpec | None = None,
) -> "LoRAModel":
"""Create a LoRAModel from a local checkpoint.
@@ -156,6 +193,11 @@ class LoRAModel:
skip_prefixes: List of module name prefixes to skip during loading.
Models can define this to skip modules not used in inference
(e.g., MTP layers). Format: ["mtp."]
moe_ep_spec: When 2D FusedMoE LoRA modules are present with
expert parallelism enabled, the (ep_rank, local, global)
slicing metadata shared across all MoE layers. Non-local
expert weights are skipped at read time instead of being
loaded and discarded later.
Returns:
Loaded LoRA Model.
@@ -227,6 +269,10 @@ class LoRAModel:
# Load tensors if there are only expected modules.
check_unexpected_modules(f)
for module in f.keys(): # noqa
if moe_ep_spec is not None and _is_remote_expert_key(
module, moe_ep_spec
):
continue
tensors[module] = f.get_tensor(module)
elif os.path.isfile(lora_bin_file_path) or os.path.isfile(lora_pt_file_path):
lora_file_path = (
@@ -236,6 +282,15 @@ class LoRAModel:
)
tensors = torch.load(lora_file_path, map_location=device, weights_only=True)
check_unexpected_modules(tensors)
if moe_ep_spec is not None:
# `.bin`/`.pt` adapters can't be lazy-loaded, but pruning
# the dict here still frees the non-local expert tensors
# before the dtype cast / pin_memory work that follows.
tensors = {
k: v
for k, v in tensors.items()
if not _is_remote_expert_key(k, moe_ep_spec)
}
else:
raise ValueError(f"{lora_dir} doesn't contain tensors")
+90 -8
View File
@@ -18,7 +18,7 @@ from vllm.lora.layers import (
LoRAMapping,
LoRAMappingType,
)
from vllm.lora.lora_model import LoRAModel
from vllm.lora.lora_model import LoRAModel, MoEEPLoadSpec
from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
from vllm.lora.punica_wrapper import PunicaWrapperBase, get_punica_wrapper
from vllm.lora.utils import (
@@ -109,8 +109,8 @@ class LoRAModelManager:
# Dict instead of a set for compatibility with LRUCache.
self._last_mapping: LoRAMapping | None = None
is_moe = is_moe_model(self.model)
# Whether the underlying model class declares 3D fused MoE weights.
self._model_is_3d_moe = is_moe and self.model.is_3d_moe_weight
self._is_moe = is_moe
# When the engine is started with enable_mixed_moe_lora_format=True
# we force the universal 2D wrapper (FusedMoEWithLoRA) regardless of
# the model's 3D flag, so 2D and 3D adapters can coexist.
@@ -118,15 +118,22 @@ class LoRAModelManager:
is_moe and lora_config.enable_mixed_moe_lora_format
)
self._is_3d_moe_model = (
self._model_is_3d_moe and not self._enable_mixed_moe_lora_format
self._is_moe
and self.model.is_3d_moe_weight
and not self._enable_mixed_moe_lora_format
)
self.packed_modules_mapping = process_packed_modules_mapping(
self.model, force_2d_moe=self._enable_mixed_moe_lora_format
)
self._is_non_gated_moe = is_moe and self.model.is_non_gated_moe
self._use_ep = bool(
vllm_config and vllm_config.parallel_config.enable_expert_parallel
)
self._init_punica_wrapper(max_num_batched_tokens, vllm_config)
self._create_lora_modules()
self.moe_ep_load_spec: MoEEPLoadSpec | None = self._build_moe_ep_load_spec()
self.model.lora_manager = self
def _init_punica_wrapper(
@@ -715,15 +722,21 @@ class LoRAModelManager:
def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None:
for module_name, new_module_names in self.packed_modules.items():
# For 2D FusedMoE modules with EP, narrow the per-expert
# sub-module list to this rank's owned experts so pack_moe
# produces a tensor sized to local_num_experts directly.
packed_module_names = new_module_names
if module_name.endswith(".experts"):
new_module_names = self._restrict_to_local_experts(
module_name, new_module_names
)
replacement_loras: list[LoRALayerWeights | None] = []
replaced_module: set[str] = set()
has_replacement = False
for r in new_module_names:
lora = self._get_lora_layer_weights(lora_model, r)
replacement_loras.append(lora)
if lora:
has_replacement = True
replaced_module.add(r)
if not has_replacement:
continue
for i in range(len(replacement_loras)):
@@ -749,8 +762,11 @@ class LoRAModelManager:
lora_model.loras[module_name] = PackedLoRALayerWeights.pack(
replacement_loras
)
# Remove the modules that have been replaced.
for module in replaced_module:
# Drop every candidate sub-module, including non-local expert
# entries that were loaded but did not contribute to the
# packed result. Without this they would keep extra CPU
# memory alive after pack_moe.
for module in packed_module_names:
lora_model.loras.pop(module, None)
for lora in lora_model.loras.values():
@@ -998,6 +1014,13 @@ class LoRAModelManager:
is active each rank only owns local_num_experts; without this slice
the CPU LoRAModel keeps the full global weight and set_lora has to
re-slice on every activation.
With the load-time / pack-time slicing in
``_restrict_to_local_experts``, the stacked tensors already match
``local_num_experts`` and the inner branch becomes a no-op. The
guard remains so checkpoints that bypassed the pre-slicing (e.g.
``.bin``/``.pt`` adapters with weights mappers we don't recognize)
still get sliced here.
"""
if not module.base_layer.use_ep:
return
@@ -1022,6 +1045,65 @@ class LoRAModelManager:
module_lora.lora_a = new_lora_a
module_lora.lora_b = new_lora_b
def _restrict_to_local_experts(
self, module_name: str, new_module_names: list[str]
) -> list[str]:
"""Narrow a flat expert-major sub-module list to this rank's experts.
``new_module_names`` is produced by
``FusedMoE.make_expert_params_mapping`` and is ordered
``[e=0,w1, e=0,w2, e=0,w3, e=1,w1, ...]`` (non-gated MoE has 2
entries per expert instead of 3). When the module is a 2D
``FusedMoEWithLoRA`` with EP enabled, we slice the list to the
contiguous block of experts owned by this rank so the downstream
``pack_moe`` call only consumes local weights and produces a
tensor sized to ``local_num_experts`` directly.
Returns the original list unchanged for non-MoE modules, the 3D
MoE path (handled separately by ``_stack_moe_lora_weights``),
modules without EP, or layouts we cannot cleanly partition.
"""
module = self.modules.get(module_name)
if not isinstance(module, FusedMoEWithLoRA):
return new_module_names
if isinstance(module, FusedMoE3DWithLoRA):
return new_module_names
if not getattr(module.base_layer, "use_ep", False):
return new_module_names
global_num_experts = module.base_layer.global_num_experts
local_num_experts = module.base_layer.local_num_experts
ep_rank = module.base_layer.ep_rank
if global_num_experts <= 0 or len(new_module_names) % global_num_experts != 0:
return new_module_names
per_expert = len(new_module_names) // global_num_experts
start = ep_rank * local_num_experts * per_expert
end = start + local_num_experts * per_expert
return new_module_names[start:end]
def _build_moe_ep_load_spec(self) -> MoEEPLoadSpec | None:
"""
Per-rank slicing metadata for 2D FusedMoE LoRA modules.
"""
if not self._use_ep or not self._is_moe:
return None
module = next(
(
m
for m in self.modules.values()
if isinstance(m, FusedMoEWithLoRA)
and not isinstance(m, FusedMoE3DWithLoRA)
),
None,
)
if module is None:
return None
base = module.base_layer
return MoEEPLoadSpec(
ep_rank=base.ep_rank,
local_num_experts=base.local_num_experts,
global_num_experts=base.global_num_experts,
)
def _get_lora_layer_weights(
self, lora_model: LoRAModel, module_name: str
) -> LoRALayerWeights | None:
+1
View File
@@ -140,6 +140,7 @@ class WorkerLoRAManager:
tensorizer_config_dict=lora_request.tensorizer_config_dict,
weights_mapper=hf_to_vllm_mapper,
skip_prefixes=lora_skip_prefixes,
moe_ep_spec=self._adapter_manager.moe_ep_load_spec,
)
# Stamp the on-disk MoE layout onto the loaded model so the
# adapter manager can route 3D-format checkpoints through the