[LoRA] Reduce memory of 2D weights when EP is set (#42737)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
End-to-end correctness test for 2D MoE LoRA expert-parallel
|
||||
load-time slicing
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.lora.lora_model import LoRAModel, MoEEPLoadSpec
|
||||
from vllm.lora.peft_helper import PEFTHelper
|
||||
|
||||
NUM_LAYERS = 48
|
||||
GLOBAL_NUM_EXPERTS = 128
|
||||
LOCAL_NUM_EXPERTS = 64 # ep_size = 2
|
||||
EXPERT_PROJECTIONS = ("down_proj", "gate_proj", "up_proj")
|
||||
NON_EXPERT_MODULES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate")
|
||||
|
||||
|
||||
def _expected_lora_modules() -> set[str]:
|
||||
"""Replicate the set ``WorkerLoRAManager._load_adapter`` would build
|
||||
from this model's ``packed_modules_mapping``."""
|
||||
expected: set[str] = set(NON_EXPERT_MODULES)
|
||||
for expert in range(GLOBAL_NUM_EXPERTS):
|
||||
for proj in EXPERT_PROJECTIONS:
|
||||
expected.add(f"experts.{expert}.{proj}")
|
||||
return expected
|
||||
|
||||
|
||||
def _load(lora_dir, peft_helper, *, moe_ep_spec, lora_id):
|
||||
return LoRAModel.from_local_checkpoint(
|
||||
lora_dir,
|
||||
_expected_lora_modules(),
|
||||
peft_helper=peft_helper,
|
||||
lora_model_id=lora_id,
|
||||
device="cpu",
|
||||
moe_ep_spec=moe_ep_spec,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("ep_rank", [0, 1])
|
||||
def test_moe_lora_ep2_real_qwen3moe(qwen3moe_lora_files, ep_rank):
|
||||
"""ep_size=2 against the real Qwen3-MoE adapter: each rank's loaded
|
||||
LoRA has the right size, the right expert membership, and the
|
||||
right tensor values."""
|
||||
peft_helper = PEFTHelper.from_local_dir(
|
||||
qwen3moe_lora_files, max_position_embeddings=4096
|
||||
)
|
||||
|
||||
# Baseline: no spec → loads every expert × projection × layer plus
|
||||
# all non-expert LoRA modules.
|
||||
ground_truth = _load(qwen3moe_lora_files, peft_helper, moe_ep_spec=None, lora_id=1)
|
||||
expected_baseline = (
|
||||
GLOBAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
|
||||
+ len(NON_EXPERT_MODULES) * NUM_LAYERS
|
||||
)
|
||||
assert len(ground_truth.loras) == expected_baseline
|
||||
|
||||
# Sliced load: only this rank's experts; non-expert LoRA is untouched.
|
||||
spec = MoEEPLoadSpec(
|
||||
ep_rank=ep_rank,
|
||||
local_num_experts=LOCAL_NUM_EXPERTS,
|
||||
global_num_experts=GLOBAL_NUM_EXPERTS,
|
||||
)
|
||||
sliced = _load(
|
||||
qwen3moe_lora_files,
|
||||
peft_helper,
|
||||
moe_ep_spec=spec,
|
||||
lora_id=100 + ep_rank,
|
||||
)
|
||||
|
||||
expected_sliced = (
|
||||
LOCAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
|
||||
+ len(NON_EXPERT_MODULES) * NUM_LAYERS
|
||||
)
|
||||
assert len(sliced.loras) == expected_sliced
|
||||
|
||||
expert_start = ep_rank * LOCAL_NUM_EXPERTS
|
||||
expert_end = expert_start + LOCAL_NUM_EXPERTS
|
||||
|
||||
for name, lora in sliced.loras.items():
|
||||
gt = ground_truth.loras[name]
|
||||
torch.testing.assert_close(lora.lora_a, gt.lora_a)
|
||||
torch.testing.assert_close(lora.lora_b, gt.lora_b)
|
||||
if ".experts." in name:
|
||||
expert_idx = int(name.split(".experts.")[-1].split(".")[0])
|
||||
assert expert_start <= expert_idx < expert_end, (
|
||||
f"non-local expert {expert_idx} leaked: {name}"
|
||||
)
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
import safetensors
|
||||
import torch
|
||||
@@ -21,6 +22,41 @@ from vllm.utils.platform_utils import is_pin_memory_available
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MoEEPLoadSpec:
|
||||
"""Per-expert-parallel slicing metadata for one FusedMoE LoRA module.
|
||||
|
||||
Threaded into the LoRA loader so per-expert weights from EP ranks
|
||||
other than this one can be skipped before they ever hit CPU memory.
|
||||
"""
|
||||
|
||||
ep_rank: int
|
||||
local_num_experts: int
|
||||
global_num_experts: int
|
||||
|
||||
|
||||
_EXPERTS_SEPARATOR = ".experts."
|
||||
|
||||
|
||||
def _is_remote_expert_key(raw_name: str, spec: "MoEEPLoadSpec") -> bool:
|
||||
"""
|
||||
Decide whether a checkpoint key belongs to a non-local expert.
|
||||
"""
|
||||
pos = raw_name.find(_EXPERTS_SEPARATOR)
|
||||
if pos < 0:
|
||||
return False
|
||||
idx_start = pos + len(_EXPERTS_SEPARATOR)
|
||||
idx_end = raw_name.find(".", idx_start)
|
||||
if idx_end < 0:
|
||||
return False
|
||||
idx_str = raw_name[idx_start:idx_end]
|
||||
if not idx_str.isdigit():
|
||||
return False
|
||||
expert_idx = int(idx_str)
|
||||
local_start = spec.ep_rank * spec.local_num_experts
|
||||
return not (local_start <= expert_idx < local_start + spec.local_num_experts)
|
||||
|
||||
|
||||
class LoRAModel:
|
||||
"""A LoRA fine-tuned model."""
|
||||
|
||||
@@ -141,6 +177,7 @@ class LoRAModel:
|
||||
weights_mapper: WeightsMapper | None = None,
|
||||
tensorizer_config_dict: dict | None = None,
|
||||
skip_prefixes: list[str] | None = None,
|
||||
moe_ep_spec: MoEEPLoadSpec | None = None,
|
||||
) -> "LoRAModel":
|
||||
"""Create a LoRAModel from a local checkpoint.
|
||||
|
||||
@@ -156,6 +193,11 @@ class LoRAModel:
|
||||
skip_prefixes: List of module name prefixes to skip during loading.
|
||||
Models can define this to skip modules not used in inference
|
||||
(e.g., MTP layers). Format: ["mtp."]
|
||||
moe_ep_spec: When 2D FusedMoE LoRA modules are present with
|
||||
expert parallelism enabled, the (ep_rank, local, global)
|
||||
slicing metadata shared across all MoE layers. Non-local
|
||||
expert weights are skipped at read time instead of being
|
||||
loaded and discarded later.
|
||||
|
||||
Returns:
|
||||
Loaded LoRA Model.
|
||||
@@ -227,6 +269,10 @@ class LoRAModel:
|
||||
# Load tensors if there are only expected modules.
|
||||
check_unexpected_modules(f)
|
||||
for module in f.keys(): # noqa
|
||||
if moe_ep_spec is not None and _is_remote_expert_key(
|
||||
module, moe_ep_spec
|
||||
):
|
||||
continue
|
||||
tensors[module] = f.get_tensor(module)
|
||||
elif os.path.isfile(lora_bin_file_path) or os.path.isfile(lora_pt_file_path):
|
||||
lora_file_path = (
|
||||
@@ -236,6 +282,15 @@ class LoRAModel:
|
||||
)
|
||||
tensors = torch.load(lora_file_path, map_location=device, weights_only=True)
|
||||
check_unexpected_modules(tensors)
|
||||
if moe_ep_spec is not None:
|
||||
# `.bin`/`.pt` adapters can't be lazy-loaded, but pruning
|
||||
# the dict here still frees the non-local expert tensors
|
||||
# before the dtype cast / pin_memory work that follows.
|
||||
tensors = {
|
||||
k: v
|
||||
for k, v in tensors.items()
|
||||
if not _is_remote_expert_key(k, moe_ep_spec)
|
||||
}
|
||||
else:
|
||||
raise ValueError(f"{lora_dir} doesn't contain tensors")
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ from vllm.lora.layers import (
|
||||
LoRAMapping,
|
||||
LoRAMappingType,
|
||||
)
|
||||
from vllm.lora.lora_model import LoRAModel
|
||||
from vllm.lora.lora_model import LoRAModel, MoEEPLoadSpec
|
||||
from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
|
||||
from vllm.lora.punica_wrapper import PunicaWrapperBase, get_punica_wrapper
|
||||
from vllm.lora.utils import (
|
||||
@@ -109,8 +109,8 @@ class LoRAModelManager:
|
||||
# Dict instead of a set for compatibility with LRUCache.
|
||||
self._last_mapping: LoRAMapping | None = None
|
||||
is_moe = is_moe_model(self.model)
|
||||
# Whether the underlying model class declares 3D fused MoE weights.
|
||||
self._model_is_3d_moe = is_moe and self.model.is_3d_moe_weight
|
||||
self._is_moe = is_moe
|
||||
|
||||
# When the engine is started with enable_mixed_moe_lora_format=True
|
||||
# we force the universal 2D wrapper (FusedMoEWithLoRA) regardless of
|
||||
# the model's 3D flag, so 2D and 3D adapters can coexist.
|
||||
@@ -118,15 +118,22 @@ class LoRAModelManager:
|
||||
is_moe and lora_config.enable_mixed_moe_lora_format
|
||||
)
|
||||
self._is_3d_moe_model = (
|
||||
self._model_is_3d_moe and not self._enable_mixed_moe_lora_format
|
||||
self._is_moe
|
||||
and self.model.is_3d_moe_weight
|
||||
and not self._enable_mixed_moe_lora_format
|
||||
)
|
||||
self.packed_modules_mapping = process_packed_modules_mapping(
|
||||
self.model, force_2d_moe=self._enable_mixed_moe_lora_format
|
||||
)
|
||||
self._is_non_gated_moe = is_moe and self.model.is_non_gated_moe
|
||||
self._use_ep = bool(
|
||||
vllm_config and vllm_config.parallel_config.enable_expert_parallel
|
||||
)
|
||||
self._init_punica_wrapper(max_num_batched_tokens, vllm_config)
|
||||
self._create_lora_modules()
|
||||
|
||||
self.moe_ep_load_spec: MoEEPLoadSpec | None = self._build_moe_ep_load_spec()
|
||||
|
||||
self.model.lora_manager = self
|
||||
|
||||
def _init_punica_wrapper(
|
||||
@@ -715,15 +722,21 @@ class LoRAModelManager:
|
||||
|
||||
def _create_merged_loras_inplace(self, lora_model: LoRAModel) -> None:
|
||||
for module_name, new_module_names in self.packed_modules.items():
|
||||
# For 2D FusedMoE modules with EP, narrow the per-expert
|
||||
# sub-module list to this rank's owned experts so pack_moe
|
||||
# produces a tensor sized to local_num_experts directly.
|
||||
packed_module_names = new_module_names
|
||||
if module_name.endswith(".experts"):
|
||||
new_module_names = self._restrict_to_local_experts(
|
||||
module_name, new_module_names
|
||||
)
|
||||
replacement_loras: list[LoRALayerWeights | None] = []
|
||||
replaced_module: set[str] = set()
|
||||
has_replacement = False
|
||||
for r in new_module_names:
|
||||
lora = self._get_lora_layer_weights(lora_model, r)
|
||||
replacement_loras.append(lora)
|
||||
if lora:
|
||||
has_replacement = True
|
||||
replaced_module.add(r)
|
||||
if not has_replacement:
|
||||
continue
|
||||
for i in range(len(replacement_loras)):
|
||||
@@ -749,8 +762,11 @@ class LoRAModelManager:
|
||||
lora_model.loras[module_name] = PackedLoRALayerWeights.pack(
|
||||
replacement_loras
|
||||
)
|
||||
# Remove the modules that have been replaced.
|
||||
for module in replaced_module:
|
||||
# Drop every candidate sub-module, including non-local expert
|
||||
# entries that were loaded but did not contribute to the
|
||||
# packed result. Without this they would keep extra CPU
|
||||
# memory alive after pack_moe.
|
||||
for module in packed_module_names:
|
||||
lora_model.loras.pop(module, None)
|
||||
|
||||
for lora in lora_model.loras.values():
|
||||
@@ -998,6 +1014,13 @@ class LoRAModelManager:
|
||||
is active each rank only owns local_num_experts; without this slice
|
||||
the CPU LoRAModel keeps the full global weight and set_lora has to
|
||||
re-slice on every activation.
|
||||
|
||||
With the load-time / pack-time slicing in
|
||||
``_restrict_to_local_experts``, the stacked tensors already match
|
||||
``local_num_experts`` and the inner branch becomes a no-op. The
|
||||
guard remains so checkpoints that bypassed the pre-slicing (e.g.
|
||||
``.bin``/``.pt`` adapters with weights mappers we don't recognize)
|
||||
still get sliced here.
|
||||
"""
|
||||
if not module.base_layer.use_ep:
|
||||
return
|
||||
@@ -1022,6 +1045,65 @@ class LoRAModelManager:
|
||||
module_lora.lora_a = new_lora_a
|
||||
module_lora.lora_b = new_lora_b
|
||||
|
||||
def _restrict_to_local_experts(
|
||||
self, module_name: str, new_module_names: list[str]
|
||||
) -> list[str]:
|
||||
"""Narrow a flat expert-major sub-module list to this rank's experts.
|
||||
|
||||
``new_module_names`` is produced by
|
||||
``FusedMoE.make_expert_params_mapping`` and is ordered
|
||||
``[e=0,w1, e=0,w2, e=0,w3, e=1,w1, ...]`` (non-gated MoE has 2
|
||||
entries per expert instead of 3). When the module is a 2D
|
||||
``FusedMoEWithLoRA`` with EP enabled, we slice the list to the
|
||||
contiguous block of experts owned by this rank so the downstream
|
||||
``pack_moe`` call only consumes local weights and produces a
|
||||
tensor sized to ``local_num_experts`` directly.
|
||||
|
||||
Returns the original list unchanged for non-MoE modules, the 3D
|
||||
MoE path (handled separately by ``_stack_moe_lora_weights``),
|
||||
modules without EP, or layouts we cannot cleanly partition.
|
||||
"""
|
||||
module = self.modules.get(module_name)
|
||||
if not isinstance(module, FusedMoEWithLoRA):
|
||||
return new_module_names
|
||||
if isinstance(module, FusedMoE3DWithLoRA):
|
||||
return new_module_names
|
||||
if not getattr(module.base_layer, "use_ep", False):
|
||||
return new_module_names
|
||||
global_num_experts = module.base_layer.global_num_experts
|
||||
local_num_experts = module.base_layer.local_num_experts
|
||||
ep_rank = module.base_layer.ep_rank
|
||||
if global_num_experts <= 0 or len(new_module_names) % global_num_experts != 0:
|
||||
return new_module_names
|
||||
per_expert = len(new_module_names) // global_num_experts
|
||||
start = ep_rank * local_num_experts * per_expert
|
||||
end = start + local_num_experts * per_expert
|
||||
return new_module_names[start:end]
|
||||
|
||||
def _build_moe_ep_load_spec(self) -> MoEEPLoadSpec | None:
|
||||
"""
|
||||
Per-rank slicing metadata for 2D FusedMoE LoRA modules.
|
||||
"""
|
||||
if not self._use_ep or not self._is_moe:
|
||||
return None
|
||||
module = next(
|
||||
(
|
||||
m
|
||||
for m in self.modules.values()
|
||||
if isinstance(m, FusedMoEWithLoRA)
|
||||
and not isinstance(m, FusedMoE3DWithLoRA)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if module is None:
|
||||
return None
|
||||
base = module.base_layer
|
||||
return MoEEPLoadSpec(
|
||||
ep_rank=base.ep_rank,
|
||||
local_num_experts=base.local_num_experts,
|
||||
global_num_experts=base.global_num_experts,
|
||||
)
|
||||
|
||||
def _get_lora_layer_weights(
|
||||
self, lora_model: LoRAModel, module_name: str
|
||||
) -> LoRALayerWeights | None:
|
||||
|
||||
@@ -140,6 +140,7 @@ class WorkerLoRAManager:
|
||||
tensorizer_config_dict=lora_request.tensorizer_config_dict,
|
||||
weights_mapper=hf_to_vllm_mapper,
|
||||
skip_prefixes=lora_skip_prefixes,
|
||||
moe_ep_spec=self._adapter_manager.moe_ep_load_spec,
|
||||
)
|
||||
# Stamp the on-disk MoE layout onto the loaded model so the
|
||||
# adapter manager can route 3D-format checkpoints through the
|
||||
|
||||
Reference in New Issue
Block a user