[LoRA] Initial EP support for LoRA (#40867)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
Jee Jee Li
2026-05-09 00:31:23 -07:00
committed by GitHub
parent d6563d693c
commit ecd0b60aad
13 changed files with 209 additions and 73 deletions
+63 -9
View File
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
from vllm.lora.layers import (
BaseLayerWithLoRA,
FusedMoE3DWithLoRA,
FusedMoEWithLoRA,
LoRAMapping,
LoRAMappingType,
)
@@ -562,6 +563,10 @@ class LoRAModelManager:
else:
parts = module_name.split(".")
replacements = self.packed_modules_mapping[parts[-1]]
if module.__class__.__name__ == "FusedMoEWithLoRA":
replacements = replacements[
: len(module.lora_a_stacked) // self.lora_slots
]
subloras: list[LoRALayerWeights | None] = []
for i, r in enumerate(replacements):
lora = LoRALayerWeights.create_dummy_lora_weights(
@@ -716,6 +721,8 @@ class LoRAModelManager:
for module_name, module in self.modules.items():
if isinstance(module, FusedMoE3DWithLoRA):
self._stack_moe_lora_weights(lora_model, module, module_name)
elif isinstance(module, FusedMoEWithLoRA):
self._slice_moe_lora_ep(lora_model, module, module_name)
first_lora: LoRALayerWeights = next(iter(lora_model.loras.values()))
assert first_lora.lora_a is not None
@@ -762,23 +769,33 @@ class LoRAModelManager:
assert gate_up_proj_lora is not None
assert down_proj_lora is not None
if self._is_3d_moe_model:
num_experts = module.w13_lora_a_stacked[0].shape[1]
local_num_experts = module.w13_lora_a_stacked[0].shape[1]
# The checkpoint holds weights for all global experts, but
# each EP rank owns only local_num_experts. Reshape against
# the adapter's actual expert count, then slice this rank's
# owned expert range before it gets copied into the local
# stacked buffer. For non-EP (local == global) this is a
# no-op slice.
global_num_experts = module.base_layer.global_num_experts
ep_rank = module.base_layer.ep_rank
expert_start = ep_rank * local_num_experts
expert_end = expert_start + local_num_experts
# (num_experts,rank,input_size)
gate_up_proj_lora.lora_a = gate_up_proj_lora.lora_a.reshape(
num_experts, -1, gate_up_proj_lora.lora_a.shape[-1]
)
global_num_experts, -1, gate_up_proj_lora.lora_a.shape[-1]
)[expert_start:expert_end].contiguous()
down_proj_lora.lora_a = down_proj_lora.lora_a.reshape(
num_experts, -1, down_proj_lora.lora_a.shape[-1]
)
global_num_experts, -1, down_proj_lora.lora_a.shape[-1]
)[expert_start:expert_end].contiguous()
# (output_size,rank,num_experts)
gate_up_proj_lora.lora_b = gate_up_proj_lora.lora_b.reshape(
gate_up_proj_lora.lora_b.shape[0], -1, num_experts
)
gate_up_proj_lora.lora_b.shape[0], -1, global_num_experts
)[..., expert_start:expert_end]
down_proj_lora.lora_b = down_proj_lora.lora_b.reshape(
down_proj_lora.lora_b.shape[0], -1, num_experts
)
down_proj_lora.lora_b.shape[0], -1, global_num_experts
)[..., expert_start:expert_end]
# (num_experts,output_size,rank)
gate_up_proj_lora.lora_b = gate_up_proj_lora.lora_b.permute(
@@ -828,6 +845,43 @@ class LoRAModelManager:
module_lora.lora_a = lora_a
module_lora.lora_b = lora_b
def _slice_moe_lora_ep(
self,
lora_model: LoRAModel,
module: FusedMoEWithLoRA,
module_name: str,
) -> None:
"""Slice the cached LoRA tensors down to this rank's local experts.
The 2D MoE checkpoint enters as a list of per-(w1/w2/w3) tensors of
shape (num_experts, rank, in) / (num_experts, out, rank). When EP
is active each rank only owns local_num_experts; without this slice
the CPU LoRAModel keeps the full global weight and set_lora has to
re-slice on every activation.
"""
if not module.base_layer.use_ep:
return
module_lora = self._get_lora_layer_weights(lora_model, module_name)
if module_lora is None or not isinstance(module_lora.lora_a, list):
return
local_num_experts = module.base_layer.local_num_experts
global_num_experts = module.base_layer.global_num_experts
ep_rank = module.base_layer.ep_rank
expert_start = ep_rank * local_num_experts
expert_end = expert_start + local_num_experts
new_lora_a: list[torch.Tensor | None] = []
new_lora_b: list[torch.Tensor | None] = []
for a, b in zip(module_lora.lora_a, module_lora.lora_b):
if a is not None and b is not None and a.shape[0] == global_num_experts:
a = a[expert_start:expert_end].contiguous()
b = b[expert_start:expert_end].contiguous()
new_lora_a.append(a)
new_lora_b.append(b)
module_lora.lora_a = new_lora_a
module_lora.lora_b = new_lora_b
def _get_lora_layer_weights(
self, lora_model: LoRAModel, module_name: str
) -> LoRALayerWeights | None: