[LoRA] Initial EP support for LoRA (#40867)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com> Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai>
This commit is contained in:
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
|
||||
from vllm.lora.layers import (
|
||||
BaseLayerWithLoRA,
|
||||
FusedMoE3DWithLoRA,
|
||||
FusedMoEWithLoRA,
|
||||
LoRAMapping,
|
||||
LoRAMappingType,
|
||||
)
|
||||
@@ -562,6 +563,10 @@ class LoRAModelManager:
|
||||
else:
|
||||
parts = module_name.split(".")
|
||||
replacements = self.packed_modules_mapping[parts[-1]]
|
||||
if module.__class__.__name__ == "FusedMoEWithLoRA":
|
||||
replacements = replacements[
|
||||
: len(module.lora_a_stacked) // self.lora_slots
|
||||
]
|
||||
subloras: list[LoRALayerWeights | None] = []
|
||||
for i, r in enumerate(replacements):
|
||||
lora = LoRALayerWeights.create_dummy_lora_weights(
|
||||
@@ -716,6 +721,8 @@ class LoRAModelManager:
|
||||
for module_name, module in self.modules.items():
|
||||
if isinstance(module, FusedMoE3DWithLoRA):
|
||||
self._stack_moe_lora_weights(lora_model, module, module_name)
|
||||
elif isinstance(module, FusedMoEWithLoRA):
|
||||
self._slice_moe_lora_ep(lora_model, module, module_name)
|
||||
|
||||
first_lora: LoRALayerWeights = next(iter(lora_model.loras.values()))
|
||||
assert first_lora.lora_a is not None
|
||||
@@ -762,23 +769,33 @@ class LoRAModelManager:
|
||||
assert gate_up_proj_lora is not None
|
||||
assert down_proj_lora is not None
|
||||
if self._is_3d_moe_model:
|
||||
num_experts = module.w13_lora_a_stacked[0].shape[1]
|
||||
local_num_experts = module.w13_lora_a_stacked[0].shape[1]
|
||||
# The checkpoint holds weights for all global experts, but
|
||||
# each EP rank owns only local_num_experts. Reshape against
|
||||
# the adapter's actual expert count, then slice this rank's
|
||||
# owned expert range before it gets copied into the local
|
||||
# stacked buffer. For non-EP (local == global) this is a
|
||||
# no-op slice.
|
||||
global_num_experts = module.base_layer.global_num_experts
|
||||
ep_rank = module.base_layer.ep_rank
|
||||
expert_start = ep_rank * local_num_experts
|
||||
expert_end = expert_start + local_num_experts
|
||||
|
||||
# (num_experts,rank,input_size)
|
||||
gate_up_proj_lora.lora_a = gate_up_proj_lora.lora_a.reshape(
|
||||
num_experts, -1, gate_up_proj_lora.lora_a.shape[-1]
|
||||
)
|
||||
global_num_experts, -1, gate_up_proj_lora.lora_a.shape[-1]
|
||||
)[expert_start:expert_end].contiguous()
|
||||
down_proj_lora.lora_a = down_proj_lora.lora_a.reshape(
|
||||
num_experts, -1, down_proj_lora.lora_a.shape[-1]
|
||||
)
|
||||
global_num_experts, -1, down_proj_lora.lora_a.shape[-1]
|
||||
)[expert_start:expert_end].contiguous()
|
||||
|
||||
# (output_size,rank,num_experts)
|
||||
gate_up_proj_lora.lora_b = gate_up_proj_lora.lora_b.reshape(
|
||||
gate_up_proj_lora.lora_b.shape[0], -1, num_experts
|
||||
)
|
||||
gate_up_proj_lora.lora_b.shape[0], -1, global_num_experts
|
||||
)[..., expert_start:expert_end]
|
||||
down_proj_lora.lora_b = down_proj_lora.lora_b.reshape(
|
||||
down_proj_lora.lora_b.shape[0], -1, num_experts
|
||||
)
|
||||
down_proj_lora.lora_b.shape[0], -1, global_num_experts
|
||||
)[..., expert_start:expert_end]
|
||||
|
||||
# (num_experts,output_size,rank)
|
||||
gate_up_proj_lora.lora_b = gate_up_proj_lora.lora_b.permute(
|
||||
@@ -828,6 +845,43 @@ class LoRAModelManager:
|
||||
module_lora.lora_a = lora_a
|
||||
module_lora.lora_b = lora_b
|
||||
|
||||
def _slice_moe_lora_ep(
|
||||
self,
|
||||
lora_model: LoRAModel,
|
||||
module: FusedMoEWithLoRA,
|
||||
module_name: str,
|
||||
) -> None:
|
||||
"""Slice the cached LoRA tensors down to this rank's local experts.
|
||||
|
||||
The 2D MoE checkpoint enters as a list of per-(w1/w2/w3) tensors of
|
||||
shape (num_experts, rank, in) / (num_experts, out, rank). When EP
|
||||
is active each rank only owns local_num_experts; without this slice
|
||||
the CPU LoRAModel keeps the full global weight and set_lora has to
|
||||
re-slice on every activation.
|
||||
"""
|
||||
if not module.base_layer.use_ep:
|
||||
return
|
||||
module_lora = self._get_lora_layer_weights(lora_model, module_name)
|
||||
if module_lora is None or not isinstance(module_lora.lora_a, list):
|
||||
return
|
||||
|
||||
local_num_experts = module.base_layer.local_num_experts
|
||||
global_num_experts = module.base_layer.global_num_experts
|
||||
ep_rank = module.base_layer.ep_rank
|
||||
expert_start = ep_rank * local_num_experts
|
||||
expert_end = expert_start + local_num_experts
|
||||
|
||||
new_lora_a: list[torch.Tensor | None] = []
|
||||
new_lora_b: list[torch.Tensor | None] = []
|
||||
for a, b in zip(module_lora.lora_a, module_lora.lora_b):
|
||||
if a is not None and b is not None and a.shape[0] == global_num_experts:
|
||||
a = a[expert_start:expert_end].contiguous()
|
||||
b = b[expert_start:expert_end].contiguous()
|
||||
new_lora_a.append(a)
|
||||
new_lora_b.append(b)
|
||||
module_lora.lora_a = new_lora_a
|
||||
module_lora.lora_b = new_lora_b
|
||||
|
||||
def _get_lora_layer_weights(
|
||||
self, lora_model: LoRAModel, module_name: str
|
||||
) -> LoRALayerWeights | None:
|
||||
|
||||
Reference in New Issue
Block a user