[Model] Add Inkling LoRA support [4/N] (#48884)
Signed-off-by: Woosuk Kwon <woosuk@inferact.ai> Co-authored-by: Bugen Zhao <i@bugenzhao.com> Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Isotr0py <mozf@inferact.ai> Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
Bugen Zhao
Giancarlo Delfin
Isotr0py
Isotr0py
Jee Jee Li
Roger Wang
Yifan Qiao
Claude Fable 5
OpenAI Codex
parent
fe784ff22e
commit
f3e9497e92
@@ -43,6 +43,11 @@ class MoELoRAContext:
|
||||
# try_get_optimal_moe_lora_config for Triton kernel tile configs.
|
||||
use_tuned_config: bool
|
||||
|
||||
# Shared MoE LoRA: w13 lora_A and w2 lora_B are shared across all experts
|
||||
# (stored collapsed with expert-dim 1). When True, LoRAExpertsMixin expands
|
||||
# them to local_num_experts via a stride-0 view before the kernel.
|
||||
enable_moe_shared_loras: bool = False
|
||||
|
||||
# Optional dual-stream support for overlapping each (base GEMM, LoRA)
|
||||
# pair. When aux_stream is None, the experts.apply() path runs the
|
||||
# original sequential schedule. When set, base GEMM runs on the default
|
||||
|
||||
@@ -52,10 +52,19 @@ class LoRAExpertsMixin:
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
]:
|
||||
w13_lora_a_stacked = lora_context.w13_lora_a_stacked
|
||||
if lora_context.enable_moe_shared_loras:
|
||||
# w13 lora_A is shared across experts (collapsed expert-dim 1);
|
||||
# broadcast to local_num_experts via a stride-0 view. The kernel
|
||||
# derives num_experts from shape[1] and indexes via stride(1)==0.
|
||||
w13_lora_a_stacked = tuple(
|
||||
a.expand(-1, lora_context.local_num_experts, -1, -1)
|
||||
for a in w13_lora_a_stacked
|
||||
)
|
||||
return lora_context.punica_wrapper.add_lora_w13(
|
||||
y,
|
||||
x,
|
||||
lora_context.w13_lora_a_stacked,
|
||||
w13_lora_a_stacked,
|
||||
lora_context.w13_lora_b_stacked,
|
||||
topk_ids,
|
||||
topk_weights,
|
||||
@@ -92,11 +101,19 @@ class LoRAExpertsMixin:
|
||||
top_k_num: int,
|
||||
add_inputs: bool = True,
|
||||
) -> None:
|
||||
w2_lora_b_stacked = lora_context.w2_lora_b_stacked
|
||||
if lora_context.enable_moe_shared_loras:
|
||||
# w2 lora_B is shared across experts (collapsed expert-dim 1);
|
||||
# broadcast to local_num_experts via a stride-0 view.
|
||||
w2_lora_b_stacked = tuple(
|
||||
b.expand(-1, lora_context.local_num_experts, -1, -1)
|
||||
for b in w2_lora_b_stacked
|
||||
)
|
||||
lora_context.punica_wrapper.add_lora_w2(
|
||||
y,
|
||||
x,
|
||||
lora_context.w2_lora_a_stacked,
|
||||
lora_context.w2_lora_b_stacked,
|
||||
w2_lora_b_stacked,
|
||||
topk_weights,
|
||||
sorted_token_ids_lora,
|
||||
expert_ids_lora,
|
||||
|
||||
Reference in New Issue
Block a user