[Model] Add Inkling LoRA support [4/N] (#48884)

Signed-off-by: Woosuk Kwon <woosuk@inferact.ai>
Co-authored-by: Bugen Zhao <i@bugenzhao.com>
Co-authored-by: Giancarlo Delfin <32987265+TheEpicDolphin@users.noreply.github.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Isotr0py <mozf@inferact.ai>
Co-authored-by: Jee Jee Li <jeejeelee@inferact.ai>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Woosuk Kwon
2026-07-17 09:52:15 +08:00
committed by GitHub
co-authored by Bugen Zhao Giancarlo Delfin Isotr0py Isotr0py Jee Jee Li Roger Wang Yifan Qiao Claude Fable 5 OpenAI Codex
parent fe784ff22e
commit f3e9497e92
12 changed files with 337 additions and 47 deletions
@@ -43,6 +43,11 @@ class MoELoRAContext:
# try_get_optimal_moe_lora_config for Triton kernel tile configs.
use_tuned_config: bool
# Shared MoE LoRA: w13 lora_A and w2 lora_B are shared across all experts
# (stored collapsed with expert-dim 1). When True, LoRAExpertsMixin expands
# them to local_num_experts via a stride-0 view before the kernel.
enable_moe_shared_loras: bool = False
# Optional dual-stream support for overlapping each (base GEMM, LoRA)
# pair. When aux_stream is None, the experts.apply() path runs the
# original sequential schedule. When set, base GEMM runs on the default
@@ -52,10 +52,19 @@ class LoRAExpertsMixin:
torch.Tensor | None,
torch.Tensor | None,
]:
w13_lora_a_stacked = lora_context.w13_lora_a_stacked
if lora_context.enable_moe_shared_loras:
# w13 lora_A is shared across experts (collapsed expert-dim 1);
# broadcast to local_num_experts via a stride-0 view. The kernel
# derives num_experts from shape[1] and indexes via stride(1)==0.
w13_lora_a_stacked = tuple(
a.expand(-1, lora_context.local_num_experts, -1, -1)
for a in w13_lora_a_stacked
)
return lora_context.punica_wrapper.add_lora_w13(
y,
x,
lora_context.w13_lora_a_stacked,
w13_lora_a_stacked,
lora_context.w13_lora_b_stacked,
topk_ids,
topk_weights,
@@ -92,11 +101,19 @@ class LoRAExpertsMixin:
top_k_num: int,
add_inputs: bool = True,
) -> None:
w2_lora_b_stacked = lora_context.w2_lora_b_stacked
if lora_context.enable_moe_shared_loras:
# w2 lora_B is shared across experts (collapsed expert-dim 1);
# broadcast to local_num_experts via a stride-0 view.
w2_lora_b_stacked = tuple(
b.expand(-1, lora_context.local_num_experts, -1, -1)
for b in w2_lora_b_stacked
)
lora_context.punica_wrapper.add_lora_w2(
y,
x,
lora_context.w2_lora_a_stacked,
lora_context.w2_lora_b_stacked,
w2_lora_b_stacked,
topk_weights,
sorted_token_ids_lora,
expert_ids_lora,