[LoRA] Add one shot triton kernel For MoE LoRA (#42290)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Jee Jee Li
2026-05-25 19:47:04 -07:00
committed by GitHub
parent 71d810bbf4
commit ec5de7fa7d
14 changed files with 1975 additions and 85 deletions
@@ -43,6 +43,16 @@ class MoELoRAContext:
# try_get_optimal_moe_lora_config for Triton kernel tile configs.
use_tuned_config: bool
# Optional dual-stream support for overlapping each (base GEMM, LoRA)
# pair. When aux_stream is None, the experts.apply() path runs the
# original sequential schedule. When set, base GEMM runs on the default
# stream and the LoRA fast-path writes the delta into a fresh buffer on
# aux_stream, which the default stream sums in afterwards.
# Events are paired one-per-overlap-pair: events[0,1] for w13,
# events[2,3] for w2, so the two pairs do not race on the same event.
aux_stream: torch.cuda.Stream | None = None
events: tuple[torch.cuda.Event, ...] | None = None
# Per-rank token→LoRA mapping after EP dispatch. Set by
# FusedMoEPrepareAndFinalizeModular.prepare() when EP+LoRA is active, read
# by LoRAExpertsMixin helpers in place of punica_wrapper's global mapping.
@@ -45,6 +45,7 @@ class LoRAExpertsMixin:
w2: torch.Tensor,
num_tokens: int,
top_k_num: int,
add_inputs: bool = True,
) -> tuple[
torch.Tensor | None,
torch.Tensor | None,
@@ -70,6 +71,7 @@ class LoRAExpertsMixin:
lora_context.w13_num_slices,
lora_context.fully_sharded,
lora_context.use_tuned_config,
add_inputs=add_inputs,
token_lora_mapping=lora_context.local_token_lora_mapping,
)
@@ -88,6 +90,7 @@ class LoRAExpertsMixin:
w1: torch.Tensor,
w2: torch.Tensor,
top_k_num: int,
add_inputs: bool = True,
) -> None:
lora_context.punica_wrapper.add_lora_w2(
y,
@@ -109,4 +112,5 @@ class LoRAExpertsMixin:
lora_context.fully_sharded,
lora_context.tp_rank,
lora_context.use_tuned_config,
add_inputs=add_inputs,
)
@@ -48,6 +48,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
)
from vllm.platforms import current_platform
from vllm.triton_utils import tl
from vllm.utils.multi_stream_utils import maybe_execute_in_parallel
class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
@@ -247,55 +248,99 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
)
)
invoke_fused_moe_triton_kernel(
hidden_states,
w1,
intermediate_cache1,
a1q_scale if a1q_scale is not None else self.a1_scale,
self.w1_scale,
None, # topk_weights
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
False, # mul_routed_weights
top_k_num,
config,
compute_type=compute_type,
use_fp8_w8a8=self.quant_config.use_fp8_w8a8,
use_int8_w8a8=self.quant_config.use_int8_w8a8,
use_int8_w8a16=self.quant_config.use_int8_w8a16,
use_int4_w4a16=self.quant_config.use_int4_w4a16,
per_channel_quant=self.per_act_token_quant,
block_shape=self.block_shape,
B_bias=self.w1_bias,
)
# LoRA w13: applied to intermediate_cache1 before activation. When
# the LoRA layer requested a dual-stream schedule, we run base w13
# GEMM on the default stream and the LoRA fast-path on aux_stream;
# the LoRA writes its delta into a fresh zero buffer (add_inputs=
# False) and we sum it into intermediate_cache1 after both finish.
# LoRA w13: applied to intermediate_cache1 before activation, using
# hidden_states as the lora_a input. moe_lora_align_block_size is
# called once here and results reused for the w2 LoRA below.
sorted_token_ids_lora = None
expert_ids_lora = None
num_tokens_post_padded_lora = None
token_lora_mapping = None
lora_context = self._lora_context
if lora_context is not None:
def _base_w13_fn():
invoke_fused_moe_triton_kernel(
hidden_states,
w1,
intermediate_cache1,
a1q_scale if a1q_scale is not None else self.a1_scale,
self.w1_scale,
None, # topk_weights
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
False, # mul_routed_weights
top_k_num,
config,
compute_type=compute_type,
use_fp8_w8a8=self.quant_config.use_fp8_w8a8,
use_int8_w8a8=self.quant_config.use_int8_w8a8,
use_int8_w8a16=self.quant_config.use_int8_w8a16,
use_int4_w4a16=self.quant_config.use_int4_w4a16,
per_channel_quant=self.per_act_token_quant,
block_shape=self.block_shape,
B_bias=self.w1_bias,
)
if lora_context is not None and lora_context.aux_stream is not None:
# add_inputs=False: kernel overwrites lora_delta_w13. zeros (not
# empty) so untouched rows -- e.g. blocks where every program
# early-exits because lora_id<0 -- stay at zero and the trailing
# add_() is a no-op there.
lora_delta_w13 = torch.zeros_like(intermediate_cache1)
def _lora_w13_fn():
return self.apply_w13_lora(
lora_context,
y=lora_delta_w13,
x=hidden_states,
topk_ids=topk_ids,
topk_weights=topk_weights,
expert_map=expert_map,
w1=w1,
w2=w2,
num_tokens=num_tokens,
top_k_num=top_k_num,
add_inputs=False,
)
assert lora_context.events is not None
_, lora_meta = maybe_execute_in_parallel(
_base_w13_fn,
_lora_w13_fn,
lora_context.events[0],
lora_context.events[1],
lora_context.aux_stream,
)
(
sorted_token_ids_lora,
expert_ids_lora,
num_tokens_post_padded_lora,
token_lora_mapping,
) = self.apply_w13_lora(
lora_context,
y=intermediate_cache1,
x=hidden_states,
topk_ids=topk_ids,
topk_weights=topk_weights,
expert_map=expert_map,
w1=w1,
w2=w2,
num_tokens=num_tokens,
top_k_num=top_k_num,
)
) = lora_meta
intermediate_cache1.add_(lora_delta_w13)
else:
_base_w13_fn()
if lora_context is not None:
(
sorted_token_ids_lora,
expert_ids_lora,
num_tokens_post_padded_lora,
token_lora_mapping,
) = self.apply_w13_lora(
lora_context,
y=intermediate_cache1,
x=hidden_states,
topk_ids=topk_ids,
topk_weights=topk_weights,
expert_map=expert_map,
w1=w1,
w2=w2,
num_tokens=num_tokens,
top_k_num=top_k_num,
)
a2q_scale: torch.Tensor | None = None
@@ -328,48 +373,82 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
quantization_emulation=self.quantization_emulation,
)
invoke_fused_moe_triton_kernel(
qintermediate_cache2,
w2,
intermediate_cache3,
a2q_scale,
self.w2_scale,
topk_weights,
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
not apply_router_weight_on_input,
1,
config,
compute_type=compute_type,
use_fp8_w8a8=self.quant_config.use_fp8_w8a8,
use_int8_w8a8=self.quant_config.use_int8_w8a8,
use_int8_w8a16=self.quant_config.use_int8_w8a16,
use_int4_w4a16=self.quant_config.use_int4_w4a16,
per_channel_quant=self.per_act_token_quant,
block_shape=self.block_shape,
B_bias=self.w2_bias,
)
# LoRA w2: applied to intermediate_cache3 before moe_sum, using the
# unquantized intermediate_cache2 as the lora_a input. Reuses the
# sorted_token_ids_lora computed above.
if lora_context is not None:
self.apply_w2_lora(
lora_context,
y=intermediate_cache3,
x=intermediate_cache2,
topk_weights=topk_weights,
sorted_token_ids_lora=sorted_token_ids_lora,
expert_ids_lora=expert_ids_lora,
num_tokens_post_padded_lora=num_tokens_post_padded_lora,
token_lora_mapping=token_lora_mapping,
num_tokens=num_tokens,
w1=w1,
w2=w2,
top_k_num=top_k_num,
# sorted_token_ids_lora computed above. Same dual-stream pattern as
# the w13 pair: base GEMM on default stream, LoRA delta on aux,
# join via .add_() into intermediate_cache3.
def _base_w2_fn():
invoke_fused_moe_triton_kernel(
qintermediate_cache2,
w2,
intermediate_cache3,
a2q_scale,
self.w2_scale,
topk_weights,
sorted_token_ids,
expert_ids,
num_tokens_post_padded,
not apply_router_weight_on_input,
1,
config,
compute_type=compute_type,
use_fp8_w8a8=self.quant_config.use_fp8_w8a8,
use_int8_w8a8=self.quant_config.use_int8_w8a8,
use_int8_w8a16=self.quant_config.use_int8_w8a16,
use_int4_w4a16=self.quant_config.use_int4_w4a16,
per_channel_quant=self.per_act_token_quant,
block_shape=self.block_shape,
B_bias=self.w2_bias,
)
if lora_context is not None and lora_context.aux_stream is not None:
lora_delta_w2 = torch.zeros_like(intermediate_cache3)
def _lora_w2_fn():
self.apply_w2_lora(
lora_context,
y=lora_delta_w2,
x=intermediate_cache2,
topk_weights=topk_weights,
sorted_token_ids_lora=sorted_token_ids_lora,
expert_ids_lora=expert_ids_lora,
num_tokens_post_padded_lora=num_tokens_post_padded_lora,
token_lora_mapping=token_lora_mapping,
num_tokens=num_tokens,
w1=w1,
w2=w2,
top_k_num=top_k_num,
add_inputs=False,
)
assert lora_context.events is not None
maybe_execute_in_parallel(
_base_w2_fn,
_lora_w2_fn,
lora_context.events[2],
lora_context.events[3],
lora_context.aux_stream,
)
intermediate_cache3.add_(lora_delta_w2)
else:
_base_w2_fn()
if lora_context is not None:
self.apply_w2_lora(
lora_context,
y=intermediate_cache3,
x=intermediate_cache2,
topk_weights=topk_weights,
sorted_token_ids_lora=sorted_token_ids_lora,
expert_ids_lora=expert_ids_lora,
num_tokens_post_padded_lora=num_tokens_post_padded_lora,
token_lora_mapping=token_lora_mapping,
num_tokens=num_tokens,
w1=w1,
w2=w2,
top_k_num=top_k_num,
)
# separate function is required for MoE + LoRA
self.moe_sum(intermediate_cache3, output)
@@ -164,6 +164,8 @@ def _moe_forward_shared_fake(
return shared_out, fused_out
# NOTE: `moe_forward` and `moe_forward_shared` being opaque custom ops is a
# load-bearing assumption for the MoE-LoRA dual-stream path.
direct_register_custom_op(
op_name="moe_forward",
op_func=_moe_forward,