Early exit for MoE LoRA kernels (#27131)
Signed-off-by: gnovack <gnovack@amazon.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
@@ -1823,6 +1823,8 @@ def moe_lora_align_block_size(
|
||||
sorted_token_ids: torch.Tensor,
|
||||
experts_ids: torch.Tensor,
|
||||
num_tokens_post_pad: torch.Tensor,
|
||||
adapter_enabled: torch.Tensor,
|
||||
lora_ids: torch.Tensor,
|
||||
) -> None:
|
||||
torch.ops._moe_C.moe_lora_align_block_size(
|
||||
topk_ids,
|
||||
@@ -1835,6 +1837,8 @@ def moe_lora_align_block_size(
|
||||
sorted_token_ids,
|
||||
experts_ids,
|
||||
num_tokens_post_pad,
|
||||
adapter_enabled,
|
||||
lora_ids,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -111,6 +111,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
config["BLOCK_SIZE_M"],
|
||||
self.base_layer.local_num_experts,
|
||||
max_loras,
|
||||
self.adapter_enabled,
|
||||
expert_map,
|
||||
)
|
||||
|
||||
@@ -138,6 +139,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
max_lora_rank,
|
||||
top_k,
|
||||
config,
|
||||
self.adapter_enabled,
|
||||
)
|
||||
|
||||
result = func(*args, **kwargs)
|
||||
@@ -196,6 +198,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
max_lora_rank,
|
||||
top_k,
|
||||
config,
|
||||
self.adapter_enabled,
|
||||
True,
|
||||
)
|
||||
|
||||
@@ -227,6 +230,10 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
) -> None:
|
||||
"""Initializes lora matrices."""
|
||||
|
||||
self.adapter_enabled = torch.tensor(
|
||||
[0] * (max_loras + 1), dtype=torch.int, device=self.device
|
||||
)
|
||||
|
||||
self.w1_lora_a_stacked = torch.zeros(
|
||||
(
|
||||
max_loras,
|
||||
@@ -313,6 +320,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
self.w3_lora_b_stacked[index] = 0
|
||||
self.w2_lora_a_stacked[index] = 0
|
||||
self.w2_lora_b_stacked[index] = 0
|
||||
self.adapter_enabled[index] = 0
|
||||
|
||||
def set_lora(
|
||||
self,
|
||||
@@ -322,8 +330,9 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
embeddings_tensor: torch.Tensor | None,
|
||||
bias: torch.Tensor | None = None,
|
||||
):
|
||||
self.reset_lora(index)
|
||||
"""Overwrites lora tensors at index."""
|
||||
self.reset_lora(index)
|
||||
self.adapter_enabled[index] = 1
|
||||
for eid in range(len(lora_a) // 3):
|
||||
w1_lora_a = lora_a[eid * 3]
|
||||
w2_lora_a = lora_a[eid * 3 + 1]
|
||||
|
||||
@@ -54,6 +54,8 @@ def _fused_moe_lora_kernel(
|
||||
EM,
|
||||
num_valid_tokens,
|
||||
num_experts,
|
||||
lora_ids,
|
||||
adapter_enabled,
|
||||
# The stride variables represent how much to increase the ptr by when
|
||||
# moving by 1 element in a particular dimension. E.g. `stride_am` is
|
||||
# how much to increase `a_ptr` by to get the element one row down
|
||||
@@ -84,6 +86,11 @@ def _fused_moe_lora_kernel(
|
||||
pid = tl.program_id(axis=0)
|
||||
slice_id = tl.program_id(axis=1)
|
||||
lora_idx = tl.program_id(axis=2)
|
||||
lora_id = tl.load(lora_ids + lora_idx)
|
||||
moe_enabled = tl.load(adapter_enabled + lora_id)
|
||||
if lora_id == -1 or moe_enabled == 0:
|
||||
# Early exit for the no-lora case.
|
||||
return
|
||||
max_loras = tl.num_programs(axis=2)
|
||||
grid_k = tl.cdiv(K, BLOCK_SIZE_K * SPLIT_K)
|
||||
|
||||
@@ -100,12 +107,12 @@ def _fused_moe_lora_kernel(
|
||||
pid_m = first_pid_m + ((pid_m_n % num_pid_in_group) % group_size_m)
|
||||
pid_n = (pid_m_n % num_pid_in_group) // group_size_m
|
||||
|
||||
num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr + lora_idx)
|
||||
num_tokens_post_padded = tl.load(num_tokens_post_padded_ptr + lora_id)
|
||||
if pid_m * BLOCK_SIZE_M >= num_tokens_post_padded:
|
||||
return
|
||||
|
||||
# get the expert_id to process curr shard
|
||||
ind = lora_idx * stride_el + pid_m
|
||||
ind = lora_id * stride_el + pid_m
|
||||
expert_id = tl.load(expert_ids_ptr + ind, ind < max_loras * stride_el, -1)
|
||||
if expert_id == -1:
|
||||
return
|
||||
@@ -119,7 +126,7 @@ def _fused_moe_lora_kernel(
|
||||
offs_k = pid_sk * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K)
|
||||
|
||||
offs_token_id = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M).to(tl.int64)
|
||||
token_ind = stride_tl * lora_idx + offs_token_id
|
||||
token_ind = stride_tl * lora_id + offs_token_id
|
||||
offs_token = tl.load(
|
||||
sorted_token_ids_ptr + token_ind, token_ind < max_loras * stride_tl, 0
|
||||
)
|
||||
@@ -132,7 +139,7 @@ def _fused_moe_lora_kernel(
|
||||
|
||||
b_ptrs = (
|
||||
cur_b_ptr
|
||||
+ lora_idx * stride_bl
|
||||
+ lora_id * stride_bl
|
||||
+ expert_id * stride_be
|
||||
+ offs_k[:, None] * stride_bk
|
||||
+ offs_bn[None, :] * stride_bn
|
||||
@@ -184,6 +191,8 @@ def _fused_moe_lora(
|
||||
num_tokens_post_padded: torch.Tensor, # (max_loras, )
|
||||
max_lora_rank: int,
|
||||
top_k_num: int,
|
||||
lora_ids: torch.Tensor,
|
||||
adapter_enabled: torch.Tensor,
|
||||
block_size_m: int,
|
||||
block_size_n: int,
|
||||
block_size_k: int,
|
||||
@@ -234,7 +243,7 @@ def _fused_moe_lora(
|
||||
num_tokens = M * top_k_num
|
||||
w1_output_dim_size = w1_lora_b_stacked.shape[2]
|
||||
|
||||
lora_intermediate_cache1 = torch.empty(
|
||||
lora_intermediate_cache1 = torch.zeros(
|
||||
(num_slices * M * top_k_num * (max_lora_rank + w1_output_dim_size)),
|
||||
dtype=output.dtype,
|
||||
device=device,
|
||||
@@ -272,6 +281,8 @@ def _fused_moe_lora(
|
||||
EM,
|
||||
num_tokens,
|
||||
num_experts,
|
||||
lora_ids,
|
||||
adapter_enabled,
|
||||
qcurr_hidden_states.stride(0),
|
||||
qcurr_hidden_states.stride(1),
|
||||
w1_lora_a_stacked.stride(0),
|
||||
@@ -319,6 +330,8 @@ def _fused_moe_lora(
|
||||
EM,
|
||||
num_tokens,
|
||||
num_experts,
|
||||
lora_ids,
|
||||
adapter_enabled,
|
||||
a_intermediate_cache1.stride(0),
|
||||
a_intermediate_cache1.stride(1),
|
||||
w1_lora_b_stacked.stride(0),
|
||||
@@ -352,6 +365,8 @@ def _fused_moe_lora_fake(
|
||||
num_tokens_post_padded: torch.Tensor,
|
||||
max_lora_rank: int,
|
||||
top_k_num: int,
|
||||
lora_ids: torch.Tensor,
|
||||
adapter_enabled: torch.Tensor,
|
||||
block_size_m: int,
|
||||
block_size_n: int,
|
||||
block_size_k: int,
|
||||
|
||||
@@ -456,6 +456,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
block_size: int,
|
||||
num_experts: int,
|
||||
max_loras: int,
|
||||
adapter_enabled: torch.Tensor,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
pad_sorted_ids: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
@@ -479,6 +480,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
max_lora_rank: int,
|
||||
top_k_num: int,
|
||||
config,
|
||||
adapter_enabled: torch.Tensor,
|
||||
mul_routed_weight=False,
|
||||
):
|
||||
"""
|
||||
|
||||
@@ -305,6 +305,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
block_size: int,
|
||||
num_experts: int,
|
||||
max_loras: int,
|
||||
adapter_enabled: torch.Tensor,
|
||||
expert_map: torch.Tensor | None = None,
|
||||
pad_sorted_ids: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
@@ -331,7 +332,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
(max_loras), dtype=torch.int32, device=topk_ids.device
|
||||
)
|
||||
|
||||
(token_lora_mapping, _, _, _, _, _) = self.token_mapping_meta.meta_args(
|
||||
(token_lora_mapping, _, _, _, lora_ids, _) = self.token_mapping_meta.meta_args(
|
||||
num_tokens
|
||||
)
|
||||
|
||||
@@ -346,6 +347,8 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
sorted_ids,
|
||||
expert_ids,
|
||||
num_tokens_post_pad,
|
||||
adapter_enabled,
|
||||
lora_ids,
|
||||
)
|
||||
if expert_map is not None:
|
||||
expert_ids = expert_map[expert_ids]
|
||||
@@ -365,11 +368,13 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
max_lora_rank: int,
|
||||
top_k_num: int,
|
||||
config,
|
||||
adapter_enabled: torch.Tensor,
|
||||
mul_routed_weight=False,
|
||||
):
|
||||
"""
|
||||
Performs a fused forward computation for LoRA of Mixture-of-Experts (MoE) layer.
|
||||
"""
|
||||
(_, _, _, _, lora_ids, _) = self.token_mapping_meta.meta_args(x.size(0))
|
||||
fused_moe_lora(
|
||||
y,
|
||||
x,
|
||||
@@ -381,6 +386,8 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
num_tokens_post_padded,
|
||||
max_lora_rank,
|
||||
top_k_num,
|
||||
lora_ids,
|
||||
adapter_enabled,
|
||||
config["BLOCK_SIZE_M"],
|
||||
config["BLOCK_SIZE_N"],
|
||||
config["BLOCK_SIZE_K"],
|
||||
|
||||
Reference in New Issue
Block a user