[Bugfix] Fix DSV4 swiglu_limit on marlin backend (#42287)
Signed-off-by: Jee Jee Li <jeejeelee@inferact.ai> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -413,6 +413,7 @@ def batched_fused_marlin_moe(
|
|||||||
is_k_full: bool = True,
|
is_k_full: bool = True,
|
||||||
output: torch.Tensor | None = None,
|
output: torch.Tensor | None = None,
|
||||||
inplace: bool = False,
|
inplace: bool = False,
|
||||||
|
clamp_limit: float | None = None,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""
|
"""
|
||||||
This function massages the inputs so the batched hidden_states can be
|
This function massages the inputs so the batched hidden_states can be
|
||||||
@@ -535,6 +536,7 @@ def batched_fused_marlin_moe(
|
|||||||
intermediate_cache2=intermediate_cache2,
|
intermediate_cache2=intermediate_cache2,
|
||||||
output=output.view(-1, K) if output is not None else output,
|
output=output.view(-1, K) if output is not None else output,
|
||||||
is_k_full=is_k_full,
|
is_k_full=is_k_full,
|
||||||
|
clamp_limit=clamp_limit,
|
||||||
)
|
)
|
||||||
|
|
||||||
output = output.view(B, BATCH_TOKENS_MAX, K)
|
output = output.view(B, BATCH_TOKENS_MAX, K)
|
||||||
@@ -768,6 +770,7 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
|
|||||||
sort_indices2=self.w2_g_idx_sort_indices,
|
sort_indices2=self.w2_g_idx_sort_indices,
|
||||||
is_k_full=self.is_k_full,
|
is_k_full=self.is_k_full,
|
||||||
input_dtype=self.input_dtype,
|
input_dtype=self.input_dtype,
|
||||||
|
clamp_limit=self.gemm1_clamp_limit,
|
||||||
)
|
)
|
||||||
return
|
return
|
||||||
|
|
||||||
@@ -970,4 +973,5 @@ class BatchedMarlinExperts(MarlinExpertsBase):
|
|||||||
sort_indices1=self.w13_g_idx_sort_indices,
|
sort_indices1=self.w13_g_idx_sort_indices,
|
||||||
sort_indices2=self.w2_g_idx_sort_indices,
|
sort_indices2=self.w2_g_idx_sort_indices,
|
||||||
is_k_full=self.is_k_full,
|
is_k_full=self.is_k_full,
|
||||||
|
clamp_limit=self.gemm1_clamp_limit,
|
||||||
)
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user