add pad-aware reduce path (#48385)

Signed-off-by: gnovack <novackgm@gmail.com>
This commit is contained in:
gnovack
2026-07-14 18:05:50 -07:00
committed by GitHub
parent 442c421e79
commit f7aadae5e5
7 changed files with 239 additions and 34 deletions
@@ -176,9 +176,6 @@ def _fused_marlin_moe(
if output is None:
output = intermediate_cache3
if expert_map is not None:
output.zero_()
a_scales2 = None
if input_dtype == torch.int8:
intermediate_cache2, a_scales2 = marlin_quant_input(
@@ -238,7 +235,7 @@ def fused_marlin_moe(
global_num_experts: int = -1,
activation: MoEActivation = MoEActivation.SILU,
activation_func: Callable[..., None] = apply_moe_activation,
moe_sum: Callable[[torch.Tensor, torch.Tensor], None] | None = None,
moe_sum: Callable[..., torch.Tensor | None] | None = None,
expert_map: torch.Tensor | None = None,
input_global_scale1: torch.Tensor | None = None,
input_global_scale2: torch.Tensor | None = None,
@@ -385,9 +382,12 @@ def fused_marlin_moe(
output = torch.empty_like(hidden_states)
if moe_sum is None:
if expert_map is not None:
ops.moe_sum(moe_output, output, topk_ids, expert_map)
return output
return torch.sum(moe_output.view(-1, topk, K), dim=1, out=output)
else:
return moe_sum(moe_output, output)
return moe_sum(moe_output, output, topk_ids, expert_map)
def batched_fused_marlin_moe(
@@ -874,7 +874,12 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
)
lora_state["cache2"] = act_output
def moe_sum_with_lora(moe_out: torch.Tensor, out: torch.Tensor) -> None:
def moe_sum_with_lora(
moe_out: torch.Tensor,
out: torch.Tensor,
topk_ids: torch.Tensor,
expert_map: torch.Tensor | None,
) -> None:
# moe_out shape: (M, topk, K)
self.apply_w2_lora(
ctx,
@@ -890,7 +895,7 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
w2=w2,
top_k_num=top_k_num,
)
self.moe_sum(moe_out, out)
self.moe_sum(moe_out, out, topk_ids, expert_map)
return fused_marlin_moe(
hidden_states=hidden_states,
@@ -929,8 +934,17 @@ class MarlinExperts(LoRAExpertsMixin, MarlinExpertsBase):
gemm1_beta=self.gemm1_beta,
)
def moe_sum(self, input: torch.Tensor, output: torch.Tensor) -> None:
ops.moe_sum(input, output)
def moe_sum(
self,
input: torch.Tensor,
output: torch.Tensor,
topk_ids: torch.Tensor,
expert_map: torch.Tensor | None,
) -> None:
if expert_map is not None:
ops.moe_sum(input, output, topk_ids, expert_map)
else:
ops.moe_sum(input, output)
class BatchedMarlinExperts(MarlinExpertsBase):