[CPU][Zen] Pass f32 weight scales to the zentorch INT8 MoE (#59434)
Signed-off-by: Ganesh R <Ganesh.R@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: Douglas Lehr <91553416+dllehr-amd@users.noreply.github.com>
This commit is contained in:
co-authored by
Cursor
Douglas Lehr
parent
e12b735109
commit
5addde2a0a
@@ -224,8 +224,9 @@ def test_zen_int8_dispatch_contract(
|
||||
call = mock_zentorch_fused_moe["fused_moe"]
|
||||
assert call["w13_scales"].shape == (EXPERT_NUM, 2 * INTERMEDIATE_SIZE)
|
||||
assert call["w2_scales"].shape == (EXPERT_NUM, HIDDEN_SIZE)
|
||||
assert call["w13_scales"].dtype == torch.bfloat16
|
||||
assert call["w2_scales"].dtype == torch.bfloat16
|
||||
# f32 scales keep the layer on zentorch's fast routed MoE executor.
|
||||
assert call["w13_scales"].dtype == torch.float32
|
||||
assert call["w2_scales"].dtype == torch.float32
|
||||
assert call["act"] == act.value.lower()
|
||||
assert call["skip_weighted"] is False
|
||||
assert call["topk_weights"].dtype == torch.float32
|
||||
|
||||
@@ -1502,15 +1502,17 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
"ZenCPUExpertsInt8 requires per-channel weight scales on the layer."
|
||||
)
|
||||
num_experts = self.w1_scale.shape[0]
|
||||
self._w1_scale_bf16 = (
|
||||
# ZenDNN's fast routed MoE kernel only accepts f32 expert scales; bf16
|
||||
# silently drops the layer onto the slower per-expert path.
|
||||
self._w1_scale_f32 = (
|
||||
self.w1_scale.detach()
|
||||
.to(torch.bfloat16)
|
||||
.to(torch.float32)
|
||||
.reshape(num_experts, -1)
|
||||
.contiguous()
|
||||
)
|
||||
self._w2_scale_bf16 = (
|
||||
self._w2_scale_f32 = (
|
||||
self.w2_scale.detach()
|
||||
.to(torch.bfloat16)
|
||||
.to(torch.float32)
|
||||
.reshape(num_experts, -1)
|
||||
.contiguous()
|
||||
)
|
||||
@@ -1637,8 +1639,8 @@ class ZenCPUExpertsInt8(mk.FusedMoEExpertsModular):
|
||||
topk_ids.to(torch.int32).contiguous(),
|
||||
apply_router_weight_on_input, # skip_weighted
|
||||
str(activation.value).lower(),
|
||||
self._w1_scale_bf16,
|
||||
self._w2_scale_bf16,
|
||||
self._w1_scale_f32,
|
||||
self._w2_scale_f32,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user