[Bugfix][Quantization] Fix INT8 W8A8 MoE crash in TritonExperts (#51411)
Signed-off-by: Djordje Ramic <djoramic@amd.com>
This commit is contained in:
@@ -252,6 +252,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
|||||||
torch.bfloat16,
|
torch.bfloat16,
|
||||||
torch.float8_e4m3fn,
|
torch.float8_e4m3fn,
|
||||||
torch.float8_e4m3fnuz,
|
torch.float8_e4m3fnuz,
|
||||||
|
torch.int8,
|
||||||
]
|
]
|
||||||
|
|
||||||
# We declared expects_unquantized_inputs (LoRA + DP/EP all2all), so the
|
# We declared expects_unquantized_inputs (LoRA + DP/EP all2all), so the
|
||||||
@@ -298,6 +299,7 @@ class TritonExperts(LoRAExpertsMixin, mk.FusedMoEExpertsModular):
|
|||||||
elif (
|
elif (
|
||||||
hidden_states.dtype == torch.float8_e4m3fn
|
hidden_states.dtype == torch.float8_e4m3fn
|
||||||
or hidden_states.dtype == torch.float8_e4m3fnuz
|
or hidden_states.dtype == torch.float8_e4m3fnuz
|
||||||
|
or hidden_states.dtype == torch.int8
|
||||||
):
|
):
|
||||||
compute_type = tl.bfloat16
|
compute_type = tl.bfloat16
|
||||||
else:
|
else:
|
||||||
|
|||||||
Reference in New Issue
Block a user