[ROCm][Quantization] GPT_OSS in amd-quark format model loading and emulations (#29008)
Signed-off-by: xuebwang-amd <xuebwang@amd.com> Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
co-authored by
Robert Shaw
Robert Shaw
parent
599e4335a4
commit
b129136c7a
@@ -20,26 +20,44 @@ SUPPORTED_OCP_MX_DTYPES = {"mxfp4", "mxfp6_e3m2", "mxfp6_e2m3"}
|
||||
|
||||
|
||||
class OCP_MX_Scheme(str, Enum):
|
||||
w_mxfp4 = "w_mxfp4"
|
||||
w_mxfp4_a_mxfp4 = "w_mxfp4_a_mxfp4"
|
||||
w_mxfp4_a_mxfp6_e3m2 = "w_mxfp4_a_mxfp6_e3m2"
|
||||
w_mxfp4_a_mxfp6_e2m3 = "w_mxfp4_a_mxfp6_e2m3"
|
||||
w_mxfp4_a_fp8 = "w_mxfp4_a_fp8"
|
||||
w_mxfp6_e3m2 = "w_mxfp6_e3m2"
|
||||
w_mxfp6_e3m2_a_mxfp6_e3m2 = "w_mxfp6_e3m2_a_mxfp6_e3m2"
|
||||
w_mxfp6_e3m2_a_fp8 = "w_mxfp6_e3m2_a_fp8"
|
||||
w_mxfp6_e2m3 = "w_mxfp6_e2m3"
|
||||
w_mxfp6_e2m3_a_mxfp6_e2m3 = "w_mxfp6_e2m3_a_mxfp6_e2m3"
|
||||
w_mxfp6_e2m3_a_fp8 = "w_mxfp6_e2m3_a_fp8"
|
||||
|
||||
@classmethod
|
||||
def from_quant_dtype(cls, input_dtype: str | None, weight_dtype: str | None):
|
||||
if input_dtype not in OCP_MX_DTYPES or weight_dtype not in OCP_MX_DTYPES:
|
||||
if input_dtype not in OCP_MX_DTYPES and weight_dtype not in OCP_MX_DTYPES:
|
||||
return None
|
||||
elif input_dtype is None and weight_dtype == "mxfp4":
|
||||
return cls.w_mxfp4
|
||||
elif input_dtype is None and weight_dtype == "mxfp6_e3m2":
|
||||
return cls.w_mxfp6_e3m2
|
||||
elif input_dtype is None and weight_dtype == "mxfp6_e2m3":
|
||||
return cls.w_mxfp6_e2m3
|
||||
elif input_dtype == "mxfp4" and weight_dtype == "mxfp4":
|
||||
return cls.w_mxfp4_a_mxfp4
|
||||
elif input_dtype == "mxfp6_e3m2" and weight_dtype == "mxfp4":
|
||||
return cls.w_mxfp4_a_mxfp6_e3m2
|
||||
elif input_dtype == "mxfp6_e2m3" and weight_dtype == "mxfp4":
|
||||
return cls.w_mxfp4_a_mxfp6_e2m3
|
||||
elif input_dtype == "fp8" and weight_dtype == "mxfp4":
|
||||
return cls.w_mxfp4_a_fp8
|
||||
elif input_dtype == "mxfp6_e3m2" and weight_dtype == "mxfp6_e3m2":
|
||||
return cls.w_mxfp6_e3m2_a_mxfp6_e3m2
|
||||
elif input_dtype == "fp8" and weight_dtype == "mxfp6_e3m2":
|
||||
return cls.w_mxfp6_e3m2_a_fp8
|
||||
elif input_dtype == "mxfp6_e2m3" and weight_dtype == "mxfp6_e2m3":
|
||||
return cls.w_mxfp6_e2m3_a_mxfp6_e2m3
|
||||
elif input_dtype == "fp8" and weight_dtype == "mxfp6_e2m3":
|
||||
return cls.w_mxfp6_e2m3_a_fp8
|
||||
else:
|
||||
logger.warning(
|
||||
"input_dtype='%s' and"
|
||||
|
||||
Reference in New Issue
Block a user