[MoE Refactor] W4a8 int8 oracle (#42789)
Signed-off-by: Bill Nell <bnell@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
This commit is contained in:
@@ -181,6 +181,31 @@ kInt8Static = QuantKey(INT8_DTYPE, scale=kInt8StaticGroupScale, symmetric=True)
|
||||
kInt8StaticChannelSym = QuantKey(torch.int8, kStaticChannelScale, symmetric=True)
|
||||
kInt8DynamicTokenSym = QuantKey(torch.int8, kDynamicTokenScale, symmetric=True)
|
||||
|
||||
# INT4 W4A8 quantization keys
|
||||
|
||||
# For group-wise quantization (e.g., group_size=128)
|
||||
# Note: group_size will be specified at runtime, this is a generic group scale
|
||||
kInt4W4A8StaticGroupScale128 = ScaleDesc(torch.bfloat16, True, GroupShape(1, 128))
|
||||
kInt4W4A8StaticGroup128Sym = QuantKey(
|
||||
torch.int8, kInt4W4A8StaticGroupScale128, symmetric=True
|
||||
)
|
||||
|
||||
kInt4W4A8StaticGroupScale64 = ScaleDesc(torch.bfloat16, True, GroupShape(1, 64))
|
||||
kInt4W4A8StaticGroup64Sym = QuantKey(
|
||||
torch.int8, kInt4W4A8StaticGroupScale64, symmetric=True
|
||||
)
|
||||
|
||||
kInt4W4A8StaticGroupScale32 = ScaleDesc(torch.bfloat16, True, GroupShape(1, 32))
|
||||
kInt4W4A8StaticGroup32Sym = QuantKey(
|
||||
torch.int8, kInt4W4A8StaticGroupScale32, symmetric=True
|
||||
)
|
||||
|
||||
# Generic group-wise with flexible group size (per-token groups)
|
||||
kInt4W4A8StaticGroupScale = ScaleDesc(torch.bfloat16, True, GroupShape(1, -1))
|
||||
kInt4W4A8StaticGroupSym = QuantKey(
|
||||
torch.int8, kInt4W4A8StaticGroupScale, symmetric=True
|
||||
)
|
||||
|
||||
|
||||
def create_fp8_quant_key(
|
||||
static: bool,
|
||||
|
||||
Reference in New Issue
Block a user