[Quantization] Let ModelOpt MXFP8 layers load pre-processed weights Purpose (#57316)
Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
@@ -27,6 +27,10 @@ class Mxfp8LinearKernel(ABC):
|
||||
Marlin, emulation).
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights: bool = False
|
||||
"""True if ``process_weights_after_loading`` only rewrites parameters, so
|
||||
weights exported by the weight cache daemon can be used as-is."""
|
||||
|
||||
def __init__(self, c: Mxfp8LinearLayerConfig) -> None:
|
||||
assert self.can_implement(c)[0]
|
||||
assert self.is_supported()[0]
|
||||
|
||||
@@ -16,6 +16,8 @@ from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
class EmulationMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""Software emulation fallback for MXFP8 (dequant to BF16)."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
|
||||
@@ -27,6 +27,8 @@ from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""MXFP8 W8A8 GEMM via FlashInfer CUTLASS (SM100+)."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
@@ -117,6 +119,8 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""MXFP8 W8A8 GEMM via FlashInfer CuTe-DSL (SM100/SM103)."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
|
||||
@@ -212,6 +212,8 @@ _DOT_SCALED_K_ALIGN = 128
|
||||
class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
|
||||
Reference in New Issue
Block a user