[Bugfix][Quantization] Fix MXFP8 startup crash on layers below mm_mxfp8 shape limits (#54223)

Signed-off-by: samuelkim7 <samuelmwkim@gmail.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Samuel Kim
2026-09-25 02:54:11 +00:00
committed by GitHub
co-authored by mgoin
parent 6936e77ba4
commit f55419c43c
16 changed files with 221 additions and 57 deletions
@@ -33,9 +33,6 @@ class CompressedTensorsW8A8Mxfp8(CompressedTensorsScheme):
- Activations dynamically quantized to MXFP8 during inference
"""
def __init__(self):
self.kernel = init_mxfp8_linear_kernel()
@classmethod
def get_min_capability(cls) -> int:
return 75
@@ -79,6 +76,8 @@ class CompressedTensorsW8A8Mxfp8(CompressedTensorsScheme):
)
layer.register_parameter("weight_scale", weight_scale)
self.kernel = init_mxfp8_linear_kernel(weight_shape=layer.weight.shape)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.kernel.process_weights_after_loading(layer)
@@ -17,9 +17,6 @@ from .inc_scheme import INCLinearScheme
class INCMxfp8LinearScheme(INCLinearScheme):
def __init__(self) -> None:
self.kernel = init_mxfp8_linear_kernel()
@classmethod
def get_min_capability(cls) -> int:
return 75
@@ -72,6 +69,8 @@ class INCMxfp8LinearScheme(INCLinearScheme):
)
layer.register_parameter("weight_scale", weight_scale)
self.kernel = init_mxfp8_linear_kernel(weight_shape=layer.weight.shape)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.kernel.process_weights_after_loading(layer)
@@ -2320,7 +2320,8 @@ def select_linear_kernel(
return init_nvfp4_linear_kernel(use_a16=spec.activation is None)
if w.scale.dtype == MXFP8_SCALE_DTYPE:
return init_mxfp8_linear_kernel(
bmm_batch_size=getattr(layer, "bmm_batch_size", None)
weight_shape=weight_shape or layer.weight.shape,
bmm_batch_size=getattr(layer, "bmm_batch_size", None),
)
# fp8 family: init_fp8 routes block-vs-plain itself off the activation key,
# and needs a real key -- weight-only fp8 is not a ModelOpt format.
@@ -2526,7 +2527,10 @@ class ModelOptLinearMethod(LinearMethodBase):
def process_weights_after_loading(self, layer) -> None:
if self.spec.weight == kMxfp8Static and getattr(layer, "is_bmm", False):
self.kernel = init_mxfp8_linear_kernel(bmm_batch_size=layer.bmm_batch_size)
self.kernel = init_mxfp8_linear_kernel(
weight_shape=(layer.weight.shape[-2], layer.weight.shape[-1]),
bmm_batch_size=layer.bmm_batch_size,
)
if is_weights_pre_processed():
if not self.supports_pre_processed_weights:
raise RuntimeError(
@@ -42,10 +42,6 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
FP8 with block-32 scales) during weight loading.
"""
def __init__(self):
super().__init__()
self.kernel = init_mxfp8_linear_kernel()
def create_weights(
self,
layer: torch.nn.Module,
@@ -73,6 +69,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
**extra_weight_attrs,
)
self.kernel = init_mxfp8_linear_kernel(weight_shape=layer.weight.shape)
def process_weights_after_loading(self, layer: Module) -> None:
if getattr(layer, "_already_called_process_weights_after_loading", False):
return