[Bugfix][Quantization] Fix MXFP8 startup crash on layers below mm_mxfp8 shape limits (#54223)
Signed-off-by: samuelkim7 <samuelmwkim@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -2320,7 +2320,8 @@ def select_linear_kernel(
|
||||
return init_nvfp4_linear_kernel(use_a16=spec.activation is None)
|
||||
if w.scale.dtype == MXFP8_SCALE_DTYPE:
|
||||
return init_mxfp8_linear_kernel(
|
||||
bmm_batch_size=getattr(layer, "bmm_batch_size", None)
|
||||
weight_shape=weight_shape or layer.weight.shape,
|
||||
bmm_batch_size=getattr(layer, "bmm_batch_size", None),
|
||||
)
|
||||
# fp8 family: init_fp8 routes block-vs-plain itself off the activation key,
|
||||
# and needs a real key -- weight-only fp8 is not a ModelOpt format.
|
||||
@@ -2526,7 +2527,10 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
|
||||
def process_weights_after_loading(self, layer) -> None:
|
||||
if self.spec.weight == kMxfp8Static and getattr(layer, "is_bmm", False):
|
||||
self.kernel = init_mxfp8_linear_kernel(bmm_batch_size=layer.bmm_batch_size)
|
||||
self.kernel = init_mxfp8_linear_kernel(
|
||||
weight_shape=(layer.weight.shape[-2], layer.weight.shape[-1]),
|
||||
bmm_batch_size=layer.bmm_batch_size,
|
||||
)
|
||||
if is_weights_pre_processed():
|
||||
if not self.supports_pre_processed_weights:
|
||||
raise RuntimeError(
|
||||
|
||||
Reference in New Issue
Block a user