[Bugfix][Quantization] Fix MXFP8 startup crash on layers below mm_mxfp8 shape limits (#54223)
Signed-off-by: samuelkim7 <samuelmwkim@gmail.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
@@ -880,10 +880,15 @@ def choose_mp_linear_kernel(
|
||||
)
|
||||
|
||||
|
||||
def init_mxfp8_linear_kernel(*, bmm_batch_size: int | None = None) -> Mxfp8LinearKernel:
|
||||
def init_mxfp8_linear_kernel(
|
||||
*, weight_shape: tuple[int, int], bmm_batch_size: int | None = None
|
||||
) -> Mxfp8LinearKernel:
|
||||
"""Select and instantiate the best MXFP8 linear kernel for the
|
||||
current platform."""
|
||||
config = Mxfp8LinearLayerConfig(bmm_batch_size=bmm_batch_size)
|
||||
current platform and `(N, K)` weight shape."""
|
||||
config = Mxfp8LinearLayerConfig(
|
||||
weight_shape=weight_shape,
|
||||
bmm_batch_size=bmm_batch_size,
|
||||
)
|
||||
|
||||
platform = current_platform._enum
|
||||
possible: list[type[Mxfp8LinearKernel]]
|
||||
|
||||
@@ -15,8 +15,13 @@ class Mxfp8LinearLayerConfig:
|
||||
|
||||
All MXFP8 layers share the same structure: FP8-E4M3 weights with
|
||||
uint8 (E8M0) per-block scales at block size 32.
|
||||
|
||||
Attributes:
|
||||
weight_shape: The layer's `(out_features, in_features)`, i.e. `(N, K)`.
|
||||
|
||||
"""
|
||||
|
||||
weight_shape: tuple[int, int]
|
||||
bmm_batch_size: int | None = None
|
||||
|
||||
|
||||
|
||||
@@ -24,6 +24,20 @@ from vllm.utils.flashinfer import has_flashinfer, has_flashinfer_cutedsl
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
|
||||
|
||||
def _check_mm_mxfp8_shape(weight_shape: tuple[int, int]) -> tuple[bool, str | None]:
|
||||
N, K = weight_shape
|
||||
if K < 128:
|
||||
return False, f"mm_mxfp8 requires K >= 128, got K={K}."
|
||||
if K % MXFP8_BLOCK_SIZE != 0:
|
||||
return (
|
||||
False,
|
||||
f"mm_mxfp8 requires K to be divisible by {MXFP8_BLOCK_SIZE}, got K={K}.",
|
||||
)
|
||||
if N < 128:
|
||||
return False, f"mm_mxfp8 requires N >= 128, got N={N}."
|
||||
return True, None
|
||||
|
||||
|
||||
class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""MXFP8 W8A8 GEMM via FlashInfer CUTLASS (SM100+)."""
|
||||
|
||||
@@ -43,6 +57,15 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
can_implement, reason = _check_mm_mxfp8_shape(c.weight_shape)
|
||||
if not can_implement:
|
||||
return False, reason
|
||||
N, _ = c.weight_shape
|
||||
if current_platform.is_device_capability_family(120) and N % 32 != 0:
|
||||
return (
|
||||
False,
|
||||
f"mm_mxfp8 on SM12x requires N to be divisible by 32, got N={N}.",
|
||||
)
|
||||
return True, None
|
||||
|
||||
def input_quant_key(self) -> QuantKey:
|
||||
@@ -72,20 +95,6 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
weight_scale = layer.weight_scale
|
||||
N, K = weight.shape
|
||||
|
||||
min_dim = 128
|
||||
|
||||
assert min_dim <= K, (
|
||||
f"mm_mxfp8 requires K >= {min_dim}, got K={K}. "
|
||||
f"in_features is too small for mm_mxfp8."
|
||||
)
|
||||
assert K % MXFP8_BLOCK_SIZE == 0, (
|
||||
f"mm_mxfp8 requires K to be divisible by {MXFP8_BLOCK_SIZE}, got K={K}."
|
||||
)
|
||||
assert min_dim <= N, (
|
||||
f"mm_mxfp8 requires N >= {min_dim}, got N={N}. "
|
||||
f"out_features is too small for mm_mxfp8."
|
||||
)
|
||||
|
||||
qa = as_quantized_activation(x, self.input_quant_key())
|
||||
if qa is not None:
|
||||
input_mxfp8, input_scale = qa.data, qa.scale
|
||||
@@ -136,7 +145,7 @@ class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
return True, None
|
||||
return _check_mm_mxfp8_shape(c.weight_shape)
|
||||
|
||||
def input_quant_key(self) -> QuantKey:
|
||||
# Activations use FlashInfer's F8_128x4 swizzled scale layout.
|
||||
@@ -166,20 +175,6 @@ class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
weight_scale = layer.weight_scale
|
||||
K, N = weight.shape
|
||||
|
||||
min_dim = 128
|
||||
|
||||
assert min_dim <= K, (
|
||||
f"mm_mxfp8 requires K >= {min_dim}, got K={K}. "
|
||||
f"in_features is too small for mm_mxfp8."
|
||||
)
|
||||
assert K % MXFP8_BLOCK_SIZE == 0, (
|
||||
f"mm_mxfp8 requires K to be divisible by {MXFP8_BLOCK_SIZE}, got K={K}."
|
||||
)
|
||||
assert min_dim <= N, (
|
||||
f"mm_mxfp8 requires N >= {min_dim}, got N={N}. "
|
||||
f"out_features is too small for mm_mxfp8."
|
||||
)
|
||||
|
||||
qa = as_quantized_activation(x, self.input_quant_key())
|
||||
if qa is not None:
|
||||
input_mxfp8, input_scale = qa.data, qa.scale
|
||||
|
||||
@@ -3,6 +3,10 @@
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
)
|
||||
|
||||
from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
|
||||
|
||||
@@ -23,6 +27,16 @@ class MarlinMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
|
||||
@classmethod
|
||||
def can_implement(cls, c: Mxfp8LinearLayerConfig) -> tuple[bool, str | None]:
|
||||
N, K = c.weight_shape
|
||||
if N <= 0 or K <= 0:
|
||||
return False, f"Marlin MXFP8 requires positive N and K, got N={N}, K={K}."
|
||||
if K % MXFP8_BLOCK_SIZE != 0:
|
||||
return (
|
||||
False,
|
||||
f"Marlin MXFP8 requires K to be divisible by {MXFP8_BLOCK_SIZE}, "
|
||||
f"got K={K}.",
|
||||
)
|
||||
# Tile misalignment is fixed by zero-padding at weight prep.
|
||||
return True, None
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
|
||||
+2
-3
@@ -33,9 +33,6 @@ class CompressedTensorsW8A8Mxfp8(CompressedTensorsScheme):
|
||||
- Activations dynamically quantized to MXFP8 during inference
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.kernel = init_mxfp8_linear_kernel()
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 75
|
||||
@@ -79,6 +76,8 @@ class CompressedTensorsW8A8Mxfp8(CompressedTensorsScheme):
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
self.kernel = init_mxfp8_linear_kernel(weight_shape=layer.weight.shape)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
@@ -17,9 +17,6 @@ from .inc_scheme import INCLinearScheme
|
||||
|
||||
|
||||
class INCMxfp8LinearScheme(INCLinearScheme):
|
||||
def __init__(self) -> None:
|
||||
self.kernel = init_mxfp8_linear_kernel()
|
||||
|
||||
@classmethod
|
||||
def get_min_capability(cls) -> int:
|
||||
return 75
|
||||
@@ -72,6 +69,8 @@ class INCMxfp8LinearScheme(INCLinearScheme):
|
||||
)
|
||||
layer.register_parameter("weight_scale", weight_scale)
|
||||
|
||||
self.kernel = init_mxfp8_linear_kernel(weight_shape=layer.weight.shape)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
@@ -2320,7 +2320,8 @@ def select_linear_kernel(
|
||||
return init_nvfp4_linear_kernel(use_a16=spec.activation is None)
|
||||
if w.scale.dtype == MXFP8_SCALE_DTYPE:
|
||||
return init_mxfp8_linear_kernel(
|
||||
bmm_batch_size=getattr(layer, "bmm_batch_size", None)
|
||||
weight_shape=weight_shape or layer.weight.shape,
|
||||
bmm_batch_size=getattr(layer, "bmm_batch_size", None),
|
||||
)
|
||||
# fp8 family: init_fp8 routes block-vs-plain itself off the activation key,
|
||||
# and needs a real key -- weight-only fp8 is not a ModelOpt format.
|
||||
@@ -2526,7 +2527,10 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
|
||||
def process_weights_after_loading(self, layer) -> None:
|
||||
if self.spec.weight == kMxfp8Static and getattr(layer, "is_bmm", False):
|
||||
self.kernel = init_mxfp8_linear_kernel(bmm_batch_size=layer.bmm_batch_size)
|
||||
self.kernel = init_mxfp8_linear_kernel(
|
||||
weight_shape=(layer.weight.shape[-2], layer.weight.shape[-1]),
|
||||
bmm_batch_size=layer.bmm_batch_size,
|
||||
)
|
||||
if is_weights_pre_processed():
|
||||
if not self.supports_pre_processed_weights:
|
||||
raise RuntimeError(
|
||||
|
||||
@@ -42,10 +42,6 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
FP8 with block-32 scales) during weight loading.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.kernel = init_mxfp8_linear_kernel()
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
@@ -73,6 +69,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
|
||||
self.kernel = init_mxfp8_linear_kernel(weight_shape=layer.weight.shape)
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user