[Quantization] Let ModelOpt MXFP8 layers load pre-processed weights Purpose (#57316)
Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
@@ -144,6 +144,36 @@ def test_modelopt_mxfp8_preserves_per_row_checkpoint_scales(dist_init, monkeypat
|
||||
assert torch.equal(linear.weight_scale, scales)
|
||||
|
||||
|
||||
def test_modelopt_mxfp8_pre_processed_weights_follow_kernel(monkeypatch):
|
||||
"""The weight cache daemon can only serve MXFP8 layers whose kernel needs
|
||||
no post-load state beyond the parameters it exports."""
|
||||
from vllm.config.quantization import QuantSpec
|
||||
from vllm.model_executor.kernels.linear.mxfp8 import Mxfp8LinearKernel
|
||||
|
||||
method = ModelOptLinearMethod.__new__(ModelOptLinearMethod)
|
||||
method.spec = QuantSpec(weight=kMxfp8Static, activation=kMxfp8Dynamic)
|
||||
method.kernel = None
|
||||
assert not method.supports_pre_processed_weights
|
||||
|
||||
method.kernel = Mock(spec=Mxfp8LinearKernel)
|
||||
method.kernel.supports_pre_processed_weights = False
|
||||
assert not method.supports_pre_processed_weights
|
||||
method.kernel.supports_pre_processed_weights = True
|
||||
assert method.supports_pre_processed_weights
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.modelopt.is_weights_pre_processed",
|
||||
lambda: True,
|
||||
)
|
||||
layer = torch.nn.Module()
|
||||
method.process_weights_after_loading(layer)
|
||||
method.kernel.process_weights_after_loading.assert_not_called()
|
||||
|
||||
method.kernel.supports_pre_processed_weights = False
|
||||
with pytest.raises(RuntimeError, match="pre-processed"):
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
def test_modelopt_fp8_updates_weight_dims_after_transpose():
|
||||
"""Humming reads weight.input_dim/output_dim. Swapping the
|
||||
ModelWeightParameter for a plain Parameter drops them, so the per-tensor
|
||||
|
||||
@@ -27,6 +27,10 @@ class Mxfp8LinearKernel(ABC):
|
||||
Marlin, emulation).
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights: bool = False
|
||||
"""True if ``process_weights_after_loading`` only rewrites parameters, so
|
||||
weights exported by the weight cache daemon can be used as-is."""
|
||||
|
||||
def __init__(self, c: Mxfp8LinearLayerConfig) -> None:
|
||||
assert self.can_implement(c)[0]
|
||||
assert self.is_supported()[0]
|
||||
|
||||
@@ -16,6 +16,8 @@ from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
class EmulationMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""Software emulation fallback for MXFP8 (dequant to BF16)."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
|
||||
@@ -27,6 +27,8 @@ from .Mxfp8LinearKernel import Mxfp8LinearKernel, Mxfp8LinearLayerConfig
|
||||
class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""MXFP8 W8A8 GEMM via FlashInfer CUTLASS (SM100+)."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
@@ -117,6 +119,8 @@ class FlashInferCutlassMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
class FlashInferCutedslMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""MXFP8 W8A8 GEMM via FlashInfer CuTe-DSL (SM100/SM103)."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
|
||||
@@ -212,6 +212,8 @@ _DOT_SCALED_K_ALIGN = 128
|
||||
class RocmDotScaledMxfp8LinearKernel(Mxfp8LinearKernel):
|
||||
"""Native CDNA4 (gfx950) MXFP8 linear via Triton ``tl.dot_scaled``."""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
@classmethod
|
||||
def is_supported(
|
||||
cls, compute_capability: int | None = None
|
||||
|
||||
@@ -2482,9 +2482,16 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
|
||||
@property
|
||||
def supports_pre_processed_weights(self) -> bool: # type: ignore[override]
|
||||
# TODO(Isotr0py): support fp8/mxfp8 ModelOpt kernels transpose/repack.
|
||||
# TODO(Isotr0py): support fp8 ModelOpt kernels transpose/repack.
|
||||
w = self.spec.weight
|
||||
return isinstance(w, QuantKey) and w.dtype == FP4_DTYPE
|
||||
return isinstance(w, QuantKey) and (
|
||||
w.dtype == FP4_DTYPE
|
||||
or (
|
||||
w == kMxfp8Static
|
||||
and self.kernel is not None
|
||||
and self.kernel.supports_pre_processed_weights
|
||||
)
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -2524,7 +2531,13 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
expose_input_quant_key(layer, self.kernel)
|
||||
|
||||
def process_weights_after_loading(self, layer) -> None:
|
||||
if self.spec.weight == kMxfp8Static and getattr(layer, "is_bmm", False):
|
||||
self.kernel = init_mxfp8_linear_kernel(bmm_batch_size=layer.bmm_batch_size)
|
||||
if is_weights_pre_processed():
|
||||
if not self.supports_pre_processed_weights:
|
||||
raise RuntimeError(
|
||||
f"{type(self.kernel).__name__} cannot use pre-processed weights"
|
||||
)
|
||||
return
|
||||
self.fmt.pre_process(layer)
|
||||
self.wkey.process(layer, WEIGHT)
|
||||
@@ -2556,8 +2569,6 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
persistent=False,
|
||||
)
|
||||
layer._nvfp4_group_size_for_gather = self.ctx.group_size
|
||||
if self.spec.weight == kMxfp8Static and getattr(layer, "is_bmm", False):
|
||||
self.kernel = init_mxfp8_linear_kernel(bmm_batch_size=layer.bmm_batch_size)
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def apply(self, layer, x, bias=None):
|
||||
|
||||
Reference in New Issue
Block a user