[Quantization] Let ModelOpt MXFP8 layers load pre-processed weights Purpose (#57316)
Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
@@ -144,6 +144,36 @@ def test_modelopt_mxfp8_preserves_per_row_checkpoint_scales(dist_init, monkeypat
|
||||
assert torch.equal(linear.weight_scale, scales)
|
||||
|
||||
|
||||
def test_modelopt_mxfp8_pre_processed_weights_follow_kernel(monkeypatch):
|
||||
"""The weight cache daemon can only serve MXFP8 layers whose kernel needs
|
||||
no post-load state beyond the parameters it exports."""
|
||||
from vllm.config.quantization import QuantSpec
|
||||
from vllm.model_executor.kernels.linear.mxfp8 import Mxfp8LinearKernel
|
||||
|
||||
method = ModelOptLinearMethod.__new__(ModelOptLinearMethod)
|
||||
method.spec = QuantSpec(weight=kMxfp8Static, activation=kMxfp8Dynamic)
|
||||
method.kernel = None
|
||||
assert not method.supports_pre_processed_weights
|
||||
|
||||
method.kernel = Mock(spec=Mxfp8LinearKernel)
|
||||
method.kernel.supports_pre_processed_weights = False
|
||||
assert not method.supports_pre_processed_weights
|
||||
method.kernel.supports_pre_processed_weights = True
|
||||
assert method.supports_pre_processed_weights
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.modelopt.is_weights_pre_processed",
|
||||
lambda: True,
|
||||
)
|
||||
layer = torch.nn.Module()
|
||||
method.process_weights_after_loading(layer)
|
||||
method.kernel.process_weights_after_loading.assert_not_called()
|
||||
|
||||
method.kernel.supports_pre_processed_weights = False
|
||||
with pytest.raises(RuntimeError, match="pre-processed"):
|
||||
method.process_weights_after_loading(layer)
|
||||
|
||||
|
||||
def test_modelopt_fp8_updates_weight_dims_after_transpose():
|
||||
"""Humming reads weight.input_dim/output_dim. Swapping the
|
||||
ModelWeightParameter for a plain Parameter drops them, so the per-tensor
|
||||
|
||||
Reference in New Issue
Block a user