[Bugfix][Quantization] Register Quark per-block FP8 scales as weight_scale (#54770)
Signed-off-by: jimmy-adams <41593649+jimmy-adams@users.noreply.github.com> Co-authored-by: fxmarty-amd <felmarty@amd.com> Co-authored-by: Shanshan Shen <467638484@qq.com>
This commit is contained in:
co-authored by
fxmarty-amd
Shanshan Shen
parent
6cbb3c154e
commit
385ba6b5b0
@@ -11,6 +11,7 @@ import importlib.metadata
|
||||
from dataclasses import dataclass
|
||||
from importlib.util import find_spec
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import huggingface_hub
|
||||
import lm_eval
|
||||
@@ -702,6 +703,68 @@ def enable_pickle(monkeypatch):
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
|
||||
def test_quark_w8a8_fp8_per_block_registers_weight_scale(monkeypatch):
|
||||
from vllm.model_executor.layers.quantization.utils.fp8_utils import (
|
||||
get_fp8_block_weight_scale,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.model_executor.layers.quantization.quark.schemes."
|
||||
"quark_w8a8_fp8.get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=SimpleNamespace(dtype=torch.bfloat16)),
|
||||
)
|
||||
scheme = QuarkW8A8Fp8PerBlock(kFp8Static128BlockSym, kFp8Dynamic128Sym)
|
||||
|
||||
layer = torch.nn.Module()
|
||||
layer.weight_scale = torch.tensor([2.0])
|
||||
assert get_fp8_block_weight_scale(layer) is None
|
||||
layer.scheme = scheme
|
||||
assert get_fp8_block_weight_scale(layer) is layer.weight_scale
|
||||
layer.weight_scale_inv = torch.tensor([3.0])
|
||||
assert get_fp8_block_weight_scale(layer) is layer.weight_scale
|
||||
layer.scheme = None
|
||||
assert get_fp8_block_weight_scale(layer) is layer.weight_scale_inv
|
||||
|
||||
loaded = torch.nn.Module()
|
||||
|
||||
def weight_loader(param, loaded_weight):
|
||||
return None
|
||||
|
||||
dummy_param = torch.nn.Parameter(torch.empty(1), requires_grad=False)
|
||||
with (
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8."
|
||||
"validate_fp8_block_shape"
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8."
|
||||
"create_fp8_weight_parameter",
|
||||
return_value=dummy_param,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8."
|
||||
"create_fp8_scale_parameter",
|
||||
return_value=dummy_param,
|
||||
),
|
||||
patch(
|
||||
"vllm.model_executor.layers.quantization.quark.schemes.quark_w8a8_fp8."
|
||||
"init_fp8_linear_kernel",
|
||||
return_value=MagicMock(),
|
||||
),
|
||||
):
|
||||
scheme.create_weights(
|
||||
loaded,
|
||||
output_partition_sizes=[256],
|
||||
input_size_per_partition=256,
|
||||
params_dtype=torch.bfloat16,
|
||||
weight_loader=weight_loader,
|
||||
input_size=256,
|
||||
output_size=256,
|
||||
)
|
||||
assert hasattr(loaded, "weight_scale")
|
||||
assert not hasattr(loaded, "weight_scale_inv")
|
||||
|
||||
|
||||
def test_quark_config_has_no_model_specific_fused_mappings():
|
||||
config = QuarkConfig({})
|
||||
|
||||
|
||||
Reference in New Issue
Block a user