[Quantization] Layer re-quantization for linear layers through online quantization API (MXFP8 -> FP8 PTPC showcase) (#55684)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
vllmellm
Tan Pin Siang
Misha Goin
parent
29f955cce0
commit
f8d92cae41
@@ -58,6 +58,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptFp8Config,
|
||||
ModelOptLinearMethod,
|
||||
ModelOptMxFp8Config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
@@ -93,12 +94,14 @@ from vllm.model_executor.layers.quantization.quark.quark import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils import quant_utils
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
amax_for_moe_weight_quant,
|
||||
amax_for_tp_weight_quant,
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
weight_amax,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
@@ -112,6 +115,7 @@ from vllm.model_executor.models.granitemoe import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx942, on_gfx950
|
||||
@@ -307,6 +311,14 @@ def _fully_quantized_modelopt_config() -> ModelOptFp8Config:
|
||||
)
|
||||
|
||||
|
||||
def _fully_quantized_modelopt_mxfp8_config() -> ModelOptMxFp8Config:
|
||||
return ModelOptMxFp8Config(
|
||||
is_checkpoint_mxfp8_serialized=True,
|
||||
kv_cache_quant_algo=None,
|
||||
exclude_modules=["lm_head"],
|
||||
)
|
||||
|
||||
|
||||
def _moe_only_compressed_tensors_config() -> CompressedTensorsConfig:
|
||||
return CompressedTensorsConfig(
|
||||
target_scheme_map={"RoutedExperts": {}},
|
||||
@@ -334,10 +346,19 @@ def _write_minimal_llama_config(
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checkpoint_config_factory,raises_conflict",
|
||||
"checkpoint_config_factory,raises_requantization_error",
|
||||
[
|
||||
pytest.param(_fully_quantized_quark_config, True, id="quark"),
|
||||
pytest.param(_fully_quantized_modelopt_config, True, id="modelopt"),
|
||||
pytest.param(
|
||||
_fully_quantized_modelopt_config,
|
||||
True,
|
||||
id="modelopt",
|
||||
),
|
||||
pytest.param(
|
||||
_fully_quantized_modelopt_mxfp8_config,
|
||||
False,
|
||||
id="modelopt_mxfp8",
|
||||
),
|
||||
pytest.param(
|
||||
_moe_only_compressed_tensors_config,
|
||||
False,
|
||||
@@ -345,9 +366,13 @@ def _write_minimal_llama_config(
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_rocm()),
|
||||
reason="MXFP8 requantization requires CUDA or ROCm.",
|
||||
)
|
||||
def test_online_prequantized_compatibility(
|
||||
checkpoint_config_factory,
|
||||
raises_conflict: bool,
|
||||
raises_requantization_error: bool,
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
) -> None:
|
||||
@@ -370,35 +395,69 @@ def test_online_prequantized_compatibility(
|
||||
"disable_tp": True,
|
||||
}
|
||||
|
||||
if raises_conflict:
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
ColumnParallelLinear(**layer_kwargs)
|
||||
layer = ColumnParallelLinear(**layer_kwargs)
|
||||
assert isinstance(layer.quant_method, Mxfp8OnlineLinearMethod)
|
||||
|
||||
if layer.weight.is_meta:
|
||||
# Weight loading.
|
||||
layer = layer.to_empty(device=DEVICE)
|
||||
for parameter in layer.parameters():
|
||||
parameter.zero_()
|
||||
else:
|
||||
layer = ColumnParallelLinear(**layer_kwargs)
|
||||
layer = layer.to(device=DEVICE)
|
||||
|
||||
if raises_requantization_error:
|
||||
with pytest.raises(
|
||||
NotImplementedError,
|
||||
match="does not implement dequantize_weight|only supported for MXFP8",
|
||||
):
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
else:
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
|
||||
if isinstance(checkpoint_config, ModelOptMxFp8Config):
|
||||
assert isinstance(layer.quant_method, Mxfp8OnlineLinearMethod)
|
||||
assert not layer.quant_method.uses_meta_device
|
||||
assert isinstance(
|
||||
layer.quant_method.requantization_source, ModelOptLinearMethod
|
||||
)
|
||||
elif not raises_requantization_error:
|
||||
assert layer.quant_method.uses_meta_device
|
||||
assert layer.quant_method.requantization_source is None
|
||||
|
||||
|
||||
def test_online_target_rejects_prequantized_layer(
|
||||
default_vllm_config, dist_init
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_rocm())
|
||||
or not is_quant_method_supported("fp8"),
|
||||
reason="Requires FP8 support on CUDA or ROCm.",
|
||||
)
|
||||
def test_online_block_fp8_requantization_removes_source_scales(
|
||||
default_vllm_config, dist_init, workspace_init
|
||||
) -> None:
|
||||
"""A targets match participates in checkpoint compatibility checks."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
prefix = "model.layers.0.self_attn.o_proj"
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
"""Blockwise FP8 replaces MXFP8 scales with its own scale parameter."""
|
||||
default_vllm_config.model_config = ModelConfig(dtype="bfloat16")
|
||||
quant_config = _fully_quantized_modelopt_mxfp8_config()
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(targets={prefix: "mxfp8"})
|
||||
QuantizationConfigArgs(linear="fp8_per_block")
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
ColumnParallelLinear(
|
||||
input_size=32,
|
||||
output_size=32,
|
||||
with set_default_torch_dtype(torch.bfloat16), torch.device(DEVICE):
|
||||
layer = ColumnParallelLinear(
|
||||
input_size=256,
|
||||
output_size=256,
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
prefix="model.layers.0.self_attn.o_proj",
|
||||
disable_tp=True,
|
||||
)
|
||||
with torch.no_grad():
|
||||
layer.weight.fill_(1)
|
||||
layer.weight_scale.fill_(127) # E8M0 encoding of scale 1.
|
||||
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
|
||||
assert not hasattr(layer, "weight_scale")
|
||||
assert layer.weight_scale_inv.numel() > 0
|
||||
|
||||
|
||||
def test_online_ignore_keeps_checkpoint_quantization_linear(
|
||||
@@ -431,17 +490,25 @@ def test_online_ignore_keeps_checkpoint_quantization_linear(
|
||||
|
||||
|
||||
def test_online_quantization_rejects_prequantized_moe(
|
||||
default_vllm_config, dist_init
|
||||
default_vllm_config, dist_init, monkeypatch
|
||||
) -> None:
|
||||
"""Online linear and MoE quantization reject a pre-quantized MoE layer."""
|
||||
"""Reject pre-quantized MoE before constructing an online target backend."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
prefix = "model.layers.0.mlp.experts"
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
quant_config_args(linear="mxfp4", moe="mxfp4")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
quant_config.online_quantization_config,
|
||||
"get_quant_method",
|
||||
lambda *args: pytest.fail("unsupported online backend must not be constructed"),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
with pytest.raises(
|
||||
NotImplementedError,
|
||||
match="Requantizing checkpoint-quantized MoE layers is not supported",
|
||||
):
|
||||
FusedMoEFactory(
|
||||
num_experts=4,
|
||||
top_k=2,
|
||||
@@ -591,7 +658,7 @@ def test_log_online_quantization_for_composable_config(monkeypatch) -> None:
|
||||
|
||||
assert log_args == [
|
||||
(
|
||||
"Quantized %d layers of types: %s",
|
||||
"Quantizing %d layers of types: %s",
|
||||
2,
|
||||
"; ".join(online_config.quantized_layer_summaries),
|
||||
)
|
||||
@@ -836,6 +903,24 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
|
||||
{},
|
||||
id="partially_prequantized_checkpoint",
|
||||
),
|
||||
pytest.param(
|
||||
PARTIALLY_PREQUANTIZED_MODEL_NAME,
|
||||
None,
|
||||
{"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}},
|
||||
Mxfp8OnlineLinearMethod,
|
||||
CompressedTensorsMoEMethod,
|
||||
{},
|
||||
id="partially_prequantized_checkpoint",
|
||||
),
|
||||
pytest.param(
|
||||
"mgoin/Qwen3-0.6B-MXFP8",
|
||||
None,
|
||||
{"linear": "fp8_per_channel"},
|
||||
Fp8PtpcOnlineLinearMethod,
|
||||
None,
|
||||
{},
|
||||
id="requantization_mxfp8_ptcp_fp8",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
@@ -888,6 +973,25 @@ def test_online_quantization(
|
||||
moe_backend=runner_kwargs.get("moe_backend", "auto"),
|
||||
),
|
||||
}
|
||||
if model_name == "mgoin/Qwen3-0.6B-MXFP8":
|
||||
original_process = Fp8PtpcOnlineLinearMethod.process_weights_after_loading
|
||||
|
||||
def assert_source_weights_released(method, layer) -> None:
|
||||
source_parameters = dict(method.requantization_source_parameters)
|
||||
assert set(source_parameters) == {"weight", "weight_scale"}
|
||||
|
||||
original_process(method, layer)
|
||||
|
||||
assert not method.requantization_source_parameters
|
||||
assert layer.weight is not source_parameters["weight"]
|
||||
assert layer.weight_scale is not source_parameters["weight_scale"]
|
||||
assert layer.weight_scale.shape[-1] == 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
Fp8PtpcOnlineLinearMethod,
|
||||
"process_weights_after_loading",
|
||||
assert_source_weights_released,
|
||||
)
|
||||
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
@@ -943,11 +1047,17 @@ def test_online_quantization(
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
|
||||
o_proj = model.model.layers[1].self_attn.o_proj
|
||||
moe = model.model.layers[0].mlp.experts
|
||||
elif model_name == "mgoin/Qwen3-0.6B-MXFP8":
|
||||
o_proj = model.model.layers[0].self_attn.o_proj
|
||||
moe = None
|
||||
else:
|
||||
o_proj = model.model.layers[0].self_attn.o_proj
|
||||
moe = model.model.layers[0].block_sparse_moe.experts
|
||||
|
||||
assert isinstance(o_proj.quant_method, expected_linear_cls)
|
||||
assert isinstance(moe._quant_method, expected_moe_cls)
|
||||
|
||||
if moe is not None:
|
||||
assert isinstance(moe._quant_method, expected_moe_cls)
|
||||
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance(
|
||||
o_proj.quant_method.kernel, MarlinMxfp8LinearKernel
|
||||
@@ -1242,7 +1352,7 @@ def test_log_online_quantization(default_vllm_config, monkeypatch) -> None:
|
||||
log_online_quantization(default_vllm_config)
|
||||
|
||||
assert logged_messages == [
|
||||
"Quantized 3 layers of types: mlp.down_proj: 2 (from linear: "
|
||||
"Quantizing 3 layers of types: mlp.down_proj: 2 (from linear: "
|
||||
"fp8_per_tensor); self_attn.qkv_proj: 1 (from targets: "
|
||||
"re:.*qkv_proj.*, mxfp4)"
|
||||
]
|
||||
@@ -1365,6 +1475,48 @@ def test_online_linear_tp_weight_quant_matches_unsharded(
|
||||
assert torch.equal(tp_scale, full_scale)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_modelopt_mxfp8_requantized_ptpc_tp_shards_match_unsharded() -> None:
|
||||
"""MXFP8 source shards convert to slices of the unsharded PTPC weight."""
|
||||
torch.manual_seed(0)
|
||||
serialized_weight = torch.randn(16, 64, device=DEVICE, dtype=torch.bfloat16).to(
|
||||
MXFP8_VALUE_DTYPE
|
||||
)
|
||||
serialized_scale = torch.randint(
|
||||
124,
|
||||
131,
|
||||
(16, 2),
|
||||
device=DEVICE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
)
|
||||
|
||||
source = ModelOptLinearMethod.__new__(ModelOptLinearMethod)
|
||||
source.wkey = SimpleNamespace(key=kMxfp8Static)
|
||||
|
||||
def dequantize(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
||||
layer = SimpleNamespace(weight=weight, weight_scale=scale)
|
||||
return source.dequantize_weight(layer)
|
||||
|
||||
full_weight = dequantize(serialized_weight, serialized_scale)
|
||||
full_scale = _fp8_channel_scale(weight_amax(full_weight, dim=-1, keepdim=True))
|
||||
full_qweight = _fp8_quant_per_channel(full_weight, full_scale)
|
||||
|
||||
# Mimic the two K-dimension shards of a row-parallel layer. PTPC's global
|
||||
# per-output-channel scale is the same on both ranks after its MAX reduce.
|
||||
for rank in range(2):
|
||||
start = rank * 32
|
||||
shard_weight = serialized_weight[:, start : start + 32].contiguous()
|
||||
shard_scale = serialized_scale[:, rank : rank + 1].contiguous()
|
||||
weight = dequantize(shard_weight, shard_scale)
|
||||
qweight = _fp8_quant_per_channel(weight, full_scale)
|
||||
|
||||
assert torch.equal(weight, full_weight[:, start : start + 32])
|
||||
assert torch.equal(qweight, full_qweight[:, start : start + 32])
|
||||
|
||||
|
||||
def _quantize_moe(weight, scheme, moe_tp_size):
|
||||
if scheme == "nvfp4":
|
||||
return _quantize_moe_weight_to_nvfp4(weight, moe_tp_size)
|
||||
|
||||
Reference in New Issue
Block a user