[Quantization] Layer re-quantization for linear layers through online quantization API (MXFP8 -> FP8 PTPC showcase) (#55684)
Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
vllmellm
Tan Pin Siang
Misha Goin
parent
29f955cce0
commit
f8d92cae41
@@ -99,6 +99,35 @@ vllm serve openai/gpt-oss-20b --quantization-config.moe.activation mxfp8
|
||||
|
||||
Combine with `--moe-backend` to pin a specific kernel family.
|
||||
|
||||
### Re-quantizing layers to a different precision
|
||||
|
||||
An online target can re-quantize a checkpoint-quantized (prequantized) layer at load time.
|
||||
|
||||
vLLM uses the checkpoint method to load and dequantize the checkpoint weight, and then applies the online quantization method. This requires the checkpoint
|
||||
method to implement weight dequantization, unsupported source and target
|
||||
combinations raise an error.
|
||||
|
||||
Currently supported combinations are:
|
||||
|
||||
| Checkpoint layer | Online target | Requirements |
|
||||
| ---------------- | ------------- | ------------ |
|
||||
| Serialized ModelOpt MXFP8 linear | any | Target-specific hardware requirements |
|
||||
|
||||
For example, serialized ModelOpt MXFP8 linears can be converted to
|
||||
per-channel-weight/per-token-activation FP8 at load time:
|
||||
|
||||
```bash
|
||||
vllm serve MiniMaxAI/MiniMax-M3-MXFP8 \
|
||||
--linear-backend auto \
|
||||
--quantization-config.linear fp8_per_channel
|
||||
```
|
||||
|
||||
!!! info
|
||||
|
||||
This is only available for linear layers at the moment.
|
||||
|
||||
Checkpoint-exclusion policy is not enforced by online quantization/re-quantization. Only exclusion through `--quantization-config.targets`/`--quantization-config.ignore` is enforced.
|
||||
|
||||
### Online quantization on unquantized layers from partially-quantized checkpoints
|
||||
|
||||
Online quantization can be used on already quantized checkpoints independently of their original `quant_method` (`modelopt`, `compressed-tensors`, `quark`, etc.), for layers that are left unquantized in the original checkpoint.
|
||||
|
||||
@@ -58,6 +58,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.modelopt import (
|
||||
ModelOptFp8Config,
|
||||
ModelOptLinearMethod,
|
||||
ModelOptMxFp8Config,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
@@ -93,12 +94,14 @@ from vllm.model_executor.layers.quantization.quark.quark import (
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils import quant_utils
|
||||
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
amax_for_moe_weight_quant,
|
||||
amax_for_tp_weight_quant,
|
||||
kMxfp8Dynamic,
|
||||
kMxfp8Static,
|
||||
weight_amax,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
@@ -112,6 +115,7 @@ from vllm.model_executor.models.granitemoe import (
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.platforms.rocm import on_gfx942, on_gfx950
|
||||
@@ -307,6 +311,14 @@ def _fully_quantized_modelopt_config() -> ModelOptFp8Config:
|
||||
)
|
||||
|
||||
|
||||
def _fully_quantized_modelopt_mxfp8_config() -> ModelOptMxFp8Config:
|
||||
return ModelOptMxFp8Config(
|
||||
is_checkpoint_mxfp8_serialized=True,
|
||||
kv_cache_quant_algo=None,
|
||||
exclude_modules=["lm_head"],
|
||||
)
|
||||
|
||||
|
||||
def _moe_only_compressed_tensors_config() -> CompressedTensorsConfig:
|
||||
return CompressedTensorsConfig(
|
||||
target_scheme_map={"RoutedExperts": {}},
|
||||
@@ -334,10 +346,19 @@ def _write_minimal_llama_config(
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"checkpoint_config_factory,raises_conflict",
|
||||
"checkpoint_config_factory,raises_requantization_error",
|
||||
[
|
||||
pytest.param(_fully_quantized_quark_config, True, id="quark"),
|
||||
pytest.param(_fully_quantized_modelopt_config, True, id="modelopt"),
|
||||
pytest.param(
|
||||
_fully_quantized_modelopt_config,
|
||||
True,
|
||||
id="modelopt",
|
||||
),
|
||||
pytest.param(
|
||||
_fully_quantized_modelopt_mxfp8_config,
|
||||
False,
|
||||
id="modelopt_mxfp8",
|
||||
),
|
||||
pytest.param(
|
||||
_moe_only_compressed_tensors_config,
|
||||
False,
|
||||
@@ -345,9 +366,13 @@ def _write_minimal_llama_config(
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_rocm()),
|
||||
reason="MXFP8 requantization requires CUDA or ROCm.",
|
||||
)
|
||||
def test_online_prequantized_compatibility(
|
||||
checkpoint_config_factory,
|
||||
raises_conflict: bool,
|
||||
raises_requantization_error: bool,
|
||||
default_vllm_config,
|
||||
dist_init,
|
||||
) -> None:
|
||||
@@ -370,35 +395,69 @@ def test_online_prequantized_compatibility(
|
||||
"disable_tp": True,
|
||||
}
|
||||
|
||||
if raises_conflict:
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
ColumnParallelLinear(**layer_kwargs)
|
||||
layer = ColumnParallelLinear(**layer_kwargs)
|
||||
assert isinstance(layer.quant_method, Mxfp8OnlineLinearMethod)
|
||||
|
||||
if layer.weight.is_meta:
|
||||
# Weight loading.
|
||||
layer = layer.to_empty(device=DEVICE)
|
||||
for parameter in layer.parameters():
|
||||
parameter.zero_()
|
||||
else:
|
||||
layer = ColumnParallelLinear(**layer_kwargs)
|
||||
layer = layer.to(device=DEVICE)
|
||||
|
||||
if raises_requantization_error:
|
||||
with pytest.raises(
|
||||
NotImplementedError,
|
||||
match="does not implement dequantize_weight|only supported for MXFP8",
|
||||
):
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
else:
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
|
||||
if isinstance(checkpoint_config, ModelOptMxFp8Config):
|
||||
assert isinstance(layer.quant_method, Mxfp8OnlineLinearMethod)
|
||||
assert not layer.quant_method.uses_meta_device
|
||||
assert isinstance(
|
||||
layer.quant_method.requantization_source, ModelOptLinearMethod
|
||||
)
|
||||
elif not raises_requantization_error:
|
||||
assert layer.quant_method.uses_meta_device
|
||||
assert layer.quant_method.requantization_source is None
|
||||
|
||||
|
||||
def test_online_target_rejects_prequantized_layer(
|
||||
default_vllm_config, dist_init
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_cuda() or current_platform.is_rocm())
|
||||
or not is_quant_method_supported("fp8"),
|
||||
reason="Requires FP8 support on CUDA or ROCm.",
|
||||
)
|
||||
def test_online_block_fp8_requantization_removes_source_scales(
|
||||
default_vllm_config, dist_init, workspace_init
|
||||
) -> None:
|
||||
"""A targets match participates in checkpoint compatibility checks."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
prefix = "model.layers.0.self_attn.o_proj"
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
"""Blockwise FP8 replaces MXFP8 scales with its own scale parameter."""
|
||||
default_vllm_config.model_config = ModelConfig(dtype="bfloat16")
|
||||
quant_config = _fully_quantized_modelopt_mxfp8_config()
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
QuantizationConfigArgs(targets={prefix: "mxfp8"})
|
||||
QuantizationConfigArgs(linear="fp8_per_block")
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
ColumnParallelLinear(
|
||||
input_size=32,
|
||||
output_size=32,
|
||||
with set_default_torch_dtype(torch.bfloat16), torch.device(DEVICE):
|
||||
layer = ColumnParallelLinear(
|
||||
input_size=256,
|
||||
output_size=256,
|
||||
bias=False,
|
||||
params_dtype=torch.bfloat16,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
prefix="model.layers.0.self_attn.o_proj",
|
||||
disable_tp=True,
|
||||
)
|
||||
with torch.no_grad():
|
||||
layer.weight.fill_(1)
|
||||
layer.weight_scale.fill_(127) # E8M0 encoding of scale 1.
|
||||
|
||||
layer.quant_method.process_weights_after_loading(layer)
|
||||
|
||||
assert not hasattr(layer, "weight_scale")
|
||||
assert layer.weight_scale_inv.numel() > 0
|
||||
|
||||
|
||||
def test_online_ignore_keeps_checkpoint_quantization_linear(
|
||||
@@ -431,17 +490,25 @@ def test_online_ignore_keeps_checkpoint_quantization_linear(
|
||||
|
||||
|
||||
def test_online_quantization_rejects_prequantized_moe(
|
||||
default_vllm_config, dist_init
|
||||
default_vllm_config, dist_init, monkeypatch
|
||||
) -> None:
|
||||
"""Online linear and MoE quantization reject a pre-quantized MoE layer."""
|
||||
"""Reject pre-quantized MoE before constructing an online target backend."""
|
||||
default_vllm_config.model_config = ModelConfig()
|
||||
prefix = "model.layers.0.mlp.experts"
|
||||
quant_config = _fully_quantized_quark_config()
|
||||
quant_config.online_quantization_config = OnlineQuantizationConfig(
|
||||
quant_config_args(linear="mxfp4", moe="mxfp4")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
quant_config.online_quantization_config,
|
||||
"get_quant_method",
|
||||
lambda *args: pytest.fail("unsupported online backend must not be constructed"),
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="pre-quantized layer"):
|
||||
with pytest.raises(
|
||||
NotImplementedError,
|
||||
match="Requantizing checkpoint-quantized MoE layers is not supported",
|
||||
):
|
||||
FusedMoEFactory(
|
||||
num_experts=4,
|
||||
top_k=2,
|
||||
@@ -591,7 +658,7 @@ def test_log_online_quantization_for_composable_config(monkeypatch) -> None:
|
||||
|
||||
assert log_args == [
|
||||
(
|
||||
"Quantized %d layers of types: %s",
|
||||
"Quantizing %d layers of types: %s",
|
||||
2,
|
||||
"; ".join(online_config.quantized_layer_summaries),
|
||||
)
|
||||
@@ -836,6 +903,24 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
|
||||
{},
|
||||
id="partially_prequantized_checkpoint",
|
||||
),
|
||||
pytest.param(
|
||||
PARTIALLY_PREQUANTIZED_MODEL_NAME,
|
||||
None,
|
||||
{"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}},
|
||||
Mxfp8OnlineLinearMethod,
|
||||
CompressedTensorsMoEMethod,
|
||||
{},
|
||||
id="partially_prequantized_checkpoint",
|
||||
),
|
||||
pytest.param(
|
||||
"mgoin/Qwen3-0.6B-MXFP8",
|
||||
None,
|
||||
{"linear": "fp8_per_channel"},
|
||||
Fp8PtpcOnlineLinearMethod,
|
||||
None,
|
||||
{},
|
||||
id="requantization_mxfp8_ptcp_fp8",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
@@ -888,6 +973,25 @@ def test_online_quantization(
|
||||
moe_backend=runner_kwargs.get("moe_backend", "auto"),
|
||||
),
|
||||
}
|
||||
if model_name == "mgoin/Qwen3-0.6B-MXFP8":
|
||||
original_process = Fp8PtpcOnlineLinearMethod.process_weights_after_loading
|
||||
|
||||
def assert_source_weights_released(method, layer) -> None:
|
||||
source_parameters = dict(method.requantization_source_parameters)
|
||||
assert set(source_parameters) == {"weight", "weight_scale"}
|
||||
|
||||
original_process(method, layer)
|
||||
|
||||
assert not method.requantization_source_parameters
|
||||
assert layer.weight is not source_parameters["weight"]
|
||||
assert layer.weight_scale is not source_parameters["weight_scale"]
|
||||
assert layer.weight_scale.shape[-1] == 1
|
||||
|
||||
monkeypatch.setattr(
|
||||
Fp8PtpcOnlineLinearMethod,
|
||||
"process_weights_after_loading",
|
||||
assert_source_weights_released,
|
||||
)
|
||||
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
|
||||
model, vllm_config = load_model_without_vllm_runner(
|
||||
@@ -943,11 +1047,17 @@ def test_online_quantization(
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
|
||||
o_proj = model.model.layers[1].self_attn.o_proj
|
||||
moe = model.model.layers[0].mlp.experts
|
||||
elif model_name == "mgoin/Qwen3-0.6B-MXFP8":
|
||||
o_proj = model.model.layers[0].self_attn.o_proj
|
||||
moe = None
|
||||
else:
|
||||
o_proj = model.model.layers[0].self_attn.o_proj
|
||||
moe = model.model.layers[0].block_sparse_moe.experts
|
||||
|
||||
assert isinstance(o_proj.quant_method, expected_linear_cls)
|
||||
assert isinstance(moe._quant_method, expected_moe_cls)
|
||||
|
||||
if moe is not None:
|
||||
assert isinstance(moe._quant_method, expected_moe_cls)
|
||||
|
||||
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance(
|
||||
o_proj.quant_method.kernel, MarlinMxfp8LinearKernel
|
||||
@@ -1242,7 +1352,7 @@ def test_log_online_quantization(default_vllm_config, monkeypatch) -> None:
|
||||
log_online_quantization(default_vllm_config)
|
||||
|
||||
assert logged_messages == [
|
||||
"Quantized 3 layers of types: mlp.down_proj: 2 (from linear: "
|
||||
"Quantizing 3 layers of types: mlp.down_proj: 2 (from linear: "
|
||||
"fp8_per_tensor); self_attn.qkv_proj: 1 (from targets: "
|
||||
"re:.*qkv_proj.*, mxfp4)"
|
||||
]
|
||||
@@ -1365,6 +1475,48 @@ def test_online_linear_tp_weight_quant_matches_unsharded(
|
||||
assert torch.equal(tp_scale, full_scale)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_quant_method_supported("fp8"),
|
||||
reason="FP8 is not supported on this GPU type.",
|
||||
)
|
||||
def test_modelopt_mxfp8_requantized_ptpc_tp_shards_match_unsharded() -> None:
|
||||
"""MXFP8 source shards convert to slices of the unsharded PTPC weight."""
|
||||
torch.manual_seed(0)
|
||||
serialized_weight = torch.randn(16, 64, device=DEVICE, dtype=torch.bfloat16).to(
|
||||
MXFP8_VALUE_DTYPE
|
||||
)
|
||||
serialized_scale = torch.randint(
|
||||
124,
|
||||
131,
|
||||
(16, 2),
|
||||
device=DEVICE,
|
||||
dtype=MXFP8_SCALE_DTYPE,
|
||||
)
|
||||
|
||||
source = ModelOptLinearMethod.__new__(ModelOptLinearMethod)
|
||||
source.wkey = SimpleNamespace(key=kMxfp8Static)
|
||||
|
||||
def dequantize(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
|
||||
layer = SimpleNamespace(weight=weight, weight_scale=scale)
|
||||
return source.dequantize_weight(layer)
|
||||
|
||||
full_weight = dequantize(serialized_weight, serialized_scale)
|
||||
full_scale = _fp8_channel_scale(weight_amax(full_weight, dim=-1, keepdim=True))
|
||||
full_qweight = _fp8_quant_per_channel(full_weight, full_scale)
|
||||
|
||||
# Mimic the two K-dimension shards of a row-parallel layer. PTPC's global
|
||||
# per-output-channel scale is the same on both ranks after its MAX reduce.
|
||||
for rank in range(2):
|
||||
start = rank * 32
|
||||
shard_weight = serialized_weight[:, start : start + 32].contiguous()
|
||||
shard_scale = serialized_scale[:, rank : rank + 1].contiguous()
|
||||
weight = dequantize(shard_weight, shard_scale)
|
||||
qweight = _fp8_quant_per_channel(weight, full_scale)
|
||||
|
||||
assert torch.equal(weight, full_weight[:, start : start + 32])
|
||||
assert torch.equal(qweight, full_qweight[:, start : start + 32])
|
||||
|
||||
|
||||
def _quantize_moe(weight, scheme, moe_tp_size):
|
||||
if scheme == "nvfp4":
|
||||
return _quantize_moe_weight_to_nvfp4(weight, moe_tp_size)
|
||||
|
||||
@@ -84,6 +84,13 @@ class QuantizeMethodBase(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
def dequantize_weight(self, layer: nn.Module) -> torch.Tensor:
|
||||
"""Materialize a serialized quantized weight for requantization."""
|
||||
raise NotImplementedError(
|
||||
f"The quantization method {type(self)} does not implement "
|
||||
"dequantize_weight. Please open an issue."
|
||||
)
|
||||
|
||||
|
||||
def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) -> bool:
|
||||
"""Not all quant methods have embedding implemented, so we need to check that
|
||||
@@ -294,13 +301,16 @@ def resolve_quant_method(
|
||||
LinearBase,
|
||||
UnquantizedLinearMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
|
||||
base_quant_method = quant_config.get_quant_method(layer, prefix)
|
||||
if quant_config.online_quantization_config is None:
|
||||
# No online configuration: retain the checkpoint method.
|
||||
return base_quant_method
|
||||
# Online quantization currently supports only LinearBase and RoutedExperts.
|
||||
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
|
||||
if not isinstance(layer, (LinearBase, RoutedExperts)):
|
||||
# Online quantization only supports linear and routed MoE layers.
|
||||
return base_quant_method
|
||||
|
||||
quant_config.online_quantization_config.packed_modules_mapping = (
|
||||
@@ -312,14 +322,34 @@ def resolve_quant_method(
|
||||
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
|
||||
layer, prefix
|
||||
)
|
||||
|
||||
if checkpoint_is_quantized:
|
||||
if online_target is not None:
|
||||
raise ValueError(
|
||||
f"Cannot apply requested online quantization {online_target[3]} to "
|
||||
f"pre-quantized layer {prefix}: {base_quant_method} was already "
|
||||
"selected by the checkpoint quantization config."
|
||||
if online_target is None:
|
||||
# The checkpoint quant method is applied as there is no online override.
|
||||
return base_quant_method
|
||||
|
||||
if isinstance(layer, RoutedExperts):
|
||||
raise NotImplementedError(
|
||||
"Requantizing checkpoint-quantized MoE layers is not supported."
|
||||
)
|
||||
return base_quant_method
|
||||
|
||||
online_quant_method = quant_config.online_quantization_config.get_quant_method(
|
||||
layer, prefix
|
||||
)
|
||||
|
||||
assert base_quant_method is not None and not isinstance(
|
||||
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
|
||||
)
|
||||
|
||||
assert isinstance(online_quant_method, OnlineLinearBase)
|
||||
online_quant_method.set_requantization_source(base_quant_method)
|
||||
|
||||
# The online method dequantizes the checkpoint method before requantizing.
|
||||
return online_quant_method
|
||||
|
||||
if online_target is None:
|
||||
# The layer is unquantized and no online target applies.
|
||||
return base_quant_method
|
||||
|
||||
# Quantize an unquantized layer with the online method.
|
||||
return quant_config.online_quantization_config.get_quant_method(layer, prefix)
|
||||
|
||||
@@ -76,6 +76,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
|
||||
MXFP8_BLOCK_SIZE,
|
||||
MXFP8_SCALE_DTYPE,
|
||||
MXFP8_VALUE_DTYPE,
|
||||
dequant_mxfp8_to_bf16,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
FP4_DTYPE,
|
||||
@@ -2572,6 +2573,17 @@ class ModelOptLinearMethod(LinearMethodBase):
|
||||
layer._nvfp4_group_size_for_gather = self.ctx.group_size
|
||||
self.kernel.process_weights_after_loading(layer)
|
||||
|
||||
def dequantize_weight(self, layer: torch.nn.Module) -> torch.Tensor:
|
||||
"""Reconstruct serialized weights for online requantization."""
|
||||
if self.wkey.key is kMxfp8Static:
|
||||
return dequant_mxfp8_to_bf16(
|
||||
layer.weight.contiguous(), layer.weight_scale.contiguous()
|
||||
)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"ModelOpt weight dequantization is only supported for MXFP8."
|
||||
)
|
||||
|
||||
def apply(self, layer, x, bias=None):
|
||||
return self.fmt.apply(
|
||||
layer,
|
||||
|
||||
@@ -156,6 +156,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
|
||||
self.args = args
|
||||
self.ignored_layers: list[str] = args.ignore
|
||||
self.quantized_layers: dict[str, tuple[str, str, str | None]] = {}
|
||||
self.online_quantization_time = 0.0
|
||||
|
||||
@property
|
||||
def quantized_layer_summaries(self) -> list[str]:
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.fusion.quant_activation import (
|
||||
from vllm.model_executor.layers.linear import (
|
||||
LinearMethodBase,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
@@ -125,6 +126,13 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
def __init__(self):
|
||||
self.out_dtype = torch.get_default_dtype()
|
||||
self.input_dtype = get_current_vllm_config().model_config.dtype
|
||||
self.requantization_source: QuantizeMethodBase | None = None
|
||||
self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {}
|
||||
|
||||
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
|
||||
"""Configure serialized-weight conversion before online quantization."""
|
||||
self.requantization_source = source_method
|
||||
self.uses_meta_device = False
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
@@ -136,6 +144,24 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
params_dtype: torch.dtype,
|
||||
**extra_weight_attrs,
|
||||
):
|
||||
if self.requantization_source is not None:
|
||||
existing_parameter_names = set(layer._parameters)
|
||||
self.requantization_source.create_weights(
|
||||
layer,
|
||||
input_size_per_partition,
|
||||
output_partition_sizes,
|
||||
input_size,
|
||||
output_size,
|
||||
params_dtype,
|
||||
**extra_weight_attrs,
|
||||
)
|
||||
self.requantization_source_parameters = {
|
||||
name: parameter
|
||||
for name, parameter in layer._parameters.items()
|
||||
if name not in existing_parameter_names and parameter is not None
|
||||
}
|
||||
return
|
||||
|
||||
output_size_per_partition = sum(output_partition_sizes)
|
||||
weight_loader = extra_weight_attrs.get("weight_loader")
|
||||
layer.logical_widths = output_partition_sizes
|
||||
@@ -159,6 +185,19 @@ class OnlineLinearBase(LinearMethodBase):
|
||||
|
||||
initialize_online_processing(layer)
|
||||
|
||||
def release_requantization_source_weights(self, layer: torch.nn.Module) -> None:
|
||||
"""Release checkpoint parameters after successful requantization."""
|
||||
for name, source_parameter in self.requantization_source_parameters.items():
|
||||
if layer._parameters.get(name) is source_parameter:
|
||||
delattr(layer, name)
|
||||
self.requantization_source_parameters.clear()
|
||||
|
||||
def get_weight_for_quantization(self, layer: Module) -> torch.Tensor:
|
||||
"""Return checkpoint weights materialized for online quantization."""
|
||||
if self.requantization_source is None:
|
||||
return layer.weight
|
||||
return self.requantization_source.dequantize_weight(layer)
|
||||
|
||||
|
||||
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
"""Online tensorwise FP8 linear quantization.
|
||||
@@ -213,10 +252,11 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
|
||||
return
|
||||
|
||||
layer.input_scale = None
|
||||
amax = weight_amax(layer.weight).reshape(1)
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
amax = weight_amax(weight).reshape(1)
|
||||
amax = amax_for_tp_weight_quant(amax, _is_tp_sharded(layer))
|
||||
weight_scale = _fp8_scale(amax)
|
||||
qweight, _ = ops.scaled_fp8_quant(layer.weight, scale=weight_scale)
|
||||
qweight, _ = ops.scaled_fp8_quant(weight, scale=weight_scale)
|
||||
|
||||
# Update layer with new values.
|
||||
replace_parameter(layer, "weight", qweight.t().data)
|
||||
@@ -312,14 +352,18 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
|
||||
|
||||
layer.input_scale = None
|
||||
block_size = self.weight_block_size
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
|
||||
qweight, weight_scale_inv = per_block_cast_to_fp8(
|
||||
layer.weight, block_size=block_size, use_ue8m0=False
|
||||
weight, block_size=block_size, use_ue8m0=False
|
||||
)
|
||||
|
||||
replace_parameter(layer, "weight", qweight.data)
|
||||
replace_parameter(layer, "weight_scale_inv", weight_scale_inv.data)
|
||||
|
||||
if self.requantization_source is not None and hasattr(layer, "weight_scale"):
|
||||
del layer.weight_scale
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
|
||||
# Prevent duplicate processing (e.g., during weight reload)
|
||||
@@ -395,18 +439,20 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
|
||||
return
|
||||
|
||||
layer.input_scale = None
|
||||
amax = weight_amax(layer.weight, dim=-1, keepdim=True)
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
amax = weight_amax(weight, dim=-1, keepdim=True)
|
||||
amax = amax_for_tp_weight_quant(
|
||||
amax, _is_tp_sharded(layer, reduces_output_dim=False)
|
||||
)
|
||||
weight_scale = _fp8_channel_scale(amax)
|
||||
qweight = _fp8_quant_per_channel(layer.weight, weight_scale)
|
||||
qweight = _fp8_quant_per_channel(weight, weight_scale)
|
||||
|
||||
replace_parameter(layer, "weight", qweight.t())
|
||||
replace_parameter(layer, "weight_scale", weight_scale)
|
||||
|
||||
self.fp8_linear.process_weights_after_loading(layer)
|
||||
expose_input_quant_key(layer, self.fp8_linear)
|
||||
self.release_requantization_source_weights(layer)
|
||||
|
||||
layer._already_called_process_weights_after_loading = True
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.fused_moe import (
|
||||
SharedExperts,
|
||||
)
|
||||
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.model_loader.reload.layerwise import (
|
||||
initialize_online_processing,
|
||||
)
|
||||
@@ -24,6 +25,12 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
|
||||
|
||||
uses_meta_device: bool = True
|
||||
|
||||
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
|
||||
"""Reject requantization from a checkpoint-quantized MoE method."""
|
||||
raise NotImplementedError(
|
||||
"Requantizing checkpoint-quantized MoE layers is not supported."
|
||||
)
|
||||
|
||||
def create_weights(
|
||||
self,
|
||||
layer: torch.nn.Module,
|
||||
|
||||
@@ -119,7 +119,8 @@ class Mxfp4OnlineLinearMethod(OnlineLinearBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
weight_fp4, weight_scale = mxfp4_quantize(layer.weight.contiguous())
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
weight_fp4, weight_scale = mxfp4_quantize(weight.contiguous())
|
||||
|
||||
layer.input_scale = None
|
||||
replace_parameter(layer, "weight", weight_fp4.data)
|
||||
|
||||
@@ -75,7 +75,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
|
||||
if getattr(layer, "_already_called_process_weights_after_loading", False):
|
||||
return
|
||||
|
||||
weight_fp8, weight_scale = mxfp8_e4m3_quantize(layer.weight.contiguous())
|
||||
weight = self.get_weight_for_quantization(layer)
|
||||
weight_fp8, weight_scale = mxfp8_e4m3_quantize(weight.contiguous())
|
||||
|
||||
layer.input_scale = None
|
||||
replace_parameter(layer, "weight", weight_fp8.data)
|
||||
|
||||
@@ -101,6 +101,7 @@ class BaseModelLoader(ABC):
|
||||
finalize_layerwise_processing(model, model_config)
|
||||
|
||||
process_weights_after_loading(model, model_config, target_device)
|
||||
log_online_quantization_time(vllm_config)
|
||||
|
||||
return model.eval()
|
||||
|
||||
@@ -115,8 +116,8 @@ def log_model_inspection(model: nn.Module) -> None:
|
||||
logger.info("vLLM model structure:\n%s", format_model_inspection(model))
|
||||
|
||||
|
||||
def log_online_quantization(vllm_config: VllmConfig) -> None:
|
||||
"""Log the online-quantized layer count and types, when applicable."""
|
||||
def _get_online_quantization_config(vllm_config: VllmConfig):
|
||||
"""Return the online quantization config when one is configured."""
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
@@ -128,15 +129,35 @@ def log_online_quantization(vllm_config: VllmConfig) -> None:
|
||||
if isinstance(online_quantization_config, OnlineQuantizationConfig):
|
||||
quant_config = online_quantization_config
|
||||
if not isinstance(quant_config, OnlineQuantizationConfig):
|
||||
return None
|
||||
return quant_config
|
||||
|
||||
|
||||
def log_online_quantization(vllm_config: VllmConfig) -> None:
|
||||
"""Log the online-quantized layer count and types, when applicable."""
|
||||
quant_config = _get_online_quantization_config(vllm_config)
|
||||
if quant_config is None:
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Quantized %d layers of types: %s",
|
||||
"Quantizing %d layers of types: %s",
|
||||
len(quant_config.quantized_layers),
|
||||
"; ".join(quant_config.quantized_layer_summaries),
|
||||
)
|
||||
|
||||
|
||||
def log_online_quantization_time(vllm_config: VllmConfig) -> None:
|
||||
"""Log online quantization processing time, when applicable."""
|
||||
quant_config = _get_online_quantization_config(vllm_config)
|
||||
if quant_config is None:
|
||||
return
|
||||
|
||||
logger.info(
|
||||
"Online quantization as part of model loading took %.2f seconds",
|
||||
quant_config.online_quantization_time,
|
||||
)
|
||||
|
||||
|
||||
def _has_online_quant(model: nn.Module):
|
||||
for module in model.modules():
|
||||
quant_method = getattr(module, "quant_method", None)
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import inspect
|
||||
from collections.abc import Callable
|
||||
from contextlib import nullcontext
|
||||
from functools import wraps
|
||||
from weakref import WeakKeyDictionary, WeakSet
|
||||
|
||||
@@ -321,7 +322,21 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
|
||||
_get_weight_loader(param)(*args.args, **args.kwargs)
|
||||
|
||||
if quant_method is not None:
|
||||
quant_method.process_weights_after_loading(layer)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import (
|
||||
online_quantization_timing_context,
|
||||
)
|
||||
|
||||
timing_context = (
|
||||
online_quantization_timing_context(layer, quant_method)
|
||||
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
|
||||
else nullcontext()
|
||||
)
|
||||
with timing_context:
|
||||
quant_method.process_weights_after_loading(layer)
|
||||
|
||||
_copy_and_restore_kernel_tensors(layer, info)
|
||||
|
||||
@@ -356,7 +371,21 @@ def _layerwise_process(layer: torch.nn.Module, info: LayerReloadingInfo):
|
||||
# Process weights (quantization, repacking, etc.)
|
||||
quant_method = getattr(layer, "quant_method", None)
|
||||
if isinstance(quant_method, QuantizeMethodBase):
|
||||
quant_method.process_weights_after_loading(layer)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
from vllm.model_executor.model_loader.utils import (
|
||||
online_quantization_timing_context,
|
||||
)
|
||||
|
||||
timing_context = (
|
||||
online_quantization_timing_context(layer, quant_method)
|
||||
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
|
||||
else nullcontext()
|
||||
)
|
||||
with timing_context:
|
||||
quant_method.process_weights_after_loading(layer)
|
||||
# Re-reconcile parameter TP state: process_weights_after_loading may
|
||||
# have re-created Parameters (stamped with the global rank), which would
|
||||
# otherwise break replicated (disable_tp) weights on a subsequent reload.
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"""Utilities for selecting and loading models."""
|
||||
|
||||
import inspect
|
||||
import time
|
||||
import warnings
|
||||
from contextlib import contextmanager, nullcontext
|
||||
from typing import Any
|
||||
@@ -141,6 +142,11 @@ def process_weights_after_loading(
|
||||
"""
|
||||
# Reclaim memory when an explicit lm_head has been
|
||||
# loaded, but it is identical to the input embeddings.
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
|
||||
maybe_retie_word_embeddings(model, model_config)
|
||||
|
||||
for name, module in model.named_modules():
|
||||
@@ -166,7 +172,12 @@ def process_weights_after_loading(
|
||||
if quant_method.requires_device_loading
|
||||
else nullcontext()
|
||||
)
|
||||
with loading_context:
|
||||
timing_context = (
|
||||
online_quantization_timing_context(module, quant_method)
|
||||
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
|
||||
else nullcontext()
|
||||
)
|
||||
with loading_context, timing_context:
|
||||
quant_method.process_weights_after_loading(module)
|
||||
# process_weights_after_loading may swap in freshly-created
|
||||
# Parameters (e.g. FP8 requantization), which are stamped with the
|
||||
@@ -245,6 +256,43 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
|
||||
p._vllm_is_uva_offloaded = True
|
||||
|
||||
|
||||
@contextmanager
|
||||
def online_quantization_timing_context(
|
||||
layer: nn.Module, quant_method: QuantizeMethodBase
|
||||
):
|
||||
"""Accumulate online quantization processing time for ``quant_method``."""
|
||||
from vllm.model_executor.layers.quantization.online.base import (
|
||||
OnlineQuantizationConfig,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
|
||||
from vllm.model_executor.layers.quantization.online.moe_base import (
|
||||
OnlineMoEMethodBase,
|
||||
)
|
||||
|
||||
assert isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
|
||||
quant_config = layer.quant_config
|
||||
assert quant_config is not None
|
||||
online_quantization_config = (
|
||||
quant_config
|
||||
if isinstance(quant_config, OnlineQuantizationConfig)
|
||||
else quant_config.online_quantization_config
|
||||
)
|
||||
# reload/layerwise.py's _layerwise_process calls this for experts_int8,
|
||||
# whose online MoE method has no OnlineQuantizationConfig.
|
||||
if online_quantization_config is None:
|
||||
yield
|
||||
return
|
||||
assert isinstance(online_quantization_config, OnlineQuantizationConfig)
|
||||
|
||||
start_time = time.perf_counter()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
online_quantization_config.online_quantization_time += (
|
||||
time.perf_counter() - start_time
|
||||
)
|
||||
|
||||
|
||||
_MODEL_ARCH_BY_HASH = dict[int, tuple[type[nn.Module], str]]()
|
||||
"""Caches the outputs of `_get_model_architecture`."""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user