[Quantization] Layer re-quantization for linear layers through online quantization API (MXFP8 -> FP8 PTPC showcase) (#55684)

Signed-off-by: Felix Marty <Felix.Marty@amd.com>
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com>
Co-authored-by: Tan Pin Siang <tanpinsiang@gmail.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
fxmarty-amd
2026-10-06 11:38:18 -05:00
committed by GitHub
co-authored by vllmellm Tan Pin Siang Misha Goin
parent 29f955cce0
commit f8d92cae41
12 changed files with 422 additions and 45 deletions
+29
View File
@@ -99,6 +99,35 @@ vllm serve openai/gpt-oss-20b --quantization-config.moe.activation mxfp8
Combine with `--moe-backend` to pin a specific kernel family.
### Re-quantizing layers to a different precision
An online target can re-quantize a checkpoint-quantized (prequantized) layer at load time.
vLLM uses the checkpoint method to load and dequantize the checkpoint weight, and then applies the online quantization method. This requires the checkpoint
method to implement weight dequantization, unsupported source and target
combinations raise an error.
Currently supported combinations are:
| Checkpoint layer | Online target | Requirements |
| ---------------- | ------------- | ------------ |
| Serialized ModelOpt MXFP8 linear | any | Target-specific hardware requirements |
For example, serialized ModelOpt MXFP8 linears can be converted to
per-channel-weight/per-token-activation FP8 at load time:
```bash
vllm serve MiniMaxAI/MiniMax-M3-MXFP8 \
--linear-backend auto \
--quantization-config.linear fp8_per_channel
```
!!! info
This is only available for linear layers at the moment.
Checkpoint-exclusion policy is not enforced by online quantization/re-quantization. Only exclusion through `--quantization-config.targets`/`--quantization-config.ignore` is enforced.
### Online quantization on unquantized layers from partially-quantized checkpoints
Online quantization can be used on already quantized checkpoints independently of their original `quant_method` (`modelopt`, `compressed-tensors`, `quark`, etc.), for layers that are left unquantized in the original checkpoint.
+178 -26
View File
@@ -58,6 +58,7 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
)
from vllm.model_executor.layers.quantization.modelopt import (
ModelOptFp8Config,
ModelOptLinearMethod,
ModelOptMxFp8Config,
)
from vllm.model_executor.layers.quantization.online.base import (
@@ -93,12 +94,14 @@ from vllm.model_executor.layers.quantization.quark.quark import (
)
from vllm.model_executor.layers.quantization.utils import quant_utils
from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
amax_for_moe_weight_quant,
amax_for_tp_weight_quant,
kMxfp8Dynamic,
kMxfp8Static,
weight_amax,
)
from vllm.model_executor.layers.vocab_parallel_embedding import (
@@ -112,6 +115,7 @@ from vllm.model_executor.models.granitemoe import (
)
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer_trtllm_fused_moe
from vllm.utils.torch_utils import set_default_torch_dtype
if current_platform.is_rocm():
from vllm.platforms.rocm import on_gfx942, on_gfx950
@@ -307,6 +311,14 @@ def _fully_quantized_modelopt_config() -> ModelOptFp8Config:
)
def _fully_quantized_modelopt_mxfp8_config() -> ModelOptMxFp8Config:
return ModelOptMxFp8Config(
is_checkpoint_mxfp8_serialized=True,
kv_cache_quant_algo=None,
exclude_modules=["lm_head"],
)
def _moe_only_compressed_tensors_config() -> CompressedTensorsConfig:
return CompressedTensorsConfig(
target_scheme_map={"RoutedExperts": {}},
@@ -334,10 +346,19 @@ def _write_minimal_llama_config(
@pytest.mark.parametrize(
"checkpoint_config_factory,raises_conflict",
"checkpoint_config_factory,raises_requantization_error",
[
pytest.param(_fully_quantized_quark_config, True, id="quark"),
pytest.param(_fully_quantized_modelopt_config, True, id="modelopt"),
pytest.param(
_fully_quantized_modelopt_config,
True,
id="modelopt",
),
pytest.param(
_fully_quantized_modelopt_mxfp8_config,
False,
id="modelopt_mxfp8",
),
pytest.param(
_moe_only_compressed_tensors_config,
False,
@@ -345,9 +366,13 @@ def _write_minimal_llama_config(
),
],
)
@pytest.mark.skipif(
not (current_platform.is_cuda() or current_platform.is_rocm()),
reason="MXFP8 requantization requires CUDA or ROCm.",
)
def test_online_prequantized_compatibility(
checkpoint_config_factory,
raises_conflict: bool,
raises_requantization_error: bool,
default_vllm_config,
dist_init,
) -> None:
@@ -370,35 +395,69 @@ def test_online_prequantized_compatibility(
"disable_tp": True,
}
if raises_conflict:
with pytest.raises(ValueError, match="pre-quantized layer"):
ColumnParallelLinear(**layer_kwargs)
layer = ColumnParallelLinear(**layer_kwargs)
assert isinstance(layer.quant_method, Mxfp8OnlineLinearMethod)
if layer.weight.is_meta:
# Weight loading.
layer = layer.to_empty(device=DEVICE)
for parameter in layer.parameters():
parameter.zero_()
else:
layer = ColumnParallelLinear(**layer_kwargs)
layer = layer.to(device=DEVICE)
if raises_requantization_error:
with pytest.raises(
NotImplementedError,
match="does not implement dequantize_weight|only supported for MXFP8",
):
layer.quant_method.process_weights_after_loading(layer)
else:
layer.quant_method.process_weights_after_loading(layer)
if isinstance(checkpoint_config, ModelOptMxFp8Config):
assert isinstance(layer.quant_method, Mxfp8OnlineLinearMethod)
assert not layer.quant_method.uses_meta_device
assert isinstance(
layer.quant_method.requantization_source, ModelOptLinearMethod
)
elif not raises_requantization_error:
assert layer.quant_method.uses_meta_device
assert layer.quant_method.requantization_source is None
def test_online_target_rejects_prequantized_layer(
default_vllm_config, dist_init
@pytest.mark.skipif(
not (current_platform.is_cuda() or current_platform.is_rocm())
or not is_quant_method_supported("fp8"),
reason="Requires FP8 support on CUDA or ROCm.",
)
def test_online_block_fp8_requantization_removes_source_scales(
default_vllm_config, dist_init, workspace_init
) -> None:
"""A targets match participates in checkpoint compatibility checks."""
default_vllm_config.model_config = ModelConfig()
prefix = "model.layers.0.self_attn.o_proj"
quant_config = _fully_quantized_quark_config()
"""Blockwise FP8 replaces MXFP8 scales with its own scale parameter."""
default_vllm_config.model_config = ModelConfig(dtype="bfloat16")
quant_config = _fully_quantized_modelopt_mxfp8_config()
quant_config.online_quantization_config = OnlineQuantizationConfig(
QuantizationConfigArgs(targets={prefix: "mxfp8"})
QuantizationConfigArgs(linear="fp8_per_block")
)
with pytest.raises(ValueError, match="pre-quantized layer"):
ColumnParallelLinear(
input_size=32,
output_size=32,
with set_default_torch_dtype(torch.bfloat16), torch.device(DEVICE):
layer = ColumnParallelLinear(
input_size=256,
output_size=256,
bias=False,
params_dtype=torch.bfloat16,
quant_config=quant_config,
prefix=prefix,
prefix="model.layers.0.self_attn.o_proj",
disable_tp=True,
)
with torch.no_grad():
layer.weight.fill_(1)
layer.weight_scale.fill_(127) # E8M0 encoding of scale 1.
layer.quant_method.process_weights_after_loading(layer)
assert not hasattr(layer, "weight_scale")
assert layer.weight_scale_inv.numel() > 0
def test_online_ignore_keeps_checkpoint_quantization_linear(
@@ -431,17 +490,25 @@ def test_online_ignore_keeps_checkpoint_quantization_linear(
def test_online_quantization_rejects_prequantized_moe(
default_vllm_config, dist_init
default_vllm_config, dist_init, monkeypatch
) -> None:
"""Online linear and MoE quantization reject a pre-quantized MoE layer."""
"""Reject pre-quantized MoE before constructing an online target backend."""
default_vllm_config.model_config = ModelConfig()
prefix = "model.layers.0.mlp.experts"
quant_config = _fully_quantized_quark_config()
quant_config.online_quantization_config = OnlineQuantizationConfig(
quant_config_args(linear="mxfp4", moe="mxfp4")
)
monkeypatch.setattr(
quant_config.online_quantization_config,
"get_quant_method",
lambda *args: pytest.fail("unsupported online backend must not be constructed"),
)
with pytest.raises(ValueError, match="pre-quantized layer"):
with pytest.raises(
NotImplementedError,
match="Requantizing checkpoint-quantized MoE layers is not supported",
):
FusedMoEFactory(
num_experts=4,
top_k=2,
@@ -591,7 +658,7 @@ def test_log_online_quantization_for_composable_config(monkeypatch) -> None:
assert log_args == [
(
"Quantized %d layers of types: %s",
"Quantizing %d layers of types: %s",
2,
"; ".join(online_config.quantized_layer_summaries),
)
@@ -836,6 +903,24 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp
{},
id="partially_prequantized_checkpoint",
),
pytest.param(
PARTIALLY_PREQUANTIZED_MODEL_NAME,
None,
{"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}},
Mxfp8OnlineLinearMethod,
CompressedTensorsMoEMethod,
{},
id="partially_prequantized_checkpoint",
),
pytest.param(
"mgoin/Qwen3-0.6B-MXFP8",
None,
{"linear": "fp8_per_channel"},
Fp8PtpcOnlineLinearMethod,
None,
{},
id="requantization_mxfp8_ptcp_fp8",
),
],
)
@pytest.mark.parametrize(
@@ -888,6 +973,25 @@ def test_online_quantization(
moe_backend=runner_kwargs.get("moe_backend", "auto"),
),
}
if model_name == "mgoin/Qwen3-0.6B-MXFP8":
original_process = Fp8PtpcOnlineLinearMethod.process_weights_after_loading
def assert_source_weights_released(method, layer) -> None:
source_parameters = dict(method.requantization_source_parameters)
assert set(source_parameters) == {"weight", "weight_scale"}
original_process(method, layer)
assert not method.requantization_source_parameters
assert layer.weight is not source_parameters["weight"]
assert layer.weight_scale is not source_parameters["weight_scale"]
assert layer.weight_scale.shape[-1] == 1
monkeypatch.setattr(
Fp8PtpcOnlineLinearMethod,
"process_weights_after_loading",
assert_source_weights_released,
)
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
model, vllm_config = load_model_without_vllm_runner(
@@ -943,11 +1047,17 @@ def test_online_quantization(
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:
o_proj = model.model.layers[1].self_attn.o_proj
moe = model.model.layers[0].mlp.experts
elif model_name == "mgoin/Qwen3-0.6B-MXFP8":
o_proj = model.model.layers[0].self_attn.o_proj
moe = None
else:
o_proj = model.model.layers[0].self_attn.o_proj
moe = model.model.layers[0].block_sparse_moe.experts
assert isinstance(o_proj.quant_method, expected_linear_cls)
assert isinstance(moe._quant_method, expected_moe_cls)
if moe is not None:
assert isinstance(moe._quant_method, expected_moe_cls)
if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME and isinstance(
o_proj.quant_method.kernel, MarlinMxfp8LinearKernel
@@ -1242,7 +1352,7 @@ def test_log_online_quantization(default_vllm_config, monkeypatch) -> None:
log_online_quantization(default_vllm_config)
assert logged_messages == [
"Quantized 3 layers of types: mlp.down_proj: 2 (from linear: "
"Quantizing 3 layers of types: mlp.down_proj: 2 (from linear: "
"fp8_per_tensor); self_attn.qkv_proj: 1 (from targets: "
"re:.*qkv_proj.*, mxfp4)"
]
@@ -1365,6 +1475,48 @@ def test_online_linear_tp_weight_quant_matches_unsharded(
assert torch.equal(tp_scale, full_scale)
@pytest.mark.skipif(
not is_quant_method_supported("fp8"),
reason="FP8 is not supported on this GPU type.",
)
def test_modelopt_mxfp8_requantized_ptpc_tp_shards_match_unsharded() -> None:
"""MXFP8 source shards convert to slices of the unsharded PTPC weight."""
torch.manual_seed(0)
serialized_weight = torch.randn(16, 64, device=DEVICE, dtype=torch.bfloat16).to(
MXFP8_VALUE_DTYPE
)
serialized_scale = torch.randint(
124,
131,
(16, 2),
device=DEVICE,
dtype=MXFP8_SCALE_DTYPE,
)
source = ModelOptLinearMethod.__new__(ModelOptLinearMethod)
source.wkey = SimpleNamespace(key=kMxfp8Static)
def dequantize(weight: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
layer = SimpleNamespace(weight=weight, weight_scale=scale)
return source.dequantize_weight(layer)
full_weight = dequantize(serialized_weight, serialized_scale)
full_scale = _fp8_channel_scale(weight_amax(full_weight, dim=-1, keepdim=True))
full_qweight = _fp8_quant_per_channel(full_weight, full_scale)
# Mimic the two K-dimension shards of a row-parallel layer. PTPC's global
# per-output-channel scale is the same on both ranks after its MAX reduce.
for rank in range(2):
start = rank * 32
shard_weight = serialized_weight[:, start : start + 32].contiguous()
shard_scale = serialized_scale[:, rank : rank + 1].contiguous()
weight = dequantize(shard_weight, shard_scale)
qweight = _fp8_quant_per_channel(weight, full_scale)
assert torch.equal(weight, full_weight[:, start : start + 32])
assert torch.equal(qweight, full_qweight[:, start : start + 32])
def _quantize_moe(weight, scheme, moe_tp_size):
if scheme == "nvfp4":
return _quantize_moe_weight_to_nvfp4(weight, moe_tp_size)
@@ -84,6 +84,13 @@ class QuantizeMethodBase(ABC):
"""
return
def dequantize_weight(self, layer: nn.Module) -> torch.Tensor:
"""Materialize a serialized quantized weight for requantization."""
raise NotImplementedError(
f"The quantization method {type(self)} does not implement "
"dequantize_weight. Please open an issue."
)
def method_has_implemented_embedding(method_class: type[QuantizeMethodBase]) -> bool:
"""Not all quant methods have embedding implemented, so we need to check that
@@ -294,13 +301,16 @@ def resolve_quant_method(
LinearBase,
UnquantizedLinearMethod,
)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
base_quant_method = quant_config.get_quant_method(layer, prefix)
if quant_config.online_quantization_config is None:
# No online configuration: retain the checkpoint method.
return base_quant_method
# Online quantization currently supports only LinearBase and RoutedExperts.
# Embeddings and ParallelLMHead retain their checkpoint quantization method.
if not isinstance(layer, (LinearBase, RoutedExperts)):
# Online quantization only supports linear and routed MoE layers.
return base_quant_method
quant_config.online_quantization_config.packed_modules_mapping = (
@@ -312,14 +322,34 @@ def resolve_quant_method(
online_target = quant_config.online_quantization_config.resolve_quant_method_cls(
layer, prefix
)
if checkpoint_is_quantized:
if online_target is not None:
raise ValueError(
f"Cannot apply requested online quantization {online_target[3]} to "
f"pre-quantized layer {prefix}: {base_quant_method} was already "
"selected by the checkpoint quantization config."
if online_target is None:
# The checkpoint quant method is applied as there is no online override.
return base_quant_method
if isinstance(layer, RoutedExperts):
raise NotImplementedError(
"Requantizing checkpoint-quantized MoE layers is not supported."
)
return base_quant_method
online_quant_method = quant_config.online_quantization_config.get_quant_method(
layer, prefix
)
assert base_quant_method is not None and not isinstance(
base_quant_method, (UnquantizedLinearMethod, UnquantizedFusedMoEMethod)
)
assert isinstance(online_quant_method, OnlineLinearBase)
online_quant_method.set_requantization_source(base_quant_method)
# The online method dequantizes the checkpoint method before requantizing.
return online_quant_method
if online_target is None:
# The layer is unquantized and no online target applies.
return base_quant_method
# Quantize an unquantized layer with the online method.
return quant_config.online_quantization_config.get_quant_method(layer, prefix)
@@ -76,6 +76,7 @@ from vllm.model_executor.layers.quantization.utils.mxfp8_utils import (
MXFP8_BLOCK_SIZE,
MXFP8_SCALE_DTYPE,
MXFP8_VALUE_DTYPE,
dequant_mxfp8_to_bf16,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
FP4_DTYPE,
@@ -2572,6 +2573,17 @@ class ModelOptLinearMethod(LinearMethodBase):
layer._nvfp4_group_size_for_gather = self.ctx.group_size
self.kernel.process_weights_after_loading(layer)
def dequantize_weight(self, layer: torch.nn.Module) -> torch.Tensor:
"""Reconstruct serialized weights for online requantization."""
if self.wkey.key is kMxfp8Static:
return dequant_mxfp8_to_bf16(
layer.weight.contiguous(), layer.weight_scale.contiguous()
)
else:
raise NotImplementedError(
"ModelOpt weight dequantization is only supported for MXFP8."
)
def apply(self, layer, x, bias=None):
return self.fmt.apply(
layer,
@@ -156,6 +156,7 @@ class OnlineQuantizationConfig(QuantizationConfig):
self.args = args
self.ignored_layers: list[str] = args.ignore
self.quantized_layers: dict[str, tuple[str, str, str | None]] = {}
self.online_quantization_time = 0.0
@property
def quantized_layer_summaries(self) -> list[str]:
@@ -34,6 +34,7 @@ from vllm.model_executor.layers.fusion.quant_activation import (
from vllm.model_executor.layers.linear import (
LinearMethodBase,
)
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
@@ -125,6 +126,13 @@ class OnlineLinearBase(LinearMethodBase):
def __init__(self):
self.out_dtype = torch.get_default_dtype()
self.input_dtype = get_current_vllm_config().model_config.dtype
self.requantization_source: QuantizeMethodBase | None = None
self.requantization_source_parameters: dict[str, torch.nn.Parameter] = {}
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
"""Configure serialized-weight conversion before online quantization."""
self.requantization_source = source_method
self.uses_meta_device = False
def create_weights(
self,
@@ -136,6 +144,24 @@ class OnlineLinearBase(LinearMethodBase):
params_dtype: torch.dtype,
**extra_weight_attrs,
):
if self.requantization_source is not None:
existing_parameter_names = set(layer._parameters)
self.requantization_source.create_weights(
layer,
input_size_per_partition,
output_partition_sizes,
input_size,
output_size,
params_dtype,
**extra_weight_attrs,
)
self.requantization_source_parameters = {
name: parameter
for name, parameter in layer._parameters.items()
if name not in existing_parameter_names and parameter is not None
}
return
output_size_per_partition = sum(output_partition_sizes)
weight_loader = extra_weight_attrs.get("weight_loader")
layer.logical_widths = output_partition_sizes
@@ -159,6 +185,19 @@ class OnlineLinearBase(LinearMethodBase):
initialize_online_processing(layer)
def release_requantization_source_weights(self, layer: torch.nn.Module) -> None:
"""Release checkpoint parameters after successful requantization."""
for name, source_parameter in self.requantization_source_parameters.items():
if layer._parameters.get(name) is source_parameter:
delattr(layer, name)
self.requantization_source_parameters.clear()
def get_weight_for_quantization(self, layer: Module) -> torch.Tensor:
"""Return checkpoint weights materialized for online quantization."""
if self.requantization_source is None:
return layer.weight
return self.requantization_source.dequantize_weight(layer)
class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
"""Online tensorwise FP8 linear quantization.
@@ -213,10 +252,11 @@ class Fp8PerTensorOnlineLinearMethod(OnlineLinearBase):
return
layer.input_scale = None
amax = weight_amax(layer.weight).reshape(1)
weight = self.get_weight_for_quantization(layer)
amax = weight_amax(weight).reshape(1)
amax = amax_for_tp_weight_quant(amax, _is_tp_sharded(layer))
weight_scale = _fp8_scale(amax)
qweight, _ = ops.scaled_fp8_quant(layer.weight, scale=weight_scale)
qweight, _ = ops.scaled_fp8_quant(weight, scale=weight_scale)
# Update layer with new values.
replace_parameter(layer, "weight", qweight.t().data)
@@ -312,14 +352,18 @@ class Fp8PerBlockOnlineLinearMethod(OnlineLinearBase):
layer.input_scale = None
block_size = self.weight_block_size
weight = self.get_weight_for_quantization(layer)
qweight, weight_scale_inv = per_block_cast_to_fp8(
layer.weight, block_size=block_size, use_ue8m0=False
weight, block_size=block_size, use_ue8m0=False
)
replace_parameter(layer, "weight", qweight.data)
replace_parameter(layer, "weight_scale_inv", weight_scale_inv.data)
if self.requantization_source is not None and hasattr(layer, "weight_scale"):
del layer.weight_scale
self.fp8_linear.process_weights_after_loading(layer)
# Prevent duplicate processing (e.g., during weight reload)
@@ -395,18 +439,20 @@ class Fp8PtpcOnlineLinearMethod(OnlineLinearBase):
return
layer.input_scale = None
amax = weight_amax(layer.weight, dim=-1, keepdim=True)
weight = self.get_weight_for_quantization(layer)
amax = weight_amax(weight, dim=-1, keepdim=True)
amax = amax_for_tp_weight_quant(
amax, _is_tp_sharded(layer, reduces_output_dim=False)
)
weight_scale = _fp8_channel_scale(amax)
qweight = _fp8_quant_per_channel(layer.weight, weight_scale)
qweight = _fp8_quant_per_channel(weight, weight_scale)
replace_parameter(layer, "weight", qweight.t())
replace_parameter(layer, "weight_scale", weight_scale)
self.fp8_linear.process_weights_after_loading(layer)
expose_input_quant_key(layer, self.fp8_linear)
self.release_requantization_source_weights(layer)
layer._already_called_process_weights_after_loading = True
@@ -11,6 +11,7 @@ from vllm.model_executor.layers.fused_moe import (
SharedExperts,
)
from vllm.model_executor.layers.fused_moe.moe_output import UnfinalizedMoEOutput
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
from vllm.model_executor.model_loader.reload.layerwise import (
initialize_online_processing,
)
@@ -24,6 +25,12 @@ class OnlineMoEMethodBase(FusedMoEMethodBase):
uses_meta_device: bool = True
def set_requantization_source(self, source_method: QuantizeMethodBase) -> None:
"""Reject requantization from a checkpoint-quantized MoE method."""
raise NotImplementedError(
"Requantizing checkpoint-quantized MoE layers is not supported."
)
def create_weights(
self,
layer: torch.nn.Module,
@@ -119,7 +119,8 @@ class Mxfp4OnlineLinearMethod(OnlineLinearBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
weight_fp4, weight_scale = mxfp4_quantize(layer.weight.contiguous())
weight = self.get_weight_for_quantization(layer)
weight_fp4, weight_scale = mxfp4_quantize(weight.contiguous())
layer.input_scale = None
replace_parameter(layer, "weight", weight_fp4.data)
@@ -75,7 +75,8 @@ class Mxfp8OnlineLinearMethod(OnlineLinearBase):
if getattr(layer, "_already_called_process_weights_after_loading", False):
return
weight_fp8, weight_scale = mxfp8_e4m3_quantize(layer.weight.contiguous())
weight = self.get_weight_for_quantization(layer)
weight_fp8, weight_scale = mxfp8_e4m3_quantize(weight.contiguous())
layer.input_scale = None
replace_parameter(layer, "weight", weight_fp8.data)
@@ -101,6 +101,7 @@ class BaseModelLoader(ABC):
finalize_layerwise_processing(model, model_config)
process_weights_after_loading(model, model_config, target_device)
log_online_quantization_time(vllm_config)
return model.eval()
@@ -115,8 +116,8 @@ def log_model_inspection(model: nn.Module) -> None:
logger.info("vLLM model structure:\n%s", format_model_inspection(model))
def log_online_quantization(vllm_config: VllmConfig) -> None:
"""Log the online-quantized layer count and types, when applicable."""
def _get_online_quantization_config(vllm_config: VllmConfig):
"""Return the online quantization config when one is configured."""
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
@@ -128,15 +129,35 @@ def log_online_quantization(vllm_config: VllmConfig) -> None:
if isinstance(online_quantization_config, OnlineQuantizationConfig):
quant_config = online_quantization_config
if not isinstance(quant_config, OnlineQuantizationConfig):
return None
return quant_config
def log_online_quantization(vllm_config: VllmConfig) -> None:
"""Log the online-quantized layer count and types, when applicable."""
quant_config = _get_online_quantization_config(vllm_config)
if quant_config is None:
return
logger.info(
"Quantized %d layers of types: %s",
"Quantizing %d layers of types: %s",
len(quant_config.quantized_layers),
"; ".join(quant_config.quantized_layer_summaries),
)
def log_online_quantization_time(vllm_config: VllmConfig) -> None:
"""Log online quantization processing time, when applicable."""
quant_config = _get_online_quantization_config(vllm_config)
if quant_config is None:
return
logger.info(
"Online quantization as part of model loading took %.2f seconds",
quant_config.online_quantization_time,
)
def _has_online_quant(model: nn.Module):
for module in model.modules():
quant_method = getattr(module, "quant_method", None)
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import inspect
from collections.abc import Callable
from contextlib import nullcontext
from functools import wraps
from weakref import WeakKeyDictionary, WeakSet
@@ -321,7 +322,21 @@ def _reload_attention_scales(layer: torch.nn.Module, info: LayerReloadingInfo) -
_get_weight_loader(param)(*args.args, **args.kwargs)
if quant_method is not None:
quant_method.process_weights_after_loading(layer)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
from vllm.model_executor.model_loader.utils import (
online_quantization_timing_context,
)
timing_context = (
online_quantization_timing_context(layer, quant_method)
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
else nullcontext()
)
with timing_context:
quant_method.process_weights_after_loading(layer)
_copy_and_restore_kernel_tensors(layer, info)
@@ -356,7 +371,21 @@ def _layerwise_process(layer: torch.nn.Module, info: LayerReloadingInfo):
# Process weights (quantization, repacking, etc.)
quant_method = getattr(layer, "quant_method", None)
if isinstance(quant_method, QuantizeMethodBase):
quant_method.process_weights_after_loading(layer)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
from vllm.model_executor.model_loader.utils import (
online_quantization_timing_context,
)
timing_context = (
online_quantization_timing_context(layer, quant_method)
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
else nullcontext()
)
with timing_context:
quant_method.process_weights_after_loading(layer)
# Re-reconcile parameter TP state: process_weights_after_loading may
# have re-created Parameters (stamped with the global rank), which would
# otherwise break replicated (disable_tp) weights on a subsequent reload.
+49 -1
View File
@@ -3,6 +3,7 @@
"""Utilities for selecting and loading models."""
import inspect
import time
import warnings
from contextlib import contextmanager, nullcontext
from typing import Any
@@ -141,6 +142,11 @@ def process_weights_after_loading(
"""
# Reclaim memory when an explicit lm_head has been
# loaded, but it is identical to the input embeddings.
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
maybe_retie_word_embeddings(model, model_config)
for name, module in model.named_modules():
@@ -166,7 +172,12 @@ def process_weights_after_loading(
if quant_method.requires_device_loading
else nullcontext()
)
with loading_context:
timing_context = (
online_quantization_timing_context(module, quant_method)
if isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
else nullcontext()
)
with loading_context, timing_context:
quant_method.process_weights_after_loading(module)
# process_weights_after_loading may swap in freshly-created
# Parameters (e.g. FP8 requantization), which are stamped with the
@@ -245,6 +256,43 @@ def device_loading_context(module: torch.nn.Module, target_device: torch.device)
p._vllm_is_uva_offloaded = True
@contextmanager
def online_quantization_timing_context(
layer: nn.Module, quant_method: QuantizeMethodBase
):
"""Accumulate online quantization processing time for ``quant_method``."""
from vllm.model_executor.layers.quantization.online.base import (
OnlineQuantizationConfig,
)
from vllm.model_executor.layers.quantization.online.fp8 import OnlineLinearBase
from vllm.model_executor.layers.quantization.online.moe_base import (
OnlineMoEMethodBase,
)
assert isinstance(quant_method, (OnlineLinearBase, OnlineMoEMethodBase))
quant_config = layer.quant_config
assert quant_config is not None
online_quantization_config = (
quant_config
if isinstance(quant_config, OnlineQuantizationConfig)
else quant_config.online_quantization_config
)
# reload/layerwise.py's _layerwise_process calls this for experts_int8,
# whose online MoE method has no OnlineQuantizationConfig.
if online_quantization_config is None:
yield
return
assert isinstance(online_quantization_config, OnlineQuantizationConfig)
start_time = time.perf_counter()
try:
yield
finally:
online_quantization_config.online_quantization_time += (
time.perf_counter() - start_time
)
_MODEL_ARCH_BY_HASH = dict[int, tuple[type[nn.Module], str]]()
"""Caches the outputs of `_get_model_architecture`."""