Fast Start (#54921)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: liusy58 <liusy58@smail.nju.edu.cn>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-09-04 06:32:06 -07:00
committed by GitHub
co-authored by liusy58 Isotr0py
parent 07ee97e023
commit a69e75b9b6
16 changed files with 1529 additions and 13 deletions
@@ -27,6 +27,13 @@ class QuantizeMethodBase(ABC):
in process_weights_after_loading, reducing peak memory during loading.
"""
supports_pre_processed_weights: bool = False
"""
Whether ``process_weights_after_loading`` supports running under
``weights_already_processed``. Methods must skip tensor transforms in
that mode; the loader driver rejects methods that do not declare support.
"""
@abstractmethod
def create_weights(
self, layer: torch.nn.Module, *weight_args, **extra_weight_attrs
+31 -1
View File
@@ -74,7 +74,11 @@ from vllm.model_executor.parameter import (
BlockQuantScaleParameter,
PerTensorScaleParameter,
)
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
from vllm.model_executor.utils import (
is_weights_pre_processed,
replace_parameter,
set_weight_attrs,
)
from vllm.platforms import current_platform
from vllm.utils.deep_gemm import (
is_deep_gemm_supported,
@@ -248,6 +252,8 @@ class Fp8LinearMethod(LinearMethodBase):
quant_config: The quantization config.
"""
supports_pre_processed_weights = True
def __init__(self, quant_config: Fp8Config):
self.quant_config = quant_config
self.is_scale_e8m0 = getattr(quant_config, "is_scale_e8m0", False)
@@ -366,6 +372,19 @@ class Fp8LinearMethod(LinearMethodBase):
self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if is_weights_pre_processed():
# Weights are already in runtime format; the only state tensor
# export cannot carry is the `input_scale = None` stamp that dynamic
# activation quantization relies on, since create_weights never
# registered that parameter.
if not self.act_q_static and not hasattr(layer, "input_scale"):
layer.input_scale = None
# Marlin reads marlin_input_dtype in apply_weights; it lives on the
# method (not exported with the weights), so restore it here too.
if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"):
self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype
return
if self.use_marlin:
if not self.block_quant:
# Canonicalize to (K, N) for the kernel.
@@ -472,6 +491,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
quant_config: The quantization config.
"""
supports_pre_processed_weights = True
def __init__(self, quant_config: Fp8Config, layer: RoutedExperts):
super().__init__(layer.moe_config)
self.quant_config = quant_config
@@ -713,6 +734,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
replace_parameter(layer, f"w13_{self.weight_scale_name}", w13_scale)
replace_parameter(layer, f"w2_{self.weight_scale_name}", w2_scale)
self._init_moe_kernel(layer)
def _init_moe_kernel(self, layer: RoutedExperts) -> None:
"""Build the MoE kernel from the layer's current (converted) weights."""
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
assert self.moe_quant_config is not None
assert self.experts_cls is not None
@@ -725,6 +750,11 @@ class Fp8MoEMethod(FusedMoEMethodBase):
)
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
if is_weights_pre_processed():
# Weights are already in kernel format; rebuild the kernel only.
self._init_moe_kernel(layer)
return
# Allow for accessing weights and scales in standard way.
w13 = layer.w13_weight
w2 = layer.w2_weight
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.utils import is_weights_pre_processed
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.kv_cache_interface import kv_cache_uses_per_token_head_scales
@@ -51,6 +52,8 @@ class BaseKVCacheMethod(QuantizeMethodBase):
quant_config: the appropriate QuantizationConfig
"""
supports_pre_processed_weights = True
def __init__(self, quant_config: QuantizationConfig):
self.quant_config = quant_config
@@ -72,6 +75,20 @@ class BaseKVCacheMethod(QuantizeMethodBase):
raise RuntimeError(f"{self.__class__.__name__}.apply should not be called.")
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if is_weights_pre_processed():
# Scale buffers were exported with post-processed values; drop the
# consumed scale placeholders and rebuild the host-side copies.
for name in ("k_scale", "v_scale", "q_scale", "prob_scale"):
if hasattr(layer, name):
delattr(layer, name)
if hasattr(layer, "_k_scale"):
layer._k_scale_float = layer._k_scale.item()
layer._v_scale_float = layer._v_scale.item()
layer._q_scale_float = layer._q_scale.item()
layer._k_scale_cpu = layer._k_scale.detach().to("cpu")
layer._v_scale_cpu = layer._v_scale.detach().to("cpu")
return
# skip if there are no weights to process (for example, weight reloading)
if not hasattr(layer, "q_scale"):
assert not hasattr(layer, "k_scale")