Fast Start (#54921)
Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: liusy58 <liusy58@smail.nju.edu.cn> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
@@ -27,6 +27,13 @@ class QuantizeMethodBase(ABC):
|
||||
in process_weights_after_loading, reducing peak memory during loading.
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights: bool = False
|
||||
"""
|
||||
Whether ``process_weights_after_loading`` supports running under
|
||||
``weights_already_processed``. Methods must skip tensor transforms in
|
||||
that mode; the loader driver rejects methods that do not declare support.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def create_weights(
|
||||
self, layer: torch.nn.Module, *weight_args, **extra_weight_attrs
|
||||
|
||||
@@ -74,7 +74,11 @@ from vllm.model_executor.parameter import (
|
||||
BlockQuantScaleParameter,
|
||||
PerTensorScaleParameter,
|
||||
)
|
||||
from vllm.model_executor.utils import replace_parameter, set_weight_attrs
|
||||
from vllm.model_executor.utils import (
|
||||
is_weights_pre_processed,
|
||||
replace_parameter,
|
||||
set_weight_attrs,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.deep_gemm import (
|
||||
is_deep_gemm_supported,
|
||||
@@ -248,6 +252,8 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
quant_config: The quantization config.
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, quant_config: Fp8Config):
|
||||
self.quant_config = quant_config
|
||||
self.is_scale_e8m0 = getattr(quant_config, "is_scale_e8m0", False)
|
||||
@@ -366,6 +372,19 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
self.use_marlin = isinstance(self.fp8_linear, MarlinFP8ScaledMMLinearKernel)
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if is_weights_pre_processed():
|
||||
# Weights are already in runtime format; the only state tensor
|
||||
# export cannot carry is the `input_scale = None` stamp that dynamic
|
||||
# activation quantization relies on, since create_weights never
|
||||
# registered that parameter.
|
||||
if not self.act_q_static and not hasattr(layer, "input_scale"):
|
||||
layer.input_scale = None
|
||||
# Marlin reads marlin_input_dtype in apply_weights; it lives on the
|
||||
# method (not exported with the weights), so restore it here too.
|
||||
if self.use_marlin and hasattr(self.fp8_linear, "marlin_input_dtype"):
|
||||
self.fp8_linear.marlin_input_dtype = self.marlin_input_dtype
|
||||
return
|
||||
|
||||
if self.use_marlin:
|
||||
if not self.block_quant:
|
||||
# Canonicalize to (K, N) for the kernel.
|
||||
@@ -472,6 +491,8 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
quant_config: The quantization config.
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, quant_config: Fp8Config, layer: RoutedExperts):
|
||||
super().__init__(layer.moe_config)
|
||||
self.quant_config = quant_config
|
||||
@@ -713,6 +734,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
replace_parameter(layer, f"w13_{self.weight_scale_name}", w13_scale)
|
||||
replace_parameter(layer, f"w2_{self.weight_scale_name}", w2_scale)
|
||||
|
||||
self._init_moe_kernel(layer)
|
||||
|
||||
def _init_moe_kernel(self, layer: RoutedExperts) -> None:
|
||||
"""Build the MoE kernel from the layer's current (converted) weights."""
|
||||
self.moe_quant_config = self.get_fused_moe_quant_config(layer)
|
||||
assert self.moe_quant_config is not None
|
||||
assert self.experts_cls is not None
|
||||
@@ -725,6 +750,11 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
|
||||
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
|
||||
if is_weights_pre_processed():
|
||||
# Weights are already in kernel format; rebuild the kernel only.
|
||||
self._init_moe_kernel(layer)
|
||||
return
|
||||
|
||||
# Allow for accessing weights and scales in standard way.
|
||||
w13 = layer.w13_weight
|
||||
w2 = layer.w2_weight
|
||||
|
||||
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.quantization.base_config import (
|
||||
QuantizationConfig,
|
||||
QuantizeMethodBase,
|
||||
)
|
||||
from vllm.model_executor.utils import is_weights_pre_processed
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import is_quantized_kv_cache
|
||||
from vllm.v1.kv_cache_interface import kv_cache_uses_per_token_head_scales
|
||||
@@ -51,6 +52,8 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
quant_config: the appropriate QuantizationConfig
|
||||
"""
|
||||
|
||||
supports_pre_processed_weights = True
|
||||
|
||||
def __init__(self, quant_config: QuantizationConfig):
|
||||
self.quant_config = quant_config
|
||||
|
||||
@@ -72,6 +75,20 @@ class BaseKVCacheMethod(QuantizeMethodBase):
|
||||
raise RuntimeError(f"{self.__class__.__name__}.apply should not be called.")
|
||||
|
||||
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
|
||||
if is_weights_pre_processed():
|
||||
# Scale buffers were exported with post-processed values; drop the
|
||||
# consumed scale placeholders and rebuild the host-side copies.
|
||||
for name in ("k_scale", "v_scale", "q_scale", "prob_scale"):
|
||||
if hasattr(layer, name):
|
||||
delattr(layer, name)
|
||||
if hasattr(layer, "_k_scale"):
|
||||
layer._k_scale_float = layer._k_scale.item()
|
||||
layer._v_scale_float = layer._v_scale.item()
|
||||
layer._q_scale_float = layer._q_scale.item()
|
||||
layer._k_scale_cpu = layer._k_scale.detach().to("cpu")
|
||||
layer._v_scale_cpu = layer._v_scale.detach().to("cpu")
|
||||
return
|
||||
|
||||
# skip if there are no weights to process (for example, weight reloading)
|
||||
if not hasattr(layer, "q_scale"):
|
||||
assert not hasattr(layer, "k_scale")
|
||||
|
||||
Reference in New Issue
Block a user