Fast Start (#54921)

Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: liusy58 <liusy58@smail.nju.edu.cn>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
siyu
2026-09-04 06:32:06 -07:00
committed by GitHub
co-authored by liusy58 Isotr0py
parent 07ee97e023
commit a69e75b9b6
16 changed files with 1529 additions and 13 deletions
@@ -8,6 +8,7 @@ from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
QuantizeMethodBase,
)
from vllm.model_executor.utils import is_weights_pre_processed
from vllm.platforms import current_platform
from vllm.utils.torch_utils import is_quantized_kv_cache
from vllm.v1.kv_cache_interface import kv_cache_uses_per_token_head_scales
@@ -51,6 +52,8 @@ class BaseKVCacheMethod(QuantizeMethodBase):
quant_config: the appropriate QuantizationConfig
"""
supports_pre_processed_weights = True
def __init__(self, quant_config: QuantizationConfig):
self.quant_config = quant_config
@@ -72,6 +75,20 @@ class BaseKVCacheMethod(QuantizeMethodBase):
raise RuntimeError(f"{self.__class__.__name__}.apply should not be called.")
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
if is_weights_pre_processed():
# Scale buffers were exported with post-processed values; drop the
# consumed scale placeholders and rebuild the host-side copies.
for name in ("k_scale", "v_scale", "q_scale", "prob_scale"):
if hasattr(layer, name):
delattr(layer, name)
if hasattr(layer, "_k_scale"):
layer._k_scale_float = layer._k_scale.item()
layer._v_scale_float = layer._v_scale.item()
layer._q_scale_float = layer._q_scale.item()
layer._k_scale_cpu = layer._k_scale.detach().to("cpu")
layer._v_scale_cpu = layer._v_scale.detach().to("cpu")
return
# skip if there are no weights to process (for example, weight reloading)
if not hasattr(layer, "q_scale"):
assert not hasattr(layer, "k_scale")