[Quantization] Consolidate dummy format logic into DummyModelLoader (#38637)
Signed-off-by: Josephasafg <ajgard7@gmail.com>
This commit is contained in:
@@ -4,8 +4,19 @@ import torch.nn as nn
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config.load import LoadConfig
|
||||
from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
|
||||
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
|
||||
from vllm.model_executor.model_loader.weight_utils import initialize_dummy_weights
|
||||
from vllm.model_executor.model_loader.reload.layerwise import (
|
||||
_get_original_loader,
|
||||
get_layerwise_info,
|
||||
)
|
||||
from vllm.model_executor.model_loader.reload.meta import materialize_layer
|
||||
from vllm.model_executor.model_loader.reload.types import LayerReloadingInfo
|
||||
from vllm.model_executor.model_loader.reload.utils import get_layer_tensors
|
||||
from vllm.model_executor.model_loader.weight_utils import (
|
||||
initialize_dummy_weights,
|
||||
initialize_single_dummy_weight,
|
||||
)
|
||||
|
||||
|
||||
class DummyModelLoader(BaseModelLoader):
|
||||
@@ -23,6 +34,31 @@ class DummyModelLoader(BaseModelLoader):
|
||||
pass # Nothing to download
|
||||
|
||||
def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
|
||||
# NOTE(woosuk): For accurate performance evaluation, we assign
|
||||
# random values to the weights.
|
||||
initialize_dummy_weights(model, model_config)
|
||||
for layer in model.modules():
|
||||
info = get_layerwise_info(layer)
|
||||
if info.can_load():
|
||||
self._process_online_quant_layer(layer, info)
|
||||
else:
|
||||
# NOTE(woosuk): For accurate performance evaluation, we assign
|
||||
# random values to the weights.
|
||||
initialize_dummy_weights(layer, model_config)
|
||||
|
||||
def _process_online_quant_layer(
|
||||
self,
|
||||
layer: nn.Module,
|
||||
info: LayerReloadingInfo,
|
||||
) -> None:
|
||||
"""Materialize, apply dummy weights, and run quantization processing."""
|
||||
materialize_layer(layer, info)
|
||||
|
||||
for tensor in get_layer_tensors(layer).values():
|
||||
initialize_single_dummy_weight(tensor)
|
||||
|
||||
for param in get_layer_tensors(layer).values():
|
||||
param.weight_loader = _get_original_loader(param)
|
||||
|
||||
quant_method = getattr(layer, "quant_method", None)
|
||||
if isinstance(quant_method, QuantizeMethodBase):
|
||||
quant_method.process_weights_after_loading(layer)
|
||||
|
||||
info.reset()
|
||||
|
||||
Reference in New Issue
Block a user