diff --git a/tests/v1/worker/test_gpu_model_runner.py b/tests/v1/worker/test_gpu_model_runner.py index 158befc150..0578768d68 100644 --- a/tests/v1/worker/test_gpu_model_runner.py +++ b/tests/v1/worker/test_gpu_model_runner.py @@ -10,6 +10,7 @@ import numpy as np import pytest import torch +import vllm.model_executor.model_loader.reload.inplace as reload_inplace_module import vllm.v1.worker.gpu_model_runner as gpu_model_runner_module from vllm.config import ( AttentionConfig, @@ -63,6 +64,7 @@ from vllm.v1.worker.gpu.mm.encoder_cache import EncoderCache from vllm.v1.worker.gpu.mm.lora import set_active_mm_loras from vllm.v1.worker.gpu_input_batch import InputBatch from vllm.v1.worker.gpu_model_runner import GPUModelRunner +from vllm.v1.worker.gpu_worker import Worker from vllm.v1.worker.utils import select_common_block_size BLOCK_SIZE = 16 @@ -1045,13 +1047,22 @@ def test_update_states_pp_async_multi_request_keeps_rank_state_consistent( ) +def _update_config(model_runner: GPUModelRunner, overrides: dict) -> None: + # Apply overrides via the worker without constructing a full worker. + worker = object.__new__(Worker) + worker.vllm_config = model_runner.vllm_config + worker.model_runner = model_runner + worker.update_config(overrides) + + def test_update_config(model_runner): # Simple update - model_runner.update_config({"load_config": {"load_format": "dummy"}}) + _update_config(model_runner, {"load_config": {"load_format": "dummy"}}) assert model_runner.load_config.load_format == "dummy" + assert model_runner.vllm_config.load_config is model_runner.load_config # Raise error on non-existing config with pytest.raises(ValueError, match="do_not_exist_config"): - model_runner.update_config({"do_not_exist_config": "dummy"}) + _update_config(model_runner, {"do_not_exist_config": "dummy"}) def test_load_model_weights_inplace(dist_init, model_runner, model_runner_2): @@ -1059,12 +1070,14 @@ def test_load_model_weights_inplace(dist_init, model_runner, model_runner_2): # model_runner_2 loads dummy weights first then load real weights inplace model_runner.load_model() original_load_format = model_runner_2.load_config.load_format - model_runner_2.update_config({"load_config": {"load_format": "dummy"}}) + _update_config(model_runner_2, {"load_config": {"load_format": "dummy"}}) model_runner_2.load_model() # Initial model loading with dummy weights assert str(model_runner.get_model().state_dict()) != str( model_runner_2.get_model().state_dict() ) - model_runner_2.update_config({"load_config": {"load_format": original_load_format}}) + _update_config( + model_runner_2, {"load_config": {"load_format": original_load_format}} + ) model_runner_2.reload_weights() # Load real weights inplace assert str(model_runner.get_model().state_dict()) == str( model_runner_2.get_model().state_dict() @@ -1083,26 +1096,34 @@ def test_reload_weights_path_replaces_object_storage_source(monkeypatch): # silently re-streams the original checkpoint. loader = Mock() loader.get_all_weights.return_value = iter(()) - monkeypatch.setattr(gpu_model_runner_module, "get_model_loader", lambda _: loader) - monkeypatch.setattr(gpu_model_runner_module, "initialize_layerwise_reload", Mock()) - monkeypatch.setattr(gpu_model_runner_module, "finalize_layerwise_reload", Mock()) + monkeypatch.setattr(reload_inplace_module, "get_model_loader", lambda _: loader) + monkeypatch.setattr(reload_inplace_module, "initialize_layerwise_reload", Mock()) + monkeypatch.setattr(reload_inplace_module, "finalize_layerwise_reload", Mock()) - runner = Mock(lora_config=None) - runner.model_config = SimpleNamespace( - model="/tmp/pulled-config-files", - model_weights="s3://bucket/original", - revision="abc123", - quantization=None, + vllm_config = SimpleNamespace( + lora_config=None, + load_config=SimpleNamespace(load_format="runai_streamer"), + model_config=SimpleNamespace( + model="/tmp/pulled-config-files", + model_weights="s3://bucket/original", + revision="abc123", + quantization=None, + ), ) - runner.get_model.return_value.named_parameters.return_value = [] - runner.get_model.return_value.load_weights.return_value = None + model = Mock() + model.named_parameters.return_value = [] + model.load_weights.return_value = None - GPUModelRunner.reload_weights(runner, weights_path="org/new-model") - - loader.get_all_weights.assert_called_once_with( - runner.model_config, runner.get_model.return_value + reload_inplace_module.reload_weights( + vllm_config, + model, + weights_iterator=None, + weights_path="org/new-model", + is_checkpoint_format=True, ) - cfg = runner.model_config + + loader.get_all_weights.assert_called_once_with(vllm_config.model_config, model) + cfg = vllm_config.model_config assert (cfg.model, cfg.model_weights, cfg.revision) == ("org/new-model", "", None) diff --git a/tests/v1/worker/test_gpu_worker_weight_transfer.py b/tests/v1/worker/test_gpu_worker_weight_transfer.py index 49ef707894..f1965cb1fa 100644 --- a/tests/v1/worker/test_gpu_worker_weight_transfer.py +++ b/tests/v1/worker/test_gpu_worker_weight_transfer.py @@ -13,7 +13,7 @@ import torch.nn as nn from vllm.config import ParallelConfig, VllmConfig, get_current_vllm_config from vllm.lora.layers import BaseLayerWithLoRA -from vllm.v1.worker.gpu_model_runner import _get_parameter_for_reload +from vllm.model_executor.model_loader.reload.inplace import get_parameter_for_reload from vllm.v1.worker.gpu_worker import Worker @@ -95,7 +95,7 @@ def test_reload_parameter_lookup_preserves_lora_module_names(): assert set(named_parameters) == {"proj.base_layer.qweight"} assert named_parameters["proj.base_layer.qweight"] is qweight assert model.get_parameter("proj.base_layer.qweight") is qweight - assert _get_parameter_for_reload(model, "proj.qweight") is qweight + assert get_parameter_for_reload(model, "proj.qweight") is qweight def test_start_update_finish_delegates_to_engine(): diff --git a/vllm/model_executor/model_loader/reload/inplace.py b/vllm/model_executor/model_loader/reload/inplace.py new file mode 100644 index 0000000000..dfe11ed4e7 --- /dev/null +++ b/vllm/model_executor/model_loader/reload/inplace.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import time +from collections.abc import Iterable +from typing import cast + +import torch +import torch.nn as nn + +from vllm.config import VllmConfig +from vllm.logger import init_logger +from vllm.model_executor.model_loader import get_model_loader + +from .layerwise import finalize_layerwise_reload, initialize_layerwise_reload + +logger = init_logger(__name__) + +__all__ = ["get_parameter_for_reload", "reload_weights"] + + +def get_parameter_for_reload(model: nn.Module, name: str) -> nn.Parameter: + """Resolve checkpoint names without changing the model's module tree.""" + from vllm.lora.layers import BaseLayerWithLoRA + + module_name, _, parameter_name = name.rpartition(".") + module = model.get_submodule(module_name) + if isinstance(module, BaseLayerWithLoRA): + module = module.base_layer + return module.get_parameter(parameter_name) + + +def reload_weights( + vllm_config: VllmConfig, + model: nn.Module, + weights_iterator: Iterable[tuple[str, torch.Tensor]] | None, + weights_path: str | None, + is_checkpoint_format: bool, +) -> None: + """Reload a model's weights in place from a weights iterator or from disk. + + Callers are responsible for invalidating any state derived from the old + weights (e.g. LoRA, encoder and multimodal caches). + + Args: + vllm_config: config of the model being reloaded. If `weights_path` is + given, `vllm_config.model_config` is updated to point at it. + model: model to load weights into + weights_iterator: weights to load into model + weights_path: path to load weights from if weights_iterator is not + provided. Use path of original model if neither is provided. + is_checkpoint_format: set to False if weights have already been + processed into kernel format (repacking, renaming, etc.) + + """ + # TODO(@kylesayrs): generalize to all runners and loaders + # argument validation + if weights_iterator is None and not is_checkpoint_format: + logger.warning( + "Reloading from disk means that weights will be in checkpoint format. " + "Please use `is_checkpoint_format=True` " + "to avoid weight reloading errors" + ) + + model_config = vllm_config.model_config + load_config = vllm_config.load_config + weights_to_load = { + name.replace(".base_layer.", ".") if vllm_config.lora_config else name + for name, _ in model.named_parameters() + } + counter_before_reloading = time.perf_counter() + + # load weights from disk if none are provided + if weights_iterator is None: + model_loader = get_model_loader(load_config) + if not hasattr(model_loader, "get_all_weights"): + raise NotImplementedError( + f"Model reloading with `{load_config.load_format}` format" + ) + + if weights_path is not None: + # The revision and any object-storage `model_weights` source + # belong to the model we are reloading away from, so they must + # not be carried over to the new path. + model_config.model = weights_path + model_config.model_weights = "" + model_config.revision = None + weights_iterator = model_loader.get_all_weights(model_config, model) + weights_iterator = cast(Iterable[tuple[str, torch.Tensor]], weights_iterator) + + # begin loading weights + logger.info_once("Reloading weights inplace...") + if is_checkpoint_format: + # load weights from checkpoint/ original model format + initialize_layerwise_reload(model) + loaded_weights = model.load_weights(weights_iterator) + finalize_layerwise_reload(model, model_config) + + else: + # load weights from kernel format + logger.warning_once( + "Reloading with `is_checkpoint_format=True` requires that " + "weights be in kernel format and already sharded", + ) + loaded_weights = set() + for name, loaded_weight in weights_iterator: + param = get_parameter_for_reload(model, name) # TODO: buffers? + param.copy_(loaded_weight) + loaded_weights.add(name) + + # logging and validation + counter_after_reloading = time.perf_counter() + diff_seconds = counter_after_reloading - counter_before_reloading + logger.info_once( + "Reloading and processing weights took %.2f seconds", + diff_seconds, + ) + if model_config.quantization is None and loaded_weights is not None: + weights_not_loaded = weights_to_load - loaded_weights + if weights_not_loaded: + logger.warning( + "Following weights were not loaded from checkpoint: %s", + weights_not_loaded, + ) diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index e04a381b94..20fa4c17bb 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -19,6 +19,7 @@ instead of embedding feature-specific logic directly. import functools import gc import time +from collections.abc import Iterable from contextlib import AbstractContextManager from copy import deepcopy from dataclasses import replace @@ -46,6 +47,7 @@ from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( initialize_mamba_ssu_backend, ) from vllm.model_executor.model_loader import get_model_loader +from vllm.model_executor.model_loader.reload.inplace import reload_weights from vllm.model_executor.models.interfaces import requires_raw_input_tokens from vllm.model_executor.offloader import ( create_offloader, @@ -563,22 +565,21 @@ class GPUModelRunner(LoRAModelRunnerMixin): return None return speculator.model - def reload_weights(self, *args, **kwargs) -> None: - # TODO(Wentao): Use full version instead of import when fully migrated to v2 - from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1 - - GPUModelRunnerV1.reload_weights(self, *args, **kwargs) # type: ignore[arg-type] - - def update_config(self, *args, **kwargs) -> None: - # TODO(Wentao): Use full version instead of import when fully migrated to v2 - from vllm.v1.worker.gpu_model_runner import GPUModelRunner as GPUModelRunnerV1 - - GPUModelRunnerV1.update_config(self, *args, **kwargs) # type: ignore[arg-type] - - # v2 reads config via self.vllm_config (e.g. in load_model), so keep it - # in sync with the attributes the v1 helper just replaced. - self.vllm_config.model_config = self.model_config - self.vllm_config.load_config = self.load_config + def reload_weights( + self, + weights_iterator: Iterable[tuple[str, torch.Tensor]] | None = None, + weights_path: str | None = None, + is_checkpoint_format: bool = True, + ) -> None: + """Reload weights in place.""" + vllm_config = self.vllm_config + model = self.get_model() + reload_weights( + vllm_config, model, weights_iterator, weights_path, is_checkpoint_format + ) + self.reset_lora_state() + self.reset_encoder_cache() + self.reset_mm_cache() @functools.cached_property def main_stream(self) -> torch.cuda.Stream: diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 90bea22c9d..4db3f00b91 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -34,7 +34,6 @@ from vllm.config import ( VllmConfig, get_layers_from_vllm_config, set_current_vllm_config, - update_config, ) from vllm.config.cache import CacheConfig from vllm.config.ec_manager_config import EncoderCacheManagerMetadata @@ -57,7 +56,7 @@ from vllm.forward_context import ( set_forward_context, ) from vllm.logger import init_logger -from vllm.lora.layers import BaseLayerWithLoRA, LoRAMapping, LoRAMappingType +from vllm.lora.layers import LoRAMapping, LoRAMappingType from vllm.model_executor.layers.attention import Attention from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase from vllm.model_executor.layers.fused_moe.all2all_utils import get_ep_all2all_manager @@ -67,10 +66,7 @@ from vllm.model_executor.layers.mamba.ops.ssu_dispatch import ( ) from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding from vllm.model_executor.model_loader import get_model_loader -from vllm.model_executor.model_loader.reload import ( - finalize_layerwise_reload, - initialize_layerwise_reload, -) +from vllm.model_executor.model_loader.reload.inplace import reload_weights from vllm.model_executor.models.interfaces import ( MixtureOfExperts, MultiModalEmbeddings, @@ -247,15 +243,6 @@ if TYPE_CHECKING: logger = init_logger(__name__) -def _get_parameter_for_reload(model: nn.Module, name: str) -> nn.Parameter: - """Resolve checkpoint names without changing the model's module tree.""" - module_name, _, parameter_name = name.rpartition(".") - module = model.get_submodule(module_name) - if isinstance(module, BaseLayerWithLoRA): - module = module.base_layer - return module.get_parameter(parameter_name) - - AttnMetadataDict: TypeAlias = dict[str, AttentionMetadata] # list when ubatching is enabled PerLayerAttnMetadata: TypeAlias = list[AttnMetadataDict] | AttnMetadataDict @@ -5203,19 +5190,6 @@ class GPUModelRunner( return draft_token_ids - def update_config(self, overrides: dict[str, Any]) -> None: - allowed_config_names = {"load_config", "model_config"} - for config_name, config_overrides in overrides.items(): - if config_name not in allowed_config_names: - allowed = ", ".join(sorted(allowed_config_names)) - raise ValueError( - f"Config override '{config_name}' is not supported. " - f"Supported configs: {allowed}" - ) - config = getattr(self, config_name) - new_config = update_config(config, config_overrides) - setattr(self, config_name, new_config) - @instrument(span_name="Loading (GPU)") def load_model( self, @@ -5472,89 +5446,16 @@ class GPUModelRunner( weights_path: str | None = None, is_checkpoint_format: bool = True, ) -> None: - """Reload weights from a weights iterator or from disk - - Args: - weights_iterator: weights to load into model - weights_path: path to load weights from if weights_iterator is not - provided. Use path of original model if neither is provided. - is_checkpoint_format: set to False if weights have already been - processed into kernel format (repacking, renaming, etc.) - - """ - # TODO(@kylesayrs): generalize to all runners and loaders - # argument validation - if weights_iterator is None and not is_checkpoint_format: - logger.warning( - "Reloading from disk means that weights will be in checkpoint format. " - "Please use `is_checkpoint_format=True` " - "to avoid weight reloading errors" - ) - - model = self.get_model() - weights_to_load = { - name.replace(".base_layer.", ".") if self.lora_config else name - for name, _ in model.named_parameters() - } - counter_before_reloading = time.perf_counter() - - # load weights from disk if none are provided - if weights_iterator is None: - model_loader = get_model_loader(self.load_config) - if not hasattr(model_loader, "get_all_weights"): - raise NotImplementedError( - f"Model reloading with `{self.load_config.load_format}` format" - ) - - if weights_path is not None: - # The revision and any object-storage `model_weights` source - # belong to the model we are reloading away from, so they must - # not be carried over to the new path. - self.model_config.model = weights_path - self.model_config.model_weights = "" - self.model_config.revision = None - weights_iterator = model_loader.get_all_weights(self.model_config, model) - weights_iterator = cast( - Iterable[tuple[str, torch.Tensor]], weights_iterator - ) - - # begin loading weights - logger.info_once("Reloading weights inplace...") - if is_checkpoint_format: - # load weights from checkpoint/ original model format - initialize_layerwise_reload(model) - loaded_weights = model.load_weights(weights_iterator) - finalize_layerwise_reload(model, self.model_config) - - else: - # load weights from kernel format - logger.warning_once( - "Reloading with `is_checkpoint_format=True` requires that " - "weights be in kernel format and already sharded", - ) - loaded_weights = set() - for name, loaded_weight in weights_iterator: - param = _get_parameter_for_reload(model, name) # TODO: buffers? - param.copy_(loaded_weight) - loaded_weights.add(name) - - self.reset_lora_state() - - # logging and validation - counter_after_reloading = time.perf_counter() - diff_seconds = counter_after_reloading - counter_before_reloading - logger.info_once( - "Reloading and processing weights took %.2f seconds", - diff_seconds, + """Reload weights in place; see `reload_weights` in + `vllm.model_executor.model_loader.reload.inplace` for details.""" + reload_weights( + self.vllm_config, + self.get_model(), + weights_iterator=weights_iterator, + weights_path=weights_path, + is_checkpoint_format=is_checkpoint_format, ) - if self.model_config.quantization is None and loaded_weights is not None: - weights_not_loaded = weights_to_load - loaded_weights - if weights_not_loaded: - logger.warning( - "Following weights were not loaded from checkpoint: %s", - weights_not_loaded, - ) - + self.reset_lora_state() self.reset_encoder_cache() self.reset_mm_cache() diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index a01e5dd5f7..37fa00bb7e 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -16,7 +16,12 @@ import torch import torch.nn as nn import vllm.envs as envs -from vllm.config import CUDAGraphMode, VllmConfig, set_current_vllm_config +from vllm.config import ( + CUDAGraphMode, + VllmConfig, + set_current_vllm_config, + update_config, +) from vllm.config.compilation import CompilationMode from vllm.config.profiler import validate_profile_prefix from vllm.device_allocator import get_mem_allocator_instance @@ -591,7 +596,21 @@ class Worker(WorkerBase): set_torch_threads_for_runtime() def update_config(self, overrides: dict[str, Any]) -> None: - self.model_runner.update_config(overrides) + """Apply config overrides to the worker, its model runner and the + shared VllmConfig.""" + allowed_config_names = {"load_config", "model_config"} + for config_name, config_overrides in overrides.items(): + if config_name not in allowed_config_names: + allowed = ", ".join(sorted(allowed_config_names)) + raise ValueError( + f"Config override '{config_name}' is not supported. " + f"Supported configs: {allowed}" + ) + config = getattr(self.vllm_config, config_name) + new_config = update_config(config, config_overrides) + setattr(self.vllm_config, config_name, new_config) + setattr(self.model_runner, config_name, new_config) + setattr(self, config_name, new_config) def reload_weights(self, *args, **kwargs) -> None: with set_current_vllm_config(self.vllm_config):