[core][model] yet another cpu offload implementation (#6496)

Co-authored-by: Michael Goin <michael@neuralmagic.com>
This commit is contained in:
youkaichao
2024-07-17 20:54:35 -07:00
committed by GitHub
co-authored by Michael Goin
parent 18fecc3559
commit 1c27d25fb5
7 changed files with 128 additions and 4 deletions
+4
View File
@@ -39,6 +39,7 @@ from vllm.model_executor.model_loader import get_model
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
from vllm.model_executor.models.interfaces import (supports_lora,
supports_vision)
from vllm.model_executor.models.utils import set_cpu_offload_max_bytes
from vllm.multimodal import (MULTIMODAL_REGISTRY, BatchedTensors,
MultiModalInputs)
from vllm.prompt_adapter.layers import PromptAdapterMapping
@@ -544,6 +545,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
self.flashinfer_prefill_workspace_buffer = None
self.flashinfer_prefill_wrapper = None
set_cpu_offload_max_bytes(
int(self.cache_config.cpu_offload_gb * 1024**3))
def load_model(self) -> None:
with CudaMemoryProfiler() as m:
self.model = get_model(model_config=self.model_config,