[Fast Start] Charge daemon-held weights against gpu_memory_utilization (#57298)
Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Xun Sun <UNIDY2002@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Xun Sun <UNIDY2002@outlook.com>
This commit is contained in:
@@ -57,6 +57,7 @@ from vllm.distributed.weight_transfer import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.model_executor.model_loader import get_model_loader
|
||||
from vllm.model_executor.warmup.kernel_warmup import kernel_warmup
|
||||
from vllm.multimodal.gpu_ipc_memory import reserve_mm_ipc_gpu_memory
|
||||
from vllm.platforms import current_platform
|
||||
@@ -487,7 +488,13 @@ class Worker(WorkerBase):
|
||||
|
||||
# take current memory snapshot
|
||||
self.init_snapshot = init_snapshot = MemorySnapshot(device=self.device)
|
||||
self.requested_memory = request_memory(init_snapshot, self.cache_config)
|
||||
# Weights from external model loader process
|
||||
external_weight_memory = get_model_loader(
|
||||
self.load_config
|
||||
).get_external_weight_memory(self.vllm_config)
|
||||
self.requested_memory = request_memory(
|
||||
init_snapshot, self.cache_config, external_weight_memory
|
||||
)
|
||||
logger.debug("worker init memory snapshot: %r", self.init_snapshot)
|
||||
logger.debug(
|
||||
"worker requested memory: %sGiB", format_gib(self.requested_memory)
|
||||
|
||||
Reference in New Issue
Block a user