[Fast Start] Charge daemon-held weights against gpu_memory_utilization (#57298)
Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Signed-off-by: Xun Sun <UNIDY2002@outlook.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Xun Sun <UNIDY2002@outlook.com>
This commit is contained in:
@@ -28,6 +28,11 @@ class BaseModelLoader(ABC):
|
||||
def __init__(self, load_config: LoadConfig):
|
||||
self.load_config = load_config
|
||||
|
||||
def get_external_weight_memory(self, vllm_config: VllmConfig) -> int:
|
||||
"""Get weights memory from external process;
|
||||
0 when the weights are not external."""
|
||||
return 0
|
||||
|
||||
@abstractmethod
|
||||
def download_model(self, model_config: ModelConfig) -> None:
|
||||
"""Download a model so that it can be immediately loaded."""
|
||||
|
||||
@@ -78,6 +78,7 @@ cached and keep loading from disk in the engine.
|
||||
|
||||
import contextlib
|
||||
import fcntl
|
||||
import gc
|
||||
import multiprocessing
|
||||
import os
|
||||
import socket
|
||||
@@ -116,6 +117,7 @@ from vllm.model_executor.model_loader.weight_cache.utils import (
|
||||
is_draft_model_cacheable,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_utils import format_gib
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
@@ -249,10 +251,17 @@ class WeightCacheDaemon:
|
||||
with set_current_vllm_config(self.vllm_config):
|
||||
ensure_model_parallel_initialized(self.tp_size, self.pp_size)
|
||||
self.model = self.get_model()
|
||||
# Loading and post-processing leave freed transients in the caching
|
||||
# allocator; return them so engines sharing the GPU can use them.
|
||||
gc.collect()
|
||||
torch.accelerator.empty_cache()
|
||||
logger.info(
|
||||
"Weight cache %s daemon rank %d loaded model",
|
||||
"Weight cache %s daemon rank %d loaded model (%s GiB allocated, "
|
||||
"%s GiB reserved)",
|
||||
self.role,
|
||||
self.global_rank,
|
||||
format_gib(torch.accelerator.memory_allocated()),
|
||||
format_gib(torch.accelerator.memory_reserved()),
|
||||
)
|
||||
|
||||
def get_model(self) -> torch.nn.Module:
|
||||
@@ -369,6 +378,14 @@ class WeightCacheDaemon:
|
||||
cmd = request.get("cmd")
|
||||
if cmd == "get_state":
|
||||
self._handle_get_state(conn, request)
|
||||
elif cmd == "get_memory":
|
||||
send_msg(
|
||||
conn,
|
||||
{
|
||||
"status": "ok",
|
||||
"memory_bytes": torch.accelerator.memory_allocated(),
|
||||
},
|
||||
)
|
||||
elif cmd == "release":
|
||||
self._handle_release(conn)
|
||||
else:
|
||||
|
||||
@@ -6,7 +6,9 @@ daemon via CUDA IPC instead of loading from disk."""
|
||||
import dataclasses
|
||||
import socket
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from copy import copy
|
||||
from typing import TypeVar
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -23,6 +25,7 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.model_executor.model_loader.utils import (
|
||||
get_draft_load_config,
|
||||
initialize_model,
|
||||
process_weights_after_loading,
|
||||
)
|
||||
@@ -40,6 +43,9 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
|
||||
send_msg,
|
||||
verify_socket_owner,
|
||||
)
|
||||
from vllm.model_executor.model_loader.weight_cache.utils import (
|
||||
is_draft_model_cacheable,
|
||||
)
|
||||
from vllm.model_executor.utils import weights_already_processed
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.torch_utils import set_default_torch_dtype
|
||||
@@ -50,6 +56,8 @@ _CONNECT_TIMEOUT_S = 5.0
|
||||
_STATE_TIMEOUT_S = 300.0
|
||||
_STARTUP_RETRY_INTERVAL_S = 0.5
|
||||
|
||||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
class IpcModelLoader(BaseModelLoader):
|
||||
"""Loads a model by mapping the weight cache daemon's tensors via CUDA IPC.
|
||||
@@ -110,6 +118,52 @@ class IpcModelLoader(BaseModelLoader):
|
||||
f"{load_config.load_format}: {sorted(extra_config)}"
|
||||
)
|
||||
|
||||
def get_external_weight_memory(self, vllm_config: VllmConfig) -> int:
|
||||
# Copy mode clones the weights into this process; nothing external.
|
||||
if self.mode != "zero_copy":
|
||||
return 0
|
||||
total = self._daemon_memory()
|
||||
if is_draft_model_cacheable(vllm_config.speculative_config):
|
||||
# The draft group is queried with the same config the engine
|
||||
# would load the draft with.
|
||||
draft_config = get_draft_load_config(vllm_config)
|
||||
if draft_config.load_format == "ipc_cache":
|
||||
draft_loader = IpcModelLoader(draft_config)
|
||||
# Only a zero-copy draft-group daemon holds external weights;
|
||||
# an explicit draft_load_config without is_draft would resolve
|
||||
# back to the target socket and double-count it.
|
||||
if draft_loader.is_draft and draft_loader.mode == "zero_copy":
|
||||
total += draft_loader._daemon_memory()
|
||||
return total
|
||||
|
||||
def _daemon_memory(self) -> int:
|
||||
if not self.fallback:
|
||||
# The loader waits for the daemon at load time, so the weights
|
||||
# will be zero-copy mapped for sure; mirror that wait here since
|
||||
# returning 0 would over-grant the memory budget.
|
||||
return self._with_startup_wait(self._query_daemon_memory)
|
||||
# An unreachable daemon means a disk load, i.e. nothing external.
|
||||
try:
|
||||
return self._query_daemon_memory()
|
||||
except (WeightCacheUnavailableError, ConnectionError, OSError) as e:
|
||||
logger.warning(
|
||||
"Cannot query weight cache daemon memory (%s); "
|
||||
"assuming the weights are not externally held",
|
||||
e,
|
||||
)
|
||||
return 0
|
||||
|
||||
def _query_daemon_memory(self) -> int:
|
||||
with self._connect(self.connect_timeout_s) as conn:
|
||||
send_msg(conn, {"cmd": "get_memory"})
|
||||
response = recv_msg(conn)
|
||||
if response.get("status") != "ok":
|
||||
raise WeightCacheUnavailableError(
|
||||
"Weight cache daemon rejected the memory query: "
|
||||
f"{response.get('message')}"
|
||||
)
|
||||
return int(response.get("memory_bytes", 0))
|
||||
|
||||
def download_model(self, model_config: ModelConfig) -> None:
|
||||
DefaultModelLoader(self._fallback_load_config()).download_model(model_config)
|
||||
|
||||
@@ -308,23 +362,25 @@ class IpcModelLoader(BaseModelLoader):
|
||||
def _request_state_with_startup_wait(
|
||||
self, cache_config: WeightCacheKey
|
||||
) -> WeightCacheState:
|
||||
return self._with_startup_wait(lambda: self._request_state(cache_config))
|
||||
|
||||
def _with_startup_wait(self, op: Callable[[], _T]) -> _T:
|
||||
"""Retry op until the daemon answers or the state timeout elapses;
|
||||
the daemon may still be loading the model when the engine starts."""
|
||||
deadline = time.monotonic() + self.state_timeout_s
|
||||
waiting_logged = False
|
||||
while True:
|
||||
try:
|
||||
return self._request_state(cache_config)
|
||||
except WeightCacheUnavailableError as e:
|
||||
return op()
|
||||
except (WeightCacheUnavailableError, ConnectionError, OSError) as e:
|
||||
if time.monotonic() >= deadline:
|
||||
raise WeightCacheUnavailableError(
|
||||
"Weight cache daemon did not become ready within "
|
||||
f"{self.state_timeout_s:.1f}s"
|
||||
f"{self.state_timeout_s:.1f}s: {e}"
|
||||
) from e
|
||||
if not waiting_logged:
|
||||
logger.info(
|
||||
"Waiting up to %.1fs for the weight cache daemon to start",
|
||||
self.state_timeout_s,
|
||||
)
|
||||
waiting_logged = True
|
||||
logger.info_once(
|
||||
"Waiting up to %.1fs for the weight cache daemon to start",
|
||||
self.state_timeout_s,
|
||||
)
|
||||
time.sleep(
|
||||
max(
|
||||
0.0,
|
||||
|
||||
Reference in New Issue
Block a user