[Fast Start] Charge daemon-held weights against gpu_memory_utilization (#57298)

Signed-off-by: liusy58 <mg21330037@smail.nju.edu.cn>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Signed-off-by: Xun Sun <UNIDY2002@outlook.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Xun Sun <UNIDY2002@outlook.com>
This commit is contained in:
siyu
2026-09-29 17:58:35 +00:00
committed by GitHub
co-authored by Isotr0py Xun Sun
parent cb4f016c5d
commit f5ed3d9c03
7 changed files with 187 additions and 14 deletions
@@ -78,6 +78,7 @@ cached and keep loading from disk in the engine.
import contextlib
import fcntl
import gc
import multiprocessing
import os
import socket
@@ -116,6 +117,7 @@ from vllm.model_executor.model_loader.weight_cache.utils import (
is_draft_model_cacheable,
)
from vllm.platforms import current_platform
from vllm.utils.mem_utils import format_gib
from vllm.utils.torch_utils import set_default_torch_dtype
from vllm.v1.worker.workspace import init_workspace_manager
@@ -249,10 +251,17 @@ class WeightCacheDaemon:
with set_current_vllm_config(self.vllm_config):
ensure_model_parallel_initialized(self.tp_size, self.pp_size)
self.model = self.get_model()
# Loading and post-processing leave freed transients in the caching
# allocator; return them so engines sharing the GPU can use them.
gc.collect()
torch.accelerator.empty_cache()
logger.info(
"Weight cache %s daemon rank %d loaded model",
"Weight cache %s daemon rank %d loaded model (%s GiB allocated, "
"%s GiB reserved)",
self.role,
self.global_rank,
format_gib(torch.accelerator.memory_allocated()),
format_gib(torch.accelerator.memory_reserved()),
)
def get_model(self) -> torch.nn.Module:
@@ -369,6 +378,14 @@ class WeightCacheDaemon:
cmd = request.get("cmd")
if cmd == "get_state":
self._handle_get_state(conn, request)
elif cmd == "get_memory":
send_msg(
conn,
{
"status": "ok",
"memory_bytes": torch.accelerator.memory_allocated(),
},
)
elif cmd == "release":
self._handle_release(conn)
else:
@@ -6,7 +6,9 @@ daemon via CUDA IPC instead of loading from disk."""
import dataclasses
import socket
import time
from collections.abc import Callable
from copy import copy
from typing import TypeVar
import torch
import torch.nn as nn
@@ -23,6 +25,7 @@ from vllm.logger import init_logger
from vllm.model_executor.model_loader.base_loader import BaseModelLoader
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.model_executor.model_loader.utils import (
get_draft_load_config,
initialize_model,
process_weights_after_loading,
)
@@ -40,6 +43,9 @@ from vllm.model_executor.model_loader.weight_cache.protocol import (
send_msg,
verify_socket_owner,
)
from vllm.model_executor.model_loader.weight_cache.utils import (
is_draft_model_cacheable,
)
from vllm.model_executor.utils import weights_already_processed
from vllm.tracing import instrument
from vllm.utils.torch_utils import set_default_torch_dtype
@@ -50,6 +56,8 @@ _CONNECT_TIMEOUT_S = 5.0
_STATE_TIMEOUT_S = 300.0
_STARTUP_RETRY_INTERVAL_S = 0.5
_T = TypeVar("_T")
class IpcModelLoader(BaseModelLoader):
"""Loads a model by mapping the weight cache daemon's tensors via CUDA IPC.
@@ -110,6 +118,52 @@ class IpcModelLoader(BaseModelLoader):
f"{load_config.load_format}: {sorted(extra_config)}"
)
def get_external_weight_memory(self, vllm_config: VllmConfig) -> int:
# Copy mode clones the weights into this process; nothing external.
if self.mode != "zero_copy":
return 0
total = self._daemon_memory()
if is_draft_model_cacheable(vllm_config.speculative_config):
# The draft group is queried with the same config the engine
# would load the draft with.
draft_config = get_draft_load_config(vllm_config)
if draft_config.load_format == "ipc_cache":
draft_loader = IpcModelLoader(draft_config)
# Only a zero-copy draft-group daemon holds external weights;
# an explicit draft_load_config without is_draft would resolve
# back to the target socket and double-count it.
if draft_loader.is_draft and draft_loader.mode == "zero_copy":
total += draft_loader._daemon_memory()
return total
def _daemon_memory(self) -> int:
if not self.fallback:
# The loader waits for the daemon at load time, so the weights
# will be zero-copy mapped for sure; mirror that wait here since
# returning 0 would over-grant the memory budget.
return self._with_startup_wait(self._query_daemon_memory)
# An unreachable daemon means a disk load, i.e. nothing external.
try:
return self._query_daemon_memory()
except (WeightCacheUnavailableError, ConnectionError, OSError) as e:
logger.warning(
"Cannot query weight cache daemon memory (%s); "
"assuming the weights are not externally held",
e,
)
return 0
def _query_daemon_memory(self) -> int:
with self._connect(self.connect_timeout_s) as conn:
send_msg(conn, {"cmd": "get_memory"})
response = recv_msg(conn)
if response.get("status") != "ok":
raise WeightCacheUnavailableError(
"Weight cache daemon rejected the memory query: "
f"{response.get('message')}"
)
return int(response.get("memory_bytes", 0))
def download_model(self, model_config: ModelConfig) -> None:
DefaultModelLoader(self._fallback_load_config()).download_model(model_config)
@@ -308,23 +362,25 @@ class IpcModelLoader(BaseModelLoader):
def _request_state_with_startup_wait(
self, cache_config: WeightCacheKey
) -> WeightCacheState:
return self._with_startup_wait(lambda: self._request_state(cache_config))
def _with_startup_wait(self, op: Callable[[], _T]) -> _T:
"""Retry op until the daemon answers or the state timeout elapses;
the daemon may still be loading the model when the engine starts."""
deadline = time.monotonic() + self.state_timeout_s
waiting_logged = False
while True:
try:
return self._request_state(cache_config)
except WeightCacheUnavailableError as e:
return op()
except (WeightCacheUnavailableError, ConnectionError, OSError) as e:
if time.monotonic() >= deadline:
raise WeightCacheUnavailableError(
"Weight cache daemon did not become ready within "
f"{self.state_timeout_s:.1f}s"
f"{self.state_timeout_s:.1f}s: {e}"
) from e
if not waiting_logged:
logger.info(
"Waiting up to %.1fs for the weight cache daemon to start",
self.state_timeout_s,
)
waiting_logged = True
logger.info_once(
"Waiting up to %.1fs for the weight cache daemon to start",
self.state_timeout_s,
)
time.sleep(
max(
0.0,