[Bugfix] Export weight-cache IPC tensors separately for each client (#56472)
Signed-off-by: 子华 <huaxi.shx@alibaba-inc.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Codex
Isotr0py
parent
17bc3e106a
commit
567f745c1b
@@ -93,11 +93,14 @@ def export_entries(
|
||||
``named_parameters``/``named_buffers`` are iterated with
|
||||
``remove_duplicate=False`` so tied weights (e.g. ``lm_head.weight`` sharing
|
||||
storage with ``embed_tokens.weight``) are not silently dropped. Each unique
|
||||
tensor is exported once; every additional name that refers to the same
|
||||
tensor is exported once per call; every additional name that refers to the same
|
||||
tensor object is recorded in the returned alias map so the client can
|
||||
re-establish the shared identity instead of allocating uninitialized
|
||||
memory for it.
|
||||
|
||||
CUDA reduction arguments must be exported separately for each consumer so
|
||||
that PyTorch registers a reference for each IPC mapping's lifetime.
|
||||
|
||||
Returns:
|
||||
A ``(entries, aliases)`` pair where ``entries`` maps a canonical name to
|
||||
its ``TensorEntry`` and ``aliases`` maps each duplicate name to its
|
||||
@@ -167,8 +170,6 @@ class WeightCacheDaemon:
|
||||
self.distributed_init_method = distributed_init_method
|
||||
self.socket_dir = socket_dir
|
||||
self.model: torch.nn.Module | None = None
|
||||
self.entries: dict[str, TensorEntry] = {}
|
||||
self.aliases: dict[str, str] = {}
|
||||
# Fingerprint before loading: process_weights_after_loading may
|
||||
# mutate hf_config.quantization_config.
|
||||
self.cache_config = WeightCacheKey.from_model_config(
|
||||
@@ -190,17 +191,11 @@ class WeightCacheDaemon:
|
||||
with set_current_vllm_config(self.vllm_config):
|
||||
ensure_model_parallel_initialized(tp_size, 1)
|
||||
self.model = get_daemon_model(self.vllm_config)
|
||||
self._export_entries()
|
||||
logger.info(
|
||||
"Weight cache daemon rank %d cached %d tensors",
|
||||
"Weight cache daemon rank %d loaded model",
|
||||
self.tp_rank,
|
||||
len(self.entries),
|
||||
)
|
||||
|
||||
def _export_entries(self) -> None:
|
||||
assert self.model is not None
|
||||
self.entries, self.aliases = export_entries(self.model)
|
||||
|
||||
def serve_forever(self, ready_callback: Callable[[], None] | None = None) -> None:
|
||||
"""Serve requests until terminated.
|
||||
|
||||
@@ -295,28 +290,28 @@ class WeightCacheDaemon:
|
||||
logger.warning("WeightCacheKey mismatch on fields: %s", mismatched)
|
||||
send_msg(conn, {"status": "mismatch", "fields": mismatched})
|
||||
return
|
||||
if not self.entries:
|
||||
if self.model is None:
|
||||
send_msg(conn, {"status": "error", "message": "Weights were released"})
|
||||
return
|
||||
gpu_uuid = get_current_device_uuid()
|
||||
entries, aliases = export_entries(self.model)
|
||||
send_msg(
|
||||
conn,
|
||||
{
|
||||
"status": "ok",
|
||||
"entries": self.entries,
|
||||
"aliases": self.aliases,
|
||||
"gpu_uuid": get_current_device_uuid(),
|
||||
"entries": entries,
|
||||
"aliases": aliases,
|
||||
"gpu_uuid": gpu_uuid,
|
||||
},
|
||||
)
|
||||
logger.info_once(
|
||||
"Weight cache daemon rank %d sent %d tensors (+%d aliases) to engine",
|
||||
self.tp_rank,
|
||||
len(self.entries),
|
||||
len(self.aliases),
|
||||
len(entries),
|
||||
len(aliases),
|
||||
)
|
||||
|
||||
def _handle_release(self, conn: socket.socket) -> None:
|
||||
self.entries.clear()
|
||||
self.aliases.clear()
|
||||
self.model = None
|
||||
torch.accelerator.empty_cache()
|
||||
logger.info("Weight cache daemon rank %d released cached weights", self.tp_rank)
|
||||
|
||||
Reference in New Issue
Block a user