[mooncake] support CUSTOM_MEM_POOL in vllm (#49300)

Signed-off-by: bruce.xu <bruce.xb@alibaba-inc.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: bruce.xu <bruce.xb@alibaba-inc.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
void
2026-09-27 00:26:05 +00:00
committed by GitHub
co-authored by bruce.xu Yifan Qiao OpenAI Codex
parent cd1daf58a9
commit eb0f2ca37f
7 changed files with 193 additions and 4 deletions
+12 -3
View File
@@ -801,11 +801,20 @@ class Worker(WorkerBase):
# related to kv cache connector (e.g. kv cache sharing layers).
ensure_kv_transfer_initialized(self.vllm_config, kv_cache_config)
# If the connector provides a custom memory pool (e.g. Mooncake
# NVLink/BAREX), use it for KV cache allocation; otherwise fall
# back to the standard CuMem pool.
mem_pool_context = (
get_kv_transfer_group().get_mem_pool_context()
if has_kv_transfer_group()
else None
)
if mem_pool_context is None:
mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache")
self.model_runner.initialize_kv_cache(
kv_cache_config,
kv_cache_allocation_context=self._maybe_get_memory_pool_context(
tag="kv_cache"
),
kv_cache_allocation_context=mem_pool_context,
)
# Build KV-zero metadata outside the CuMem pool so the bookkeeping