[mooncake] support CUSTOM_MEM_POOL in vllm (#49300)
Signed-off-by: bruce.xu <bruce.xb@alibaba-inc.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: bruce.xu <bruce.xb@alibaba-inc.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
bruce.xu
Yifan Qiao
OpenAI Codex
parent
cd1daf58a9
commit
eb0f2ca37f
@@ -801,11 +801,20 @@ class Worker(WorkerBase):
|
||||
# related to kv cache connector (e.g. kv cache sharing layers).
|
||||
ensure_kv_transfer_initialized(self.vllm_config, kv_cache_config)
|
||||
|
||||
# If the connector provides a custom memory pool (e.g. Mooncake
|
||||
# NVLink/BAREX), use it for KV cache allocation; otherwise fall
|
||||
# back to the standard CuMem pool.
|
||||
mem_pool_context = (
|
||||
get_kv_transfer_group().get_mem_pool_context()
|
||||
if has_kv_transfer_group()
|
||||
else None
|
||||
)
|
||||
if mem_pool_context is None:
|
||||
mem_pool_context = self._maybe_get_memory_pool_context(tag="kv_cache")
|
||||
|
||||
self.model_runner.initialize_kv_cache(
|
||||
kv_cache_config,
|
||||
kv_cache_allocation_context=self._maybe_get_memory_pool_context(
|
||||
tag="kv_cache"
|
||||
),
|
||||
kv_cache_allocation_context=mem_pool_context,
|
||||
)
|
||||
|
||||
# Build KV-zero metadata outside the CuMem pool so the bookkeeping
|
||||
|
||||
Reference in New Issue
Block a user