Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Chao Lei <leichao139636@163.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Chao Lei <leichao139636@163.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Chao Lei
Claude Fable 5
Matthew Bonanni
OpenAI Codex
OpenAI Codex
parent
ebe1dec2da
commit
e19a3e172e
+44
-27
@@ -120,27 +120,32 @@ def init_hisparse_kv_cache(
|
||||
block_tables: "BlockTables",
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""Allocate and bind HiSparse caches within the caller's allocation context."""
|
||||
host_pool = HiSparseHostPool()
|
||||
kv_caches = allocate_hisparse_kv_caches(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
host_pool,
|
||||
)
|
||||
cache_handles = bind_hisparse_kv_caches(
|
||||
forward_context=forward_context,
|
||||
kv_cache_config=kv_cache_config,
|
||||
kv_caches=kv_caches,
|
||||
block_tables=block_tables,
|
||||
host_pool=host_pool,
|
||||
)
|
||||
initialize_hisparse_runtime_buffers(
|
||||
cache_handles,
|
||||
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
|
||||
max_num_batched_tokens=vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
)
|
||||
return kv_caches
|
||||
host_pool = HiSparseHostPool(vllm_config, kv_cache_config)
|
||||
try:
|
||||
kv_caches = allocate_hisparse_kv_caches(
|
||||
kv_cache_config,
|
||||
device,
|
||||
vllm_config.cache_config.get_resolved_kv_cache_layout(),
|
||||
kernel_block_sizes,
|
||||
host_pool,
|
||||
)
|
||||
cache_handles = bind_hisparse_kv_caches(
|
||||
forward_context=forward_context,
|
||||
kv_cache_config=kv_cache_config,
|
||||
kv_caches=kv_caches,
|
||||
block_tables=block_tables,
|
||||
host_pool=host_pool,
|
||||
)
|
||||
initialize_hisparse_runtime_buffers(
|
||||
cache_handles,
|
||||
max_num_reqs=vllm_config.scheduler_config.max_num_seqs,
|
||||
max_num_batched_tokens=vllm_config.scheduler_config.max_num_batched_tokens,
|
||||
)
|
||||
return kv_caches
|
||||
except Exception:
|
||||
if host_pool.shared_region is not None:
|
||||
host_pool.shared_region.cleanup()
|
||||
raise
|
||||
|
||||
|
||||
def _get_hisparse_cache(
|
||||
@@ -166,13 +171,24 @@ def release_hisparse_profiling_cache(forward_context: dict[str, Any]) -> None:
|
||||
if not runtimes:
|
||||
return
|
||||
|
||||
registered_pools = list(
|
||||
{
|
||||
runtime.registered_host_pool.data_ptr(): runtime.registered_host_pool
|
||||
for runtime in runtimes.values()
|
||||
}.values()
|
||||
shared_regions = {
|
||||
id(runtime.shared_host_region): runtime.shared_host_region
|
||||
for runtime in runtimes.values()
|
||||
if runtime.shared_host_region is not None
|
||||
}
|
||||
assert len(shared_regions) <= 1
|
||||
shared_region = next(iter(shared_regions.values()), None)
|
||||
registered_pools = (
|
||||
[]
|
||||
if shared_region is not None
|
||||
else list(
|
||||
{
|
||||
runtime.registered_host_pool.data_ptr(): runtime.registered_host_pool
|
||||
for runtime in runtimes.values()
|
||||
}.values()
|
||||
)
|
||||
)
|
||||
release_pinned_state(list(runtimes.values()), registered_pools)
|
||||
release_pinned_state(list(runtimes.values()), registered_pools, shared_region)
|
||||
for cache in cache_handles:
|
||||
cache.mirror_staging_cache = None
|
||||
cache.mirror_staging_slots = None
|
||||
@@ -257,6 +273,7 @@ def bind_hisparse_kv_caches(
|
||||
assert source_cache.untyped_storage().data_ptr() == (
|
||||
host_pool.registered.untyped_storage().data_ptr()
|
||||
)
|
||||
cache_handle.runtime.shared_host_region = host_pool.shared_region
|
||||
cache_handle.runtime.bind_source_cache(
|
||||
source_cache,
|
||||
registered_host_pool=host_pool.registered,
|
||||
|
||||
Reference in New Issue
Block a user