[5/N] Share HiSparse host cache across TP ranks (reopens #52760) (#56629)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Chao Lei <leichao139636@163.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Chao Lei <leichao139636@163.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Lucas Wilkinson
2026-09-12 19:48:25 -04:00
committed by GitHub
co-authored by Chao Lei Claude Fable 5 Matthew Bonanni OpenAI Codex OpenAI Codex
parent ebe1dec2da
commit e19a3e172e
14 changed files with 878 additions and 85 deletions
+24 -2
View File
@@ -3,6 +3,7 @@
import copy
import hashlib
import importlib
import mmap
import subprocess
import sys
from collections.abc import Callable
@@ -15,6 +16,7 @@ import pytest
import torch
import vllm.v1.core.kv_cache_utils as kv_cache_utils
import vllm.v1.hisparse.runtime as hisparse_runtime_module
from vllm.config import (
CacheConfig,
KVTransferConfig,
@@ -90,7 +92,13 @@ pytestmark = pytest.mark.cpu_test
@pytest.mark.parametrize("gpu_block_size", [32, 64])
def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
@pytest.mark.parametrize("shared_host_pool", [False, True])
def test_hisparse_hma_uses_resolved_gpu_block_size(
monkeypatch, gpu_block_size, shared_host_pool
):
monkeypatch.setattr(
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
)
specs = {
"model.layers.0.self_attn": MLAAttentionSpec(
block_size=gpu_block_size,
@@ -116,7 +124,15 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
hf_config=SimpleNamespace(index_topk=128),
max_model_len=gpu_block_size,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
parallel_config=SimpleNamespace(
tensor_parallel_size=2 if shared_host_pool else 1,
pipeline_parallel_size=1,
prefill_context_parallel_size=1,
decode_context_parallel_size=1,
world_size=2 if shared_host_pool else 1,
distributed_executor_backend="mp",
nnodes_within_dp=1,
),
cache_config=SimpleNamespace(
num_gpu_blocks_override=7,
prefix_cache_retention_interval=None,
@@ -137,6 +153,12 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
assert cache_config.hisparse_host_num_blocks > 7
host_group, indexer_group, *auxiliary_groups = cache_config.kv_cache_groups
assert cache_config.hisparse_shared_host_pool is shared_host_pool
host_page = host_group.kv_cache_spec.page_size_bytes
alignment = mmap.PAGESIZE if shared_host_pool else 1
expected_host_stride = (host_page + alignment - 1) // alignment * alignment
assert cache_config.hisparse_host_block_stride == expected_host_stride
assert cache_config.hisparse_host_num_blocks == 2**30 // expected_host_stride
assert host_group.host_resident
assert not any(group.host_resident for group in [indexer_group, *auxiliary_groups])
host_layers = set(host_group.layer_names)