Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Chao Lei <leichao139636@163.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Chao Lei <leichao139636@163.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Chao Lei
Claude Fable 5
Matthew Bonanni
OpenAI Codex
OpenAI Codex
parent
ebe1dec2da
commit
e19a3e172e
@@ -3,6 +3,7 @@
|
||||
import copy
|
||||
import hashlib
|
||||
import importlib
|
||||
import mmap
|
||||
import subprocess
|
||||
import sys
|
||||
from collections.abc import Callable
|
||||
@@ -15,6 +16,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
import vllm.v1.hisparse.runtime as hisparse_runtime_module
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
KVTransferConfig,
|
||||
@@ -90,7 +92,13 @@ pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gpu_block_size", [32, 64])
|
||||
def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
|
||||
@pytest.mark.parametrize("shared_host_pool", [False, True])
|
||||
def test_hisparse_hma_uses_resolved_gpu_block_size(
|
||||
monkeypatch, gpu_block_size, shared_host_pool
|
||||
):
|
||||
monkeypatch.setattr(
|
||||
hisparse_runtime_module.current_platform, "is_cuda_alike", lambda: True
|
||||
)
|
||||
specs = {
|
||||
"model.layers.0.self_attn": MLAAttentionSpec(
|
||||
block_size=gpu_block_size,
|
||||
@@ -116,7 +124,15 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
|
||||
hf_config=SimpleNamespace(index_topk=128),
|
||||
max_model_len=gpu_block_size,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
parallel_config=SimpleNamespace(
|
||||
tensor_parallel_size=2 if shared_host_pool else 1,
|
||||
pipeline_parallel_size=1,
|
||||
prefill_context_parallel_size=1,
|
||||
decode_context_parallel_size=1,
|
||||
world_size=2 if shared_host_pool else 1,
|
||||
distributed_executor_backend="mp",
|
||||
nnodes_within_dp=1,
|
||||
),
|
||||
cache_config=SimpleNamespace(
|
||||
num_gpu_blocks_override=7,
|
||||
prefix_cache_retention_interval=None,
|
||||
@@ -137,6 +153,12 @@ def test_hisparse_hma_uses_resolved_gpu_block_size(monkeypatch, gpu_block_size):
|
||||
assert cache_config.hisparse_host_num_blocks > 7
|
||||
|
||||
host_group, indexer_group, *auxiliary_groups = cache_config.kv_cache_groups
|
||||
assert cache_config.hisparse_shared_host_pool is shared_host_pool
|
||||
host_page = host_group.kv_cache_spec.page_size_bytes
|
||||
alignment = mmap.PAGESIZE if shared_host_pool else 1
|
||||
expected_host_stride = (host_page + alignment - 1) // alignment * alignment
|
||||
assert cache_config.hisparse_host_block_stride == expected_host_stride
|
||||
assert cache_config.hisparse_host_num_blocks == 2**30 // expected_host_stride
|
||||
assert host_group.host_resident
|
||||
assert not any(group.host_resident for group in [indexer_group, *auxiliary_groups])
|
||||
host_layers = set(host_group.layer_names)
|
||||
|
||||
Reference in New Issue
Block a user