[KVCache] Make KVCacheSpec hashable (#21791)
Signed-off-by: Chen Zhang <zhangch99@outlook.com>
This commit is contained in:
@@ -17,7 +17,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
estimate_max_model_len, generate_block_hash_extra_keys,
|
||||
get_kv_cache_config, get_max_concurrency_for_kv_cache_config,
|
||||
hash_block_tokens, hash_request_tokens, init_none_hash,
|
||||
unify_kv_cache_configs)
|
||||
is_kv_cache_type_uniform, unify_kv_cache_configs)
|
||||
from vllm.v1.kv_cache_interface import (FullAttentionSpec, KVCacheConfig,
|
||||
KVCacheGroupSpec, KVCacheTensor,
|
||||
SlidingWindowSpec)
|
||||
@@ -685,6 +685,38 @@ def test_merge_kv_cache_spec():
|
||||
assert merged_layer_spec.sliding_window == 1
|
||||
|
||||
|
||||
def test_is_kv_cache_type_uniform():
|
||||
kv_cache_spec = {
|
||||
"layer_1": new_kv_cache_spec(num_kv_heads=32),
|
||||
"layer_2": new_kv_cache_spec(num_kv_heads=32),
|
||||
}
|
||||
assert is_kv_cache_type_uniform(kv_cache_spec)
|
||||
|
||||
kv_cache_spec = {
|
||||
"layer_1": new_kv_cache_spec(num_kv_heads=32),
|
||||
"layer_2": new_kv_cache_spec(num_kv_heads=32, sliding_window=1),
|
||||
}
|
||||
assert is_kv_cache_type_uniform(kv_cache_spec)
|
||||
|
||||
kv_cache_spec = {
|
||||
"layer_1": new_kv_cache_spec(num_kv_heads=32),
|
||||
"layer_2": new_sliding_window_spec(num_kv_heads=32, sliding_window=1),
|
||||
}
|
||||
assert not is_kv_cache_type_uniform(kv_cache_spec)
|
||||
|
||||
kv_cache_spec = {
|
||||
"layer_1": new_sliding_window_spec(num_kv_heads=32, sliding_window=1),
|
||||
"layer_2": new_sliding_window_spec(num_kv_heads=32, sliding_window=1),
|
||||
}
|
||||
assert is_kv_cache_type_uniform(kv_cache_spec)
|
||||
|
||||
kv_cache_spec = {
|
||||
"layer_1": new_sliding_window_spec(num_kv_heads=32, sliding_window=1),
|
||||
"layer_2": new_sliding_window_spec(num_kv_heads=32, sliding_window=2),
|
||||
}
|
||||
assert not is_kv_cache_type_uniform(kv_cache_spec)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_id", "max_model_len", "want_estimated_max_len"), [
|
||||
("Qwen/Qwen1.5-7B", 16385, 16384),
|
||||
|
||||
@@ -30,7 +30,9 @@ model_config = {
|
||||
])
|
||||
@pytest.mark.parametrize("batch_size", [5])
|
||||
@pytest.mark.parametrize("seed", [1])
|
||||
def test_sliding_window_retrieval(monkeypatch, model, batch_size, seed):
|
||||
@pytest.mark.parametrize("disable_hybrid_kv_cache_manager", [True, False])
|
||||
def test_sliding_window_retrieval(monkeypatch, model, batch_size, seed,
|
||||
disable_hybrid_kv_cache_manager):
|
||||
"""
|
||||
The test does a bunch of assignments "x1 = 10\nx2 = 33\n..." and then
|
||||
asks for value of one of them (which is outside the sliding window).
|
||||
@@ -42,7 +44,9 @@ def test_sliding_window_retrieval(monkeypatch, model, batch_size, seed):
|
||||
|
||||
test_config = model_config[model]
|
||||
|
||||
llm = LLM(model=model)
|
||||
llm = LLM(
|
||||
model=model,
|
||||
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager)
|
||||
sampling_params = SamplingParams(temperature=0.0, max_tokens=100)
|
||||
|
||||
prompts, answer, indices = prep_prompts(batch_size,
|
||||
|
||||
Reference in New Issue
Block a user