[Core][Hybrid allocator + kv connector 1/n] Enable hybrid allocator + KV cache connector (#25712)
Signed-off-by: KuntaiDu <kuntai@uchicago.edu> Signed-off-by: Kuntai Du <kuntai@uchicago.edu>
This commit is contained in:
@@ -899,6 +899,7 @@ def test_kv_connector_basic():
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
use_kv_connector=True,
|
||||
disable_hybrid_kv_cache_manager=True,
|
||||
)
|
||||
NUM_TOTAL_BLOCKS = scheduler.kv_cache_manager.block_pool.get_num_free_blocks()
|
||||
BLOCK_SIZE = scheduler.cache_config.block_size
|
||||
@@ -1024,6 +1025,7 @@ def test_external_prefix_cache_metrics():
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=False,
|
||||
use_kv_connector=True,
|
||||
disable_hybrid_kv_cache_manager=True,
|
||||
)
|
||||
|
||||
# Mock connector to simulate a partial external cache hit
|
||||
@@ -1088,6 +1090,7 @@ def test_kv_connector_unable_to_allocate():
|
||||
use_kv_connector=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_blocks=NUM_BLOCKS,
|
||||
disable_hybrid_kv_cache_manager=True,
|
||||
)
|
||||
NUM_MATCHED_NEW_TOKENS = BLOCK_SIZE * 2
|
||||
scheduler.connector.get_num_new_matched_tokens = Mock(name="method")
|
||||
@@ -1171,6 +1174,7 @@ def test_kv_connector_handles_preemption():
|
||||
use_kv_connector=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
num_blocks=NUM_BLOCKS,
|
||||
disable_hybrid_kv_cache_manager=True,
|
||||
)
|
||||
|
||||
NUM_MATCHED_NEW_TOKENS = BLOCK_SIZE
|
||||
@@ -1387,6 +1391,7 @@ def create_scheduler_with_priority(
|
||||
block_size: int = 16,
|
||||
max_model_len: int | None = None,
|
||||
num_speculative_tokens: int | None = None,
|
||||
disable_hybrid_kv_cache_manager: bool = False,
|
||||
) -> Scheduler:
|
||||
"""Create scheduler with priority policy enabled.
|
||||
|
||||
@@ -1411,6 +1416,7 @@ def create_scheduler_with_priority(
|
||||
disable_chunked_mm_input=disable_chunked_mm_input,
|
||||
enable_chunked_prefill=True,
|
||||
policy="priority", # Enable priority scheduling
|
||||
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager,
|
||||
)
|
||||
model_config = ModelConfig(
|
||||
model=model,
|
||||
@@ -2018,6 +2024,7 @@ def test_priority_scheduling_preemption_and_resumption_when_out_of_kv():
|
||||
num_blocks=5, # Can hold 64 tokens (first block is null)
|
||||
block_size=16, # Standard block size
|
||||
use_kv_connector=True,
|
||||
disable_hybrid_kv_cache_manager=True,
|
||||
)
|
||||
|
||||
# Create a request and schedule it
|
||||
|
||||
@@ -46,6 +46,7 @@ def create_scheduler(
|
||||
num_speculative_tokens: int | None = None,
|
||||
skip_tokenizer_init: bool = False,
|
||||
async_scheduling: bool = False,
|
||||
disable_hybrid_kv_cache_manager: bool = False,
|
||||
) -> Scheduler | AsyncScheduler:
|
||||
"""Create scheduler under test.
|
||||
|
||||
@@ -70,6 +71,7 @@ def create_scheduler(
|
||||
disable_chunked_mm_input=disable_chunked_mm_input,
|
||||
enable_chunked_prefill=True,
|
||||
async_scheduling=async_scheduling,
|
||||
disable_hybrid_kv_cache_manager=disable_hybrid_kv_cache_manager,
|
||||
)
|
||||
model_config = ModelConfig(
|
||||
model=model,
|
||||
|
||||
@@ -136,6 +136,7 @@ run_tests_for_model() {
|
||||
vllm serve $model_name \
|
||||
--port $PORT \
|
||||
--enforce-eager \
|
||||
--disable-hybrid-kv-cache-manager \
|
||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||
--tensor-parallel-size $PREFILLER_TP_SIZE \
|
||||
--kv-transfer-config '$KV_CONFIG'"
|
||||
@@ -178,6 +179,7 @@ run_tests_for_model() {
|
||||
--port $PORT \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization $GPU_MEMORY_UTILIZATION \
|
||||
--disable-hybrid-kv-cache-manager \
|
||||
--kv-transfer-config '$KV_CONFIG'"
|
||||
|
||||
# DP-EP attention mode
|
||||
|
||||
@@ -85,6 +85,7 @@ run_tests_for_model() {
|
||||
--port $PREFILL_PORT \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.2 \
|
||||
--disable-hybrid-kv-cache-manager \
|
||||
--kv-transfer-config '$KV_CONFIG'"
|
||||
|
||||
if [ -n "$model_args" ]; then
|
||||
@@ -103,6 +104,7 @@ run_tests_for_model() {
|
||||
--port $DECODE_PORT \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.2 \
|
||||
--disable-hybrid-kv-cache-manager \
|
||||
--kv-transfer-config '$KV_CONFIG'"
|
||||
|
||||
if [ -n "$model_args" ]; then
|
||||
|
||||
@@ -114,6 +114,7 @@ def test_multi_shared_storage_connector_consistency():
|
||||
enforce_eager=True,
|
||||
gpu_memory_utilization=0.5,
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
disable_hybrid_kv_cache_manager=True,
|
||||
)
|
||||
# Run generation - this should trigger saving KV cache
|
||||
_ = llm.generate(PROMPTS, SAMPLING_PARAMS)
|
||||
|
||||
@@ -932,6 +932,7 @@ def test_abort_timeout_on_prefiller(monkeypatch, distributed_executor_backend):
|
||||
"gpu_memory_utilization": 0.5,
|
||||
"kv_transfer_config": kv_transfer_config,
|
||||
"distributed_executor_backend": distributed_executor_backend,
|
||||
"disable_hybrid_kv_cache_manager": True,
|
||||
}
|
||||
|
||||
timeout = 6
|
||||
|
||||
@@ -132,6 +132,7 @@ def test_shared_storage_connector_hashes(tmp_path):
|
||||
enforce_eager=True,
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
limit_mm_per_prompt={"image": 2},
|
||||
disable_hybrid_kv_cache_manager=True,
|
||||
)
|
||||
|
||||
# don't put this import at the top level
|
||||
|
||||
@@ -91,6 +91,9 @@ def create_vllm_config(
|
||||
max_num_batched_tokens=max_num_batched_tokens,
|
||||
max_model_len=max_model_len,
|
||||
enable_chunked_prefill=enable_chunked_prefill,
|
||||
# Disable hybrid KV cache manager for testing
|
||||
# Should be removed after we support hybrid KV cache manager-based testing.
|
||||
disable_hybrid_kv_cache_manager=True,
|
||||
)
|
||||
model_config = ModelConfig(
|
||||
model=model,
|
||||
|
||||
@@ -27,6 +27,7 @@ def test_cpu_offloading(cpu_block_size: int) -> None:
|
||||
model="meta-llama/Llama-3.2-1B-Instruct",
|
||||
gpu_memory_utilization=0.5,
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
disable_hybrid_kv_cache_manager=True,
|
||||
)
|
||||
|
||||
prompts = ["Hi " * 100]
|
||||
|
||||
Reference in New Issue
Block a user