[Frontend][Core] Add release_kv_cache_memory() API (#44890)

Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: AlanFokCo <alanfok2868@gmail.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com>
Co-authored-by: AlanFokCo <alanfok2868@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Dakai An
2026-09-18 07:12:12 +08:00
committed by GitHub
co-authored by AlanFokCo aoshen02 OpenAI Codex
parent a524f80b1f
commit 092bdd6d57
24 changed files with 574 additions and 44 deletions
+85
View File
@@ -109,6 +109,91 @@ def test_basic_cumem():
assert torch.allclose(output, torch.ones_like(output) * 3)
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
kv_cache_memory_bytes = 256 * 1024 * 1024
llm = LLM(
"Qwen/Qwen3-0.6B",
enable_sleep_mode=True,
enforce_eager=True,
kv_cache_memory_bytes=kv_cache_memory_bytes,
max_model_len=1024,
max_num_seqs=4,
gpu_memory_utilization=0.05,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
def get_mapped_bytes(worker):
return mapped_usage(get_mem_allocator_instance())
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
# Utility RPC transports the engine error as a plain Exception.
with pytest.raises(Exception, match="requires a completed pause"):
llm.release_kv_cache_memory()
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
assert not llm.llm_engine.is_sleeping()
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
llm.sleep(level=0)
llm.release_kv_cache_memory()
mapped_after = llm.collective_rpc(get_mapped_bytes)[0]
assert mapped_before - mapped_after >= kv_cache_memory_bytes * 0.99
assert mapped_after > 0
assert llm.llm_engine.is_sleeping()
if full_sleep:
llm.sleep(level=full_sleep)
assert llm.collective_rpc(get_mapped_bytes)[0] == 0
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
if full_sleep == 2:
llm.collective_rpc("reload_weights")
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
assert not llm.llm_engine.is_sleeping()
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
assert actual == expected
@pytest.mark.parametrize("first_level", [1, 2])
@pytest.mark.parametrize("second_level", [1, 2])
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
llm = LLM(
"Qwen/Qwen3-0.6B",
enable_sleep_mode=True,
enforce_eager=True,
kv_cache_memory_bytes=256 * 1024 * 1024,
max_model_len=1024,
max_num_seqs=4,
gpu_memory_utilization=0.05,
)
prompt = "How are you?"
sampling_params = SamplingParams(temperature=0, max_tokens=10)
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
def get_mapped_bytes(worker):
return mapped_usage(get_mem_allocator_instance())
llm.sleep(level=first_level)
llm.wake_up(tags=["kv_cache"])
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
assert mapped_before > 0
llm.sleep(level=second_level)
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
llm.wake_up(tags=["weights"])
if first_level == 2:
llm.collective_rpc("reload_weights")
assert not llm.llm_engine.is_sleeping()
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
assert actual == expected
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
def test_discard_tags():
"""Test that discard(tags) selectively frees GPU memory for specific