[Frontend][Core] Add release_kv_cache_memory() API (#44890)
Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: AlanFokCo <alanfok2868@gmail.com> Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Dakai An <77474977+andakai@users.noreply.github.com> Co-authored-by: AlanFokCo <alanfok2868@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
AlanFokCo
aoshen02
OpenAI Codex
parent
a524f80b1f
commit
092bdd6d57
@@ -109,6 +109,91 @@ def test_basic_cumem():
|
||||
assert torch.allclose(output, torch.ones_like(output) * 3)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("full_sleep", [0, 1, 2], ids=["kv-only", "sleep-1", "sleep-2"])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_release_kv_cache_memory_preserves_generation(full_sleep, monkeypatch):
|
||||
"""Rejected release and KV-only/full sleep cycles preserve greedy tokens."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
kv_cache_memory_bytes = 256 * 1024 * 1024
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
kv_cache_memory_bytes=kv_cache_memory_bytes,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=4,
|
||||
gpu_memory_utilization=0.05,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
|
||||
def get_mapped_bytes(worker):
|
||||
return mapped_usage(get_mem_allocator_instance())
|
||||
|
||||
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
|
||||
# Utility RPC transports the engine error as a plain Exception.
|
||||
with pytest.raises(Exception, match="requires a completed pause"):
|
||||
llm.release_kv_cache_memory()
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
assert llm.generate(prompt, sampling_params)[0].outputs[0].token_ids == expected
|
||||
|
||||
llm.sleep(level=0)
|
||||
llm.release_kv_cache_memory()
|
||||
mapped_after = llm.collective_rpc(get_mapped_bytes)[0]
|
||||
assert mapped_before - mapped_after >= kv_cache_memory_bytes * 0.99
|
||||
assert mapped_after > 0
|
||||
assert llm.llm_engine.is_sleeping()
|
||||
|
||||
if full_sleep:
|
||||
llm.sleep(level=full_sleep)
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == 0
|
||||
llm.wake_up(tags=None if full_sleep else ["kv_cache"])
|
||||
if full_sleep == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("first_level", [1, 2])
|
||||
@pytest.mark.parametrize("second_level", [1, 2])
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_sleep_with_only_weights_asleep(first_level, second_level, monkeypatch):
|
||||
"""Repeated sleep after KV-only wake preserves mappings and recoverability."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
llm = LLM(
|
||||
"Qwen/Qwen3-0.6B",
|
||||
enable_sleep_mode=True,
|
||||
enforce_eager=True,
|
||||
kv_cache_memory_bytes=256 * 1024 * 1024,
|
||||
max_model_len=1024,
|
||||
max_num_seqs=4,
|
||||
gpu_memory_utilization=0.05,
|
||||
)
|
||||
prompt = "How are you?"
|
||||
sampling_params = SamplingParams(temperature=0, max_tokens=10)
|
||||
expected = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
|
||||
def get_mapped_bytes(worker):
|
||||
return mapped_usage(get_mem_allocator_instance())
|
||||
|
||||
llm.sleep(level=first_level)
|
||||
llm.wake_up(tags=["kv_cache"])
|
||||
mapped_before = llm.collective_rpc(get_mapped_bytes)[0]
|
||||
assert mapped_before > 0
|
||||
llm.sleep(level=second_level)
|
||||
assert llm.collective_rpc(get_mapped_bytes)[0] == mapped_before
|
||||
llm.wake_up(tags=["weights"])
|
||||
if first_level == 2:
|
||||
llm.collective_rpc("reload_weights")
|
||||
assert not llm.llm_engine.is_sleeping()
|
||||
actual = llm.generate(prompt, sampling_params)[0].outputs[0].token_ids
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@create_new_process_for_each_test("fork" if current_platform.is_cuda() else "spawn")
|
||||
def test_discard_tags():
|
||||
"""Test that discard(tags) selectively frees GPU memory for specific
|
||||
|
||||
Reference in New Issue
Block a user