[Core] Support reseting all running requests' KV while calling reset_prefix_cache (#28827)

Signed-off-by: Zhuohan Li <zhuohan123@gmail.com>
Signed-off-by: Nick Hill <nhill@redhat.com>
Co-authored-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Zhuohan Li
2025-12-02 02:25:05 +00:00
committed by GitHub
co-authored by Nick Hill
parent fa8804ad9c
commit d0cd728907
16 changed files with 315 additions and 35 deletions
@@ -0,0 +1,66 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm import EngineArgs, LLMEngine, SamplingParams
PROMPTS = [
"A robot may not injure a human being ",
"To be or not to be,",
"What is the meaning of life?",
"What does the fox say? " * 20, # Test long prompt
]
def test_reset_prefix_cache_e2e():
engine_args = EngineArgs(
model="Qwen/Qwen3-0.6B",
gpu_memory_utilization=0.2,
async_scheduling=True,
max_num_batched_tokens=32,
max_model_len=2048,
compilation_config={"mode": 0},
)
engine = LLMEngine.from_engine_args(engine_args)
sampling_params = SamplingParams(
temperature=0.0,
max_tokens=16,
)
# No preempt case:
for i, prompt in enumerate(PROMPTS):
engine.add_request("ground_truth_" + str(i), prompt, sampling_params)
ground_truth_results = {}
while engine.has_unfinished_requests():
request_outputs = engine.step()
for request_output in request_outputs:
if request_output.finished:
ground_truth_results[request_output.request_id] = request_output
# Preempt case:
for i, prompt in enumerate(PROMPTS):
engine.add_request("preempted_" + str(i), prompt, sampling_params)
step_id = 0
preempted_results = {}
while engine.has_unfinished_requests():
if step_id == 10:
engine.reset_prefix_cache(reset_running_requests=True)
request_outputs = engine.step()
for request_output in request_outputs:
if request_output.finished:
preempted_results[request_output.request_id] = request_output
step_id += 1
for i in range(len(PROMPTS)):
assert (
ground_truth_results["ground_truth_" + str(i)].outputs[0].text
== preempted_results["preempted_" + str(i)].outputs[0].text
), (
f"ground_truth_results['ground_truth_{i}'].outputs[0].text="
f"{ground_truth_results['ground_truth_' + str(i)].outputs[0].text} "
f"preempted_results['preempted_{i}'].outputs[0].text="
f"{preempted_results['preempted_' + str(i)].outputs[0].text}"
)
+31
View File
@@ -728,6 +728,37 @@ def test_preempt_during_execution():
assert requests[1].output_token_ids[0] == 42
def test_scheduler_reset_prefix_cache():
scheduler = create_scheduler(enable_prefix_caching=True)
requests = create_requests(num_requests=10)
for request in requests:
scheduler.add_request(request)
# Initial scheduling, requests should be at the running state now
_ = scheduler.schedule()
# Verify requests moved from waiting to running
assert len(scheduler.waiting) == 0
assert len(scheduler.running) == len(requests)
for i, request in enumerate(requests):
assert scheduler.running[i] == request
# Reset prefix cache should fail since there are still running requests
# and they are taking KV cache
assert not scheduler.reset_prefix_cache()
# Reset prefix cache with reset_running_requests=True. All running requests
# Should be pushed back to the waiting queue and kv cache should be freed
assert scheduler.reset_prefix_cache(reset_running_requests=True)
# Verify requests moved from running to waiting
assert len(scheduler.waiting) == len(requests)
assert len(scheduler.running) == 0
for i, request in enumerate(requests):
assert scheduler.waiting[i] == request
# Note - these test cases mirror some of those in test_rejection_sampler.py
@pytest.mark.parametrize(
"spec_tokens,output_tokens,expected",