[KV Offload] Per-job store completion for CPU offloading connector (#39186)

Signed-off-by: Itay Etelis <itay.etelis@ibm.com>
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
Co-authored-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Itay Etelis
2026-04-29 08:52:55 +03:00
committed by GitHub
co-authored by Itay Etelis Or Ozeri Or Ozeri
parent a269744e9f
commit 8a8c9b564e
8 changed files with 428 additions and 143 deletions
@@ -232,6 +232,9 @@ def test_request_preemption(request_runner, async_scheduling: bool):
expected_stored_gpu_block_indexes=(9, 10, 11),
)
# All stores completed before request_finished -> fence index empty.
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling: bool):
@@ -292,6 +295,9 @@ def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling:
# second request will use the GPU prefix cache
assert transfer_jobs == list(runner.offloading_spec.handler.transfer_specs)
# Fence index drained: stores completed before request_finished ran.
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_abort_loading_requests(request_runner, async_scheduling: bool):
@@ -534,3 +540,131 @@ def test_do_remote_decode_stores_all_blocks(request_runner, async_scheduling: bo
decoded_tokens=[EOS_TOKEN_ID],
expected_stored_gpu_block_indexes=(0, 1, 2, 3, 4, 5),
)
# All stores completed before request_finished -> fence index empty.
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
# ---------------------------------------------------------------------------
# Tests for the per-job-store-completion design and fence invariants.
# ---------------------------------------------------------------------------
def test_loads_do_not_populate_fence_index(request_runner):
"""Loads don't populate _block_id_to_pending_jobs (protected by
delay_free_blocks while in flight)."""
runner = request_runner(
offloaded_block_size=12,
gpu_block_size=4,
num_gpu_blocks=100,
async_scheduling=False,
)
runner.new_request(token_ids=[0] * 12)
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
runner.run(decoded_tokens=[], complete_transfers=False)
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
def test_fence_at_update_state_after_alloc(request_runner):
"""A load reusing a finished request's pending-store block triggers
a flush via update_state_after_alloc's fence.
num_gpu_blocks=2 forces the BlockPool to give req2 the same block
req1 just freed.
"""
runner = request_runner(
offloaded_block_size=4,
gpu_block_size=4,
num_gpu_blocks=2,
async_scheduling=False,
)
runner.new_request(token_ids=[0] * 4)
runner.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output(keys)
)
runner.run(decoded_tokens=[EOS_TOKEN_ID], complete_transfers=False)
assert runner.connector_scheduler._block_id_to_pending_jobs
runner.scheduler.reset_prefix_cache()
runner.new_request(token_ids=[0] * 4)
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
runner.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output([])
)
runner.run(
decoded_tokens=[],
complete_transfers=False,
expected_stored_gpu_block_indexes=(0,),
expected_flushed_gpu_block_indexes=(0,),
)
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
def test_fence_at_build_store_jobs(request_runner):
"""A new prefill (no load -> update_state_after_alloc returns early)
reusing a finished request's pending-store block is flushed by
_build_store_jobs's fence."""
runner = request_runner(
offloaded_block_size=4,
gpu_block_size=4,
num_gpu_blocks=2,
async_scheduling=False,
)
runner.new_request(token_ids=[0] * 4)
runner.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output(keys)
)
runner.run(decoded_tokens=[EOS_TOKEN_ID], complete_transfers=False)
assert runner.connector_scheduler._block_id_to_pending_jobs
runner.scheduler.reset_prefix_cache()
runner.new_request(token_ids=[1] * 4)
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 0
runner.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output([])
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored_gpu_block_indexes=(0,),
expected_flushed_gpu_block_indexes=(0,),
)
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
"""complete_store fires per-job, not deferred to request finish.
Each call carries only that store's keys."""
offloaded_block_size = 12
runner = request_runner(
offloaded_block_size=offloaded_block_size,
gpu_block_size=4,
num_gpu_blocks=100,
async_scheduling=async_scheduling,
)
runner.new_request(token_ids=[0] * offloaded_block_size)
runner.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output(keys)
)
# First store: fires when block 0 is fully populated.
runner.run(decoded_tokens=[0, 0], expected_stored_gpu_block_indexes=(0, 1, 2))
assert runner.manager.complete_store.call_count == 1
first_call_keys = set(runner.manager.complete_store.call_args.args[0])
assert len(first_call_keys) == 1
runner.manager.complete_store.reset_mock()
# Second store: fires when block 1 is fully populated, with different keys.
runner.run(
decoded_tokens=[0] * (offloaded_block_size + 1),
expected_stored_gpu_block_indexes=(3, 4, 5),
)
assert runner.manager.complete_store.call_count == 1
second_call_keys = set(runner.manager.complete_store.call_args.args[0])
assert first_call_keys != second_call_keys
runner.manager.complete_store.reset_mock()
# Finish: no store pending -> no further call.
runner.run(decoded_tokens=[EOS_TOKEN_ID])
assert runner.manager.complete_store.call_count == 0
@@ -0,0 +1,32 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingWorkerMetadata,
)
pytestmark = pytest.mark.cpu_test
def test_aggregate_sums_counts():
meta1 = OffloadingWorkerMetadata(completed_jobs={42: 1, 7: 1})
meta2 = OffloadingWorkerMetadata(completed_jobs={42: 1, 7: 1})
result = meta1.aggregate(meta2)
assert result.completed_jobs == {42: 2, 7: 2}
def test_aggregate_disjoint_jobs():
meta1 = OffloadingWorkerMetadata(completed_jobs={42: 1, 7: 1})
meta2 = OffloadingWorkerMetadata(completed_jobs={43: 1, 8: 1})
result = meta1.aggregate(meta2)
assert result.completed_jobs == {42: 1, 7: 1, 43: 1, 8: 1}
def test_aggregate_multiple_workers():
meta1 = OffloadingWorkerMetadata(completed_jobs={42: 1, 43: 1, 7: 1})
meta2 = OffloadingWorkerMetadata(completed_jobs={42: 1, 7: 1, 8: 1})
meta3 = OffloadingWorkerMetadata(completed_jobs={42: 1, 43: 1, 8: 1})
result = meta1.aggregate(meta2).aggregate(meta3)
assert result.completed_jobs == {42: 3, 43: 2, 7: 2, 8: 2}
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import copy
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
from typing import Any
@@ -19,6 +18,7 @@ from vllm.config import KVTransferConfig, VllmConfig, set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
OffloadingWorkerMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
OffloadingConnector,
@@ -51,7 +51,6 @@ from vllm.v1.kv_offload.worker.worker import (
TransferResult,
TransferSpec,
)
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT, KVConnectorOutput
from vllm.v1.request import Request
from vllm.v1.structured_output import StructuredOutputManager
@@ -369,7 +368,12 @@ class RequestRunner:
prev_scheduler_output = None
prev_model_runner_output = None
while True:
assert self.scheduler.requests
# Strict-always-False frees the request immediately on EOS, but
# the worker may still have a deferred store queued. In production
# the next request's step drains it; in single-request tests we
# must keep stepping until the scheduler sees no in-flight jobs.
if not self.scheduler.requests and not self.connector_scheduler._jobs:
break
scheduler_output = self.scheduler.schedule()
self._update_gpu_block_idx()
@@ -392,6 +396,10 @@ class RequestRunner:
finished_sending, finished_recving = self.worker_connector.get_finished(
scheduler_output.finished_req_ids
)
worker_meta = (
self.worker_connector.build_connector_worker_meta()
or OffloadingWorkerMetadata()
)
self.worker_connector.clear_connector_metadata()
@@ -400,6 +408,7 @@ class RequestRunner:
finished_sending=finished_sending,
finished_recving=finished_recving,
token_id=token_id or 0,
kv_connector_worker_meta=worker_meta,
)
prev_token_id = token_id
@@ -420,7 +429,7 @@ class RequestRunner:
if (
prev_token_id == EOS_TOKEN_ID
and prev_token_id != token_id
and self.scheduler.requests
and (self.scheduler.requests or self.connector_scheduler._jobs)
):
# continue for one more step to allow offloading to kick off
continue
@@ -435,26 +444,9 @@ class RequestRunner:
self._parse_transfers()
# run one more step to update finished stored
if EOS_TOKEN_ID in decoded_tokens:
assert not self.scheduler.running
while self.scheduler.requests:
scheduler_output = self.scheduler.schedule()
finished_sending, finished_recving = self.worker_connector.get_finished(
scheduler_output.finished_req_ids
)
assert not finished_recving
model_runner_output = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT)
model_runner_output.kv_connector_output = KVConnectorOutput(
finished_sending=finished_sending
)
self.scheduler.update_from_output(scheduler_output, model_runner_output)
def run(
self,
decoded_tokens: list[int],
+4
View File
@@ -24,6 +24,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorWorkerMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import ( # noqa
ExampleConnector,
@@ -249,6 +250,7 @@ def create_model_runner_output(
invalid_block_ids: set[int] | None = None,
use_eos: bool = False,
token_id: int = 0,
kv_connector_worker_meta: KVConnectorWorkerMetadata | None = None,
) -> ModelRunnerOutput:
"""Make dummy model runner output for testing."""
@@ -266,11 +268,13 @@ def create_model_runner_output(
finished_sending is None
and finished_recving is None
and invalid_block_ids is None
and kv_connector_worker_meta is None
)
else KVConnectorOutput(
finished_sending=finished_sending,
finished_recving=finished_recving,
invalid_block_ids=invalid_block_ids or set(),
kv_connector_worker_meta=kv_connector_worker_meta,
)
)