[KV Offload] Per-job store completion for CPU offloading connector (#39186)
Signed-off-by: Itay Etelis <itay.etelis@ibm.com> Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Itay Etelis <itay.etelis@ibm.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Itay Etelis
Or Ozeri
Or Ozeri
parent
a269744e9f
commit
8a8c9b564e
@@ -232,6 +232,9 @@ def test_request_preemption(request_runner, async_scheduling: bool):
|
||||
expected_stored_gpu_block_indexes=(9, 10, 11),
|
||||
)
|
||||
|
||||
# All stores completed before request_finished -> fence index empty.
|
||||
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling: bool):
|
||||
@@ -292,6 +295,9 @@ def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling:
|
||||
# second request will use the GPU prefix cache
|
||||
assert transfer_jobs == list(runner.offloading_spec.handler.transfer_specs)
|
||||
|
||||
# Fence index drained: stores completed before request_finished ran.
|
||||
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_abort_loading_requests(request_runner, async_scheduling: bool):
|
||||
@@ -534,3 +540,131 @@ def test_do_remote_decode_stores_all_blocks(request_runner, async_scheduling: bo
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored_gpu_block_indexes=(0, 1, 2, 3, 4, 5),
|
||||
)
|
||||
# All stores completed before request_finished -> fence index empty.
|
||||
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests for the per-job-store-completion design and fence invariants.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_loads_do_not_populate_fence_index(request_runner):
|
||||
"""Loads don't populate _block_id_to_pending_jobs (protected by
|
||||
delay_free_blocks while in flight)."""
|
||||
runner = request_runner(
|
||||
offloaded_block_size=12,
|
||||
gpu_block_size=4,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
)
|
||||
runner.new_request(token_ids=[0] * 12)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.run(decoded_tokens=[], complete_transfers=False)
|
||||
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
|
||||
|
||||
|
||||
def test_fence_at_update_state_after_alloc(request_runner):
|
||||
"""A load reusing a finished request's pending-store block triggers
|
||||
a flush via update_state_after_alloc's fence.
|
||||
|
||||
num_gpu_blocks=2 forces the BlockPool to give req2 the same block
|
||||
req1 just freed.
|
||||
"""
|
||||
runner = request_runner(
|
||||
offloaded_block_size=4,
|
||||
gpu_block_size=4,
|
||||
num_gpu_blocks=2,
|
||||
async_scheduling=False,
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[0] * 4)
|
||||
runner.manager.prepare_store.side_effect = (
|
||||
lambda keys, req_context: generate_store_output(keys)
|
||||
)
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID], complete_transfers=False)
|
||||
assert runner.connector_scheduler._block_id_to_pending_jobs
|
||||
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * 4)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.manager.prepare_store.side_effect = (
|
||||
lambda keys, req_context: generate_store_output([])
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[],
|
||||
complete_transfers=False,
|
||||
expected_stored_gpu_block_indexes=(0,),
|
||||
expected_flushed_gpu_block_indexes=(0,),
|
||||
)
|
||||
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
|
||||
|
||||
|
||||
def test_fence_at_build_store_jobs(request_runner):
|
||||
"""A new prefill (no load -> update_state_after_alloc returns early)
|
||||
reusing a finished request's pending-store block is flushed by
|
||||
_build_store_jobs's fence."""
|
||||
runner = request_runner(
|
||||
offloaded_block_size=4,
|
||||
gpu_block_size=4,
|
||||
num_gpu_blocks=2,
|
||||
async_scheduling=False,
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[0] * 4)
|
||||
runner.manager.prepare_store.side_effect = (
|
||||
lambda keys, req_context: generate_store_output(keys)
|
||||
)
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID], complete_transfers=False)
|
||||
assert runner.connector_scheduler._block_id_to_pending_jobs
|
||||
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[1] * 4)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 0
|
||||
runner.manager.prepare_store.side_effect = (
|
||||
lambda keys, req_context: generate_store_output([])
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored_gpu_block_indexes=(0,),
|
||||
expected_flushed_gpu_block_indexes=(0,),
|
||||
)
|
||||
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_complete_store_called_per_job(request_runner, async_scheduling: bool):
|
||||
"""complete_store fires per-job, not deferred to request finish.
|
||||
Each call carries only that store's keys."""
|
||||
offloaded_block_size = 12
|
||||
runner = request_runner(
|
||||
offloaded_block_size=offloaded_block_size,
|
||||
gpu_block_size=4,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=async_scheduling,
|
||||
)
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size)
|
||||
runner.manager.prepare_store.side_effect = (
|
||||
lambda keys, req_context: generate_store_output(keys)
|
||||
)
|
||||
|
||||
# First store: fires when block 0 is fully populated.
|
||||
runner.run(decoded_tokens=[0, 0], expected_stored_gpu_block_indexes=(0, 1, 2))
|
||||
assert runner.manager.complete_store.call_count == 1
|
||||
first_call_keys = set(runner.manager.complete_store.call_args.args[0])
|
||||
assert len(first_call_keys) == 1
|
||||
runner.manager.complete_store.reset_mock()
|
||||
|
||||
# Second store: fires when block 1 is fully populated, with different keys.
|
||||
runner.run(
|
||||
decoded_tokens=[0] * (offloaded_block_size + 1),
|
||||
expected_stored_gpu_block_indexes=(3, 4, 5),
|
||||
)
|
||||
assert runner.manager.complete_store.call_count == 1
|
||||
second_call_keys = set(runner.manager.complete_store.call_args.args[0])
|
||||
assert first_call_keys != second_call_keys
|
||||
runner.manager.complete_store.reset_mock()
|
||||
|
||||
# Finish: no store pending -> no further call.
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
assert runner.manager.complete_store.call_count == 0
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingWorkerMetadata,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.cpu_test
|
||||
|
||||
|
||||
def test_aggregate_sums_counts():
|
||||
meta1 = OffloadingWorkerMetadata(completed_jobs={42: 1, 7: 1})
|
||||
meta2 = OffloadingWorkerMetadata(completed_jobs={42: 1, 7: 1})
|
||||
result = meta1.aggregate(meta2)
|
||||
assert result.completed_jobs == {42: 2, 7: 2}
|
||||
|
||||
|
||||
def test_aggregate_disjoint_jobs():
|
||||
meta1 = OffloadingWorkerMetadata(completed_jobs={42: 1, 7: 1})
|
||||
meta2 = OffloadingWorkerMetadata(completed_jobs={43: 1, 8: 1})
|
||||
result = meta1.aggregate(meta2)
|
||||
assert result.completed_jobs == {42: 1, 7: 1, 43: 1, 8: 1}
|
||||
|
||||
|
||||
def test_aggregate_multiple_workers():
|
||||
meta1 = OffloadingWorkerMetadata(completed_jobs={42: 1, 43: 1, 7: 1})
|
||||
meta2 = OffloadingWorkerMetadata(completed_jobs={42: 1, 7: 1, 8: 1})
|
||||
meta3 = OffloadingWorkerMetadata(completed_jobs={42: 1, 43: 1, 8: 1})
|
||||
result = meta1.aggregate(meta2).aggregate(meta3)
|
||||
assert result.completed_jobs == {42: 3, 43: 2, 7: 2, 8: 2}
|
||||
@@ -1,6 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import copy
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
@@ -19,6 +18,7 @@ from vllm.config import KVTransferConfig, VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingConnectorMetadata,
|
||||
OffloadingWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading_connector import (
|
||||
OffloadingConnector,
|
||||
@@ -51,7 +51,6 @@ from vllm.v1.kv_offload.worker.worker import (
|
||||
TransferResult,
|
||||
TransferSpec,
|
||||
)
|
||||
from vllm.v1.outputs import EMPTY_MODEL_RUNNER_OUTPUT, KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
@@ -369,7 +368,12 @@ class RequestRunner:
|
||||
prev_scheduler_output = None
|
||||
prev_model_runner_output = None
|
||||
while True:
|
||||
assert self.scheduler.requests
|
||||
# Strict-always-False frees the request immediately on EOS, but
|
||||
# the worker may still have a deferred store queued. In production
|
||||
# the next request's step drains it; in single-request tests we
|
||||
# must keep stepping until the scheduler sees no in-flight jobs.
|
||||
if not self.scheduler.requests and not self.connector_scheduler._jobs:
|
||||
break
|
||||
|
||||
scheduler_output = self.scheduler.schedule()
|
||||
self._update_gpu_block_idx()
|
||||
@@ -392,6 +396,10 @@ class RequestRunner:
|
||||
finished_sending, finished_recving = self.worker_connector.get_finished(
|
||||
scheduler_output.finished_req_ids
|
||||
)
|
||||
worker_meta = (
|
||||
self.worker_connector.build_connector_worker_meta()
|
||||
or OffloadingWorkerMetadata()
|
||||
)
|
||||
|
||||
self.worker_connector.clear_connector_metadata()
|
||||
|
||||
@@ -400,6 +408,7 @@ class RequestRunner:
|
||||
finished_sending=finished_sending,
|
||||
finished_recving=finished_recving,
|
||||
token_id=token_id or 0,
|
||||
kv_connector_worker_meta=worker_meta,
|
||||
)
|
||||
|
||||
prev_token_id = token_id
|
||||
@@ -420,7 +429,7 @@ class RequestRunner:
|
||||
if (
|
||||
prev_token_id == EOS_TOKEN_ID
|
||||
and prev_token_id != token_id
|
||||
and self.scheduler.requests
|
||||
and (self.scheduler.requests or self.connector_scheduler._jobs)
|
||||
):
|
||||
# continue for one more step to allow offloading to kick off
|
||||
continue
|
||||
@@ -435,26 +444,9 @@ class RequestRunner:
|
||||
|
||||
self._parse_transfers()
|
||||
|
||||
# run one more step to update finished stored
|
||||
if EOS_TOKEN_ID in decoded_tokens:
|
||||
assert not self.scheduler.running
|
||||
|
||||
while self.scheduler.requests:
|
||||
scheduler_output = self.scheduler.schedule()
|
||||
|
||||
finished_sending, finished_recving = self.worker_connector.get_finished(
|
||||
scheduler_output.finished_req_ids
|
||||
)
|
||||
|
||||
assert not finished_recving
|
||||
|
||||
model_runner_output = copy.deepcopy(EMPTY_MODEL_RUNNER_OUTPUT)
|
||||
model_runner_output.kv_connector_output = KVConnectorOutput(
|
||||
finished_sending=finished_sending
|
||||
)
|
||||
|
||||
self.scheduler.update_from_output(scheduler_output, model_runner_output)
|
||||
|
||||
def run(
|
||||
self,
|
||||
decoded_tokens: list[int],
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.example_connector import ( # noqa
|
||||
ExampleConnector,
|
||||
@@ -249,6 +250,7 @@ def create_model_runner_output(
|
||||
invalid_block_ids: set[int] | None = None,
|
||||
use_eos: bool = False,
|
||||
token_id: int = 0,
|
||||
kv_connector_worker_meta: KVConnectorWorkerMetadata | None = None,
|
||||
) -> ModelRunnerOutput:
|
||||
"""Make dummy model runner output for testing."""
|
||||
|
||||
@@ -266,11 +268,13 @@ def create_model_runner_output(
|
||||
finished_sending is None
|
||||
and finished_recving is None
|
||||
and invalid_block_ids is None
|
||||
and kv_connector_worker_meta is None
|
||||
)
|
||||
else KVConnectorOutput(
|
||||
finished_sending=finished_sending,
|
||||
finished_recving=finished_recving,
|
||||
invalid_block_ids=invalid_block_ids or set(),
|
||||
kv_connector_worker_meta=kv_connector_worker_meta,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user