[KV Offload] Support self-describing KV events with TieringOffloadingSpec (#48679)
Signed-off-by: Change72 <changg@nvidia.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
0a5069e4e3
commit
f007cceb42
@@ -75,7 +75,7 @@ vllm serve <model> \
|
||||
| `max_tracker_size` | no | `64000` | single-tier | Max entries in the lookup tracker. |
|
||||
| `secondary_tiers` | no | `[]` | multi-tier | List of secondary tier configs (see below). |
|
||||
| `offload_prompt_only` | no | `true` | both | If `true`, only prompt (prefill) blocks are offloaded; decode blocks are skipped. |
|
||||
| `self_describing_kv_events` | no | `false` | single-tier | Opt-in. When `true` *and* KV cache events are enabled (`--kv-events-config` with `enable_kv_cache_events`), the connector emits self-describing block-granular `BlockStored`/`BlockRemoved` payloads (constituent block hashes, whole-chunk `token_ids`, per-block `block_size`, parent hash, LoRA + group/cache-spec metadata) instead of the placeholder fallback, so external KV-event consumers can index offloaded blocks. Inert unless events are enabled. Currently rejected by `TieringOffloadingSpec`. Full-attention groups only; sliding-window/SSM groups keep the placeholder fallback. In chunk mode (`block_size` > GPU block size, or `blocks_per_chunk` > 1), overlapping chunks re-announce shared per-block hashes, so consumers must reference-count (deduplicate) repeated store/remove announcements. |
|
||||
| `self_describing_kv_events` | no | `false` | both | Opt-in. When `true` *and* KV cache events are enabled (`--kv-events-config` with `enable_kv_cache_events`), the connector emits self-describing block-granular `BlockStored`/`BlockRemoved` payloads (constituent block hashes, whole-chunk `token_ids`, per-block `block_size`, parent hash, LoRA + group/cache-spec metadata) instead of the placeholder fallback, so external KV-event consumers can index offloaded blocks. Inert unless events are enabled. With `TieringOffloadingSpec`, a CPU promotion is self-describing when a local request observes its primary-tier `HIT` before event translation; otherwise its stored event may retain the placeholder, while a later `HIT` can backfill metadata for removal. Pending-removal/re-promotion races and externally initiated promotions may also produce placeholders, and consumers must ignore removals for unknown hashes. Full-attention groups only; sliding-window/SSM groups keep the placeholder fallback. In chunk mode (`block_size` > GPU block size, or `blocks_per_chunk` > 1), overlapping chunks re-announce shared per-block hashes, so consumers must reference-count (deduplicate) repeated store/remove announcements. |
|
||||
| `spec_module_path` | no | — | both | Python import path for a custom `OffloadingSpec` not in the built-in registry. Required only when `spec_name` is not built-in (advanced). |
|
||||
|
||||
## Secondary Tiers
|
||||
|
||||
@@ -7,7 +7,13 @@ import torch
|
||||
|
||||
from tests.v1.kv_connector.unit.utils import create_vllm_config
|
||||
from vllm.config import KVEventsConfig, KVTransferConfig
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU, MEDIUM_FS, BlockRemoved, BlockStored
|
||||
from vllm.distributed.kv_events import (
|
||||
MEDIUM_CPU,
|
||||
MEDIUM_FS,
|
||||
MEDIUM_OBJ,
|
||||
BlockRemoved,
|
||||
BlockStored,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
|
||||
build_offloading_config,
|
||||
)
|
||||
@@ -62,8 +68,9 @@ def _wire_hash(block_hash: BlockHash):
|
||||
return maybe_convert_block_hash(block_hash)
|
||||
|
||||
|
||||
def _request(*, block_hashes: list[BlockHash], token_count: int):
|
||||
def _request(*, block_hashes: list[BlockHash], token_count: int, req_id: str = "req"):
|
||||
req = MagicMock()
|
||||
req.request_id = req_id
|
||||
req.block_hashes = block_hashes
|
||||
req.all_token_ids = list(range(1, token_count + 1))
|
||||
req.lora_request = None
|
||||
@@ -104,10 +111,32 @@ def _record_chunks(
|
||||
return keys
|
||||
|
||||
|
||||
def _record_lookup_chunks(
|
||||
tracker: OffloadingEventsTracker,
|
||||
req,
|
||||
group_config: GroupOffloadConfig,
|
||||
num_chunks: int,
|
||||
) -> list[OffloadKey]:
|
||||
keys: list[OffloadKey] = []
|
||||
hbf = group_config.hashes_per_chunk
|
||||
for chunk_idx in range(num_chunks):
|
||||
tail_hash = req.block_hashes[(chunk_idx + 1) * hbf - 1]
|
||||
assert tail_hash is not None
|
||||
key = make_offload_key(tail_hash, group_config.group_idx)
|
||||
tracker.record_lookup(
|
||||
req,
|
||||
group_config,
|
||||
chunk_idx,
|
||||
key,
|
||||
)
|
||||
keys.append(key)
|
||||
return keys
|
||||
|
||||
|
||||
def _stored_event(
|
||||
keys: list[OffloadKey],
|
||||
locality: Locality | None = None,
|
||||
medium: str = _CPU_MEDIUM,
|
||||
locality: Locality | None = None,
|
||||
) -> OffloadingEvent:
|
||||
return OffloadingEvent(
|
||||
keys=keys,
|
||||
@@ -119,8 +148,8 @@ def _stored_event(
|
||||
|
||||
def _removed_event(
|
||||
keys: list[OffloadKey],
|
||||
locality: Locality | None = None,
|
||||
medium: str = _CPU_MEDIUM,
|
||||
locality: Locality | None = None,
|
||||
) -> OffloadingEvent:
|
||||
return OffloadingEvent(
|
||||
keys=keys,
|
||||
@@ -130,6 +159,21 @@ def _removed_event(
|
||||
)
|
||||
|
||||
|
||||
def _lookup_chunk() -> tuple[
|
||||
OffloadingEventsTracker, MagicMock, GroupOffloadConfig, OffloadKey
|
||||
]:
|
||||
tracker = _tracker()
|
||||
req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
group_config = _group_config()
|
||||
key = _record_lookup_chunks(
|
||||
tracker,
|
||||
req,
|
||||
group_config,
|
||||
num_chunks=1,
|
||||
)[0]
|
||||
return tracker, req, group_config, key
|
||||
|
||||
|
||||
def test_take_events_forwards_locality_to_rich_store():
|
||||
tracker = _tracker()
|
||||
req = _request(block_hashes=[_hash(0)], token_count=4)
|
||||
@@ -220,18 +264,37 @@ def test_take_events_publishes_routable_block_stored():
|
||||
assert len(tracker._pending_event_metadata) == 6
|
||||
|
||||
|
||||
def test_take_events_factor_gt_1_chunk_store_and_remove():
|
||||
def test_promotion_emits_full_cpu_stored_event():
|
||||
tracker, _, _, key = _lookup_chunk()
|
||||
|
||||
[event] = tracker.take_events([_stored_event([key])])
|
||||
|
||||
assert isinstance(event, BlockStored)
|
||||
assert event.medium == MEDIUM_CPU
|
||||
assert event.block_hashes == [_wire_hash(_hash(0))]
|
||||
assert event.parent_block_hash is None
|
||||
assert event.token_ids == [1, 2, 3, 4]
|
||||
assert event.block_size == 4
|
||||
assert event.lora_id is None
|
||||
assert event.lora_name is None
|
||||
assert event.extra_keys is None
|
||||
assert event.group_idx == 0
|
||||
assert event.kv_cache_spec_kind == KVCacheSpecKind.FULL_ATTENTION.value
|
||||
assert event.kv_cache_spec_sliding_window is None
|
||||
|
||||
|
||||
def test_lookup_promotion_factor_gt_1_store_and_remove():
|
||||
block_size = 4
|
||||
blocks_per_chunk = 3
|
||||
blocks_per_chunk = 2
|
||||
tracker = _tracker()
|
||||
group_config = _group_config(
|
||||
block_size=block_size, blocks_per_chunk=blocks_per_chunk
|
||||
)
|
||||
req = _request(
|
||||
block_hashes=[_hash(i) for i in range(6)],
|
||||
block_hashes=[_hash(i) for i in range(4)],
|
||||
token_count=block_size * blocks_per_chunk * 2,
|
||||
)
|
||||
keys = _record_chunks(tracker, req, group_config, num_chunks=2)
|
||||
keys = _record_lookup_chunks(tracker, req, group_config, num_chunks=2)
|
||||
|
||||
stored = list(tracker.take_events([_stored_event(keys)]))
|
||||
assert len(stored) == 2
|
||||
@@ -293,6 +356,7 @@ def test_take_events_opt_out_keeps_placeholders():
|
||||
group_config = _group_config()
|
||||
req = _request(block_hashes=[_hash(i) for i in range(3)], token_count=12)
|
||||
keys = _record_chunks(tracker, req, group_config, num_chunks=3)
|
||||
_record_lookup_chunks(tracker, req, group_config, num_chunks=3)
|
||||
|
||||
assert not tracker.self_describing_enabled
|
||||
assert not tracker._pending_event_metadata
|
||||
@@ -315,11 +379,21 @@ def test_take_events_opt_out_keeps_placeholders():
|
||||
assert len(events[3].block_hashes) == 3
|
||||
|
||||
|
||||
def test_record_store_skips_sliding_window_group():
|
||||
@pytest.mark.parametrize(
|
||||
"sliding_window_size_in_chunks",
|
||||
[1, 2],
|
||||
ids=["ssm", "sliding-window"],
|
||||
)
|
||||
def test_event_metadata_skips_non_full_attention_group(
|
||||
sliding_window_size_in_chunks: int,
|
||||
):
|
||||
tracker = _tracker()
|
||||
group_config = _group_config(sliding_window_size_in_chunks=2)
|
||||
group_config = _group_config(
|
||||
sliding_window_size_in_chunks=sliding_window_size_in_chunks
|
||||
)
|
||||
req = _request(block_hashes=[_hash(i) for i in range(3)], token_count=12)
|
||||
keys = _record_chunks(tracker, req, group_config, num_chunks=3)
|
||||
_record_lookup_chunks(tracker, req, group_config, num_chunks=3)
|
||||
|
||||
assert not tracker._pending_event_metadata
|
||||
|
||||
@@ -329,6 +403,57 @@ def test_record_store_skips_sliding_window_group():
|
||||
assert events[0].block_size == 0
|
||||
|
||||
|
||||
def test_pending_cpu_removal_consumes_hit_backfill_until_next_hit():
|
||||
tracker = _tracker()
|
||||
block_hashes = [_hash(0), _hash(1)]
|
||||
req = _request(block_hashes=block_hashes, token_count=8)
|
||||
group_config = _group_config(blocks_per_chunk=2)
|
||||
key = _record_chunks(tracker, req, group_config, num_chunks=1)[0]
|
||||
confirmed_meta = tracker._pending_event_metadata[key]
|
||||
lookup_req = _request(
|
||||
block_hashes=block_hashes,
|
||||
token_count=8,
|
||||
req_id="new-request",
|
||||
)
|
||||
|
||||
tracker.record_lookup(
|
||||
lookup_req,
|
||||
group_config,
|
||||
0,
|
||||
key,
|
||||
)
|
||||
assert tracker._pending_event_metadata[key] is confirmed_meta
|
||||
|
||||
removed = list(tracker.take_events([_removed_event([key])]))
|
||||
assert len(removed) == 1
|
||||
assert removed[0].block_hashes == [
|
||||
_wire_hash(_hash(0)),
|
||||
_wire_hash(_hash(1)),
|
||||
]
|
||||
|
||||
stored = list(tracker.take_events([_stored_event([key])]))
|
||||
assert len(stored) == 1
|
||||
assert stored[0].block_size == 0
|
||||
assert stored[0].token_ids == []
|
||||
|
||||
tracker.record_lookup(lookup_req, group_config, 0, key)
|
||||
removed = list(tracker.take_events([_removed_event([key])]))
|
||||
assert removed[0].block_hashes == [
|
||||
_wire_hash(_hash(0)),
|
||||
_wire_hash(_hash(1)),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("medium", [MEDIUM_FS, MEDIUM_OBJ])
|
||||
def test_secondary_stored_event_does_not_mutate_cpu_metadata(medium: str):
|
||||
tracker, _, _, key = _lookup_chunk()
|
||||
expected_metadata = dict(tracker._pending_event_metadata)
|
||||
|
||||
stored = list(tracker.take_events([_stored_event([key], medium)]))
|
||||
assert stored[0].token_ids == [1, 2, 3, 4]
|
||||
assert tracker._pending_event_metadata == expected_metadata
|
||||
|
||||
|
||||
def test_take_events_groups_removed_hashes_by_kv_group():
|
||||
tracker = _tracker()
|
||||
group0_config = _group_config(group_idx=0, blocks_per_chunk=2)
|
||||
@@ -378,7 +503,7 @@ def test_reset_cache_clears_side_table():
|
||||
tracker = _tracker()
|
||||
group_config = _group_config()
|
||||
req = _request(block_hashes=[_hash(i) for i in range(3)], token_count=12)
|
||||
_record_chunks(tracker, req, group_config, num_chunks=3)
|
||||
_record_lookup_chunks(tracker, req, group_config, num_chunks=3)
|
||||
|
||||
assert tracker._pending_event_metadata
|
||||
|
||||
@@ -387,7 +512,7 @@ def test_reset_cache_clears_side_table():
|
||||
assert not tracker._pending_event_metadata
|
||||
|
||||
|
||||
def test_tiering_rejects_self_describing_kv_events():
|
||||
def test_tiering_accepts_self_describing_kv_events():
|
||||
vllm_config = create_vllm_config(
|
||||
block_size=4,
|
||||
max_num_batched_tokens=16,
|
||||
@@ -423,5 +548,9 @@ def test_tiering_rejects_self_describing_kv_events():
|
||||
],
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="TieringOffloadingSpec"):
|
||||
TieringOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
|
||||
spec = TieringOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
|
||||
tracker = OffloadingEventsTracker(spec.kv_events_config)
|
||||
|
||||
assert spec.kv_events_config.enable_kv_cache_events
|
||||
assert spec.kv_events_config.self_describing_kv_events
|
||||
assert tracker.self_describing_enabled
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, call
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -11,6 +11,7 @@ from tests.v1.kv_connector.unit.offloading_connector.utils import (
|
||||
to_keys,
|
||||
)
|
||||
from tests.v1.kv_connector.unit.utils import EOS_TOKEN_ID
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU, BlockRemoved, BlockStored
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
_ConnectorMetricName,
|
||||
@@ -19,6 +20,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
OffloadingConnectorScheduler,
|
||||
RequestOffloadState,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
@@ -26,6 +28,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadingEvent,
|
||||
OffloadingManager,
|
||||
OffloadPolicy,
|
||||
ReqContext,
|
||||
@@ -143,6 +146,159 @@ def test_scheduler_reports_lookup_async_delay_on_resolve(request_runner):
|
||||
assert reduced[f"{_ConnectorMetricName.LOOKUP_ASYNC_DELAY}_sum"] > 0
|
||||
|
||||
|
||||
def test_max_offload_tokens_zero_does_not_record_pending_lookups(request_runner):
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
)
|
||||
runner.manager.lookup.return_value = LookupResult.RETRY
|
||||
runner.manager.take_events.return_value = []
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
runner.new_request(
|
||||
token_ids=[1] * 12,
|
||||
kv_transfer_params={"max_offload_tokens": 0},
|
||||
)
|
||||
runner.run(decoded_tokens=[])
|
||||
|
||||
tracker = runner.connector_scheduler._events_tracker
|
||||
assert runner.manager.lookup.call_count == 3
|
||||
assert not tracker._pending_event_metadata
|
||||
assert list(runner.connector_scheduler.take_events()) == []
|
||||
|
||||
runner.manager.lookup.return_value = LookupResult.MISS
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID])
|
||||
|
||||
assert not tracker._pending_event_metadata
|
||||
assert list(runner.connector_scheduler.take_events()) == []
|
||||
|
||||
|
||||
def test_abort_before_hit_uses_placeholder_then_later_hit_heals_removal(
|
||||
request_runner,
|
||||
):
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
blocks_per_chunk=2,
|
||||
)
|
||||
raw_events: list[OffloadingEvent] = []
|
||||
|
||||
def take_raw_events():
|
||||
yield from raw_events
|
||||
raw_events.clear()
|
||||
|
||||
runner.manager.lookup.return_value = LookupResult.RETRY
|
||||
runner.manager.take_events.side_effect = take_raw_events
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[1] * 8)
|
||||
runner.run(decoded_tokens=[])
|
||||
|
||||
tracker = runner.connector_scheduler._events_tracker
|
||||
assert not tracker._pending_event_metadata
|
||||
key = runner.manager.lookup.call_args.args[0]
|
||||
req_id = str(runner.req_id)
|
||||
req_status = runner.connector_scheduler._req_status[req_id]
|
||||
|
||||
runner.scheduler.finish_requests((req_id,), RequestStatus.FINISHED_ABORTED)
|
||||
|
||||
assert not tracker._pending_event_metadata
|
||||
|
||||
raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=False))
|
||||
events = list(runner.connector_scheduler.take_events())
|
||||
assert len(events) == 1
|
||||
assert isinstance(events[0], BlockStored)
|
||||
assert events[0].block_size == 0
|
||||
assert events[0].token_ids == []
|
||||
|
||||
runner.manager.lookup.return_value = LookupResult.HIT
|
||||
group_config = runner.connector_scheduler.config.kv_group_configs[0]
|
||||
assert (
|
||||
runner.connector_scheduler._maximal_prefix_lookup(
|
||||
[key],
|
||||
req_status.req_context,
|
||||
req_status.req,
|
||||
group_config,
|
||||
0,
|
||||
)
|
||||
== 1
|
||||
)
|
||||
assert key in tracker._pending_event_metadata
|
||||
|
||||
raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=True))
|
||||
[event] = runner.connector_scheduler.take_events()
|
||||
assert isinstance(event, BlockRemoved)
|
||||
assert event.medium == MEDIUM_CPU
|
||||
assert len(event.block_hashes) == 2
|
||||
assert key not in tracker._pending_event_metadata
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blocks_per_chunk", [1, 2])
|
||||
def test_promotion_hit_precedes_stored_event_translation(
|
||||
request_runner,
|
||||
blocks_per_chunk: int,
|
||||
):
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
num_gpu_blocks=10,
|
||||
async_scheduling=False,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
token_ids = [1] * 4 * blocks_per_chunk
|
||||
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.new_request(token_ids=token_ids)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=tuple(range(blocks_per_chunk)),
|
||||
)
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.connector_scheduler._events_tracker.reset()
|
||||
|
||||
raw_events: list[OffloadingEvent] = []
|
||||
|
||||
def lookup(key, req_context):
|
||||
raw_events.append(OffloadingEvent(keys=[key], medium=MEDIUM_CPU, removed=False))
|
||||
return LookupResult.HIT
|
||||
|
||||
def take_raw_events():
|
||||
yield from raw_events
|
||||
raw_events.clear()
|
||||
|
||||
runner.manager.lookup.side_effect = lookup
|
||||
runner.manager.take_events.side_effect = take_raw_events
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
publisher = MagicMock()
|
||||
runner.scheduler.kv_event_publisher = publisher
|
||||
|
||||
runner.new_request(token_ids=token_ids)
|
||||
runner.run(
|
||||
decoded_tokens=[],
|
||||
expected_loaded=tuple(range(blocks_per_chunk)),
|
||||
)
|
||||
|
||||
events = [
|
||||
event
|
||||
for publish_call in publisher.publish.call_args_list
|
||||
for event in publish_call.args[0].events
|
||||
if isinstance(event, BlockStored) and event.medium == MEDIUM_CPU
|
||||
]
|
||||
assert len(events) == 1
|
||||
assert len(events[0].block_hashes) == blocks_per_chunk
|
||||
assert events[0].block_size == 4
|
||||
assert events[0].token_ids == token_ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
block_size = 4
|
||||
@@ -241,7 +397,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_loaded=(0, 1, 2))
|
||||
|
||||
# single block lookup with a hit in a middle block
|
||||
@@ -249,7 +405,7 @@ def test_offloading_connector(request_runner, async_scheduling: bool):
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1
|
||||
runner.run(decoded_tokens=[EOS_TOKEN_ID], expected_loaded=(3, 4, 5))
|
||||
|
||||
|
||||
@@ -307,7 +463,7 @@ def test_request_preemption(request_runner, async_scheduling: bool):
|
||||
|
||||
# request should now return from preemption
|
||||
# re-load [0, ..., 8] from the CPU and store [9, 10, 11]
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 3
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 3
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -427,7 +583,7 @@ def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling:
|
||||
# start a request to load the first block, but don't complete
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1
|
||||
runner.run(
|
||||
decoded_tokens=[],
|
||||
complete_transfers=False,
|
||||
@@ -439,7 +595,7 @@ def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling:
|
||||
|
||||
# start a new request to load the same first block
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1
|
||||
runner.run(
|
||||
decoded_tokens=[],
|
||||
complete_transfers=False,
|
||||
@@ -491,7 +647,7 @@ def test_abort_loading_requests(request_runner, async_scheduling: bool):
|
||||
# start a request to load the first block, but don't complete
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1
|
||||
runner.run(
|
||||
decoded_tokens=[],
|
||||
complete_transfers=False,
|
||||
@@ -793,73 +949,144 @@ def _make_scheduler_with_lookup(
|
||||
|
||||
scheduler = object.__new__(OffloadingConnectorScheduler)
|
||||
scheduler.manager = manager
|
||||
scheduler._events_tracker = MagicMock()
|
||||
return scheduler
|
||||
|
||||
|
||||
_EMPTY_REQ_CTX = ReqContext(req_id="")
|
||||
_LOOKUP_REQ = MagicMock()
|
||||
_LOOKUP_REQ.request_id = "req"
|
||||
_LOOKUP_GROUP_CONFIG = MagicMock()
|
||||
|
||||
|
||||
def _maximal_lookup(sched, keys, start_chunk_idx: int = 0):
|
||||
return sched._maximal_prefix_lookup(
|
||||
keys,
|
||||
_EMPTY_REQ_CTX,
|
||||
_LOOKUP_REQ,
|
||||
_LOOKUP_GROUP_CONFIG,
|
||||
start_chunk_idx,
|
||||
)
|
||||
|
||||
|
||||
class TestMaximalPrefixLookup:
|
||||
def test_all_hit(self):
|
||||
sched = _make_scheduler_with_lookup({1: LookupResult.HIT, 2: LookupResult.HIT})
|
||||
assert sched._maximal_prefix_lookup(to_keys([1, 2]), _EMPTY_REQ_CTX) == 2
|
||||
assert _maximal_lookup(sched, to_keys([1, 2])) == 2
|
||||
|
||||
def test_records_absolute_chunk_indices(self):
|
||||
keys = to_keys([1, 2])
|
||||
sched = _make_scheduler_with_lookup({1: LookupResult.HIT, 2: LookupResult.HIT})
|
||||
|
||||
assert _maximal_lookup(sched, keys, start_chunk_idx=3) == 2
|
||||
assert sched._events_tracker.record_lookup.call_args_list == [
|
||||
call(
|
||||
_LOOKUP_REQ,
|
||||
_LOOKUP_GROUP_CONFIG,
|
||||
3,
|
||||
keys[0],
|
||||
),
|
||||
call(
|
||||
_LOOKUP_REQ,
|
||||
_LOOKUP_GROUP_CONFIG,
|
||||
4,
|
||||
keys[1],
|
||||
),
|
||||
]
|
||||
|
||||
def test_all_miss(self):
|
||||
sched = _make_scheduler_with_lookup({})
|
||||
assert sched._maximal_prefix_lookup(to_keys([1, 2]), _EMPTY_REQ_CTX) == 0
|
||||
assert _maximal_lookup(sched, to_keys([1, 2])) == 0
|
||||
sched._events_tracker.record_lookup.assert_not_called()
|
||||
|
||||
def test_partial_prefix(self):
|
||||
sched = _make_scheduler_with_lookup({1: LookupResult.HIT, 2: LookupResult.HIT})
|
||||
assert sched._maximal_prefix_lookup(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) == 2
|
||||
assert _maximal_lookup(sched, to_keys([1, 2, 3])) == 2
|
||||
|
||||
def test_miss_then_hit(self):
|
||||
sched = _make_scheduler_with_lookup({2: LookupResult.HIT})
|
||||
assert sched._maximal_prefix_lookup(to_keys([1, 2]), _EMPTY_REQ_CTX) == 0
|
||||
assert _maximal_lookup(sched, to_keys([1, 2])) == 0
|
||||
|
||||
def test_single_hit(self):
|
||||
sched = _make_scheduler_with_lookup({1: LookupResult.HIT})
|
||||
assert sched._maximal_prefix_lookup(to_keys([1]), _EMPTY_REQ_CTX) == 1
|
||||
assert _maximal_lookup(sched, to_keys([1])) == 1
|
||||
|
||||
def test_empty(self):
|
||||
sched = _make_scheduler_with_lookup({})
|
||||
assert sched._maximal_prefix_lookup([], _EMPTY_REQ_CTX) == 0
|
||||
assert _maximal_lookup(sched, []) == 0
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pending_result",
|
||||
[LookupResult.RETRY, LookupResult.HIT_PENDING],
|
||||
)
|
||||
def test_pending_result_is_not_recorded(
|
||||
self,
|
||||
pending_result: LookupResult,
|
||||
):
|
||||
sched = _make_scheduler_with_lookup({1: pending_result})
|
||||
|
||||
assert _maximal_lookup(sched, to_keys([1])) is None
|
||||
sched._events_tracker.record_lookup.assert_not_called()
|
||||
|
||||
def test_retry_defers(self):
|
||||
keys = to_keys([1, 2])
|
||||
sched = _make_scheduler_with_lookup(
|
||||
{1: LookupResult.RETRY, 2: LookupResult.HIT}
|
||||
)
|
||||
assert sched._maximal_prefix_lookup(to_keys([1, 2]), _EMPTY_REQ_CTX) is None
|
||||
assert _maximal_lookup(sched, keys) is None
|
||||
assert sched.manager.lookup.call_count == 2
|
||||
sched._events_tracker.record_lookup.assert_called_once_with(
|
||||
_LOOKUP_REQ,
|
||||
_LOOKUP_GROUP_CONFIG,
|
||||
1,
|
||||
keys[1],
|
||||
)
|
||||
|
||||
def test_retry_after_hit_defers(self):
|
||||
keys = to_keys([1, 2])
|
||||
sched = _make_scheduler_with_lookup(
|
||||
{1: LookupResult.HIT, 2: LookupResult.RETRY}
|
||||
)
|
||||
assert sched._maximal_prefix_lookup(to_keys([1, 2]), _EMPTY_REQ_CTX) is None
|
||||
assert _maximal_lookup(sched, keys) is None
|
||||
sched._events_tracker.record_lookup.assert_called_once_with(
|
||||
_LOOKUP_REQ,
|
||||
_LOOKUP_GROUP_CONFIG,
|
||||
0,
|
||||
keys[0],
|
||||
)
|
||||
|
||||
def test_hit_pending_defers(self):
|
||||
keys = to_keys([1, 2])
|
||||
sched = _make_scheduler_with_lookup(
|
||||
{1: LookupResult.HIT_PENDING, 2: LookupResult.HIT}
|
||||
)
|
||||
assert sched._maximal_prefix_lookup(to_keys([1, 2]), _EMPTY_REQ_CTX) is None
|
||||
assert _maximal_lookup(sched, keys) is None
|
||||
assert sched.manager.lookup.call_count == 2
|
||||
sched._events_tracker.record_lookup.assert_called_once_with(
|
||||
_LOOKUP_REQ,
|
||||
_LOOKUP_GROUP_CONFIG,
|
||||
1,
|
||||
keys[1],
|
||||
)
|
||||
|
||||
def test_hit_pending_does_not_stop_scan(self):
|
||||
"""HIT_PENDING defers but does not break — scan continues until miss."""
|
||||
sched = _make_scheduler_with_lookup(
|
||||
{1: LookupResult.HIT_PENDING, 2: LookupResult.MISS, 3: LookupResult.HIT}
|
||||
)
|
||||
assert sched._maximal_prefix_lookup(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) is None
|
||||
assert _maximal_lookup(sched, to_keys([1, 2, 3])) is None
|
||||
assert sched.manager.lookup.call_count == 2
|
||||
sched._events_tracker.record_lookup.assert_not_called()
|
||||
|
||||
def test_retry_stops_at_miss(self):
|
||||
"""RETRY is treated as hit for iteration, but miss stops the scan."""
|
||||
sched = _make_scheduler_with_lookup(
|
||||
{1: LookupResult.RETRY, 2: LookupResult.MISS, 3: LookupResult.HIT}
|
||||
)
|
||||
assert sched._maximal_prefix_lookup(to_keys([1, 2, 3]), _EMPTY_REQ_CTX) is None
|
||||
assert _maximal_lookup(sched, to_keys([1, 2, 3])) is None
|
||||
# lookup should have been called for blocks 1 and 2 (stops at miss)
|
||||
assert sched.manager.lookup.call_count == 2
|
||||
sched._events_tracker.record_lookup.assert_not_called()
|
||||
|
||||
|
||||
class TestSlidingWindowLookup:
|
||||
@@ -1011,7 +1238,7 @@ def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling
|
||||
|
||||
# New request with 2 offloaded chunks; first matches what's in CPU.
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk * 2)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -1042,7 +1269,7 @@ def test_loads_do_not_populate_fence_index(request_runner):
|
||||
async_scheduling=False,
|
||||
)
|
||||
runner.new_request(token_ids=[0] * 12)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1
|
||||
runner.run(decoded_tokens=[], complete_transfers=False)
|
||||
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
|
||||
|
||||
@@ -1088,7 +1315,7 @@ def test_fence_at_update_state_after_alloc(request_runner):
|
||||
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * 4)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
@@ -1139,7 +1366,7 @@ def test_fence_at_build_store_jobs(request_runner):
|
||||
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[1] * 4)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 0
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 0
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
@@ -1365,7 +1592,7 @@ def test_reset_cache(request_runner, async_scheduling: bool):
|
||||
# Leave the load in-flight so that reset_cache must flush it.
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 1
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output([])
|
||||
)
|
||||
@@ -1553,9 +1780,7 @@ def test_async_preempt_readmit_before_transfer_output_is_deferred(request_runner
|
||||
# preemption batch's ModelRunnerOutput is consumed by update_from_output().
|
||||
free_block_queue.num_free_blocks = num_free_blocks_empty
|
||||
assert runner.scheduler.reset_prefix_cache()
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: len(
|
||||
key
|
||||
)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: len(keys)
|
||||
|
||||
readmit_output = runner.scheduler.schedule()
|
||||
|
||||
@@ -1669,7 +1894,7 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.new_request(token_ids=[0] * num_tokens + [1])
|
||||
runner.manager.lookup.return_value = LookupResult.HIT
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 2
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 2
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
# Group 0: full prefix lookup hits 2 offloaded chunks
|
||||
@@ -1840,6 +2065,13 @@ class TestEagle:
|
||||
req.request_id = "test-req"
|
||||
req.num_tokens = num_tokens
|
||||
req.kv_transfer_params = None
|
||||
num_hash_blocks = max(
|
||||
len(hashes) * scheduler.config.kv_group_configs[idx].hashes_per_chunk
|
||||
for idx, hashes in enumerate(offload_keys_per_group)
|
||||
)
|
||||
req.block_hashes = [BlockHash(str(i).encode()) for i in range(num_hash_blocks)]
|
||||
req.all_token_ids = list(range(num_tokens))
|
||||
req.lora_request = None
|
||||
|
||||
state = RequestOffloadState(
|
||||
config=scheduler.config,
|
||||
|
||||
@@ -61,9 +61,9 @@ def get_offloading_event_group_spec(
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _OffloadEventMetadata:
|
||||
"""BlockStored payload snapshot for one OffloadKey, captured at store
|
||||
time and kept until the matching eviction event. ``medium`` is forwarded
|
||||
from the OffloadingEvent."""
|
||||
"""BlockStored payload snapshot for one OffloadKey, captured while the
|
||||
Request is available and kept until the matching eviction event. ``medium``
|
||||
is forwarded from the OffloadingEvent."""
|
||||
|
||||
# The chunk's constituent block hashes; the last one is the OffloadKey.
|
||||
block_hashes: tuple[BlockHash, ...]
|
||||
@@ -81,10 +81,11 @@ class _OffloadEventMetadata:
|
||||
class OffloadingEventsTracker:
|
||||
"""Tracks offloaded chunks' KV event payloads from store to eviction.
|
||||
|
||||
The scheduler calls :meth:`record_store` from ``_build_store_jobs``
|
||||
while the ``Request`` is available, and routes the manager's raw
|
||||
:class:`OffloadingEvent` stream through :meth:`take_events`. All state
|
||||
is bounded by the CPU pool capacity and cleared by :meth:`reset`.
|
||||
The scheduler calls :meth:`record_store` from ``_build_store_jobs`` and
|
||||
:meth:`record_lookup` for ready primary-tier hits while the ``Request`` is
|
||||
available. Deferred and missing lookups add no state. Under the connector's
|
||||
supported success-only transfer model, entries follow primary allocations
|
||||
until CPU removal translation or :meth:`reset`.
|
||||
"""
|
||||
|
||||
def __init__(self, config: OffloadingKVEventsConfig):
|
||||
@@ -93,8 +94,7 @@ class OffloadingEventsTracker:
|
||||
config.enable_kv_cache_events and config.self_describing_kv_events
|
||||
)
|
||||
|
||||
# OffloadKey -> payload snapshot, kept until the eviction event so
|
||||
# BlockRemoved can fan out. Bounded: one entry per offloaded chunk.
|
||||
# OffloadKey -> payload snapshot, kept until CPU removal or reset.
|
||||
self._pending_event_metadata: dict[OffloadKey, _OffloadEventMetadata] = {}
|
||||
|
||||
def record_store(
|
||||
@@ -116,6 +116,23 @@ class OffloadingEventsTracker:
|
||||
meta = self._build_event_metadata(req, group_config, chunk_idx)
|
||||
self._pending_event_metadata[offload_key] = meta
|
||||
|
||||
def record_lookup(
|
||||
self,
|
||||
req: Request,
|
||||
group_config: "GroupOffloadConfig",
|
||||
chunk_idx: int,
|
||||
offload_key: OffloadKey,
|
||||
) -> None:
|
||||
"""Snapshot metadata for a ready primary-tier lookup hit."""
|
||||
if not self.self_describing_enabled:
|
||||
return
|
||||
if group_config.sliding_window_size_in_chunks is not None:
|
||||
return
|
||||
if offload_key not in self._pending_event_metadata:
|
||||
self._pending_event_metadata[offload_key] = self._build_event_metadata(
|
||||
req, group_config, chunk_idx
|
||||
)
|
||||
|
||||
def take_events(self, events: Iterable[OffloadingEvent]) -> Iterable[KVCacheEvent]:
|
||||
"""Translate raw OffloadingEvents into self-describing KV events.
|
||||
|
||||
@@ -165,7 +182,7 @@ class OffloadingEventsTracker:
|
||||
assert len(chunk_hashes) == hbf
|
||||
|
||||
if group_config.sliding_window_size_in_chunks is not None:
|
||||
# record_store filters these out before calling this helper.
|
||||
# The recording methods filter these out before calling this helper.
|
||||
raise AssertionError("self-describing events only support full attention")
|
||||
|
||||
parent_block_hash: BlockHash | None
|
||||
@@ -232,7 +249,8 @@ class OffloadingEventsTracker:
|
||||
"OffloadingEventsTracker: no event metadata for "
|
||||
"offload key during BlockStored emission; emitting a "
|
||||
"placeholder payload. Expected for non-full-attention "
|
||||
"groups; otherwise indicates a missing populate path."
|
||||
"groups and promotions not observed as a primary-tier "
|
||||
"hit before translation."
|
||||
)
|
||||
yield self._placeholder_stored(key, event.medium, locality)
|
||||
continue
|
||||
|
||||
@@ -463,15 +463,27 @@ class OffloadingConnectorScheduler:
|
||||
del self._req_status[req_id]
|
||||
|
||||
def _maximal_prefix_lookup(
|
||||
self, keys: Iterable[OffloadKey], req_context: ReqContext
|
||||
self,
|
||||
keys: Iterable[OffloadKey],
|
||||
req_context: ReqContext,
|
||||
req: Request,
|
||||
group_config: GroupOffloadConfig,
|
||||
start_chunk_idx: int,
|
||||
) -> int | None:
|
||||
"""Return the number of consecutive offloaded chunks from the start,
|
||||
or None if the backend deferred a lookup."""
|
||||
hit_count = 0
|
||||
defer_lookup = False
|
||||
for key in keys:
|
||||
match self.manager.lookup(key, req_context):
|
||||
for local_idx, key in enumerate(keys):
|
||||
result = self.manager.lookup(key, req_context)
|
||||
match result:
|
||||
case LookupResult.HIT:
|
||||
self._events_tracker.record_lookup(
|
||||
req,
|
||||
group_config,
|
||||
start_chunk_idx + local_idx,
|
||||
key,
|
||||
)
|
||||
hit_count += 1
|
||||
case LookupResult.HIT_PENDING:
|
||||
defer_lookup = True
|
||||
@@ -616,7 +628,11 @@ class OffloadingConnectorScheduler:
|
||||
num_hit_chunks: int | None
|
||||
if sliding_window_size_in_chunks is None:
|
||||
num_hit_chunks = self._maximal_prefix_lookup(
|
||||
offload_keys, req_status.req_context
|
||||
offload_keys,
|
||||
req_status.req_context,
|
||||
req_status.req,
|
||||
group_config,
|
||||
start_chunk_idx,
|
||||
)
|
||||
else:
|
||||
required_window = sliding_window_size_in_chunks
|
||||
|
||||
@@ -136,14 +136,6 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
super().__init__(config)
|
||||
# Redeclare for mypy: parent sets this but `--follow-imports skip` hides it
|
||||
self._manager: OffloadingManager | None = None
|
||||
if self.kv_events_config.self_describing_kv_events:
|
||||
raise ValueError(
|
||||
"self_describing_kv_events is not supported by "
|
||||
"TieringOffloadingSpec. Tier promotions can emit primary-tier "
|
||||
"store events that do not correspond to GPU store jobs, so the "
|
||||
"current self-describing side table cannot describe them "
|
||||
"correctly."
|
||||
)
|
||||
|
||||
# Parse secondary tier configurations
|
||||
self.secondary_tier_configs = self.extra_config.get("secondary_tiers", [])
|
||||
|
||||
Reference in New Issue
Block a user