[KV Offloading] Support partial-tail prefix reuse with fine-grained prefix matching (#50507)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -6,7 +6,12 @@ from unittest.mock import MagicMock, call
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.kv_connector.unit.offloading_connector.test_config import (
|
||||
_make_mamba_hybrid_kv_cache_config,
|
||||
_make_vllm_config,
|
||||
)
|
||||
from tests.v1.kv_connector.unit.offloading_connector.utils import (
|
||||
MockOffloadingSpec,
|
||||
generate_store_output,
|
||||
to_keys,
|
||||
)
|
||||
@@ -16,6 +21,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingConnectorMetadata,
|
||||
OffloadingWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
|
||||
build_offloading_config,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
_ConnectorMetricName,
|
||||
@@ -25,13 +33,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import (
|
||||
RequestOffloadState,
|
||||
is_store_reachable_swa_chunk,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
GPULoadStoreSpec,
|
||||
LookupResult,
|
||||
Medium,
|
||||
OffloadingEvent,
|
||||
@@ -40,12 +50,36 @@ from vllm.v1.kv_offload.base import (
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
get_offload_block_hash,
|
||||
get_offload_group_idx,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
|
||||
def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler:
|
||||
vllm_config = _make_vllm_config()
|
||||
vllm_config.cache_config.prefix_match_unit = 4
|
||||
vllm_config.speculative_config = None
|
||||
kv_cache_config = _make_mamba_hybrid_kv_cache_config()
|
||||
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
|
||||
return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
|
||||
|
||||
|
||||
def _make_partial_tail_request(
|
||||
scheduler: OffloadingConnectorScheduler,
|
||||
) -> MagicMock:
|
||||
request = MagicMock()
|
||||
request.request_id = "req"
|
||||
request.kv_transfer_params = None
|
||||
request.num_prompt_tokens = 30
|
||||
request.num_tokens = 30
|
||||
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
|
||||
request.is_finished.return_value = False
|
||||
scheduler.on_new_request(request)
|
||||
return request
|
||||
|
||||
|
||||
def _reduce_kv_connector_stats(runner):
|
||||
reduced: dict[str, int | float] = {}
|
||||
for payload in runner.kv_connector_stats:
|
||||
@@ -59,6 +93,79 @@ def _reduce_kv_connector_stats(runner):
|
||||
return reduced
|
||||
|
||||
|
||||
def test_partial_tail_store_uses_attention_and_recurrent_cow_sources():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
_make_partial_tail_request(scheduler)
|
||||
req_status = scheduler._req_status["req"]
|
||||
req_status.group_states[0].block_ids[:] = [11, 12]
|
||||
req_status.group_states[1].block_ids[:] = [0, 21]
|
||||
scheduler.manager.prepare_store.side_effect = (
|
||||
lambda keys, req_context: generate_store_output(keys)
|
||||
)
|
||||
|
||||
output = SimpleNamespace(partial_tail_offloads={"req": [(1, 99, 28)]})
|
||||
jobs = scheduler._build_partial_tail_store_jobs(output)
|
||||
|
||||
assert len(jobs) == 1
|
||||
[job_id] = jobs
|
||||
src_spec = jobs[job_id].src_spec
|
||||
assert isinstance(src_spec, GPULoadStoreSpec)
|
||||
assert src_spec.block_ids.tolist() == [12, 99]
|
||||
assert src_spec.group_sizes == [1, 1]
|
||||
assert src_spec.block_indices == [1, 1]
|
||||
assert scheduler._block_id_to_pending_jobs == {
|
||||
12: {job_id},
|
||||
99: {job_id},
|
||||
}
|
||||
|
||||
|
||||
def test_partial_lookup_returns_exact_boundary_and_group_load_keys():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(scheduler)
|
||||
req_status = scheduler._req_status["req"]
|
||||
req_status.num_locally_computed_tokens = 0
|
||||
req_status.update_offload_keys()
|
||||
|
||||
scheduler.manager.lookup.return_value = LookupResult.HIT
|
||||
assert scheduler._lookup(req_status) == 28
|
||||
|
||||
assert req_status.partial_tail_boundary == 28
|
||||
|
||||
scheduler.update_state_after_alloc(
|
||||
request,
|
||||
KVCacheBlocks(
|
||||
(
|
||||
[KVCacheBlock(31), KVCacheBlock(32)],
|
||||
[KVCacheBlock(0, is_null=True), KVCacheBlock(41)],
|
||||
)
|
||||
),
|
||||
num_external_tokens=28,
|
||||
)
|
||||
[load_job] = scheduler._current_batch_load_jobs.values()
|
||||
dst_spec = load_job.dst_spec
|
||||
assert isinstance(dst_spec, GPULoadStoreSpec)
|
||||
assert dst_spec.block_ids.tolist() == [31, 32, 41]
|
||||
assert dst_spec.group_sizes == [2, 1]
|
||||
assert dst_spec.block_indices == [0, 1]
|
||||
assert req_status.partial_tail_boundary is None
|
||||
|
||||
|
||||
def test_partial_lookup_requires_every_cache_group():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
_make_partial_tail_request(scheduler)
|
||||
req_status = scheduler._req_status["req"]
|
||||
req_status.update_offload_keys()
|
||||
|
||||
def lookup(key, req_context):
|
||||
if get_offload_group_idx(key) == 1 and get_offload_block_hash(key) != b"h3":
|
||||
return LookupResult.MISS
|
||||
return LookupResult.HIT
|
||||
|
||||
scheduler.manager.lookup.side_effect = lookup
|
||||
assert scheduler._lookup(req_status) == 16
|
||||
assert req_status.partial_tail_boundary is None
|
||||
|
||||
|
||||
def test_scheduler_reports_allocation_failure(request_runner):
|
||||
runner = request_runner(
|
||||
block_size=4,
|
||||
@@ -2141,7 +2248,7 @@ def test_stale_sliding_window_block_after_prepare_store_failure(
|
||||
# Now prepare_store succeeds.
|
||||
# Without the fix, the request would try to offload the stale block_id
|
||||
# at position 0 (now reused at position 3), causing a duplicate in
|
||||
# sliding_window_block_ids and eventually a KeyError.
|
||||
# fenced_block_ids and eventually a KeyError.
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
@@ -2968,7 +3075,7 @@ class TestEagle:
|
||||
def test_request_finished_with_pending_stores_populates_fence(request_runner):
|
||||
"""When a request finishes with in-flight store jobs, the fence index
|
||||
(_block_id_to_pending_jobs) is correctly populated with the store jobs'
|
||||
non_sliding_window_block_ids.
|
||||
deferred_fence_block_ids.
|
||||
|
||||
This prevents data corruption when a subsequent request reuses the same
|
||||
GPU blocks before the store completes.
|
||||
@@ -3002,7 +3109,7 @@ def test_request_finished_with_pending_stores_populates_fence(request_runner):
|
||||
)
|
||||
for js in runner.connector_scheduler._jobs.values():
|
||||
if js.is_store:
|
||||
job_block_ids.update(js.non_sliding_window_block_ids or [])
|
||||
job_block_ids.update(js.deferred_fence_block_ids or [])
|
||||
|
||||
# Run 1: create store job, finish request, populate fence.
|
||||
# With non-blocking drain (#45595), the job stays in-flight.
|
||||
@@ -3104,7 +3211,7 @@ def test_request_finished_mixed_full_attn_and_sliding_window(
|
||||
request_runner,
|
||||
):
|
||||
"""With both FullAttention and SlidingWindow groups, a single store job
|
||||
has both non_sliding_window_block_ids and sliding_window_block_ids.
|
||||
has both deferred_fence_block_ids and fenced_block_ids.
|
||||
|
||||
request_finished only registers non-SW blocks in the fence.
|
||||
SW blocks were already registered at store creation time.
|
||||
@@ -3160,8 +3267,8 @@ def test_request_finished_mixed_full_attn_and_sliding_window(
|
||||
)
|
||||
for js in runner.connector_scheduler._jobs.values():
|
||||
if js.is_store:
|
||||
sw_block_ids.update(js.sliding_window_block_ids or [])
|
||||
non_sw_block_ids.update(js.non_sliding_window_block_ids or [])
|
||||
sw_block_ids.update(js.fenced_block_ids or [])
|
||||
non_sw_block_ids.update(js.deferred_fence_block_ids or [])
|
||||
|
||||
# Run 1: create store job, finish request, populate fence.
|
||||
runner.run(
|
||||
|
||||
@@ -74,12 +74,10 @@ class TransferJobStatus:
|
||||
# Offload keys this job covers; passed to manager.complete_*().
|
||||
keys: set[OffloadKey]
|
||||
is_store: bool
|
||||
# Store src block IDs whose ref_cnt protects them while the request
|
||||
# runs. Only registered in _block_id_to_pending_jobs on request_finished.
|
||||
non_sliding_window_block_ids: list[int] | None = None
|
||||
# Store src block IDs that may be freed before the request finishes.
|
||||
# Registered in _block_id_to_pending_jobs at store creation time.
|
||||
sliding_window_block_ids: list[int] | None = None
|
||||
# Store source blocks fenced after the request finishes.
|
||||
deferred_fence_block_ids: list[int] | None = None
|
||||
# Store source blocks fenced when the transfer is created.
|
||||
fenced_block_ids: list[int] | None = None
|
||||
|
||||
|
||||
class GroupOffloadConfig(NamedTuple):
|
||||
@@ -92,6 +90,9 @@ class GroupOffloadConfig(NamedTuple):
|
||||
kv_event_group_spec: OffloadingEventGroupSpec
|
||||
# None below means full attention
|
||||
sliding_window_size_in_chunks: int | None
|
||||
# Partial-tail data for this group comes from the scheduler's CoW hand-off
|
||||
# rather than the request block table.
|
||||
requires_cow_source: bool = False
|
||||
# Number of this group's offloaded chunks per full-attention alignment
|
||||
# segment. Used to skip storing SWA chunks that can never serve a load
|
||||
# hit (e.g. DeepSeek V4 where SWA groups have much smaller block sizes
|
||||
@@ -162,8 +163,10 @@ def resolve_mamba_align_size(
|
||||
class SchedulerOffloadConfig(NamedTuple):
|
||||
kv_group_configs: tuple[GroupOffloadConfig, ...]
|
||||
blocks_per_chunk: int
|
||||
tokens_per_hash: int
|
||||
num_workers: int
|
||||
offload_prompt_only: bool
|
||||
supports_partial_tail: bool
|
||||
|
||||
@classmethod
|
||||
def from_spec(
|
||||
@@ -226,9 +229,14 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
sorted(eagle_groups),
|
||||
)
|
||||
|
||||
return cls(
|
||||
num_workers=vllm_config.parallel_config.world_size,
|
||||
kv_group_configs=tuple(
|
||||
kv_group_configs_list: list[GroupOffloadConfig] = []
|
||||
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
|
||||
kv_cache_group = kv_cache_config.kv_cache_groups[idx]
|
||||
kv_spec = kv_cache_group.kv_cache_spec
|
||||
sw = get_sliding_window_size_in_chunks(
|
||||
kv_spec, tokens_per_block * spec.blocks_per_chunk
|
||||
)
|
||||
kv_group_configs_list.append(
|
||||
GroupOffloadConfig(
|
||||
group_idx=idx,
|
||||
tokens_per_block=tokens_per_block,
|
||||
@@ -237,24 +245,52 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
(tokens_per_block * spec.blocks_per_chunk)
|
||||
// spec.tokens_per_hash
|
||||
),
|
||||
sliding_window_size_in_chunks=(
|
||||
sw := get_sliding_window_size_in_chunks(
|
||||
kv_cache_config.kv_cache_groups[idx].kv_cache_spec,
|
||||
tokens_per_block * spec.blocks_per_chunk,
|
||||
)
|
||||
),
|
||||
sliding_window_size_in_chunks=sw,
|
||||
alignment_chunk_count=_alignment_chunk_count(
|
||||
tokens_per_block * spec.blocks_per_chunk, sw
|
||||
),
|
||||
kv_event_group_spec=get_offloading_event_group_spec(
|
||||
kv_cache_config.kv_cache_groups[idx]
|
||||
),
|
||||
kv_event_group_spec=get_offloading_event_group_spec(kv_cache_group),
|
||||
is_eagle_group=idx in eagle_groups,
|
||||
requires_cow_source=(
|
||||
isinstance(kv_spec, MambaSpec)
|
||||
and kv_spec.mamba_cache_mode == "align"
|
||||
),
|
||||
)
|
||||
for idx, tokens_per_block in enumerate(spec.tokens_per_block)
|
||||
),
|
||||
)
|
||||
kv_group_configs = tuple(kv_group_configs_list)
|
||||
group_block_sizes = {config.tokens_per_block for config in kv_group_configs}
|
||||
has_partial_recurrent_group = any(
|
||||
config.requires_cow_source
|
||||
and config.tokens_per_block > spec.tokens_per_hash
|
||||
for config in kv_group_configs
|
||||
)
|
||||
# Partial tails currently require one physical block per offload chunk
|
||||
# and uniform, non-windowed groups so one boundary identifies every
|
||||
# group's source. EAGLE and DCP need additional hand-off semantics.
|
||||
supports_partial_tail = (
|
||||
spec.blocks_per_chunk == 1
|
||||
and len(group_block_sizes) == 1
|
||||
and has_partial_recurrent_group
|
||||
and not (
|
||||
spec.kv_events_config.enable_kv_cache_events
|
||||
and spec.kv_events_config.self_describing_kv_events
|
||||
)
|
||||
and all(
|
||||
config.sliding_window_size_in_chunks is None
|
||||
or config.requires_cow_source
|
||||
for config in kv_group_configs
|
||||
)
|
||||
and not any(config.is_eagle_group for config in kv_group_configs)
|
||||
and vllm_config.parallel_config.decode_context_parallel_size == 1
|
||||
)
|
||||
|
||||
return cls(
|
||||
num_workers=vllm_config.parallel_config.world_size,
|
||||
kv_group_configs=kv_group_configs,
|
||||
blocks_per_chunk=spec.blocks_per_chunk,
|
||||
tokens_per_hash=spec.tokens_per_hash,
|
||||
offload_prompt_only=spec.offload_prompt_only,
|
||||
supports_partial_tail=supports_partial_tail,
|
||||
)
|
||||
|
||||
|
||||
@@ -286,6 +322,9 @@ class RequestOffloadState:
|
||||
# time.monotonic() of this request's first deferred offload lookup;
|
||||
# None once consumed (observed) or while no lookup is pending.
|
||||
deferred_lookup_start_time: float | None = None
|
||||
# Fine-grained token boundary selected beyond the last complete offload
|
||||
# chunk. It is consumed when the corresponding load is scheduled.
|
||||
partial_tail_boundary: int | None = None
|
||||
# True once on_request_finished has been signaled to the manager.
|
||||
finished_signaled: bool = False
|
||||
|
||||
@@ -479,6 +518,16 @@ class OffloadingConnectorScheduler:
|
||||
self._mamba_align_size: int | None = resolve_mamba_align_size(
|
||||
spec, kv_cache_config
|
||||
)
|
||||
self._partial_tail_block_size = (
|
||||
self.config.kv_group_configs[0].tokens_per_block
|
||||
if self.config.supports_partial_tail
|
||||
else 0
|
||||
)
|
||||
self._cow_source_groups = frozenset(
|
||||
config.group_idx
|
||||
for config in self.config.kv_group_configs
|
||||
if config.requires_cow_source
|
||||
)
|
||||
|
||||
self._req_status: dict[ReqId, RequestOffloadState] = {}
|
||||
self._current_batch_load_jobs: dict[int, TransferJob] = {}
|
||||
@@ -627,8 +676,20 @@ class OffloadingConnectorScheduler:
|
||||
group_state.offload_keys[chunks_to_skip:],
|
||||
req_status.req_context,
|
||||
)
|
||||
if req_status.partial_tail_boundary is not None:
|
||||
self.manager.touch(
|
||||
tuple(
|
||||
self._make_boundary_key(
|
||||
req_status.req,
|
||||
group.group_idx,
|
||||
req_status.partial_tail_boundary,
|
||||
)
|
||||
for group in self.config.kv_group_configs
|
||||
),
|
||||
req_status.req_context,
|
||||
)
|
||||
|
||||
def _lookup(self, req_status: RequestOffloadState) -> int | None:
|
||||
def _lookup_complete_chunks(self, req_status: RequestOffloadState) -> int | None:
|
||||
"""
|
||||
Find how many tokens beyond num_locally_computed_tokens can be loaded.
|
||||
|
||||
@@ -801,6 +862,52 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
return num_hit_tokens
|
||||
|
||||
def _make_boundary_key(
|
||||
self, request: Request, group_idx: int, boundary_tokens: int
|
||||
) -> OffloadKey:
|
||||
hash_idx = boundary_tokens // self.config.tokens_per_hash - 1
|
||||
return make_offload_key(request.block_hashes[hash_idx], group_idx)
|
||||
|
||||
def _lookup(self, req_status: RequestOffloadState) -> int | None:
|
||||
complete_hit = self._lookup_complete_chunks(req_status)
|
||||
req_status.partial_tail_boundary = None
|
||||
if complete_hit is None or not self.config.supports_partial_tail:
|
||||
return complete_hit
|
||||
|
||||
local_tokens = req_status.num_locally_computed_tokens
|
||||
complete_boundary = local_tokens + complete_hit
|
||||
tokens_per_hash = self.config.tokens_per_hash
|
||||
block_end = complete_boundary + self._partial_tail_block_size
|
||||
max_boundary = round_down(
|
||||
min(req_status.req.num_prompt_tokens - 1, block_end - 1), tokens_per_hash
|
||||
)
|
||||
if max_boundary <= complete_boundary:
|
||||
return complete_hit
|
||||
|
||||
pending = False
|
||||
for boundary in range(max_boundary, complete_boundary, -tokens_per_hash):
|
||||
boundary_pending = False
|
||||
boundary_missed = False
|
||||
for group_config in self.config.kv_group_configs:
|
||||
key = self._make_boundary_key(
|
||||
req_status.req, group_config.group_idx, boundary
|
||||
)
|
||||
result = self.manager.lookup(key, req_status.req_context)
|
||||
if result is LookupResult.MISS:
|
||||
boundary_missed = True
|
||||
break
|
||||
if result in (LookupResult.HIT_PENDING, LookupResult.RETRY):
|
||||
boundary_pending = True
|
||||
|
||||
pending |= boundary_pending
|
||||
if not boundary_missed and not boundary_pending:
|
||||
req_status.partial_tail_boundary = boundary
|
||||
return boundary - local_tokens
|
||||
|
||||
if pending and complete_hit == 0:
|
||||
return None
|
||||
return complete_hit
|
||||
|
||||
def on_new_request(self, request: Request) -> None:
|
||||
"""Called when a new request is added to the scheduler."""
|
||||
req_context = _create_req_context(request)
|
||||
@@ -880,6 +987,9 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
num_locally_computed_tokens = req_status.num_locally_computed_tokens
|
||||
num_cached_tokens = num_locally_computed_tokens + num_external_tokens
|
||||
partial_tail_boundary = req_status.partial_tail_boundary
|
||||
if partial_tail_boundary is not None:
|
||||
assert partial_tail_boundary == num_cached_tokens
|
||||
|
||||
keys_to_load: list[OffloadKey] = []
|
||||
dst_block_ids: list[int] = []
|
||||
@@ -923,12 +1033,19 @@ class OffloadingConnectorScheduler:
|
||||
)
|
||||
|
||||
num_chunks = cdiv(num_cached_tokens, tokens_per_chunk)
|
||||
assert len(offload_keys) >= num_chunks
|
||||
if num_pending_gpu_blocks:
|
||||
start_chunk_idx = (
|
||||
num_locally_computed_gpu_blocks // self.config.blocks_per_chunk
|
||||
)
|
||||
keys_to_load.extend(offload_keys[start_chunk_idx:num_chunks])
|
||||
end_chunk_idx = num_chunks - (partial_tail_boundary is not None)
|
||||
assert len(offload_keys) >= end_chunk_idx
|
||||
keys_to_load.extend(offload_keys[start_chunk_idx:end_chunk_idx])
|
||||
if partial_tail_boundary is not None:
|
||||
keys_to_load.append(
|
||||
self._make_boundary_key(
|
||||
request, group_config.group_idx, partial_tail_boundary
|
||||
)
|
||||
)
|
||||
|
||||
dst_block_ids.extend(
|
||||
block.block_id
|
||||
@@ -968,6 +1085,7 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
if self._chunks_being_loaded is not None:
|
||||
self._chunks_being_loaded.update(keys_to_load)
|
||||
req_status.partial_tail_boundary = None
|
||||
|
||||
def _update_req_states(self, scheduler_output: SchedulerOutput) -> None:
|
||||
"""
|
||||
@@ -1020,6 +1138,94 @@ class OffloadingConnectorScheduler:
|
||||
):
|
||||
group_state.block_ids[j] = 0
|
||||
|
||||
def _build_partial_tail_store_jobs(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> dict[int, TransferJob]:
|
||||
handoffs = scheduler_output.partial_tail_offloads
|
||||
if not self.config.supports_partial_tail or not handoffs:
|
||||
return {}
|
||||
|
||||
store_jobs: dict[int, TransferJob] = {}
|
||||
for req_id, entries in handoffs.items():
|
||||
req_status = self._req_status.get(req_id)
|
||||
assert req_status is not None
|
||||
assert entries
|
||||
boundaries = {boundary for _, _, boundary in entries}
|
||||
assert len(boundaries) == 1
|
||||
boundary = boundaries.pop()
|
||||
req = req_status.req
|
||||
max_boundary = min(
|
||||
req.num_prompt_tokens,
|
||||
req_status.max_offload_tokens or req.num_prompt_tokens,
|
||||
)
|
||||
assert boundary > 0
|
||||
assert boundary % self.config.tokens_per_hash == 0
|
||||
assert boundary <= max_boundary
|
||||
|
||||
cow_blocks = {group_idx: block_id for group_idx, block_id, _ in entries}
|
||||
assert self._cow_source_groups.issubset(cow_blocks)
|
||||
|
||||
assert boundary % self._partial_tail_block_size != 0
|
||||
block_idx = boundary // self._partial_tail_block_size
|
||||
if any(
|
||||
group.group_idx not in self._cow_source_groups
|
||||
and block_idx >= len(req_status.group_states[group.group_idx].block_ids)
|
||||
for group in self.config.kv_group_configs
|
||||
):
|
||||
continue
|
||||
keys = [
|
||||
self._make_boundary_key(req, group.group_idx, boundary)
|
||||
for group in self.config.kv_group_configs
|
||||
]
|
||||
block_ids = [
|
||||
cow_blocks[group.group_idx]
|
||||
if group.group_idx in self._cow_source_groups
|
||||
else req_status.group_states[group.group_idx].block_ids[block_idx]
|
||||
for group in self.config.kv_group_configs
|
||||
]
|
||||
assert all(block_id != 0 for block_id in block_ids)
|
||||
|
||||
store_output = self.manager.prepare_store(keys, req_status.req_context)
|
||||
if store_output is None:
|
||||
self._connector_stats.increase_counter(
|
||||
_ConnectorMetricName.ALLOCATION_FAILURE
|
||||
)
|
||||
continue
|
||||
if not store_output.keys_to_store:
|
||||
continue
|
||||
|
||||
group_by_key = {key: idx for idx, key in enumerate(keys)}
|
||||
accepted_groups = [group_by_key[key] for key in store_output.keys_to_store]
|
||||
group_sizes = [0] * len(self.config.kv_group_configs)
|
||||
block_indices = [0] * len(self.config.kv_group_configs)
|
||||
for group_idx in accepted_groups:
|
||||
group_sizes[group_idx] = 1
|
||||
block_indices[group_idx] = block_idx
|
||||
source_blocks = [block_ids[group_idx] for group_idx in accepted_groups]
|
||||
|
||||
job_id = self._generate_job_id()
|
||||
req_status.transfer_jobs.add(job_id)
|
||||
for block_id in source_blocks:
|
||||
self._block_id_to_pending_jobs.setdefault(block_id, set()).add(job_id)
|
||||
self._jobs[job_id] = TransferJobStatus(
|
||||
req_id=req_id,
|
||||
pending_count=self.config.num_workers,
|
||||
keys=set(store_output.keys_to_store),
|
||||
is_store=True,
|
||||
fenced_block_ids=source_blocks,
|
||||
)
|
||||
store_jobs[job_id] = TransferJob(
|
||||
req_id=req_id,
|
||||
src_spec=GPULoadStoreSpec(
|
||||
source_blocks,
|
||||
group_sizes=group_sizes,
|
||||
block_indices=block_indices,
|
||||
),
|
||||
dst_spec=store_output.store_spec,
|
||||
)
|
||||
|
||||
return store_jobs
|
||||
|
||||
def _build_store_jobs(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
@@ -1119,8 +1325,8 @@ class OffloadingConnectorScheduler:
|
||||
group_sizes: list[int] = []
|
||||
block_indices: list[int] = []
|
||||
src_block_ids: list[int] = []
|
||||
sliding_window_block_ids: list[int] = []
|
||||
non_sliding_window_block_ids: list[int] = []
|
||||
fenced_block_ids: list[int] = []
|
||||
deferred_fence_block_ids: list[int] = []
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs, req_status.group_states
|
||||
):
|
||||
@@ -1156,9 +1362,9 @@ class OffloadingConnectorScheduler:
|
||||
src_block_ids.append(block_id)
|
||||
num_group_blocks += 1
|
||||
if is_sliding_window:
|
||||
sliding_window_block_ids.append(block_id)
|
||||
fenced_block_ids.append(block_id)
|
||||
else:
|
||||
non_sliding_window_block_ids.append(block_id)
|
||||
deferred_fence_block_ids.append(block_id)
|
||||
|
||||
group_sizes.append(num_group_blocks)
|
||||
block_indices.append(start_gpu_block_idx or 0)
|
||||
@@ -1180,7 +1386,7 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
# Watch sliding window blocks as they may get evicted
|
||||
# before the request finishes
|
||||
for bid in sliding_window_block_ids or ():
|
||||
for bid in fenced_block_ids:
|
||||
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
|
||||
|
||||
# the non-sliding window blocks will be watched only
|
||||
@@ -1190,8 +1396,8 @@ class OffloadingConnectorScheduler:
|
||||
pending_count=self.config.num_workers,
|
||||
keys=set(keys_to_store),
|
||||
is_store=True,
|
||||
non_sliding_window_block_ids=non_sliding_window_block_ids,
|
||||
sliding_window_block_ids=sliding_window_block_ids or None,
|
||||
deferred_fence_block_ids=deferred_fence_block_ids,
|
||||
fenced_block_ids=fenced_block_ids or None,
|
||||
)
|
||||
|
||||
store_jobs[job_id] = TransferJob(
|
||||
@@ -1208,7 +1414,7 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
if req.is_finished():
|
||||
# Register non-sliding-window blocks for flush detection.
|
||||
for bid in non_sliding_window_block_ids:
|
||||
for bid in deferred_fence_block_ids:
|
||||
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
|
||||
if bid in self._current_batch_allocated_block_ids:
|
||||
self._current_batch_jobs_to_flush.add(job_id)
|
||||
@@ -1248,9 +1454,11 @@ class OffloadingConnectorScheduler:
|
||||
for jid in self._block_id_to_pending_jobs[bid]
|
||||
)
|
||||
|
||||
partial_store_jobs = self._build_partial_tail_store_jobs(scheduler_output)
|
||||
normal_store_jobs = self._build_store_jobs(scheduler_output)
|
||||
meta = OffloadingConnectorMetadata(
|
||||
load_jobs=self._current_batch_load_jobs,
|
||||
store_jobs=self._build_store_jobs(scheduler_output),
|
||||
store_jobs=partial_store_jobs | normal_store_jobs,
|
||||
jobs_to_flush=self._current_batch_jobs_to_flush,
|
||||
)
|
||||
|
||||
@@ -1344,12 +1552,12 @@ class OffloadingConnectorScheduler:
|
||||
if self._block_id_to_pending_jobs:
|
||||
# Sliding window blocks are tracked from store creation
|
||||
# and must be cleaned up unconditionally.
|
||||
self._remove_pending_job(job_id, job_status.sliding_window_block_ids)
|
||||
self._remove_pending_job(job_id, job_status.fenced_block_ids)
|
||||
# Non-sliding-window blocks are only tracked after
|
||||
# request_finished, so only clean up for finished requests.
|
||||
if req_status.req.is_finished():
|
||||
self._remove_pending_job(
|
||||
job_id, job_status.non_sliding_window_block_ids
|
||||
job_id, job_status.deferred_fence_block_ids
|
||||
)
|
||||
|
||||
del self._jobs[job_id]
|
||||
@@ -1404,11 +1612,11 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
# Keep req_status alive: _build_store_jobs will process finished_req_ids
|
||||
# on the next step and handle cleanup after creating store jobs.
|
||||
# Register non_sliding_window_block_ids so future block reuse triggers
|
||||
# a flush via _block_id_to_pending_jobs.
|
||||
# Register deferred fences so future block reuse triggers a flush via
|
||||
# _block_id_to_pending_jobs.
|
||||
for job_id in req_status.transfer_jobs:
|
||||
job_status = self._jobs[job_id]
|
||||
for bid in job_status.non_sliding_window_block_ids or ():
|
||||
for bid in job_status.deferred_fence_block_ids or ():
|
||||
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
|
||||
|
||||
return False, None
|
||||
@@ -1451,6 +1659,7 @@ class OffloadingConnectorScheduler:
|
||||
for group_state in status.group_states:
|
||||
group_state.next_stored_chunk_idx = 0
|
||||
status.transfer_jobs.clear()
|
||||
status.partial_tail_boundary = None
|
||||
|
||||
# Discard jobs and save job_counter to be able to discard worker responses
|
||||
self._stale_job_threshold = self._job_counter
|
||||
|
||||
Reference in New Issue
Block a user