diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 661078041f..3ab354cd19 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -6,7 +6,12 @@ from unittest.mock import MagicMock, call import pytest import torch +from tests.v1.kv_connector.unit.offloading_connector.test_config import ( + _make_mamba_hybrid_kv_cache_config, + _make_vllm_config, +) from tests.v1.kv_connector.unit.offloading_connector.utils import ( + MockOffloadingSpec, generate_store_output, to_keys, ) @@ -16,6 +21,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import ( OffloadingConnectorMetadata, OffloadingWorkerMetadata, ) +from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import ( + build_offloading_config, +) from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import ( OffloadingConnectorStats, _ConnectorMetricName, @@ -25,13 +33,15 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.scheduler import ( RequestOffloadState, is_store_reachable_swa_chunk, ) -from vllm.v1.core.kv_cache_utils import BlockHash +from vllm.v1.core.kv_cache_manager import KVCacheBlocks +from vllm.v1.core.kv_cache_utils import BlockHash, KVCacheBlock from vllm.v1.kv_cache_interface import ( FullAttentionSpec, KVCacheGroupSpec, SlidingWindowSpec, ) from vllm.v1.kv_offload.base import ( + GPULoadStoreSpec, LookupResult, Medium, OffloadingEvent, @@ -40,12 +50,36 @@ from vllm.v1.kv_offload.base import ( ReqContext, RequestOffloadingContext, get_offload_block_hash, + get_offload_group_idx, make_offload_key, ) from vllm.v1.outputs import KVConnectorOutput from vllm.v1.request import RequestStatus +def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler: + vllm_config = _make_vllm_config() + vllm_config.cache_config.prefix_match_unit = 4 + vllm_config.speculative_config = None + kv_cache_config = _make_mamba_hybrid_kv_cache_config() + spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config)) + return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config) + + +def _make_partial_tail_request( + scheduler: OffloadingConnectorScheduler, +) -> MagicMock: + request = MagicMock() + request.request_id = "req" + request.kv_transfer_params = None + request.num_prompt_tokens = 30 + request.num_tokens = 30 + request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)] + request.is_finished.return_value = False + scheduler.on_new_request(request) + return request + + def _reduce_kv_connector_stats(runner): reduced: dict[str, int | float] = {} for payload in runner.kv_connector_stats: @@ -59,6 +93,79 @@ def _reduce_kv_connector_stats(runner): return reduced +def test_partial_tail_store_uses_attention_and_recurrent_cow_sources(): + scheduler = _make_partial_tail_scheduler() + _make_partial_tail_request(scheduler) + req_status = scheduler._req_status["req"] + req_status.group_states[0].block_ids[:] = [11, 12] + req_status.group_states[1].block_ids[:] = [0, 21] + scheduler.manager.prepare_store.side_effect = ( + lambda keys, req_context: generate_store_output(keys) + ) + + output = SimpleNamespace(partial_tail_offloads={"req": [(1, 99, 28)]}) + jobs = scheduler._build_partial_tail_store_jobs(output) + + assert len(jobs) == 1 + [job_id] = jobs + src_spec = jobs[job_id].src_spec + assert isinstance(src_spec, GPULoadStoreSpec) + assert src_spec.block_ids.tolist() == [12, 99] + assert src_spec.group_sizes == [1, 1] + assert src_spec.block_indices == [1, 1] + assert scheduler._block_id_to_pending_jobs == { + 12: {job_id}, + 99: {job_id}, + } + + +def test_partial_lookup_returns_exact_boundary_and_group_load_keys(): + scheduler = _make_partial_tail_scheduler() + request = _make_partial_tail_request(scheduler) + req_status = scheduler._req_status["req"] + req_status.num_locally_computed_tokens = 0 + req_status.update_offload_keys() + + scheduler.manager.lookup.return_value = LookupResult.HIT + assert scheduler._lookup(req_status) == 28 + + assert req_status.partial_tail_boundary == 28 + + scheduler.update_state_after_alloc( + request, + KVCacheBlocks( + ( + [KVCacheBlock(31), KVCacheBlock(32)], + [KVCacheBlock(0, is_null=True), KVCacheBlock(41)], + ) + ), + num_external_tokens=28, + ) + [load_job] = scheduler._current_batch_load_jobs.values() + dst_spec = load_job.dst_spec + assert isinstance(dst_spec, GPULoadStoreSpec) + assert dst_spec.block_ids.tolist() == [31, 32, 41] + assert dst_spec.group_sizes == [2, 1] + assert dst_spec.block_indices == [0, 1] + assert req_status.partial_tail_boundary is None + + +def test_partial_lookup_requires_every_cache_group(): + scheduler = _make_partial_tail_scheduler() + _make_partial_tail_request(scheduler) + req_status = scheduler._req_status["req"] + req_status.update_offload_keys() + + def lookup(key, req_context): + if get_offload_group_idx(key) == 1 and get_offload_block_hash(key) != b"h3": + return LookupResult.MISS + return LookupResult.HIT + + scheduler.manager.lookup.side_effect = lookup + assert scheduler._lookup(req_status) == 16 + assert req_status.partial_tail_boundary is None + + def test_scheduler_reports_allocation_failure(request_runner): runner = request_runner( block_size=4, @@ -2141,7 +2248,7 @@ def test_stale_sliding_window_block_after_prepare_store_failure( # Now prepare_store succeeds. # Without the fix, the request would try to offload the stale block_id # at position 0 (now reused at position 3), causing a duplicate in - # sliding_window_block_ids and eventually a KeyError. + # fenced_block_ids and eventually a KeyError. runner.manager.prepare_store.side_effect = lambda keys, req_context: ( generate_store_output(keys) ) @@ -2968,7 +3075,7 @@ class TestEagle: def test_request_finished_with_pending_stores_populates_fence(request_runner): """When a request finishes with in-flight store jobs, the fence index (_block_id_to_pending_jobs) is correctly populated with the store jobs' - non_sliding_window_block_ids. + deferred_fence_block_ids. This prevents data corruption when a subsequent request reuses the same GPU blocks before the store completes. @@ -3002,7 +3109,7 @@ def test_request_finished_with_pending_stores_populates_fence(request_runner): ) for js in runner.connector_scheduler._jobs.values(): if js.is_store: - job_block_ids.update(js.non_sliding_window_block_ids or []) + job_block_ids.update(js.deferred_fence_block_ids or []) # Run 1: create store job, finish request, populate fence. # With non-blocking drain (#45595), the job stays in-flight. @@ -3104,7 +3211,7 @@ def test_request_finished_mixed_full_attn_and_sliding_window( request_runner, ): """With both FullAttention and SlidingWindow groups, a single store job - has both non_sliding_window_block_ids and sliding_window_block_ids. + has both deferred_fence_block_ids and fenced_block_ids. request_finished only registers non-SW blocks in the fence. SW blocks were already registered at store creation time. @@ -3160,8 +3267,8 @@ def test_request_finished_mixed_full_attn_and_sliding_window( ) for js in runner.connector_scheduler._jobs.values(): if js.is_store: - sw_block_ids.update(js.sliding_window_block_ids or []) - non_sw_block_ids.update(js.non_sliding_window_block_ids or []) + sw_block_ids.update(js.fenced_block_ids or []) + non_sw_block_ids.update(js.deferred_fence_block_ids or []) # Run 1: create store job, finish request, populate fence. runner.run( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index fecce3d28f..8abd0aff40 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -74,12 +74,10 @@ class TransferJobStatus: # Offload keys this job covers; passed to manager.complete_*(). keys: set[OffloadKey] is_store: bool - # Store src block IDs whose ref_cnt protects them while the request - # runs. Only registered in _block_id_to_pending_jobs on request_finished. - non_sliding_window_block_ids: list[int] | None = None - # Store src block IDs that may be freed before the request finishes. - # Registered in _block_id_to_pending_jobs at store creation time. - sliding_window_block_ids: list[int] | None = None + # Store source blocks fenced after the request finishes. + deferred_fence_block_ids: list[int] | None = None + # Store source blocks fenced when the transfer is created. + fenced_block_ids: list[int] | None = None class GroupOffloadConfig(NamedTuple): @@ -92,6 +90,9 @@ class GroupOffloadConfig(NamedTuple): kv_event_group_spec: OffloadingEventGroupSpec # None below means full attention sliding_window_size_in_chunks: int | None + # Partial-tail data for this group comes from the scheduler's CoW hand-off + # rather than the request block table. + requires_cow_source: bool = False # Number of this group's offloaded chunks per full-attention alignment # segment. Used to skip storing SWA chunks that can never serve a load # hit (e.g. DeepSeek V4 where SWA groups have much smaller block sizes @@ -162,8 +163,10 @@ def resolve_mamba_align_size( class SchedulerOffloadConfig(NamedTuple): kv_group_configs: tuple[GroupOffloadConfig, ...] blocks_per_chunk: int + tokens_per_hash: int num_workers: int offload_prompt_only: bool + supports_partial_tail: bool @classmethod def from_spec( @@ -226,9 +229,14 @@ class SchedulerOffloadConfig(NamedTuple): sorted(eagle_groups), ) - return cls( - num_workers=vllm_config.parallel_config.world_size, - kv_group_configs=tuple( + kv_group_configs_list: list[GroupOffloadConfig] = [] + for idx, tokens_per_block in enumerate(spec.tokens_per_block): + kv_cache_group = kv_cache_config.kv_cache_groups[idx] + kv_spec = kv_cache_group.kv_cache_spec + sw = get_sliding_window_size_in_chunks( + kv_spec, tokens_per_block * spec.blocks_per_chunk + ) + kv_group_configs_list.append( GroupOffloadConfig( group_idx=idx, tokens_per_block=tokens_per_block, @@ -237,24 +245,52 @@ class SchedulerOffloadConfig(NamedTuple): (tokens_per_block * spec.blocks_per_chunk) // spec.tokens_per_hash ), - sliding_window_size_in_chunks=( - sw := get_sliding_window_size_in_chunks( - kv_cache_config.kv_cache_groups[idx].kv_cache_spec, - tokens_per_block * spec.blocks_per_chunk, - ) - ), + sliding_window_size_in_chunks=sw, alignment_chunk_count=_alignment_chunk_count( tokens_per_block * spec.blocks_per_chunk, sw ), - kv_event_group_spec=get_offloading_event_group_spec( - kv_cache_config.kv_cache_groups[idx] - ), + kv_event_group_spec=get_offloading_event_group_spec(kv_cache_group), is_eagle_group=idx in eagle_groups, + requires_cow_source=( + isinstance(kv_spec, MambaSpec) + and kv_spec.mamba_cache_mode == "align" + ), ) - for idx, tokens_per_block in enumerate(spec.tokens_per_block) - ), + ) + kv_group_configs = tuple(kv_group_configs_list) + group_block_sizes = {config.tokens_per_block for config in kv_group_configs} + has_partial_recurrent_group = any( + config.requires_cow_source + and config.tokens_per_block > spec.tokens_per_hash + for config in kv_group_configs + ) + # Partial tails currently require one physical block per offload chunk + # and uniform, non-windowed groups so one boundary identifies every + # group's source. EAGLE and DCP need additional hand-off semantics. + supports_partial_tail = ( + spec.blocks_per_chunk == 1 + and len(group_block_sizes) == 1 + and has_partial_recurrent_group + and not ( + spec.kv_events_config.enable_kv_cache_events + and spec.kv_events_config.self_describing_kv_events + ) + and all( + config.sliding_window_size_in_chunks is None + or config.requires_cow_source + for config in kv_group_configs + ) + and not any(config.is_eagle_group for config in kv_group_configs) + and vllm_config.parallel_config.decode_context_parallel_size == 1 + ) + + return cls( + num_workers=vllm_config.parallel_config.world_size, + kv_group_configs=kv_group_configs, blocks_per_chunk=spec.blocks_per_chunk, + tokens_per_hash=spec.tokens_per_hash, offload_prompt_only=spec.offload_prompt_only, + supports_partial_tail=supports_partial_tail, ) @@ -286,6 +322,9 @@ class RequestOffloadState: # time.monotonic() of this request's first deferred offload lookup; # None once consumed (observed) or while no lookup is pending. deferred_lookup_start_time: float | None = None + # Fine-grained token boundary selected beyond the last complete offload + # chunk. It is consumed when the corresponding load is scheduled. + partial_tail_boundary: int | None = None # True once on_request_finished has been signaled to the manager. finished_signaled: bool = False @@ -479,6 +518,16 @@ class OffloadingConnectorScheduler: self._mamba_align_size: int | None = resolve_mamba_align_size( spec, kv_cache_config ) + self._partial_tail_block_size = ( + self.config.kv_group_configs[0].tokens_per_block + if self.config.supports_partial_tail + else 0 + ) + self._cow_source_groups = frozenset( + config.group_idx + for config in self.config.kv_group_configs + if config.requires_cow_source + ) self._req_status: dict[ReqId, RequestOffloadState] = {} self._current_batch_load_jobs: dict[int, TransferJob] = {} @@ -627,8 +676,20 @@ class OffloadingConnectorScheduler: group_state.offload_keys[chunks_to_skip:], req_status.req_context, ) + if req_status.partial_tail_boundary is not None: + self.manager.touch( + tuple( + self._make_boundary_key( + req_status.req, + group.group_idx, + req_status.partial_tail_boundary, + ) + for group in self.config.kv_group_configs + ), + req_status.req_context, + ) - def _lookup(self, req_status: RequestOffloadState) -> int | None: + def _lookup_complete_chunks(self, req_status: RequestOffloadState) -> int | None: """ Find how many tokens beyond num_locally_computed_tokens can be loaded. @@ -801,6 +862,52 @@ class OffloadingConnectorScheduler: return num_hit_tokens + def _make_boundary_key( + self, request: Request, group_idx: int, boundary_tokens: int + ) -> OffloadKey: + hash_idx = boundary_tokens // self.config.tokens_per_hash - 1 + return make_offload_key(request.block_hashes[hash_idx], group_idx) + + def _lookup(self, req_status: RequestOffloadState) -> int | None: + complete_hit = self._lookup_complete_chunks(req_status) + req_status.partial_tail_boundary = None + if complete_hit is None or not self.config.supports_partial_tail: + return complete_hit + + local_tokens = req_status.num_locally_computed_tokens + complete_boundary = local_tokens + complete_hit + tokens_per_hash = self.config.tokens_per_hash + block_end = complete_boundary + self._partial_tail_block_size + max_boundary = round_down( + min(req_status.req.num_prompt_tokens - 1, block_end - 1), tokens_per_hash + ) + if max_boundary <= complete_boundary: + return complete_hit + + pending = False + for boundary in range(max_boundary, complete_boundary, -tokens_per_hash): + boundary_pending = False + boundary_missed = False + for group_config in self.config.kv_group_configs: + key = self._make_boundary_key( + req_status.req, group_config.group_idx, boundary + ) + result = self.manager.lookup(key, req_status.req_context) + if result is LookupResult.MISS: + boundary_missed = True + break + if result in (LookupResult.HIT_PENDING, LookupResult.RETRY): + boundary_pending = True + + pending |= boundary_pending + if not boundary_missed and not boundary_pending: + req_status.partial_tail_boundary = boundary + return boundary - local_tokens + + if pending and complete_hit == 0: + return None + return complete_hit + def on_new_request(self, request: Request) -> None: """Called when a new request is added to the scheduler.""" req_context = _create_req_context(request) @@ -880,6 +987,9 @@ class OffloadingConnectorScheduler: num_locally_computed_tokens = req_status.num_locally_computed_tokens num_cached_tokens = num_locally_computed_tokens + num_external_tokens + partial_tail_boundary = req_status.partial_tail_boundary + if partial_tail_boundary is not None: + assert partial_tail_boundary == num_cached_tokens keys_to_load: list[OffloadKey] = [] dst_block_ids: list[int] = [] @@ -923,12 +1033,19 @@ class OffloadingConnectorScheduler: ) num_chunks = cdiv(num_cached_tokens, tokens_per_chunk) - assert len(offload_keys) >= num_chunks if num_pending_gpu_blocks: start_chunk_idx = ( num_locally_computed_gpu_blocks // self.config.blocks_per_chunk ) - keys_to_load.extend(offload_keys[start_chunk_idx:num_chunks]) + end_chunk_idx = num_chunks - (partial_tail_boundary is not None) + assert len(offload_keys) >= end_chunk_idx + keys_to_load.extend(offload_keys[start_chunk_idx:end_chunk_idx]) + if partial_tail_boundary is not None: + keys_to_load.append( + self._make_boundary_key( + request, group_config.group_idx, partial_tail_boundary + ) + ) dst_block_ids.extend( block.block_id @@ -968,6 +1085,7 @@ class OffloadingConnectorScheduler: if self._chunks_being_loaded is not None: self._chunks_being_loaded.update(keys_to_load) + req_status.partial_tail_boundary = None def _update_req_states(self, scheduler_output: SchedulerOutput) -> None: """ @@ -1020,6 +1138,94 @@ class OffloadingConnectorScheduler: ): group_state.block_ids[j] = 0 + def _build_partial_tail_store_jobs( + self, scheduler_output: SchedulerOutput + ) -> dict[int, TransferJob]: + handoffs = scheduler_output.partial_tail_offloads + if not self.config.supports_partial_tail or not handoffs: + return {} + + store_jobs: dict[int, TransferJob] = {} + for req_id, entries in handoffs.items(): + req_status = self._req_status.get(req_id) + assert req_status is not None + assert entries + boundaries = {boundary for _, _, boundary in entries} + assert len(boundaries) == 1 + boundary = boundaries.pop() + req = req_status.req + max_boundary = min( + req.num_prompt_tokens, + req_status.max_offload_tokens or req.num_prompt_tokens, + ) + assert boundary > 0 + assert boundary % self.config.tokens_per_hash == 0 + assert boundary <= max_boundary + + cow_blocks = {group_idx: block_id for group_idx, block_id, _ in entries} + assert self._cow_source_groups.issubset(cow_blocks) + + assert boundary % self._partial_tail_block_size != 0 + block_idx = boundary // self._partial_tail_block_size + if any( + group.group_idx not in self._cow_source_groups + and block_idx >= len(req_status.group_states[group.group_idx].block_ids) + for group in self.config.kv_group_configs + ): + continue + keys = [ + self._make_boundary_key(req, group.group_idx, boundary) + for group in self.config.kv_group_configs + ] + block_ids = [ + cow_blocks[group.group_idx] + if group.group_idx in self._cow_source_groups + else req_status.group_states[group.group_idx].block_ids[block_idx] + for group in self.config.kv_group_configs + ] + assert all(block_id != 0 for block_id in block_ids) + + store_output = self.manager.prepare_store(keys, req_status.req_context) + if store_output is None: + self._connector_stats.increase_counter( + _ConnectorMetricName.ALLOCATION_FAILURE + ) + continue + if not store_output.keys_to_store: + continue + + group_by_key = {key: idx for idx, key in enumerate(keys)} + accepted_groups = [group_by_key[key] for key in store_output.keys_to_store] + group_sizes = [0] * len(self.config.kv_group_configs) + block_indices = [0] * len(self.config.kv_group_configs) + for group_idx in accepted_groups: + group_sizes[group_idx] = 1 + block_indices[group_idx] = block_idx + source_blocks = [block_ids[group_idx] for group_idx in accepted_groups] + + job_id = self._generate_job_id() + req_status.transfer_jobs.add(job_id) + for block_id in source_blocks: + self._block_id_to_pending_jobs.setdefault(block_id, set()).add(job_id) + self._jobs[job_id] = TransferJobStatus( + req_id=req_id, + pending_count=self.config.num_workers, + keys=set(store_output.keys_to_store), + is_store=True, + fenced_block_ids=source_blocks, + ) + store_jobs[job_id] = TransferJob( + req_id=req_id, + src_spec=GPULoadStoreSpec( + source_blocks, + group_sizes=group_sizes, + block_indices=block_indices, + ), + dst_spec=store_output.store_spec, + ) + + return store_jobs + def _build_store_jobs( self, scheduler_output: SchedulerOutput, @@ -1119,8 +1325,8 @@ class OffloadingConnectorScheduler: group_sizes: list[int] = [] block_indices: list[int] = [] src_block_ids: list[int] = [] - sliding_window_block_ids: list[int] = [] - non_sliding_window_block_ids: list[int] = [] + fenced_block_ids: list[int] = [] + deferred_fence_block_ids: list[int] = [] for group_config, group_state in zip( self.config.kv_group_configs, req_status.group_states ): @@ -1156,9 +1362,9 @@ class OffloadingConnectorScheduler: src_block_ids.append(block_id) num_group_blocks += 1 if is_sliding_window: - sliding_window_block_ids.append(block_id) + fenced_block_ids.append(block_id) else: - non_sliding_window_block_ids.append(block_id) + deferred_fence_block_ids.append(block_id) group_sizes.append(num_group_blocks) block_indices.append(start_gpu_block_idx or 0) @@ -1180,7 +1386,7 @@ class OffloadingConnectorScheduler: # Watch sliding window blocks as they may get evicted # before the request finishes - for bid in sliding_window_block_ids or (): + for bid in fenced_block_ids: self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id) # the non-sliding window blocks will be watched only @@ -1190,8 +1396,8 @@ class OffloadingConnectorScheduler: pending_count=self.config.num_workers, keys=set(keys_to_store), is_store=True, - non_sliding_window_block_ids=non_sliding_window_block_ids, - sliding_window_block_ids=sliding_window_block_ids or None, + deferred_fence_block_ids=deferred_fence_block_ids, + fenced_block_ids=fenced_block_ids or None, ) store_jobs[job_id] = TransferJob( @@ -1208,7 +1414,7 @@ class OffloadingConnectorScheduler: if req.is_finished(): # Register non-sliding-window blocks for flush detection. - for bid in non_sliding_window_block_ids: + for bid in deferred_fence_block_ids: self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id) if bid in self._current_batch_allocated_block_ids: self._current_batch_jobs_to_flush.add(job_id) @@ -1248,9 +1454,11 @@ class OffloadingConnectorScheduler: for jid in self._block_id_to_pending_jobs[bid] ) + partial_store_jobs = self._build_partial_tail_store_jobs(scheduler_output) + normal_store_jobs = self._build_store_jobs(scheduler_output) meta = OffloadingConnectorMetadata( load_jobs=self._current_batch_load_jobs, - store_jobs=self._build_store_jobs(scheduler_output), + store_jobs=partial_store_jobs | normal_store_jobs, jobs_to_flush=self._current_batch_jobs_to_flush, ) @@ -1344,12 +1552,12 @@ class OffloadingConnectorScheduler: if self._block_id_to_pending_jobs: # Sliding window blocks are tracked from store creation # and must be cleaned up unconditionally. - self._remove_pending_job(job_id, job_status.sliding_window_block_ids) + self._remove_pending_job(job_id, job_status.fenced_block_ids) # Non-sliding-window blocks are only tracked after # request_finished, so only clean up for finished requests. if req_status.req.is_finished(): self._remove_pending_job( - job_id, job_status.non_sliding_window_block_ids + job_id, job_status.deferred_fence_block_ids ) del self._jobs[job_id] @@ -1404,11 +1612,11 @@ class OffloadingConnectorScheduler: # Keep req_status alive: _build_store_jobs will process finished_req_ids # on the next step and handle cleanup after creating store jobs. - # Register non_sliding_window_block_ids so future block reuse triggers - # a flush via _block_id_to_pending_jobs. + # Register deferred fences so future block reuse triggers a flush via + # _block_id_to_pending_jobs. for job_id in req_status.transfer_jobs: job_status = self._jobs[job_id] - for bid in job_status.non_sliding_window_block_ids or (): + for bid in job_status.deferred_fence_block_ids or (): self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id) return False, None @@ -1451,6 +1659,7 @@ class OffloadingConnectorScheduler: for group_state in status.group_states: group_state.next_stored_chunk_idx = 0 status.transfer_jobs.clear() + status.partial_tail_boundary = None # Discard jobs and save job_counter to be able to discard worker responses self._stale_job_threshold = self._job_counter