[Bugfix][HiSparse][NIXL] Import full blocks without tail prefill on D (#57049)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
parent
41f9104fa6
commit
67e5b0acc9
@@ -325,7 +325,7 @@ def test_hisparse_reports_when_context_is_fully_resident():
|
||||
|
||||
|
||||
def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
|
||||
"""Keep indexer-only imports host-backed in the resident block table."""
|
||||
"""Restore the final host page when completing an indexer-only import."""
|
||||
manager = make_hisparse_kv_cache_manager(
|
||||
32,
|
||||
16,
|
||||
@@ -360,11 +360,15 @@ def test_hisparse_host_prefix_can_be_completed_by_indexer_offload():
|
||||
source, indexer, resident, hot = manager.get_blocks(resumed.request_id).blocks
|
||||
assert len(source) == len(indexer) == len(resident) == 4
|
||||
assert len(hot) == 2
|
||||
# The local prefix is adopted from shadow pages (GPU-resident), while the
|
||||
# externally imported page stays host-backed until its tail allocation.
|
||||
assert not any(block.is_null for block in resident[:2])
|
||||
assert resident[2].is_null
|
||||
assert not resident[2].is_null
|
||||
assert not resident[3].is_null
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
assert not coordinator.build_offload_command().page_transfers
|
||||
coordinator.finish_host_import(resumed.request_id, failed=False)
|
||||
transfers = coordinator.build_offload_command().page_transfers
|
||||
assert len(transfers) == 1 and transfers[0].restore
|
||||
assert transfers[0].resident_block_ids == (resident[2].block_id,)
|
||||
|
||||
|
||||
def test_hisparse_indexer_offload_is_capped_by_missing_host_prefix():
|
||||
@@ -465,7 +469,8 @@ def test_connector_completes_partial_prefix_without_importing_missing_host_kv(
|
||||
)
|
||||
allocated = manager.get_blocks(resumed.request_id).blocks
|
||||
assert len(allocated[0]) == host_blocks
|
||||
assert allocated[2][1].is_null == (external > 0)
|
||||
assert allocated[2][1].is_null == (host_blocks > 2)
|
||||
assert not allocated[2][-1].is_null
|
||||
manager.free(resumed)
|
||||
|
||||
# A surviving indexer offload must not make absent host KV look computed.
|
||||
@@ -490,6 +495,85 @@ def allocate_external_prefix(
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("num_tokens", [1, 15, 16, 17, 31, 32, 33, 127])
|
||||
def test_nixl_hisparse_full_block_import_keeps_a_writable_tail(num_tokens):
|
||||
"""Import every real token and restore the page used by last-token replay."""
|
||||
from tests.v1.kv_connector.unit.utils import create_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.pull_scheduler import (
|
||||
NixlPullConnectorScheduler,
|
||||
)
|
||||
|
||||
manager = make_hisparse_kv_cache_manager(32, 16, enable_caching=True)
|
||||
config = create_vllm_config(block_size=HISPARSE_BLOCK_SIZE)
|
||||
connector = NixlPullConnectorScheduler(config, "test", manager.kv_cache_config)
|
||||
request = make_request(
|
||||
"import", list(range(num_tokens)), HISPARSE_BLOCK_SIZE, sha256
|
||||
)
|
||||
request.kv_transfer_params = {"do_remote_prefill": True}
|
||||
count, is_async = connector.get_num_new_matched_tokens(request, 0)
|
||||
assert count == num_tokens and is_async
|
||||
assert allocate_external_prefix(manager, request, count) is not None
|
||||
source, _, resident, _ = manager.get_blocks(request.request_id).blocks
|
||||
assert len(source) == len(resident) == (num_tokens + 15) // 16
|
||||
assert all(block.is_null for block in resident[:-1])
|
||||
assert not resident[-1].is_null
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
assert not coordinator.build_offload_command().page_transfers
|
||||
|
||||
coordinator.finish_host_import(request.request_id, failed=False)
|
||||
transfers = coordinator.build_offload_command().page_transfers
|
||||
assert len(transfers) == 1
|
||||
restore = transfers[0]
|
||||
assert restore.restore and not restore.after_forward
|
||||
assert restore.host_block_id == source[-1].block_id
|
||||
assert restore.resident_block_ids == (resident[-1].block_id,)
|
||||
request.num_computed_tokens = count
|
||||
scheduler = SimpleNamespace(
|
||||
connector=connector,
|
||||
kv_cache_manager=manager,
|
||||
failed_recving_kv_req_ids=set(),
|
||||
finished_recving_kv_req_ids={request.request_id},
|
||||
)
|
||||
Scheduler._update_waiting_for_remote_kv(scheduler, request)
|
||||
assert request.num_tokens - request.num_computed_tokens == 1
|
||||
assert manager.allocate_slots(request, num_new_tokens=1) is not None
|
||||
assert coordinator.build_row_mirrors([(request.request_id, num_tokens - 1, 1)])
|
||||
# A pending restore must not expose uninitialized GPU copies to prefix hits.
|
||||
assert all(resident[-1] not in copies for copies in coordinator.copies.values())
|
||||
|
||||
coordinator.update_spills({restore.transfer_id: 2}, {restore.transfer_id: 1})
|
||||
assert resident[-1].ref_cnt == 2
|
||||
assert all(resident[-1] not in copies for copies in coordinator.copies.values())
|
||||
coordinator.update_spills({}, {restore.transfer_id: 1})
|
||||
assert resident[-1].ref_cnt == 1
|
||||
assert coordinator.request_states[request.request_id].valid_pages == set(
|
||||
range(num_tokens // HISPARSE_BLOCK_SIZE)
|
||||
)
|
||||
if num_tokens % HISPARSE_BLOCK_SIZE:
|
||||
assert source[-1].block_hash is None
|
||||
else:
|
||||
assert coordinator.copies[source[-1].block_hash] == (resident[-1],)
|
||||
|
||||
|
||||
def test_hisparse_aborted_tail_restore_retains_both_endpoints():
|
||||
"""An abort cannot recycle storage still used by a queued tail restore."""
|
||||
manager = make_hisparse_kv_cache_manager(16, 16)
|
||||
request = make_request("import", list(range(17)), HISPARSE_BLOCK_SIZE, sha256)
|
||||
assert allocate_external_prefix(manager, request, 17) is not None
|
||||
source, _, resident, _ = manager.get_blocks(request.request_id).blocks
|
||||
host_tail, gpu_tail = source[-1], resident[-1]
|
||||
coordinator = get_hisparse_coordinator(manager)
|
||||
coordinator.finish_host_import(request.request_id, failed=False)
|
||||
restore = coordinator.build_offload_command().page_transfers[0]
|
||||
manager.free(request)
|
||||
assert host_tail.ref_cnt == gpu_tail.ref_cnt == 1
|
||||
coordinator.update_spills({restore.transfer_id: 2}, {restore.transfer_id: 1})
|
||||
assert host_tail.ref_cnt == gpu_tail.ref_cnt == 1
|
||||
coordinator.update_spills({}, {restore.transfer_id: 1})
|
||||
assert host_tail.ref_cnt == gpu_tail.ref_cnt == 0
|
||||
assert not coordinator.has_pending_work()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("enable_caching", [False, True])
|
||||
def test_hisparse_keeps_resident_pages_until_hot_buffer_is_allocated(enable_caching):
|
||||
"""Later admissions must not remove a scheduled request's only readable tier."""
|
||||
@@ -669,7 +753,7 @@ def test_hisparse_host_cow_copy_is_drained_without_a_gpu_pool():
|
||||
def test_hisparse_inflight_host_import_reserves_remaining_gpu_pages():
|
||||
"""An in-flight host import must reserve its unwritten resident pages."""
|
||||
manager = make_hisparse_kv_cache_manager(
|
||||
11,
|
||||
12,
|
||||
16,
|
||||
transfer_device_cache=True,
|
||||
)
|
||||
@@ -1127,9 +1211,10 @@ def test_hisparse_external_import_uses_hard_gpu_footprint():
|
||||
assert len(source) == num_prompt_blocks
|
||||
assert len(indexer) == num_prompt_blocks
|
||||
assert len(resident) == num_prompt_blocks
|
||||
assert all(block.is_null for block in resident)
|
||||
assert all(block.is_null for block in resident[:-1])
|
||||
assert not resident[-1].is_null
|
||||
assert len(hot) == 2
|
||||
assert manager.block_pool.get_num_free_blocks() == 1
|
||||
assert manager.block_pool.get_num_free_blocks() == 0
|
||||
|
||||
|
||||
def test_hisparse_external_import_survives_capacity_retry():
|
||||
@@ -1149,7 +1234,8 @@ def test_hisparse_external_import_survives_capacity_retry():
|
||||
manager.free(first)
|
||||
assert allocate_external_prefix(manager, second, len(tokens)) is not None
|
||||
_, _, resident, hot = manager.get_blocks(second.request_id).blocks
|
||||
assert all(block.is_null for block in resident)
|
||||
assert all(block.is_null for block in resident[:-1])
|
||||
assert not resident[-1].is_null
|
||||
assert len(hot) == 2
|
||||
|
||||
|
||||
|
||||
@@ -531,6 +531,71 @@ def test_hisparse_eager_mirror_records_transfer_without_page_copy():
|
||||
worker._enqueue_transfers.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_host_writer", [False, True])
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
def test_hisparse_tail_restore_preserves_imported_rows(
|
||||
monkeypatch, request, is_host_writer, device
|
||||
):
|
||||
"""Restore externally pinned host rows on every rank under sync checking."""
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA is required for the device copy check")
|
||||
worker = _make_hisparse_worker()
|
||||
worker.is_host_writer = is_host_writer
|
||||
worker.kernel_block_size = 4
|
||||
worker.host_caches = tuple(
|
||||
torch.arange(24, dtype=torch.float32).reshape(12, 2) + layer * 100
|
||||
for layer in range(2)
|
||||
)
|
||||
if device == "cuda":
|
||||
# Match production's cudaHostRegister pool, which torch.is_pinned()
|
||||
# does not recognize, rather than torch's caching pinned allocator.
|
||||
host_pool, registered = hisparse_runtime_module.allocate_pinned_host_pool(
|
||||
sum(cache.nbytes for cache in worker.host_caches)
|
||||
)
|
||||
request.addfinalizer(
|
||||
lambda: hisparse_runtime_module.release_pinned_state([], [registered])
|
||||
)
|
||||
host_caches = host_pool.view(torch.float32).view(2, 12, 2).unbind()
|
||||
for destination, source in zip(host_caches, worker.host_caches):
|
||||
destination.copy_(source)
|
||||
worker.host_caches = host_caches
|
||||
worker.resident_caches = tuple(
|
||||
torch.full((4, 4, 2), -1.0, device=device) for _ in range(2)
|
||||
)
|
||||
worker.cache_handles = [
|
||||
SimpleNamespace(
|
||||
runtime=SimpleNamespace(resident_source_index=i, eager_host_mirror=True)
|
||||
)
|
||||
for i in range(2)
|
||||
]
|
||||
worker._record_transfer_completion = MagicMock()
|
||||
stream = torch.cuda.current_stream() if device == "cuda" else MagicMock()
|
||||
monkeypatch.setattr(hisparse_worker_module, "current_stream", lambda: stream)
|
||||
transfer = SparseKVPageTransfer(7, 2, (1, 3), False, restore=True)
|
||||
|
||||
restore_pages = worker._restore_pages
|
||||
if device == "cuda":
|
||||
import vllm.utils.gpu_sync_debug as gpu_sync_debug
|
||||
|
||||
monkeypatch.setattr(gpu_sync_debug, "_SYNC_CHECK_MODE", "error")
|
||||
monkeypatch.setattr(gpu_sync_debug, "_sync_check_enabled", False)
|
||||
gpu_sync_debug.enable_gpu_sync_check()
|
||||
restore_pages = gpu_sync_debug.with_gpu_sync_check(restore_pages)
|
||||
restore_pages([transfer])
|
||||
|
||||
# Appending into unused capacity must preserve every imported prompt row.
|
||||
for cache, host, block in zip(
|
||||
worker.resident_caches, worker.host_caches, transfer.resident_block_ids
|
||||
):
|
||||
assert torch.equal(cache[block].cpu(), host[8:12])
|
||||
cache[block, 2:] = 999
|
||||
assert torch.equal(cache[block, :2].cpu(), host[8:10])
|
||||
assert torch.all(cache[0] == -1)
|
||||
worker._record_transfer_completion.assert_called_once_with(
|
||||
[transfer], stream=stream
|
||||
)
|
||||
|
||||
|
||||
def test_hisparse_lazy_mirror_copies_transfer_pages():
|
||||
worker = _make_hisparse_worker()
|
||||
worker.cache_handles = [
|
||||
|
||||
@@ -22,6 +22,7 @@ from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
|
||||
@@ -414,11 +415,18 @@ class HiSparseConnectorWorker:
|
||||
transfers = (
|
||||
metadata.command.page_transfers if metadata.command is not None else []
|
||||
)
|
||||
self._restore_pages([transfer for transfer in transfers if transfer.restore])
|
||||
self._post_forward_transfers = [
|
||||
transfer for transfer in transfers if transfer.after_forward
|
||||
transfer
|
||||
for transfer in transfers
|
||||
if not transfer.restore and transfer.after_forward
|
||||
]
|
||||
self._submit_transfers(
|
||||
[transfer for transfer in transfers if not transfer.after_forward]
|
||||
[
|
||||
transfer
|
||||
for transfer in transfers
|
||||
if not transfer.restore and not transfer.after_forward
|
||||
]
|
||||
)
|
||||
self._pending_invalid_block_ids.extend(metadata.source_block_ids)
|
||||
if request_state_indices is not None:
|
||||
@@ -708,11 +716,17 @@ class HiSparseConnectorWorker:
|
||||
self._submitted_mirror_layers.update(pending_layers)
|
||||
|
||||
def _record_transfer_completion(
|
||||
self, transfers: list[SparseKVPageTransfer]
|
||||
self,
|
||||
transfers: list[SparseKVPageTransfer],
|
||||
*,
|
||||
stream: torch.Stream | None = None,
|
||||
) -> None:
|
||||
if not transfers or not self.is_host_writer:
|
||||
if not transfers:
|
||||
return
|
||||
stream = self.dma_stream
|
||||
if stream is None:
|
||||
if not self.is_host_writer:
|
||||
return
|
||||
stream = self.dma_stream
|
||||
assert stream is not None
|
||||
completion_event = torch.Event()
|
||||
completion_event.record(stream)
|
||||
@@ -720,6 +734,30 @@ class HiSparseConnectorWorker:
|
||||
self._pending_transfer_events.append((completion_event, transfer_ids))
|
||||
self._enqueued_transfer_ids.extend(transfer_ids)
|
||||
|
||||
def _restore_pages(self, transfers: list[SparseKVPageTransfer]) -> None:
|
||||
"""Restore imported tails on every rank before its forward or graph replay."""
|
||||
if not transfers:
|
||||
return
|
||||
for layer_index, cache in enumerate(self.cache_handles):
|
||||
source_index = cache.runtime.resident_source_index
|
||||
source = self.host_caches[layer_index]
|
||||
destination = self.resident_caches[layer_index]
|
||||
for transfer in transfers:
|
||||
host_start = transfer.host_block_id * self.kernel_block_size
|
||||
resident_block = transfer.resident_block_ids[source_index]
|
||||
# Each import pays one H2D page copy and enqueue per layer/rank
|
||||
# instead of tail prefill on D. Restore the prompt rows before
|
||||
# decode appends to this page; unused rows do not extend the
|
||||
# logical sequence.
|
||||
# cudaHostRegister pins this pool outside PyTorch's allocator,
|
||||
# so the sync checker cannot recognize it via is_pinned().
|
||||
with gpu_sync_allowed():
|
||||
destination[resident_block].copy_(
|
||||
source[host_start : host_start + self.kernel_block_size],
|
||||
non_blocking=True,
|
||||
)
|
||||
self._record_transfer_completion(transfers, stream=current_stream())
|
||||
|
||||
def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
|
||||
if self.cache_handles[0].runtime.eager_host_mirror:
|
||||
self._record_transfer_completion(transfers)
|
||||
@@ -733,15 +771,12 @@ class HiSparseConnectorWorker:
|
||||
descriptor_count = len(transfers) * num_layers
|
||||
descriptors = self._acquire_dma_descriptors(descriptor_count)
|
||||
destination_rows = np.fromiter(
|
||||
(
|
||||
transfer.destination_block_id * self.kernel_block_size
|
||||
for transfer in transfers
|
||||
),
|
||||
(transfer.host_block_id * self.kernel_block_size for transfer in transfers),
|
||||
dtype=np.int64,
|
||||
count=len(transfers),
|
||||
)
|
||||
source_blocks_by_transfer = np.asarray(
|
||||
[transfer.source_block_ids for transfer in transfers], dtype=np.int64
|
||||
[transfer.resident_block_ids for transfer in transfers], dtype=np.int64
|
||||
)
|
||||
if source_blocks_by_transfer.ndim != 2:
|
||||
raise RuntimeError(
|
||||
|
||||
@@ -3278,7 +3278,8 @@ class NixlBaseConnectorWorker:
|
||||
"""Pair an uncached decode suffix with the same prefill regions."""
|
||||
assert len(decode_block_ids) == len(prefill_block_ids)
|
||||
if not any(decode_block_ids):
|
||||
return [], prefill_block_ids
|
||||
empty_regions: list[list[int]] = [[] for _ in decode_block_ids]
|
||||
return empty_regions, empty_regions.copy()
|
||||
|
||||
if num_computed_blocks is not None:
|
||||
assert num_remote_blocks is not None
|
||||
|
||||
@@ -168,8 +168,14 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
local_block_ids = meta.local_physical_block_ids
|
||||
remote_region_groups = self.dst_region_group_ids[engine_id]
|
||||
local_region_groups = self.region_group_ids or remote_region_groups
|
||||
groups_differ = local_region_groups != remote_region_groups
|
||||
if groups_differ:
|
||||
if not local_block_ids:
|
||||
# Region expansion cannot index empty groups. Pass empty specs to
|
||||
# _read_blocks so its existing cache-hit notification path runs.
|
||||
read_specs = [
|
||||
ReadSpec(remote_rank=rank, local_block_ids=[], remote_block_ids=[])
|
||||
for rank in plan.all_source_ranks
|
||||
]
|
||||
elif local_region_groups != remote_region_groups:
|
||||
if not self.use_mla or self._has_mamba:
|
||||
raise NotImplementedError(
|
||||
"Different NIXL cache-group layouts are only supported for "
|
||||
|
||||
@@ -2498,12 +2498,14 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
|
||||
del num_tokens_main_model
|
||||
assert not new_computed_blocks
|
||||
if total_computed_tokens > num_local_computed_tokens:
|
||||
if total_computed_tokens % self.block_size != 0:
|
||||
raise ValueError(
|
||||
"A host-only HiSparse import must end on a cache-block boundary."
|
||||
)
|
||||
imported_pages = total_computed_tokens // self.block_size
|
||||
return max(cdiv(num_tokens, self.block_size) - imported_pages, 0)
|
||||
# Keep the last imported page writable, including the last-token
|
||||
# replay on a full prompt hit. Only earlier pages stay host-only.
|
||||
imported_pages = cdiv(total_computed_tokens, self.block_size)
|
||||
existing = len(self.req_to_blocks.get(request_id, ()))
|
||||
needs_tail = self.get_resident_page(request_id, imported_pages - 1) is None
|
||||
return int(needs_tail) + max(
|
||||
cdiv(num_tokens, self.block_size) - max(imported_pages, existing), 0
|
||||
)
|
||||
existing = len(self.req_to_blocks.get(request_id, ()))
|
||||
host_pages = cdiv(num_local_computed_tokens, self.block_size)
|
||||
required = cdiv(num_tokens, self.block_size)
|
||||
@@ -2518,13 +2520,15 @@ class HiSparseResidentManager(_HiSparseAuxiliaryManager):
|
||||
num_local_computed_tokens: int,
|
||||
num_external_computed_tokens: int,
|
||||
) -> None:
|
||||
"""Represent imported host history with null resident pages."""
|
||||
"""Reserve a writable final page; earlier imported pages stay on host."""
|
||||
assert num_external_computed_tokens > 0
|
||||
num_tokens = num_local_computed_tokens + num_external_computed_tokens
|
||||
blocks = self.req_to_blocks[request_id]
|
||||
tail_page = cdiv(num_tokens, self.block_size) - 1
|
||||
tail_page = (num_tokens - 1) // self.block_size
|
||||
if len(blocks) <= tail_page:
|
||||
blocks.extend([self._null_block] * (tail_page + 1 - len(blocks)))
|
||||
if blocks[tail_page].is_null:
|
||||
blocks[tail_page] = self.block_pool.get_new_blocks(1)[0]
|
||||
|
||||
def add_local_computed_blocks(
|
||||
self,
|
||||
|
||||
@@ -73,6 +73,7 @@ class _PendingSpill:
|
||||
request_state: _HiSparseRequestState
|
||||
host_block: KVCacheBlock
|
||||
resident_blocks: tuple[KVCacheBlock, ...]
|
||||
restore: bool = False
|
||||
expected_worker_completions: int = 0
|
||||
worker_completions: int = 0
|
||||
enqueue_applied: bool = False
|
||||
@@ -271,25 +272,29 @@ class HiSparseCoordinator:
|
||||
num_computed_tokens // self.host_manager.block_size, len(host_blocks)
|
||||
)
|
||||
for host_idx in range(state.copies_recorded_blocks, num_host_blocks):
|
||||
host_block = host_blocks[host_idx]
|
||||
if host_block.is_null or host_block.block_hash is None:
|
||||
if host_idx in state.pending_pages:
|
||||
continue
|
||||
blocks: list[KVCacheBlock] = []
|
||||
for manager in self.resident_managers:
|
||||
block = manager.get_resident_page(request_id, host_idx)
|
||||
if block is None:
|
||||
break
|
||||
blocks.append(block)
|
||||
if len(blocks) != len(self.resident_managers):
|
||||
continue
|
||||
self._drop_copy(host_block.block_hash)
|
||||
self.copies[host_block.block_hash] = tuple(blocks)
|
||||
for block in blocks:
|
||||
self._copy_by_block[block.block_id] = host_block.block_hash
|
||||
self._record_copy(request_id, host_idx, host_blocks[host_idx])
|
||||
state.copies_recorded_blocks = max(
|
||||
state.copies_recorded_blocks, num_host_blocks
|
||||
)
|
||||
|
||||
def _record_copy(
|
||||
self, request_id: str, page_idx: int, host_block: KVCacheBlock
|
||||
) -> None:
|
||||
if host_block.is_null or host_block.block_hash is None:
|
||||
return
|
||||
blocks: list[KVCacheBlock] = []
|
||||
for manager in self.resident_managers:
|
||||
block = manager.get_resident_page(request_id, page_idx)
|
||||
if block is None:
|
||||
return
|
||||
blocks.append(block)
|
||||
self._drop_copy(host_block.block_hash)
|
||||
self.copies[host_block.block_hash] = tuple(blocks)
|
||||
for block in blocks:
|
||||
self._copy_by_block[block.block_id] = host_block.block_hash
|
||||
|
||||
def _drop_copy(self, host_hash: BlockHashWithGroupId) -> None:
|
||||
blocks = self.copies.pop(host_hash, None)
|
||||
if blocks is None:
|
||||
@@ -406,9 +411,14 @@ class HiSparseCoordinator:
|
||||
assert self.host_manager is not None
|
||||
host_block_size = self.host_manager.block_size
|
||||
num_pages = num_computed_tokens // host_block_size
|
||||
importing_pages = cdiv(
|
||||
self._pending_imports.get(request_id, 0), host_block_size
|
||||
)
|
||||
state = self._get_request_state(request_id)
|
||||
budget = max(self.max_spill_pages - len(self.spills_to_send), 0)
|
||||
for page_idx in range(num_pages):
|
||||
if page_idx < importing_pages:
|
||||
continue
|
||||
if budget == 0:
|
||||
break
|
||||
if page_idx in state.pending_pages:
|
||||
@@ -420,7 +430,7 @@ class HiSparseCoordinator:
|
||||
for manager in self.resident_managers
|
||||
):
|
||||
continue
|
||||
if self._plan_spill(request_id, page_idx, after_forward=True):
|
||||
if self._plan_page_transfer(request_id, page_idx, after_forward=True):
|
||||
budget -= 1
|
||||
|
||||
def publish_when_ready(
|
||||
@@ -491,17 +501,27 @@ class HiSparseCoordinator:
|
||||
state = self._get_request_state(request_id)
|
||||
state.valid_pages = set(range(num_pages))
|
||||
state.ready_prefix_pages = num_pages
|
||||
if num_computed_tokens:
|
||||
self._plan_page_transfer(
|
||||
request_id,
|
||||
(num_computed_tokens - 1) // block_size,
|
||||
after_forward=False,
|
||||
restore=True,
|
||||
)
|
||||
self._publish_host_blocks_if_ready(request_id)
|
||||
|
||||
def _plan_spill(
|
||||
def _plan_page_transfer(
|
||||
self,
|
||||
request_id: str,
|
||||
page_idx: int,
|
||||
*,
|
||||
after_forward: bool,
|
||||
restore: bool = False,
|
||||
) -> bool:
|
||||
assert self.host_manager is not None
|
||||
state = self._get_request_state(request_id)
|
||||
if page_idx in state.pending_pages:
|
||||
return False
|
||||
host_blocks = self.host_manager.req_to_blocks.get(request_id)
|
||||
if host_blocks is None or page_idx >= len(host_blocks):
|
||||
return False
|
||||
@@ -523,9 +543,10 @@ class HiSparseCoordinator:
|
||||
self.next_spill_id += 1
|
||||
plan = SparseKVPageTransfer(
|
||||
transfer_id=spill_id,
|
||||
destination_block_id=host_block.block_id,
|
||||
source_block_ids=tuple(block.block_id for block in blocks),
|
||||
host_block_id=host_block.block_id,
|
||||
resident_block_ids=tuple(block.block_id for block in blocks),
|
||||
after_forward=after_forward,
|
||||
restore=restore,
|
||||
)
|
||||
self.pending_spills[spill_id] = _PendingSpill(
|
||||
transfer_id=spill_id,
|
||||
@@ -533,6 +554,7 @@ class HiSparseCoordinator:
|
||||
request_state=state,
|
||||
host_block=host_block,
|
||||
resident_blocks=tuple(blocks),
|
||||
restore=restore,
|
||||
)
|
||||
state.pending_pages[page_idx] = spill_id
|
||||
self.spills_to_send.append(plan)
|
||||
@@ -666,7 +688,7 @@ class HiSparseCoordinator:
|
||||
self._complete_host_writes()
|
||||
|
||||
def _apply_enqueued_spills(self) -> None:
|
||||
"""Drop the DMA pins once the worker has enqueued a spill."""
|
||||
"""Drop the GPU pins once every worker has completed the transfer."""
|
||||
for pending in self.pending_spills.values():
|
||||
if (
|
||||
pending.enqueue_applied
|
||||
@@ -692,8 +714,12 @@ class HiSparseCoordinator:
|
||||
request_id, page_idx = pending.page
|
||||
pending.request_state.pending_pages.pop(page_idx, None)
|
||||
if self.request_states.get(request_id) is pending.request_state:
|
||||
pending.request_state.valid_pages.add(page_idx)
|
||||
self._page_became_clean(request_id, pending.request_state, page_idx)
|
||||
if not pending.restore:
|
||||
pending.request_state.valid_pages.add(page_idx)
|
||||
if page_idx in pending.request_state.valid_pages:
|
||||
self._page_became_clean(request_id, pending.request_state, page_idx)
|
||||
if pending.restore:
|
||||
self._record_copy(request_id, page_idx, pending.host_block)
|
||||
completed_request_ids.add(request_id)
|
||||
assert self.host_manager is not None
|
||||
self.host_manager.block_pool.free_blocks([pending.host_block])
|
||||
|
||||
@@ -9,9 +9,10 @@ class SparseKVPageTransfer:
|
||||
"""Copy one logical KV page between cache-manager and worker-owned tiers."""
|
||||
|
||||
transfer_id: int
|
||||
destination_block_id: int
|
||||
source_block_ids: tuple[int, ...]
|
||||
host_block_id: int
|
||||
resident_block_ids: tuple[int, ...]
|
||||
after_forward: bool
|
||||
restore: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
||||
Reference in New Issue
Block a user