[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-09 21:25:44 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9bd074a996
commit d5f356dae0
15 changed files with 429 additions and 128 deletions
@@ -36,7 +36,11 @@ from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
from vllm.v1.hisparse.layout import get_hisparse_steady_state_concurrency
from vllm.v1.hisparse.types import SparseKVOffloadCommand, SparseKVRowMirror
from vllm.v1.hisparse.types import (
SparseKVOffloadCommand,
SparseKVResidencyUpdate,
SparseKVRowMirror,
)
from vllm.v1.outputs import KVConnectorOutput
if TYPE_CHECKING:
@@ -56,6 +60,7 @@ class HiSparseConnectorMetadata(KVConnectorMetadata):
source_block_ids: tuple[int, ...]
row_mirrors: dict[str, tuple[SparseKVRowMirror, ...]]
all_context_pages_resident: bool
residency_updates: dict[str, SparseKVResidencyUpdate]
@dataclass
@@ -143,7 +148,7 @@ class HiSparseConnectorScheduler:
scheduler_output.num_scheduled_tokens.items()
)
)
# Runs first: it plans this step's transfers and table updates.
# Runs first: it plans this step's transfers and residency changes.
self.coordinator.advance_scheduled(
(
request_id,
@@ -151,9 +156,6 @@ class HiSparseConnectorScheduler:
)
for request_id, start, count in scheduled_requests
)
scheduler_output.block_table_updates = (
self.coordinator.take_block_table_updates() or None
)
command = self.coordinator.build_offload_command()
host_block_copies = self.coordinator.take_host_block_copies()
source_group_id = self.coordinator.host_group_id
@@ -192,6 +194,9 @@ class HiSparseConnectorScheduler:
tuple(source_block_ids),
row_mirrors,
self.coordinator.all_context_pages_resident(scheduled_requests),
self.coordinator.take_residency_updates(
scheduler_output.num_scheduled_tokens
),
)
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
@@ -26,7 +26,11 @@ from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.torch_utils import current_stream
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
from vllm.v1.hisparse.runtime import HiSparseCacheHandle, release_pinned_state
from vllm.v1.hisparse.runtime import (
HiSparseCacheHandle,
release_pinned_state,
update_hisparse_residency,
)
from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror
from vllm.v1.kv_cache_interface import (
HiSparseHotSpec,
@@ -334,6 +338,10 @@ class HiSparseConnectorWorker:
"HiSparse request-state mapping does not match max_num_seqs."
)
self.hot_backing = hot_backing
self.residency: torch.Tensor | None = next(
(cache.residency for cache in cache_handles if cache.residency is not None),
None,
)
self._pending_invalid_block_ids: list[int] = []
# Destination block ids of host copies this worker has run.
self._completed_host_copy_dst_ids: list[int] = []
@@ -437,6 +445,15 @@ class HiSparseConnectorWorker:
self._pending_invalid_block_ids.extend(metadata.source_block_ids)
if request_state_indices is not None:
self.set_request_state_indices(request_state_indices)
if metadata.residency_updates:
assert self.residency is not None
assert request_state_indices is not None and request_ids is not None
update_hisparse_residency(
self.residency,
metadata.residency_updates,
request_ids,
request_state_indices,
)
def _clear_forward_mirror_state(self) -> None:
self._per_layer_mirrored.clear()