[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
9bd074a996
commit
d5f356dae0
@@ -36,7 +36,11 @@ from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.hisparse.layout import get_hisparse_steady_state_concurrency
|
||||
from vllm.v1.hisparse.types import SparseKVOffloadCommand, SparseKVRowMirror
|
||||
from vllm.v1.hisparse.types import (
|
||||
SparseKVOffloadCommand,
|
||||
SparseKVResidencyUpdate,
|
||||
SparseKVRowMirror,
|
||||
)
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -56,6 +60,7 @@ class HiSparseConnectorMetadata(KVConnectorMetadata):
|
||||
source_block_ids: tuple[int, ...]
|
||||
row_mirrors: dict[str, tuple[SparseKVRowMirror, ...]]
|
||||
all_context_pages_resident: bool
|
||||
residency_updates: dict[str, SparseKVResidencyUpdate]
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -143,7 +148,7 @@ class HiSparseConnectorScheduler:
|
||||
scheduler_output.num_scheduled_tokens.items()
|
||||
)
|
||||
)
|
||||
# Runs first: it plans this step's transfers and table updates.
|
||||
# Runs first: it plans this step's transfers and residency changes.
|
||||
self.coordinator.advance_scheduled(
|
||||
(
|
||||
request_id,
|
||||
@@ -151,9 +156,6 @@ class HiSparseConnectorScheduler:
|
||||
)
|
||||
for request_id, start, count in scheduled_requests
|
||||
)
|
||||
scheduler_output.block_table_updates = (
|
||||
self.coordinator.take_block_table_updates() or None
|
||||
)
|
||||
command = self.coordinator.build_offload_command()
|
||||
host_block_copies = self.coordinator.take_host_block_copies()
|
||||
source_group_id = self.coordinator.host_group_id
|
||||
@@ -192,6 +194,9 @@ class HiSparseConnectorScheduler:
|
||||
tuple(source_block_ids),
|
||||
row_mirrors,
|
||||
self.coordinator.all_context_pages_resident(scheduled_requests),
|
||||
self.coordinator.take_residency_updates(
|
||||
scheduler_output.num_scheduled_tokens
|
||||
),
|
||||
)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
|
||||
@@ -26,7 +26,11 @@ from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
|
||||
from vllm.v1.hisparse.runtime import HiSparseCacheHandle, release_pinned_state
|
||||
from vllm.v1.hisparse.runtime import (
|
||||
HiSparseCacheHandle,
|
||||
release_pinned_state,
|
||||
update_hisparse_residency,
|
||||
)
|
||||
from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
HiSparseHotSpec,
|
||||
@@ -334,6 +338,10 @@ class HiSparseConnectorWorker:
|
||||
"HiSparse request-state mapping does not match max_num_seqs."
|
||||
)
|
||||
self.hot_backing = hot_backing
|
||||
self.residency: torch.Tensor | None = next(
|
||||
(cache.residency for cache in cache_handles if cache.residency is not None),
|
||||
None,
|
||||
)
|
||||
self._pending_invalid_block_ids: list[int] = []
|
||||
# Destination block ids of host copies this worker has run.
|
||||
self._completed_host_copy_dst_ids: list[int] = []
|
||||
@@ -437,6 +445,15 @@ class HiSparseConnectorWorker:
|
||||
self._pending_invalid_block_ids.extend(metadata.source_block_ids)
|
||||
if request_state_indices is not None:
|
||||
self.set_request_state_indices(request_state_indices)
|
||||
if metadata.residency_updates:
|
||||
assert self.residency is not None
|
||||
assert request_state_indices is not None and request_ids is not None
|
||||
update_hisparse_residency(
|
||||
self.residency,
|
||||
metadata.residency_updates,
|
||||
request_ids,
|
||||
request_state_indices,
|
||||
)
|
||||
|
||||
def _clear_forward_mirror_state(self) -> None:
|
||||
self._per_layer_mirrored.clear()
|
||||
|
||||
Reference in New Issue
Block a user