[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com> Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
Lucas Wilkinson
Lucas Wilkinson
Nicolò Lucchesi
OpenAI Codex
Claude Opus 5
Lucas Wilkinson
parent
120ec4ebd2
commit
d43bb2f37f
@@ -197,6 +197,12 @@ KVConnectorFactory.register_connector(
|
||||
"MultiConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"HiSparseConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector",
|
||||
"HiSparseConnector",
|
||||
)
|
||||
|
||||
KVConnectorFactory.register_connector(
|
||||
"MoRIIOConnector",
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector",
|
||||
|
||||
@@ -60,6 +60,7 @@ class KVOutputAggregator:
|
||||
# [req_id -> n_remaining_workers]
|
||||
self._recv_remaining_count = dict[str, int]()
|
||||
self._send_remaining_count = dict[str, int]()
|
||||
self._failed_recving_pending = set[str]()
|
||||
self._expected_finished_count = expected_finished_count
|
||||
|
||||
@classmethod
|
||||
@@ -156,6 +157,10 @@ class KVOutputAggregator:
|
||||
combined_kv_cache_events.increment_workers(1)
|
||||
|
||||
invalid_block_ids |= kv_output.invalid_block_ids
|
||||
self._failed_recving_pending |= kv_output.failed_recving
|
||||
|
||||
failed_recving = self._failed_recving_pending & finished_recving
|
||||
self._failed_recving_pending -= failed_recving
|
||||
|
||||
# select output of the worker specified by output_rank
|
||||
output = outputs[output_rank]
|
||||
@@ -168,6 +173,7 @@ class KVOutputAggregator:
|
||||
kv_cache_events=combined_kv_cache_events or None,
|
||||
kv_connector_worker_meta=aggregated_kv_connector_worker_meta or None,
|
||||
invalid_block_ids=invalid_block_ids,
|
||||
failed_recving=failed_recving,
|
||||
expected_finished_count=self._expected_finished_count,
|
||||
)
|
||||
|
||||
|
||||
@@ -34,8 +34,8 @@ The class provides the following primitives:
|
||||
save_kv_layer() - starts saving KV for layer i (maybe async)
|
||||
wait_for_save() - blocks until all saves are done
|
||||
|
||||
get_finished() - called with ids of finished requests, returns
|
||||
ids of requests that have completed async sending/recving.
|
||||
get_transfer_results() - returns async send/receive completions and
|
||||
receive failures in one snapshot.
|
||||
build_connector_worker_meta() - builds metadata to be sent
|
||||
back to the scheduler-side connector
|
||||
"""
|
||||
@@ -43,6 +43,7 @@ The class provides the following primitives:
|
||||
import enum
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
import torch
|
||||
@@ -63,7 +64,7 @@ if TYPE_CHECKING:
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
@@ -82,6 +83,19 @@ CopyBlocksOp = Callable[
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class KVConnectorTransferResults:
|
||||
"""Asynchronous transfer completions from one worker snapshot.
|
||||
|
||||
Failed receives also appear in ``finished_recving`` so the scheduler can
|
||||
release the request from its transfer wait state.
|
||||
"""
|
||||
|
||||
finished_sending: set[str] = field(default_factory=set)
|
||||
finished_recving: set[str] = field(default_factory=set)
|
||||
failed_recving: set[str] = field(default_factory=set)
|
||||
|
||||
|
||||
class SupportsHMA(ABC):
|
||||
"""
|
||||
The class that indicates the corresponding connector supports hybrid memory
|
||||
@@ -211,6 +225,7 @@ class KVConnectorBase_V1(ABC):
|
||||
else:
|
||||
raise ValueError("kv_transfer_config must be set for KVConnectorBase_V1")
|
||||
self._kv_cache_config = kv_cache_config
|
||||
self._kv_cache_manager: KVCacheManager | None = None
|
||||
self._role = role
|
||||
|
||||
@property
|
||||
@@ -278,6 +293,15 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return
|
||||
|
||||
# TODO(NickLucche): group model-runner lifecycle hooks in the interface.
|
||||
def finish_forward(self) -> None:
|
||||
"""Notify the connector that the model no longer reads this step's KV."""
|
||||
return
|
||||
|
||||
def reset_capture_state(self) -> None:
|
||||
"""Reset worker state mutated while capturing CUDA graphs."""
|
||||
return
|
||||
|
||||
def handle_preemptions(self, kv_connector_metadata: KVConnectorMetadata):
|
||||
"""
|
||||
Handle preempted requests or evicted blocks BEFORE they are overwritten.
|
||||
@@ -368,6 +392,16 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return None, None
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
"""Return completed sends, receives, and receive failures together."""
|
||||
finished_sending, finished_recving = self.get_finished(finished_req_ids)
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=set(finished_sending or ()),
|
||||
finished_recving=set(finished_recving or ()),
|
||||
)
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""
|
||||
Get the set of block IDs that failed to load.
|
||||
@@ -380,9 +414,9 @@ class KVConnectorBase_V1(ABC):
|
||||
- Applies to both sync- and async-loading requests.
|
||||
- Async loading: failed blocks may be reported in any forward pass
|
||||
up to and including the pass where the request ID is returned by
|
||||
`get_finished()`. Even if failures occur, the request must still
|
||||
be reported via `get_finished()`, and the failed block IDs must
|
||||
appear here no later than that same pass.
|
||||
`get_transfer_results()`. Even if failures occur, the request
|
||||
must still be reported as finished receiving, and the failed
|
||||
block IDs must appear here no later than that same pass.
|
||||
- Sync loading: failed blocks should be reported in the forward
|
||||
pass in which they are detected.
|
||||
"""
|
||||
@@ -436,6 +470,11 @@ class KVConnectorBase_V1(ABC):
|
||||
# Scheduler-side methods
|
||||
# ==============================
|
||||
|
||||
def bind_kv_cache_manager(self, kv_cache_manager: "KVCacheManager") -> None:
|
||||
"""Bind the scheduler's cache manager after it has been constructed."""
|
||||
self._kv_cache_manager = kv_cache_manager
|
||||
self.bind_gpu_block_pool(kv_cache_manager.block_pool)
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None:
|
||||
"""
|
||||
Bind the GPU block pool to the connector for per-GPU block status tracking.
|
||||
@@ -583,6 +622,10 @@ class KVConnectorBase_V1(ABC):
|
||||
"""
|
||||
return ()
|
||||
|
||||
def has_pending_block_frees(self) -> bool:
|
||||
"""Whether pending transfers can release blocks instead of preemption."""
|
||||
return False
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
"""Return True if the connector has push-mode work that requires
|
||||
the engine main loop to keep stepping (e.g. a P-side request whose
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,329 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Connector boundary for HiSparse scheduler and worker state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.worker import (
|
||||
HiSparseConnectorWorker,
|
||||
)
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.hisparse.coordinator import get_hisparse_coordinator
|
||||
from vllm.v1.hisparse.types import SparseKVOffloadCommand, SparseKVRowMirror
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.hisparse.coordinator import HiSparseCoordinator
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
|
||||
@dataclass
|
||||
class HiSparseConnectorMetadata(KVConnectorMetadata):
|
||||
command: SparseKVOffloadCommand | None
|
||||
host_block_copies: tuple[KVCacheBlockCopy, ...]
|
||||
source_block_ids: tuple[int, ...]
|
||||
row_mirrors: dict[str, tuple[SparseKVRowMirror, ...]]
|
||||
all_context_pages_resident: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class HiSparseConnectorWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
enqueued_transfer_counts: dict[int, int]
|
||||
completed_transfer_counts: dict[int, int]
|
||||
completed_host_copy_dst_ids: tuple[int, ...] = ()
|
||||
|
||||
def aggregate(self, other: KVConnectorWorkerMetadata) -> KVConnectorWorkerMetadata:
|
||||
assert isinstance(other, HiSparseConnectorWorkerMetadata)
|
||||
|
||||
def add_counts(first: dict[int, int], second: dict[int, int]) -> dict[int, int]:
|
||||
combined = first.copy()
|
||||
for transfer_id, count in second.items():
|
||||
combined[transfer_id] = combined.get(transfer_id, 0) + count
|
||||
return combined
|
||||
|
||||
return HiSparseConnectorWorkerMetadata(
|
||||
enqueued_transfer_counts=add_counts(
|
||||
self.enqueued_transfer_counts, other.enqueued_transfer_counts
|
||||
),
|
||||
completed_transfer_counts=add_counts(
|
||||
self.completed_transfer_counts, other.completed_transfer_counts
|
||||
),
|
||||
# Every worker runs the same copies, so a union is the ack.
|
||||
completed_host_copy_dst_ids=tuple(
|
||||
{*self.completed_host_copy_dst_ids, *other.completed_host_copy_dst_ids}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class HiSparseConnectorScheduler:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
async_speculative: bool,
|
||||
draft_kv_lookahead: int = 0,
|
||||
) -> None:
|
||||
self.coordinator: HiSparseCoordinator | None = None
|
||||
self.async_speculative = async_speculative
|
||||
self.draft_kv_lookahead = draft_kv_lookahead
|
||||
self.requests: dict[str, Request] = {}
|
||||
|
||||
def bind_coordinator(self, coordinator: HiSparseCoordinator) -> None:
|
||||
assert self.coordinator is None
|
||||
self.coordinator = coordinator
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> HiSparseConnectorMetadata:
|
||||
assert self.coordinator is not None
|
||||
# HiSparse rebinds worker state every step, so the load must start
|
||||
# before the forward rather than being deferred to post-forward.
|
||||
scheduler_output.has_sync_kv_loads = True
|
||||
scheduler_output.block_table_updates = (
|
||||
self.coordinator.take_block_table_updates() or None
|
||||
)
|
||||
command = self.coordinator.build_offload_command()
|
||||
host_block_copies = self.coordinator.take_host_block_copies()
|
||||
source_group_id = self.coordinator.host_group_id
|
||||
assert source_group_id is not None
|
||||
source_block_ids = [
|
||||
block_id
|
||||
for request in scheduler_output.scheduled_new_reqs
|
||||
for block_id in request.block_ids[source_group_id]
|
||||
]
|
||||
for new_block_ids in scheduler_output.scheduled_cached_reqs.new_block_ids:
|
||||
if new_block_ids is not None:
|
||||
source_block_ids.extend(new_block_ids[source_group_id])
|
||||
num_computed_tokens = {
|
||||
request.req_id: request.num_computed_tokens
|
||||
for request in scheduler_output.scheduled_new_reqs
|
||||
}
|
||||
num_computed_tokens.update(
|
||||
zip(
|
||||
scheduler_output.scheduled_cached_reqs.req_ids,
|
||||
scheduler_output.scheduled_cached_reqs.num_computed_tokens,
|
||||
)
|
||||
)
|
||||
scheduled_requests = tuple(
|
||||
(
|
||||
request_id,
|
||||
num_computed_tokens[request_id],
|
||||
scheduled_count,
|
||||
)
|
||||
for request_id, scheduled_count in (
|
||||
scheduler_output.num_scheduled_tokens.items()
|
||||
)
|
||||
)
|
||||
row_mirrors = {}
|
||||
for request_id, scheduled_start, scheduled_count in scheduled_requests:
|
||||
mirror_start = scheduled_start
|
||||
if self.async_speculative:
|
||||
mirror_start = max(
|
||||
0,
|
||||
scheduled_start - self.requests[request_id].num_output_placeholders,
|
||||
)
|
||||
row_mirrors[request_id] = self.coordinator.build_row_mirrors(
|
||||
(
|
||||
(
|
||||
request_id,
|
||||
mirror_start,
|
||||
scheduled_count
|
||||
+ scheduled_start
|
||||
- mirror_start
|
||||
+ self.draft_kv_lookahead,
|
||||
),
|
||||
),
|
||||
)
|
||||
return HiSparseConnectorMetadata(
|
||||
command,
|
||||
host_block_copies,
|
||||
tuple(source_block_ids),
|
||||
row_mirrors,
|
||||
self.coordinator.all_context_pages_resident(scheduled_requests),
|
||||
)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
assert self.coordinator is not None
|
||||
for request_id in connector_output.finished_recving or ():
|
||||
self.coordinator.finish_host_import(
|
||||
request_id, failed=request_id in connector_output.failed_recving
|
||||
)
|
||||
metadata = connector_output.kv_connector_worker_meta
|
||||
if metadata is None:
|
||||
return
|
||||
assert isinstance(metadata, HiSparseConnectorWorkerMetadata)
|
||||
self.coordinator.release_completed_host_copies(
|
||||
metadata.completed_host_copy_dst_ids
|
||||
)
|
||||
self.coordinator.update_spills(
|
||||
metadata.enqueued_transfer_counts,
|
||||
metadata.completed_transfer_counts,
|
||||
)
|
||||
|
||||
|
||||
class HiSparseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
"""Join the scheduler coordinator to the worker's transfer engine."""
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str:
|
||||
return "BLHNC"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
role: KVConnectorRole,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> None:
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
if kv_cache_config.hisparse_host_num_blocks is None:
|
||||
raise ValueError("HiSparseConnector requires a HiSparse host pool")
|
||||
self.connector_scheduler: HiSparseConnectorScheduler | None = None
|
||||
self.connector_worker: HiSparseConnectorWorker | None = None
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
speculative_config = vllm_config.speculative_config
|
||||
self.connector_scheduler = HiSparseConnectorScheduler(
|
||||
async_speculative=bool(
|
||||
vllm_config.scheduler_config.async_scheduling
|
||||
and speculative_config is not None
|
||||
),
|
||||
draft_kv_lookahead=vllm_config.num_lookahead_tokens,
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.connector_worker = HiSparseConnectorWorker(
|
||||
vllm_config, kv_cache_config
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported KV connector role: {role}")
|
||||
|
||||
def bind_kv_cache_manager(self, kv_cache_manager: KVCacheManager) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.bind_coordinator(
|
||||
get_hisparse_coordinator(kv_cache_manager)
|
||||
)
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
return False
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
assert self.connector_scheduler.coordinator is not None
|
||||
return self.connector_scheduler.coordinator.has_pending_work()
|
||||
|
||||
def has_pending_block_frees(self) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
assert self.connector_scheduler.coordinator is not None
|
||||
return self.connector_scheduler.coordinator.has_pending_reclamation()
|
||||
|
||||
def finish_forward(self) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.finish_forward()
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.register_kv_caches(kv_caches)
|
||||
|
||||
def reset_capture_state(self) -> None:
|
||||
assert self.connector_worker is not None
|
||||
self.connector_worker.reset_hot_state()
|
||||
|
||||
def start_load_kv(self, forward_context: ForwardContext, **kwargs: Any) -> None:
|
||||
assert self.connector_worker is not None
|
||||
metadata = self._get_connector_metadata()
|
||||
assert isinstance(metadata, HiSparseConnectorMetadata)
|
||||
request_state_indices = kwargs.get("request_state_indices")
|
||||
assert request_state_indices is None or isinstance(
|
||||
request_state_indices, torch.Tensor
|
||||
)
|
||||
request_ids = kwargs.get("request_ids")
|
||||
assert request_ids is None or isinstance(request_ids, list)
|
||||
attn_metadata = kwargs.get("attn_metadata")
|
||||
assert attn_metadata is None or isinstance(attn_metadata, dict)
|
||||
self.connector_worker.start_step(
|
||||
metadata,
|
||||
request_state_indices,
|
||||
request_ids,
|
||||
num_tokens=int(kwargs.get("num_tokens") or 0),
|
||||
)
|
||||
self.connector_worker.prepare_forward(attn_metadata)
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
return
|
||||
|
||||
def save_kv_layer(
|
||||
self,
|
||||
layer_name: str,
|
||||
kv_layer: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
return
|
||||
|
||||
def wait_for_save(self) -> None:
|
||||
return
|
||||
|
||||
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
|
||||
assert self.connector_worker is not None
|
||||
enqueued, completed = self.connector_worker.take_transfer_updates()
|
||||
host_copies = self.connector_worker.take_completed_host_copies()
|
||||
if not enqueued and not completed and not host_copies:
|
||||
return None
|
||||
return HiSparseConnectorWorkerMetadata(
|
||||
enqueued_transfer_counts={transfer_id: 1 for transfer_id in enqueued},
|
||||
completed_transfer_counts={transfer_id: 1 for transfer_id in completed},
|
||||
completed_host_copy_dst_ids=tuple(host_copies),
|
||||
)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.connector_worker is not None:
|
||||
self.connector_worker.shutdown()
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
return 0, False
|
||||
|
||||
def update_state_after_alloc(
|
||||
self,
|
||||
request: Request,
|
||||
blocks: KVCacheBlocks,
|
||||
num_external_tokens: int,
|
||||
) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.requests[request.request_id] = request
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
def request_finished_all_groups(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.requests.pop(request.request_id, None)
|
||||
return False, None
|
||||
@@ -0,0 +1,832 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HiSparse worker-side host/hot state and data movement."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import deque
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.parallel_state import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.utils.torch_utils import current_stream
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.hisparse.layout import HISPARSE_HOT_SUFFIX
|
||||
from vllm.v1.hisparse.runtime import HiSparseCacheHandle, release_pinned_state
|
||||
from vllm.v1.hisparse.types import SparseKVPageTransfer, SparseKVRowMirror
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
HiSparseHotSpec,
|
||||
KVCacheConfig,
|
||||
)
|
||||
from vllm.v1.worker.utils import copy_kv_cache_blocks_inplace
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.hisparse.connector import (
|
||||
HiSparseConnectorMetadata,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
|
||||
|
||||
class _DMADescriptors(NamedTuple):
|
||||
src: torch.Tensor
|
||||
dst: torch.Tensor
|
||||
sizes: torch.Tensor
|
||||
src_np: np.ndarray
|
||||
dst_np: np.ndarray
|
||||
sizes_np: np.ndarray
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SlotMappingStaging:
|
||||
stream: torch.Stream
|
||||
event: torch.Event
|
||||
slots: torch.Tensor
|
||||
candidates: tuple[SparseKVRowMirror, ...] = ()
|
||||
num_tokens: int = 0
|
||||
source_index: int = 0
|
||||
|
||||
|
||||
def _allocate_dma_descriptors(size: int) -> _DMADescriptors:
|
||||
src, dst, sizes = (torch.empty(size, dtype=torch.int64) for _ in range(3))
|
||||
return _DMADescriptors(
|
||||
src,
|
||||
dst,
|
||||
sizes,
|
||||
src.numpy(),
|
||||
dst.numpy(),
|
||||
sizes.numpy(),
|
||||
)
|
||||
|
||||
|
||||
def _get_hisparse_cache(
|
||||
forward_context: dict[str, Any], layer_name: str
|
||||
) -> HiSparseCacheHandle:
|
||||
attention_layer = forward_context[layer_name]
|
||||
hisparse_cache = attention_layer.hisparse_cache
|
||||
assert hisparse_cache is not None
|
||||
return hisparse_cache
|
||||
|
||||
|
||||
def _flatten_row_mirrors(
|
||||
row_mirrors: Mapping[str, tuple[SparseKVRowMirror, ...]],
|
||||
request_ids: Sequence[str] | None,
|
||||
) -> tuple[SparseKVRowMirror, ...]:
|
||||
ordered_ids = row_mirrors if request_ids is None else request_ids
|
||||
return tuple(
|
||||
mirror
|
||||
for request_id in ordered_ids
|
||||
for mirror in row_mirrors.get(request_id, ())
|
||||
)
|
||||
|
||||
|
||||
def _select_written_row_mirrors(
|
||||
candidates: tuple[SparseKVRowMirror, ...],
|
||||
source_slots: np.ndarray,
|
||||
source_index: int,
|
||||
) -> tuple[SparseKVRowMirror, ...]:
|
||||
"""Select and coalesce GPU-written rows from a scheduler-owned envelope."""
|
||||
source_slots = source_slots[source_slots >= 0]
|
||||
if source_slots.size == 0 or not candidates:
|
||||
return ()
|
||||
starts = np.asarray([mirror.source_starts for mirror in candidates], dtype=np.int64)
|
||||
counts = np.fromiter((mirror.num_rows for mirror in candidates), dtype=np.int64)
|
||||
destinations = np.fromiter(
|
||||
[mirror.destination_start for mirror in candidates], dtype=np.int64
|
||||
)
|
||||
if source_index >= starts.shape[1]:
|
||||
raise RuntimeError("HiSparse row-mirror source index is out of range.")
|
||||
source_starts = starts[:, source_index]
|
||||
order = np.argsort(source_starts)
|
||||
matches = np.searchsorted(source_starts[order], source_slots, side="right") - 1
|
||||
candidate_ids = order[np.maximum(matches, 0)]
|
||||
offsets = source_slots - source_starts[candidate_ids]
|
||||
valid = (matches >= 0) & (offsets < counts[candidate_ids])
|
||||
if not np.all(valid):
|
||||
# A page the scheduler could not map has no host destination, so its
|
||||
# rows are simply not mirrorable this step.
|
||||
source_slots = source_slots[valid]
|
||||
candidate_ids = candidate_ids[valid]
|
||||
offsets = offsets[valid]
|
||||
if source_slots.size == 0:
|
||||
return ()
|
||||
sources = starts[candidate_ids] + offsets[:, None]
|
||||
destinations = destinations[candidate_ids] + offsets
|
||||
boundaries = (
|
||||
np.flatnonzero(
|
||||
(candidate_ids[1:] != candidate_ids[:-1])
|
||||
| (np.diff(destinations) != 1)
|
||||
| np.any(np.diff(sources, axis=0) != 1, axis=1)
|
||||
)
|
||||
+ 1
|
||||
)
|
||||
run_starts = np.concatenate(([0], boundaries))
|
||||
run_ends = np.concatenate((boundaries, [destinations.size]))
|
||||
return tuple(
|
||||
SparseKVRowMirror(
|
||||
source_starts=tuple(int(value) for value in sources[start]),
|
||||
destination_start=int(destinations[start]),
|
||||
num_rows=int(end - start),
|
||||
)
|
||||
for start, end in zip(run_starts, run_ends, strict=True)
|
||||
)
|
||||
|
||||
|
||||
def _create_hisparse_host_events(
|
||||
shared_host_region: SharedOffloadRegion | None,
|
||||
is_host_writer: bool,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Event, torch.Event]:
|
||||
if shared_host_region is None:
|
||||
return torch.Event(), torch.Event()
|
||||
|
||||
events: tuple[torch.Event, torch.Event] | None = None
|
||||
ipc_handles = None
|
||||
if is_host_writer:
|
||||
events = (
|
||||
torch.cuda.Event(interprocess=True),
|
||||
torch.cuda.Event(interprocess=True),
|
||||
)
|
||||
stream = current_stream()
|
||||
for event in events:
|
||||
event.record(stream)
|
||||
ipc_handles = tuple(event.ipc_handle() for event in events)
|
||||
ipc_handles = get_tp_group().broadcast_object(ipc_handles, src=0)
|
||||
if events is None:
|
||||
events = (
|
||||
torch.cuda.Event.from_ipc_handle(device, ipc_handles[0]),
|
||||
torch.cuda.Event.from_ipc_handle(device, ipc_handles[1]),
|
||||
)
|
||||
return events
|
||||
|
||||
|
||||
class HiSparseConnectorWorker:
|
||||
"""Own HiSparse host/hot state and execute its worker-side transfers."""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig, kv_cache_config: KVCacheConfig) -> None:
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self._initialized = False
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]) -> None:
|
||||
forward_context = self.vllm_config.compilation_config.static_forward_context
|
||||
cache_handles: list[HiSparseCacheHandle] = []
|
||||
cache_layer_names: list[str] = []
|
||||
for group in self.kv_cache_config.kv_cache_groups:
|
||||
if not isinstance(group.kv_cache_spec, HiSparseHotSpec):
|
||||
continue
|
||||
for cache_name in group.layer_names:
|
||||
assert cache_name.endswith(HISPARSE_HOT_SUFFIX)
|
||||
layer_name = cache_name[: -len(HISPARSE_HOT_SUFFIX)]
|
||||
cache_handles.append(_get_hisparse_cache(forward_context, layer_name))
|
||||
cache_layer_names.append(layer_name)
|
||||
|
||||
if not cache_handles:
|
||||
raise RuntimeError("HiSparse connector found no hot-cache handles.")
|
||||
hot_backings: dict[int, torch.Tensor] = {}
|
||||
registered_host_pools: dict[int, torch.Tensor] = {}
|
||||
for cache in cache_handles:
|
||||
hot_backing = cache.runtime.hot_backing
|
||||
hot_backings[hot_backing.untyped_storage().data_ptr()] = hot_backing
|
||||
registered_pool = cache.runtime.registered_host_pool
|
||||
registered_host_pools[registered_pool.data_ptr()] = registered_pool
|
||||
if len(hot_backings) != 1:
|
||||
raise RuntimeError("HiSparse hot tensors must share one GPU backing.")
|
||||
hot_backing = next(iter(hot_backings.values()))
|
||||
pinned_host_pools = list(registered_host_pools.values())
|
||||
|
||||
resident = cache_handles[0].view
|
||||
assert resident is not None
|
||||
host_num_blocks = self.kv_cache_config.hisparse_host_num_blocks
|
||||
assert host_num_blocks is not None
|
||||
try:
|
||||
self.initialize(
|
||||
cache_handles,
|
||||
cache_layer_names,
|
||||
hot_backing,
|
||||
self.vllm_config.scheduler_config.max_num_seqs,
|
||||
host_num_blocks,
|
||||
hot_backing.device,
|
||||
pinned_host_pools,
|
||||
)
|
||||
except Exception:
|
||||
release_pinned_state(
|
||||
[cache.runtime for cache in cache_handles],
|
||||
pinned_host_pools,
|
||||
)
|
||||
raise
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
cache_handles: list[HiSparseCacheHandle],
|
||||
cache_layer_names: list[str],
|
||||
hot_backing: torch.Tensor,
|
||||
max_num_reqs: int,
|
||||
host_num_blocks: int,
|
||||
device: torch.device,
|
||||
pinned_host_pools: list[torch.Tensor],
|
||||
*,
|
||||
shared_host_region: SharedOffloadRegion | None = None,
|
||||
is_host_writer: bool = True,
|
||||
) -> None:
|
||||
if self._initialized:
|
||||
raise RuntimeError("HiSparse connector worker is already initialized.")
|
||||
resident = cache_handles[0].view
|
||||
assert resident is not None
|
||||
self.kernel_block_size = resident.block_size
|
||||
self.host_num_blocks = host_num_blocks
|
||||
self.pinned_host_pools = pinned_host_pools
|
||||
self.shared_host_region = shared_host_region
|
||||
self.is_host_writer = is_host_writer
|
||||
self.dma_stream = (
|
||||
torch.cuda.Stream(device=device) if self.is_host_writer else None
|
||||
)
|
||||
self._slot_mapping_staging = None
|
||||
if self.is_host_writer:
|
||||
max_mirror_rows = (
|
||||
self.vllm_config.scheduler_config.max_num_batched_tokens
|
||||
+ max_num_reqs * (self.vllm_config.num_lookahead_tokens + 1)
|
||||
)
|
||||
self._slot_mapping_staging = _SlotMappingStaging(
|
||||
stream=torch.Stream(device=device),
|
||||
event=torch.Event(),
|
||||
slots=torch.empty(
|
||||
max_mirror_rows,
|
||||
dtype=torch.int64,
|
||||
pin_memory=True,
|
||||
),
|
||||
)
|
||||
self._dma_free_descriptors: list[_DMADescriptors] = []
|
||||
self._pending_dma_descriptors: deque[tuple[torch.Event, _DMADescriptors]] = (
|
||||
deque()
|
||||
)
|
||||
self._dma_submitted = False
|
||||
self._per_layer_mirrored: set[int] = set()
|
||||
self._submitted_mirror_layers: set[int] = set()
|
||||
self._layer_ready_events = tuple(torch.Event() for _ in cache_handles)
|
||||
self._forward_ready_event = torch.Event()
|
||||
self._set_row_mirrors(())
|
||||
self.cache_layer_names = cache_layer_names
|
||||
self._group_leaders = tuple(
|
||||
(layer_name, cache)
|
||||
for layer_name, cache in zip(cache_layer_names, cache_handles, strict=True)
|
||||
if cache.runtime.is_group_leader
|
||||
)
|
||||
self.host_write_events = _create_hisparse_host_events(
|
||||
shared_host_region, is_host_writer, device
|
||||
)
|
||||
self.host_write_event = self.host_write_events[1]
|
||||
self._next_host_write_event = 0
|
||||
self.cache_handles = cache_handles
|
||||
self.leader_runtimes = [cache.runtime for _, cache in self._group_leaders]
|
||||
request_state_indices = {
|
||||
indices.data_ptr(): indices
|
||||
for cache in cache_handles
|
||||
if (indices := cache.runtime.request_state_indices) is not None
|
||||
}
|
||||
if len(request_state_indices) != 1:
|
||||
raise RuntimeError(
|
||||
"HiSparse runtimes must share one request-state mapping."
|
||||
)
|
||||
self.request_state_indices = next(iter(request_state_indices.values()))
|
||||
if self.request_state_indices.numel() != max_num_reqs:
|
||||
raise RuntimeError(
|
||||
"HiSparse request-state mapping does not match max_num_seqs."
|
||||
)
|
||||
self.hot_backing = hot_backing
|
||||
self._pending_invalid_block_ids: list[int] = []
|
||||
# Destination block ids of host copies this worker has run.
|
||||
self._completed_host_copy_dst_ids: list[int] = []
|
||||
self._post_forward_transfers: list[SparseKVPageTransfer] = []
|
||||
self._enqueued_transfer_ids: list[int] = []
|
||||
self._pending_transfer_events: deque[tuple[torch.Event, tuple[int, ...]]] = (
|
||||
deque()
|
||||
)
|
||||
self._init_dma()
|
||||
if self.is_host_writer:
|
||||
for layer_index, handle in enumerate(cache_handles):
|
||||
handle.submit_layer_mirror = partial(
|
||||
self._enqueue_layer_mirror, layer_index
|
||||
)
|
||||
self._initialized = True
|
||||
|
||||
def set_request_state_indices(self, indices: torch.Tensor) -> None:
|
||||
if indices.numel() > self.request_state_indices.numel():
|
||||
raise ValueError(
|
||||
"HiSparse request-state mapping exceeds max_num_seqs: "
|
||||
f"{indices.numel()} > {self.request_state_indices.numel()}."
|
||||
)
|
||||
if torch.cuda.is_current_stream_capturing():
|
||||
return
|
||||
# Attention indexes persistent request state by input-batch row. Refresh
|
||||
# that indirection after every batch compaction or reorder.
|
||||
self.request_state_indices.fill_(-1)
|
||||
self.request_state_indices[: indices.numel()].copy_(indices)
|
||||
if self._pending_invalid_block_ids:
|
||||
self.invalidate_blocks(self._pending_invalid_block_ids, indices)
|
||||
self._pending_invalid_block_ids.clear()
|
||||
|
||||
def _init_dma(self) -> None:
|
||||
host_caches = tuple(cache.runtime.host_cache for cache in self.cache_handles)
|
||||
resident_caches = []
|
||||
for cache in self.cache_handles:
|
||||
assert cache.view is not None and cache.slot_mapping is not None
|
||||
resident_caches.append(cache.view.cache)
|
||||
self.host_caches = host_caches
|
||||
self.resident_caches = tuple(resident_caches)
|
||||
|
||||
for resident_cache, host_cache in zip(resident_caches, host_caches):
|
||||
row_bytes = resident_cache.shape[-1] * resident_cache.element_size()
|
||||
if (
|
||||
resident_cache.ndim != 3
|
||||
or resident_cache.shape[1] != self.kernel_block_size
|
||||
or resident_cache.stride(1) * resident_cache.element_size() != row_bytes
|
||||
):
|
||||
raise RuntimeError("HiSparse DMA requires contiguous resident rows.")
|
||||
if (
|
||||
host_cache.ndim != 2
|
||||
or not host_cache.is_contiguous()
|
||||
or host_cache.shape[1] * host_cache.element_size() != row_bytes
|
||||
):
|
||||
raise RuntimeError("HiSparse DMA requires contiguous host rows.")
|
||||
|
||||
def start_step(
|
||||
self,
|
||||
metadata: HiSparseConnectorMetadata,
|
||||
request_state_indices: torch.Tensor | None,
|
||||
request_ids: list[str] | None = None,
|
||||
num_tokens: int = 0,
|
||||
) -> None:
|
||||
self._stage_row_mirror_mapping(num_tokens)
|
||||
previous_host_write_event = self.host_write_event
|
||||
self.host_write_event = self.host_write_events[self._next_host_write_event]
|
||||
self._next_host_write_event ^= 1
|
||||
current_stream().wait_event(previous_host_write_event)
|
||||
self._release_completed_dma_descriptors()
|
||||
mirrors = _flatten_row_mirrors(metadata.row_mirrors, request_ids)
|
||||
if self._slot_mapping_staging is not None:
|
||||
self._slot_mapping_staging.candidates = mirrors
|
||||
self._set_row_mirrors(mirrors)
|
||||
self._dma_submitted = False
|
||||
self._clear_forward_mirror_state()
|
||||
for handle in self.cache_handles:
|
||||
handle.all_context_pages_resident = metadata.all_context_pages_resident
|
||||
handle.mirror_from_resident = True
|
||||
self._copy_host_blocks(metadata.host_block_copies, previous_host_write_event)
|
||||
transfers = (
|
||||
metadata.command.page_transfers if metadata.command is not None else []
|
||||
)
|
||||
self._post_forward_transfers = [
|
||||
transfer for transfer in transfers if transfer.after_forward
|
||||
]
|
||||
self._submit_transfers(
|
||||
[transfer for transfer in transfers if not transfer.after_forward]
|
||||
)
|
||||
self._pending_invalid_block_ids.extend(metadata.source_block_ids)
|
||||
if request_state_indices is not None:
|
||||
self.set_request_state_indices(request_state_indices)
|
||||
|
||||
def _clear_forward_mirror_state(self) -> None:
|
||||
self._per_layer_mirrored.clear()
|
||||
self._submitted_mirror_layers.clear()
|
||||
for handle in self.cache_handles:
|
||||
handle.decode_batch = False
|
||||
handle.host_mirror_required = False
|
||||
handle.num_actual_tokens = 0
|
||||
handle.num_decode_tokens = 0
|
||||
handle.req_id_per_token = None
|
||||
|
||||
def prepare_forward(self, attn_metadata: Mapping[str, Any] | None) -> None:
|
||||
if attn_metadata is None:
|
||||
return
|
||||
for layer_name, handle in self._group_leaders:
|
||||
metadata = attn_metadata.get(layer_name)
|
||||
if metadata is not None:
|
||||
handle.prepare_group_for_batch(metadata)
|
||||
|
||||
def _stage_row_mirror_mapping(self, num_tokens: int) -> None:
|
||||
"""Snapshot the rows this forward will write, off the compute stream.
|
||||
|
||||
The resident slot mapping is a persistent view bound at registration
|
||||
time, so the rows are already staged by the time the forward launches.
|
||||
"""
|
||||
state = self._slot_mapping_staging
|
||||
if state is None or not num_tokens:
|
||||
return
|
||||
handle = self.cache_handles[0]
|
||||
slots = handle.slot_mapping
|
||||
assert slots is not None
|
||||
source_index = handle.runtime.resident_source_index
|
||||
start = state.num_tokens
|
||||
end = start + num_tokens
|
||||
if end > state.slots.shape[0]:
|
||||
raise ValueError(
|
||||
"HiSparse row mapping exceeds staging capacity: "
|
||||
f"{end} > {state.slots.shape[0]}."
|
||||
)
|
||||
main_stream = current_stream()
|
||||
state.stream.wait_stream(main_stream)
|
||||
with torch.cuda.stream(state.stream):
|
||||
state.slots[start:end].copy_(
|
||||
slots[:num_tokens],
|
||||
non_blocking=True,
|
||||
)
|
||||
state.event.record(state.stream)
|
||||
state.num_tokens = end
|
||||
state.source_index = source_index
|
||||
|
||||
def _resolve_row_mirrors(self, state: _SlotMappingStaging) -> None:
|
||||
"""Compute the final row mirrors from the staged GPU slot mapping."""
|
||||
self._set_row_mirrors(
|
||||
_select_written_row_mirrors(
|
||||
state.candidates,
|
||||
state.slots[: state.num_tokens].numpy(),
|
||||
state.source_index,
|
||||
)
|
||||
)
|
||||
state.num_tokens = 0
|
||||
|
||||
def _copy_host_blocks(
|
||||
self,
|
||||
host_block_copies: Sequence[KVCacheBlockCopy],
|
||||
previous_host_write_event: torch.Event,
|
||||
) -> None:
|
||||
if not host_block_copies:
|
||||
return
|
||||
self._completed_host_copy_dst_ids.extend(
|
||||
copy.dst_block_id for copy in host_block_copies
|
||||
)
|
||||
if self.shared_host_region is None or get_tensor_model_parallel_rank() == 0:
|
||||
if self.host_caches:
|
||||
previous_host_write_event.synchronize()
|
||||
copy_kv_cache_blocks_inplace(
|
||||
self.host_caches,
|
||||
self.host_num_blocks,
|
||||
host_block_copies,
|
||||
)
|
||||
if self.shared_host_region is not None:
|
||||
get_tp_group().barrier()
|
||||
|
||||
def invalidate_blocks(
|
||||
self, block_ids: list[int], request_state_indices: torch.Tensor
|
||||
) -> None:
|
||||
"""Invalidate recycled host slots in this worker's leader runtimes."""
|
||||
if not block_ids:
|
||||
return
|
||||
device = self.cache_handles[0].runtime.device
|
||||
staging = torch.tensor(block_ids, dtype=torch.int32, pin_memory=True)
|
||||
blocks = staging.to(device, dtype=torch.int32, non_blocking=True)
|
||||
offsets = torch.arange(self.kernel_block_size, dtype=torch.int32, device=device)
|
||||
slots = (blocks[:, None] * self.kernel_block_size + offsets[None, :]).flatten()
|
||||
sorted_slots = torch.sort(slots).values
|
||||
state_indices = request_state_indices.to(device=device, dtype=torch.long)
|
||||
for runtime in self.leader_runtimes:
|
||||
runtime.invalidate_sorted_slots(sorted_slots, state_indices)
|
||||
|
||||
def reset_hot_state(self) -> None:
|
||||
for runtime in self.leader_runtimes:
|
||||
runtime.reset_hot_state()
|
||||
|
||||
def _release_completed_dma_descriptors(self) -> None:
|
||||
pending = self._pending_dma_descriptors
|
||||
while pending and pending[0][0].query():
|
||||
_, descriptors = pending.popleft()
|
||||
self._dma_free_descriptors.append(descriptors)
|
||||
|
||||
def _acquire_dma_descriptors(self, size: int) -> _DMADescriptors:
|
||||
for index, descriptors in enumerate(self._dma_free_descriptors):
|
||||
if descriptors.src.numel() >= size:
|
||||
return self._dma_free_descriptors.pop(index)
|
||||
return _allocate_dma_descriptors(size)
|
||||
|
||||
def _submit_dma_descriptors(
|
||||
self,
|
||||
descriptors: _DMADescriptors,
|
||||
descriptor_count: int,
|
||||
transfer_ids: tuple[int, ...] = (),
|
||||
ready_event: torch.Event | None = None,
|
||||
) -> None:
|
||||
stream = self.dma_stream
|
||||
assert stream is not None
|
||||
if ready_event is None:
|
||||
stream.wait_stream(current_stream())
|
||||
else:
|
||||
stream.wait_event(ready_event)
|
||||
completion_event = torch.Event()
|
||||
with torch.cuda.stream(stream):
|
||||
ops.swap_blocks_batch(
|
||||
descriptors.src[:descriptor_count],
|
||||
descriptors.dst[:descriptor_count],
|
||||
descriptors.sizes[:descriptor_count],
|
||||
)
|
||||
self.host_write_event.record(stream)
|
||||
completion_event.record(stream)
|
||||
self._pending_dma_descriptors.append((completion_event, descriptors))
|
||||
self._dma_submitted = True
|
||||
if transfer_ids:
|
||||
self._pending_transfer_events.append((completion_event, transfer_ids))
|
||||
self._enqueued_transfer_ids.extend(transfer_ids)
|
||||
|
||||
def _set_row_mirrors(self, mirrors: tuple[SparseKVRowMirror, ...]) -> None:
|
||||
self._row_mirrors = mirrors
|
||||
self._row_mirror_destination_starts = np.fromiter(
|
||||
(mirror.destination_start for mirror in mirrors),
|
||||
dtype=np.int64,
|
||||
count=len(mirrors),
|
||||
)
|
||||
self._row_mirror_counts = np.fromiter(
|
||||
(mirror.num_rows for mirror in mirrors),
|
||||
dtype=np.int64,
|
||||
count=len(mirrors),
|
||||
)
|
||||
if mirrors:
|
||||
self._row_mirror_source_starts = np.asarray(
|
||||
[mirror.source_starts for mirror in mirrors], dtype=np.int64
|
||||
)
|
||||
if self._row_mirror_source_starts.ndim != 2:
|
||||
raise RuntimeError("HiSparse DMA source mappings must be rectangular.")
|
||||
else:
|
||||
self._row_mirror_source_starts = np.empty((0, 0), dtype=np.int64)
|
||||
self._row_mirror_num_rows = int(self._row_mirror_counts.sum())
|
||||
|
||||
def _enqueue_row_dma(
|
||||
self, layer_indices: Sequence[int], ready_event: torch.Event | None = None
|
||||
) -> None:
|
||||
if (
|
||||
not layer_indices
|
||||
or not self._row_mirror_num_rows
|
||||
or not self.is_host_writer
|
||||
):
|
||||
return
|
||||
mirrors = self._row_mirrors
|
||||
num_layers = len(layer_indices)
|
||||
descriptor_count = len(mirrors) * num_layers
|
||||
descriptors = self._acquire_dma_descriptors(descriptor_count)
|
||||
destination_starts = self._row_mirror_destination_starts
|
||||
row_counts = self._row_mirror_counts
|
||||
for descriptor_offset, layer_index in enumerate(layer_indices):
|
||||
cache = self.cache_handles[layer_index]
|
||||
source_index = cache.runtime.resident_source_index
|
||||
if source_index >= self._row_mirror_source_starts.shape[1]:
|
||||
raise RuntimeError("HiSparse row DMA source index is out of range.")
|
||||
source_rows = self._row_mirror_source_starts[:, source_index]
|
||||
source = self.resident_caches[layer_index]
|
||||
destination = self.host_caches[layer_index]
|
||||
row_bytes = source.shape[-1] * source.element_size()
|
||||
if (
|
||||
source.stride(1) * source.element_size() != row_bytes
|
||||
or destination.shape[1] * destination.element_size() != row_bytes
|
||||
):
|
||||
raise RuntimeError("HiSparse row DMA requires contiguous rows.")
|
||||
source_blocks, source_row_offsets = np.divmod(
|
||||
source_rows, self.kernel_block_size
|
||||
)
|
||||
source_out_of_range = (
|
||||
np.any(source_blocks < 0)
|
||||
or np.any(source_blocks >= source.shape[0])
|
||||
or np.any(source_row_offsets + row_counts > self.kernel_block_size)
|
||||
)
|
||||
if source_out_of_range:
|
||||
raise RuntimeError("HiSparse row DMA source is out of range.")
|
||||
if np.any(destination_starts < 0) or np.any(
|
||||
destination_starts + row_counts > destination.shape[0]
|
||||
):
|
||||
raise RuntimeError("HiSparse row DMA index is out of range.")
|
||||
descriptor_slice = slice(descriptor_offset, descriptor_count, num_layers)
|
||||
descriptors.src_np[descriptor_slice] = (
|
||||
source.data_ptr()
|
||||
+ source_blocks * source.stride(0) * source.element_size()
|
||||
+ source_row_offsets * row_bytes
|
||||
)
|
||||
descriptors.dst_np[descriptor_slice] = (
|
||||
destination.data_ptr() + destination_starts * row_bytes
|
||||
)
|
||||
descriptors.sizes_np[descriptor_slice] = row_counts * row_bytes
|
||||
|
||||
self._submit_dma_descriptors(
|
||||
descriptors, descriptor_count, ready_event=ready_event
|
||||
)
|
||||
|
||||
def _enqueue_layer_mirror(self, layer_index: int) -> None:
|
||||
handle = self.cache_handles[layer_index]
|
||||
if not handle.host_mirror_required:
|
||||
return
|
||||
if layer_index in self._per_layer_mirrored:
|
||||
raise RuntimeError(f"HiSparse layer {layer_index} mirrored twice.")
|
||||
state = self._slot_mapping_staging
|
||||
if state is not None and state.num_tokens:
|
||||
# The staging stream waited on the compute stream before the
|
||||
# forward launched, so this only drains pre-forward work that is
|
||||
# already queued ahead of the layer kernels: no GPU bubble.
|
||||
state.event.synchronize()
|
||||
self._resolve_row_mirrors(state)
|
||||
self._per_layer_mirrored.add(layer_index)
|
||||
next_layer = layer_index + 1
|
||||
if (
|
||||
next_layer < len(self.cache_handles)
|
||||
and self.cache_handles[next_layer].runtime.resident_source_index
|
||||
== handle.runtime.resident_source_index
|
||||
):
|
||||
return
|
||||
ready_event = self._layer_ready_events[layer_index]
|
||||
ready_event.record()
|
||||
pending_layers = tuple(
|
||||
sorted(self._per_layer_mirrored - self._submitted_mirror_layers)
|
||||
)
|
||||
self._enqueue_row_dma(
|
||||
pending_layers,
|
||||
ready_event=ready_event,
|
||||
)
|
||||
self._submitted_mirror_layers.update(pending_layers)
|
||||
|
||||
def _record_transfer_completion(
|
||||
self, transfers: list[SparseKVPageTransfer]
|
||||
) -> None:
|
||||
if not transfers or not self.is_host_writer:
|
||||
return
|
||||
stream = self.dma_stream
|
||||
assert stream is not None
|
||||
completion_event = torch.Event()
|
||||
completion_event.record(stream)
|
||||
transfer_ids = tuple(transfer.transfer_id for transfer in transfers)
|
||||
self._pending_transfer_events.append((completion_event, transfer_ids))
|
||||
self._enqueued_transfer_ids.extend(transfer_ids)
|
||||
|
||||
def _submit_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
|
||||
if self.cache_handles[0].runtime.eager_host_mirror:
|
||||
self._record_transfer_completion(transfers)
|
||||
else:
|
||||
self._enqueue_transfers(transfers)
|
||||
|
||||
def _enqueue_transfers(self, transfers: list[SparseKVPageTransfer]) -> None:
|
||||
if not transfers or not self.is_host_writer:
|
||||
return
|
||||
num_layers = len(self.cache_handles)
|
||||
descriptor_count = len(transfers) * num_layers
|
||||
descriptors = self._acquire_dma_descriptors(descriptor_count)
|
||||
destination_rows = np.fromiter(
|
||||
(
|
||||
transfer.destination_block_id * self.kernel_block_size
|
||||
for transfer in transfers
|
||||
),
|
||||
dtype=np.int64,
|
||||
count=len(transfers),
|
||||
)
|
||||
source_blocks_by_transfer = np.asarray(
|
||||
[transfer.source_block_ids for transfer in transfers], dtype=np.int64
|
||||
)
|
||||
if source_blocks_by_transfer.ndim != 2:
|
||||
raise RuntimeError(
|
||||
"HiSparse spill DMA source mappings must be rectangular."
|
||||
)
|
||||
for layer_index, cache in enumerate(self.cache_handles):
|
||||
source_index = cache.runtime.resident_source_index
|
||||
if source_index >= source_blocks_by_transfer.shape[1]:
|
||||
raise RuntimeError("HiSparse spill DMA source index is out of range.")
|
||||
source_blocks = source_blocks_by_transfer[:, source_index]
|
||||
source = self.resident_caches[layer_index]
|
||||
destination = self.host_caches[layer_index]
|
||||
if np.any(source_blocks < 0) or np.any(source_blocks >= source.shape[0]):
|
||||
raise RuntimeError("HiSparse spill DMA source is out of range.")
|
||||
if np.any(destination_rows < 0) or np.any(
|
||||
destination_rows + self.kernel_block_size > destination.shape[0]
|
||||
):
|
||||
raise RuntimeError("HiSparse spill DMA destination is out of range.")
|
||||
row_bytes = source.shape[-1] * source.element_size()
|
||||
descriptor_slice = slice(layer_index, descriptor_count, num_layers)
|
||||
descriptors.src_np[descriptor_slice] = (
|
||||
source.data_ptr()
|
||||
+ source_blocks * source.stride(0) * source.element_size()
|
||||
)
|
||||
descriptors.dst_np[descriptor_slice] = (
|
||||
destination.data_ptr() + destination_rows * row_bytes
|
||||
)
|
||||
descriptors.sizes_np[descriptor_slice] = self.kernel_block_size * row_bytes
|
||||
self._submit_dma_descriptors(
|
||||
descriptors,
|
||||
descriptor_count,
|
||||
transfer_ids=tuple(transfer.transfer_id for transfer in transfers),
|
||||
)
|
||||
|
||||
def _enqueue_host_mirror(
|
||||
self,
|
||||
ready_event: torch.Event | None = None,
|
||||
) -> None:
|
||||
active = [
|
||||
(index, handle)
|
||||
for index, handle in enumerate(self.cache_handles)
|
||||
if handle.num_actual_tokens != 0
|
||||
]
|
||||
if not active:
|
||||
return
|
||||
|
||||
def mirror_key(handle: HiSparseCacheHandle) -> tuple:
|
||||
slots = handle.mirror_slot_mapping
|
||||
return (
|
||||
handle.num_actual_tokens,
|
||||
handle.num_decode_tokens,
|
||||
handle.decode_batch,
|
||||
handle.host_mirror_required,
|
||||
handle.runtime.eager_host_mirror,
|
||||
None if slots is None else slots.data_ptr(),
|
||||
)
|
||||
|
||||
keys = {mirror_key(handle) for _, handle in active}
|
||||
if len(keys) > 1:
|
||||
raise RuntimeError(
|
||||
f"HiSparse cache layers disagree on mirror metadata: {list(keys)}."
|
||||
)
|
||||
cache = active[0][1]
|
||||
if not cache.host_mirror_required:
|
||||
return
|
||||
dst_slots = cache.mirror_slot_mapping
|
||||
if dst_slots is None:
|
||||
raise RuntimeError("HiSparse host mirror has no source slot mapping.")
|
||||
num_rows = min(cache.num_actual_tokens, dst_slots.numel())
|
||||
if num_rows == 0:
|
||||
return
|
||||
if self.is_host_writer:
|
||||
expected_layers = {index for index, _ in active}
|
||||
if self._per_layer_mirrored and self._per_layer_mirrored != expected_layers:
|
||||
raise RuntimeError(
|
||||
"HiSparse per-layer DMA did not mirror every active layer: "
|
||||
f"expected {sorted(expected_layers)}, got "
|
||||
f"{sorted(self._per_layer_mirrored)}."
|
||||
)
|
||||
pending_layers = tuple(
|
||||
sorted(expected_layers - self._submitted_mirror_layers)
|
||||
)
|
||||
if pending_layers:
|
||||
self._enqueue_row_dma(pending_layers, ready_event=ready_event)
|
||||
self._submitted_mirror_layers.update(pending_layers)
|
||||
assert cache.req_id_per_token is not None
|
||||
for _, handle in active:
|
||||
if handle.runtime.is_group_leader:
|
||||
handle.runtime.invalidate_written_slots(
|
||||
dst_slots[:num_rows],
|
||||
cache.req_id_per_token[:num_rows],
|
||||
)
|
||||
|
||||
def _finish_mirror_phase(self, ready_event: torch.Event | None = None) -> None:
|
||||
state = self._slot_mapping_staging
|
||||
if state is not None and state.num_tokens:
|
||||
# Measured free: blocking here costs no throughput (26.7 vs 27.6
|
||||
# gen tok/s) and lets every step mirror exactly the written rows.
|
||||
state.event.synchronize()
|
||||
self._resolve_row_mirrors(state)
|
||||
self._enqueue_host_mirror(ready_event)
|
||||
self._clear_forward_mirror_state()
|
||||
|
||||
def finish_forward(self) -> None:
|
||||
compute_stream = current_stream()
|
||||
self._forward_ready_event.record()
|
||||
self._finish_mirror_phase(self._forward_ready_event)
|
||||
transfers = self._post_forward_transfers
|
||||
self._post_forward_transfers = []
|
||||
self._submit_transfers(transfers)
|
||||
if self.is_host_writer:
|
||||
if self._dma_submitted:
|
||||
compute_stream.wait_event(self.host_write_event)
|
||||
self._dma_submitted = False
|
||||
else:
|
||||
self.host_write_event.record(compute_stream)
|
||||
self._release_completed_dma_descriptors()
|
||||
|
||||
def take_completed_host_copies(self) -> list[int]:
|
||||
"""Drain host copies this worker has enqueued for this step."""
|
||||
completed = self._completed_host_copy_dst_ids
|
||||
self._completed_host_copy_dst_ids = []
|
||||
return completed
|
||||
|
||||
def take_transfer_updates(self) -> tuple[list[int], list[int]]:
|
||||
enqueued = self._enqueued_transfer_ids
|
||||
self._enqueued_transfer_ids = []
|
||||
completed: list[int] = []
|
||||
while self._pending_transfer_events:
|
||||
event, transfer_ids = self._pending_transfer_events[0]
|
||||
if not event.query():
|
||||
break
|
||||
self._pending_transfer_events.popleft()
|
||||
completed.extend(transfer_ids)
|
||||
return enqueued, completed
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if not self._initialized:
|
||||
return
|
||||
if self._slot_mapping_staging is not None:
|
||||
self._slot_mapping_staging.stream.synchronize()
|
||||
if self.dma_stream is not None:
|
||||
self.dma_stream.synchronize()
|
||||
release_pinned_state(
|
||||
[cache.runtime for cache in self.cache_handles], self.pinned_host_pools
|
||||
)
|
||||
self._initialized = False
|
||||
@@ -702,9 +702,9 @@ class LMCacheMPConnectorUpstream(KVConnectorBase_V1):
|
||||
- Applies to both sync- and async-loading requests.
|
||||
- Async loading: failed blocks may be reported in any forward pass
|
||||
up to and including the pass where the request ID is returned by
|
||||
`get_finished()`. Even if failures occur, the request must still
|
||||
be reported via `get_finished()`, and the failed block IDs must
|
||||
appear here no later than that same pass.
|
||||
`get_transfer_results()`. Even if failures occur, the request
|
||||
must still be reported as finished receiving, and the failed
|
||||
block IDs must appear here no later than that same pass.
|
||||
- Sync loading: failed blocks should be reported in the forward
|
||||
pass in which they are detected.
|
||||
"""
|
||||
|
||||
@@ -635,6 +635,7 @@ class MooncakeConnectorScheduler:
|
||||
):
|
||||
self.vllm_config = vllm_config
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.kv_cache_config = kv_cache_config
|
||||
|
||||
assert vllm_config.kv_transfer_config
|
||||
self.is_kv_producer: bool = (
|
||||
@@ -649,7 +650,7 @@ class MooncakeConnectorScheduler:
|
||||
not vllm_config.scheduler_config.disable_hybrid_kv_cache_manager
|
||||
and any(
|
||||
not isinstance(g.kv_cache_spec, FullAttentionSpec)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
# GDN is represented as a MambaSpec in vLLM. This Mooncake MambaSpec
|
||||
@@ -670,7 +671,7 @@ class MooncakeConnectorScheduler:
|
||||
(g.kv_cache_spec.sliding_window, g.kv_cache_spec.block_size)
|
||||
if isinstance(g.kv_cache_spec, SlidingWindowSpec)
|
||||
else (0, self.block_size)
|
||||
for g in kv_cache_config.kv_cache_groups
|
||||
for g in kv_cache_config.transfer_groups
|
||||
]
|
||||
# cdiv(n_tokens, block_size) gives blocks/window; add 1 to
|
||||
# conservatively account for boundary overlap.
|
||||
@@ -684,11 +685,17 @@ class MooncakeConnectorScheduler:
|
||||
block_ids: tuple[list[int], ...] | list[list[int]],
|
||||
) -> list[list[int]]:
|
||||
"""Clip per-group block IDs to sliding window size."""
|
||||
if len(block_ids) == 0 or not self._is_hma_required:
|
||||
return list(block_ids)
|
||||
if len(block_ids) == 0:
|
||||
return []
|
||||
if len(block_ids) == len(self.kv_cache_config.transfer_group_ids):
|
||||
selected = list(block_ids)
|
||||
else:
|
||||
selected = list(self.kv_cache_config.select_transfer_block_ids(block_ids))
|
||||
if len(selected) == 0 or not self._is_hma_required:
|
||||
return selected
|
||||
return [
|
||||
blocks[-self.blocks_per_sw[i] :] if self.blocks_per_sw[i] > 0 else blocks
|
||||
for i, blocks in enumerate(block_ids)
|
||||
for i, blocks in enumerate(selected)
|
||||
]
|
||||
|
||||
def _get_remote_prefill_token_count(self, num_prompt_tokens: int) -> int:
|
||||
@@ -1046,18 +1053,13 @@ class MooncakeConnectorWorker:
|
||||
|
||||
self._tp_size: dict[EngineId, int] = {self.engine_id: self.tp_size}
|
||||
self._layer_specs: dict[str, KVCacheSpec] = {}
|
||||
for group in kv_cache_config.kv_cache_groups:
|
||||
for group in kv_cache_config.transfer_groups:
|
||||
group_spec = group.kv_cache_spec
|
||||
specs_by_layer = getattr(group_spec, "kv_cache_specs", {})
|
||||
for layer_name in group.layer_names:
|
||||
self._layer_specs[layer_name] = specs_by_layer.get(
|
||||
layer_name, group_spec
|
||||
)
|
||||
self._layer_group_indices: dict[str, int] = {
|
||||
layer: group_index
|
||||
for group_index, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
for layer in group.layer_names
|
||||
}
|
||||
self.transfer_topo = TransferTopology(
|
||||
tp_rank=self.tp_rank,
|
||||
tp_size=self.tp_size,
|
||||
@@ -1441,7 +1443,7 @@ class MooncakeConnectorWorker:
|
||||
block_arange = np.arange(self._physical_blocks_per_logical_kv_block).reshape(
|
||||
1, -1
|
||||
)
|
||||
group_specs = self.kv_cache_config.kv_cache_groups
|
||||
group_specs = self.kv_cache_config.transfer_groups
|
||||
return [
|
||||
BlockTable.map_to_kernel_blocks(
|
||||
np.array(group),
|
||||
@@ -1494,7 +1496,7 @@ class MooncakeConnectorWorker:
|
||||
local_block_ids_by_group: list[list[int]] = []
|
||||
remote_block_ids_by_group: list[list[int]] = []
|
||||
has_block_error = False
|
||||
group_specs = self.kv_cache_config.kv_cache_groups
|
||||
group_specs = self.kv_cache_config.transfer_groups
|
||||
for group_index, (local_group, remote_group) in enumerate(
|
||||
zip(send_meta.local_block_ids, remote_block_ids_per_group)
|
||||
):
|
||||
@@ -1750,7 +1752,7 @@ class MooncakeConnectorWorker:
|
||||
self.registered_layer_names.append(layer_name)
|
||||
self.registered_layer_indices.append(layer_index)
|
||||
self.registered_group_indices.append(
|
||||
self._layer_group_indices[layer_name]
|
||||
self.kv_cache_config.transfer_group_index_by_layer[layer_name]
|
||||
)
|
||||
storage = cache.untyped_storage()
|
||||
storage_addr = storage.data_ptr()
|
||||
@@ -2121,7 +2123,7 @@ class MooncakeConnectorWorker:
|
||||
) -> list[TransferRegion]:
|
||||
if not group_indices:
|
||||
group_indices = [
|
||||
self._layer_group_indices.get(layer_name, 0)
|
||||
self.kv_cache_config.transfer_group_index_by_layer.get(layer_name, 0)
|
||||
for layer_name in layer_names
|
||||
]
|
||||
return _expand_transfer_regions(
|
||||
|
||||
@@ -3,18 +3,20 @@
|
||||
"""External-store cache-hit coordinator for MooncakeStoreConnector."""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import cast
|
||||
from typing import NamedTuple, cast
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
chunk_hashes_for_block_size,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_coordinator import SpecGroup
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
KVCacheBlock,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
SingleTypeKVCacheManager,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
@@ -25,6 +27,13 @@ from vllm.v1.kv_cache_interface import (
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
|
||||
|
||||
class StoreSpecGroup(NamedTuple):
|
||||
spec: KVCacheSpec
|
||||
group_ids: list[int]
|
||||
manager_cls: type[SingleTypeKVCacheManager]
|
||||
use_eagle: bool
|
||||
|
||||
|
||||
class ExternalCachedBlockPool:
|
||||
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
|
||||
|
||||
@@ -121,7 +130,7 @@ class MooncakeStoreCoordinator:
|
||||
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
|
||||
dispatches via spec_manager_map (we don't allocate managers).
|
||||
"""
|
||||
attention_groups: list[SpecGroup] = []
|
||||
attention_groups: list[StoreSpecGroup] = []
|
||||
for i, g in enumerate(self.kv_cache_groups):
|
||||
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
|
||||
# kpool tail): per-request scratch, never shareable, so they must
|
||||
@@ -143,7 +152,7 @@ class MooncakeStoreCoordinator:
|
||||
break
|
||||
else:
|
||||
attention_groups.append(
|
||||
SpecGroup(spec, [i], manager_cls, g.is_eagle_group)
|
||||
StoreSpecGroup(spec, [i], manager_cls, g.is_eagle_group)
|
||||
)
|
||||
# Full attention first (matches upstream convergence ordering).
|
||||
attention_groups.sort(key=lambda g: not isinstance(g.spec, FullAttentionSpec))
|
||||
|
||||
@@ -67,6 +67,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.network_utils import get_ip, make_zmq_socket
|
||||
from vllm.utils.torch_utils import is_non_overlapping_and_dense
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
@@ -1909,21 +1910,31 @@ class MooncakeStoreWorker:
|
||||
logger.warning("No KV caches to offload.")
|
||||
return
|
||||
|
||||
# Resolve each entry to a representative tensor for storage
|
||||
# deduplication. For attention layers the value is already a tensor;
|
||||
# for Mamba layers it is a list of tensors that all share the same
|
||||
# underlying raw storage, so we take the first one.
|
||||
def _repr_tensor(v: torch.Tensor | list[torch.Tensor]) -> torch.Tensor:
|
||||
assert isinstance(v, torch.Tensor | list)
|
||||
return v if isinstance(v, torch.Tensor) else v[0]
|
||||
|
||||
assert self.cache_config.num_gpu_blocks is not None
|
||||
self.num_blocks = self.cache_config.num_gpu_blocks
|
||||
use_group_regions = self._kv_cache_config.hisparse_host_num_blocks is not None
|
||||
|
||||
seen_storage_ptrs: set[int] = set()
|
||||
cache_tensors: list[torch.Tensor] = []
|
||||
|
||||
for cache in kv_caches.values():
|
||||
cache = group_kernel_blocks(cache, self.num_blocks)
|
||||
cache_tensors.append(cache)
|
||||
cache_storage = cache.untyped_storage()
|
||||
base_addr = cache_storage.data_ptr()
|
||||
region_len = cache_storage.nbytes()
|
||||
|
||||
if base_addr not in seen_storage_ptrs:
|
||||
if not use_group_regions:
|
||||
seen_storage_ptrs: set[int] = set()
|
||||
cache_tensors = [
|
||||
group_kernel_blocks(_repr_tensor(cache), self.num_blocks)
|
||||
for cache in kv_caches.values()
|
||||
]
|
||||
for cache in cache_tensors:
|
||||
cache_storage = cache.untyped_storage()
|
||||
base_addr = cache_storage.data_ptr()
|
||||
if base_addr in seen_storage_ptrs:
|
||||
continue
|
||||
seen_storage_ptrs.add(base_addr)
|
||||
region_len = cache_storage.nbytes()
|
||||
ret = self.store.register_buffer(base_addr, region_len)
|
||||
if ret != 0:
|
||||
logger.error(
|
||||
@@ -1932,17 +1943,110 @@ class MooncakeStoreWorker:
|
||||
region_len,
|
||||
ret,
|
||||
)
|
||||
for db in self.token_dbs:
|
||||
db.store_layout.register_kv_caches(cache_tensors, self.num_blocks)
|
||||
|
||||
registered_buffers: dict[int, int] = {}
|
||||
num_host_segments = 0
|
||||
num_device_segments = 0
|
||||
|
||||
for group, db in zip(self._kv_cache_groups, self.token_dbs, strict=True):
|
||||
if not use_group_regions:
|
||||
continue
|
||||
missing_layers = [
|
||||
layer_name
|
||||
for layer_name in group.layer_names
|
||||
if layer_name not in kv_caches
|
||||
]
|
||||
if missing_layers:
|
||||
raise ValueError(
|
||||
f"Missing transferable KV caches for layers: {missing_layers}"
|
||||
)
|
||||
group_caches = [
|
||||
(layer_name, kv_caches[layer_name]) for layer_name in group.layer_names
|
||||
]
|
||||
|
||||
seen_group_ptrs: set[int] = set()
|
||||
addrs: list[int] = []
|
||||
block_lens: list[int] = []
|
||||
for layer_name, value in group_caches:
|
||||
cache = _repr_tensor(value)
|
||||
is_host_resident = group.host_resident
|
||||
if is_host_resident:
|
||||
num_blocks = self._kv_cache_config.hisparse_host_num_blocks
|
||||
assert num_blocks is not None
|
||||
else:
|
||||
num_blocks = self._kv_cache_config.num_blocks
|
||||
cache = group_kernel_blocks(cache, num_blocks)
|
||||
cache_storage = cache.untyped_storage()
|
||||
if is_host_resident:
|
||||
base_addr = cache.data_ptr()
|
||||
region_len = cache.nbytes
|
||||
else:
|
||||
base_addr = cache_storage.data_ptr()
|
||||
region_len = cache_storage.nbytes()
|
||||
if region_len % num_blocks != 0:
|
||||
raise ValueError(
|
||||
f"KV cache region for {layer_name} has size {region_len}, "
|
||||
f"which is not divisible by its {num_blocks} blocks."
|
||||
)
|
||||
|
||||
registered_len = registered_buffers.get(base_addr)
|
||||
if registered_len is None:
|
||||
ret = self.store.register_buffer(base_addr, region_len)
|
||||
if ret != 0:
|
||||
mem_kind = "host" if is_host_resident else "device"
|
||||
raise RuntimeError(
|
||||
"Mooncake register_buffer failed for addr "
|
||||
f"{base_addr:#x} len {region_len} ({mem_kind}): {ret}"
|
||||
)
|
||||
registered_buffers[base_addr] = region_len
|
||||
elif registered_len != region_len:
|
||||
raise ValueError(
|
||||
f"KV cache views at {base_addr:#x} expose inconsistent "
|
||||
f"region sizes: {registered_len} and {region_len}."
|
||||
)
|
||||
|
||||
regions: list[tuple[int, int]] = []
|
||||
if not is_non_overlapping_and_dense(cache[0]):
|
||||
for head_idx in range(cache.shape[1]):
|
||||
head_cache = cache[:, head_idx]
|
||||
assert is_non_overlapping_and_dense(head_cache[0])
|
||||
regions.append(
|
||||
(
|
||||
head_cache.data_ptr(),
|
||||
head_cache.stride(0) * head_cache.element_size(),
|
||||
)
|
||||
)
|
||||
elif cache.stride(0) * cache.element_size() * num_blocks == region_len:
|
||||
regions.append((base_addr, region_len // num_blocks))
|
||||
else:
|
||||
regions.append(
|
||||
(cache.data_ptr(), cache.stride(0) * cache.element_size())
|
||||
)
|
||||
|
||||
for region_addr, block_len in regions:
|
||||
if region_addr in seen_group_ptrs:
|
||||
continue
|
||||
seen_group_ptrs.add(region_addr)
|
||||
addrs.append(region_addr)
|
||||
block_lens.append(block_len)
|
||||
if is_host_resident:
|
||||
num_host_segments += 1
|
||||
else:
|
||||
num_device_segments += 1
|
||||
|
||||
db.set_kv_caches_base_addr(addrs)
|
||||
db.set_block_len(block_lens)
|
||||
|
||||
logger.info(
|
||||
"Registered KV caches: num_groups=%d, num_tensors=%d, num_blocks=%d",
|
||||
"Registered KV caches: num_groups=%d, num_segments=%d (host=%d, device=%d)",
|
||||
len(self.token_dbs),
|
||||
len(cache_tensors),
|
||||
self.num_blocks,
|
||||
num_host_segments + num_device_segments,
|
||||
num_host_segments,
|
||||
num_device_segments,
|
||||
)
|
||||
|
||||
for db in self.token_dbs:
|
||||
db.store_layout.register_kv_caches(cache_tensors, self.num_blocks)
|
||||
|
||||
# Start transfer threads
|
||||
if self.can_put:
|
||||
ready_event_sending = threading.Event()
|
||||
@@ -2316,6 +2420,19 @@ class MooncakeStoreWorker:
|
||||
if store is None:
|
||||
return
|
||||
self.store = None
|
||||
deadline = time.monotonic() + 5.0
|
||||
for thread in (self.kv_send_thread, *self.kv_recv_threads):
|
||||
if thread is None:
|
||||
continue
|
||||
while thread.request_queue.unfinished_tasks and time.monotonic() < deadline:
|
||||
time.sleep(0.05)
|
||||
if thread.request_queue.unfinished_tasks:
|
||||
logger.warning(
|
||||
"Mooncake store %s still has %d in-flight requests at "
|
||||
"close; closing anyway.",
|
||||
thread.name,
|
||||
thread.request_queue.unfinished_tasks,
|
||||
)
|
||||
try:
|
||||
store.close()
|
||||
except Exception as e:
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorHandshakeMetadata,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
)
|
||||
@@ -35,7 +36,7 @@ if TYPE_CHECKING:
|
||||
from vllm.distributed.kv_events import KVCacheEvent
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.request import Request
|
||||
|
||||
@@ -208,6 +209,10 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
# Propagated from scheduler to worker side via the connector metadata.
|
||||
self._extra_async_saves: dict[str, int] = {}
|
||||
|
||||
@property
|
||||
def sub_connectors(self) -> list[KVConnectorBase_V1]:
|
||||
return list(self._connectors)
|
||||
|
||||
@property
|
||||
def supports_divergent_local_hybrid_hits(self) -> bool:
|
||||
return bool(self._connectors) and all(
|
||||
@@ -248,6 +253,10 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
for c in self._connectors:
|
||||
c.register_kv_caches(kv_caches)
|
||||
|
||||
def bind_kv_cache_manager(self, kv_cache_manager: "KVCacheManager") -> None:
|
||||
for connector in self._connectors:
|
||||
connector.bind_kv_cache_manager(kv_cache_manager)
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: "BlockPool") -> None:
|
||||
for c in self._connectors:
|
||||
c.bind_gpu_block_pool(gpu_block_pool)
|
||||
@@ -292,6 +301,14 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
for c in self._connectors:
|
||||
c.start_load_kv(forward_context, **kwargs)
|
||||
|
||||
def finish_forward(self) -> None:
|
||||
for c in self._connectors:
|
||||
c.finish_forward()
|
||||
|
||||
def reset_capture_state(self) -> None:
|
||||
for c in self._connectors:
|
||||
c.reset_capture_state()
|
||||
|
||||
def wait_for_layer_load(self, layer_name: str) -> None:
|
||||
for c in self._connectors:
|
||||
c.wait_for_layer_load(layer_name)
|
||||
@@ -313,29 +330,31 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
def get_finished(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> tuple[set[str] | None, set[str] | None]:
|
||||
finished_sending: set[str] = set()
|
||||
finished_recving: set[str] = set()
|
||||
for c in self._connectors:
|
||||
sending, recving = c.get_finished(finished_req_ids)
|
||||
if not recving and not sending:
|
||||
continue
|
||||
# Aggregate finished recving request ids.
|
||||
finished_recving.update(recving or ())
|
||||
# Aggregate finished sending request ids - only include
|
||||
# once we've drained the "extra" count (for cases where
|
||||
# more than one connector is async-saving the same request).
|
||||
for req_id in sending or ():
|
||||
results = self.get_transfer_results(finished_req_ids)
|
||||
return (
|
||||
results.finished_sending or None,
|
||||
results.finished_recving or None,
|
||||
)
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
results = KVConnectorTransferResults()
|
||||
for connector in self._connectors:
|
||||
child_results = connector.get_transfer_results(finished_req_ids)
|
||||
results.finished_recving.update(child_results.finished_recving)
|
||||
results.failed_recving.update(child_results.failed_recving)
|
||||
for req_id in child_results.finished_sending:
|
||||
extra_pending = self._extra_async_saves.get(req_id)
|
||||
if extra_pending is None:
|
||||
finished_sending.add(req_id)
|
||||
continue
|
||||
assert extra_pending > 0
|
||||
if extra_pending == 1:
|
||||
del self._extra_async_saves[req_id]
|
||||
results.finished_sending.add(req_id)
|
||||
else:
|
||||
self._extra_async_saves[req_id] = extra_pending - 1
|
||||
|
||||
return finished_sending or None, finished_recving or None
|
||||
assert extra_pending > 0
|
||||
if extra_pending == 1:
|
||||
del self._extra_async_saves[req_id]
|
||||
else:
|
||||
self._extra_async_saves[req_id] = extra_pending - 1
|
||||
return results
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
agg_block_ids: set[int] = set()
|
||||
@@ -563,6 +582,9 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
for c in self._connectors:
|
||||
yield from c.take_events()
|
||||
|
||||
def has_pending_block_frees(self) -> bool:
|
||||
return any(c.has_pending_block_frees() for c in self._connectors)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
return any(c.has_pending_push_work() for c in self._connectors)
|
||||
|
||||
@@ -630,7 +652,7 @@ class MultiConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
return MultiKVConnectorStats(data=reconstructed_data)
|
||||
|
||||
def get_kv_connector_stats(self) -> MultiKVConnectorStats | None:
|
||||
def get_kv_connector_stats(self) -> KVConnectorStats | None:
|
||||
# Group connector stats by connector type.
|
||||
stats_by_connector: MultiKVConnectorStats | None = None
|
||||
for c in self._connectors:
|
||||
|
||||
@@ -93,10 +93,7 @@ class NixlBaseConnectorScheduler:
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
)
|
||||
self._has_mamba = any(
|
||||
isinstance(g.kv_cache_spec, MambaSpec)
|
||||
for g in kv_cache_config.transfer_groups
|
||||
)
|
||||
self._has_mamba = kv_cache_config.has_mamba_layers
|
||||
|
||||
logger.info("Initializing NIXL Scheduler %s", engine_id)
|
||||
if vllm_config.scheduler_config.disable_hybrid_kv_cache_manager:
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Base worker-side logic for the NIXL connector."""
|
||||
|
||||
import contextlib
|
||||
import itertools
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import queue
|
||||
import threading
|
||||
@@ -31,7 +33,10 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
kv_postprocess_blksize_on_receive,
|
||||
kv_postprocess_layout_on_receive,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import CopyBlocksOp
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
CopyBlocksOp,
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import KVConnectorStats
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
GET_META_MSG,
|
||||
@@ -186,23 +191,28 @@ class NixlBaseConnectorWorker:
|
||||
num_blocks = dst_num_blocks
|
||||
if block_size_ratio is not None:
|
||||
num_blocks = int(num_blocks * block_size_ratio)
|
||||
num_regions = self.num_regions or len(self.block_len_per_layer)
|
||||
if region_num_blocks is None:
|
||||
region_num_blocks = [num_blocks] * self.num_regions
|
||||
region_num_blocks = [num_blocks] * num_regions
|
||||
elif block_size_ratio is not None:
|
||||
region_num_blocks = [
|
||||
int(count * block_size_ratio) for count in region_num_blocks
|
||||
]
|
||||
assert len(region_num_blocks) == self.num_regions
|
||||
if region_num_blocks:
|
||||
num_regions = len(region_num_blocks)
|
||||
assert len(region_num_blocks) == num_regions
|
||||
region_offsets = np.cumsum([0, *region_num_blocks[:-1]])
|
||||
num_fa_descs = sum(region_num_blocks)
|
||||
|
||||
# All-attention fast path: single vectorized broadcast.
|
||||
# All-attention fast path.
|
||||
if num_ssm_regions == 0:
|
||||
if region_group_ids is None:
|
||||
region_group_ids = self.region_group_ids
|
||||
if uses_region_group_mapping is None:
|
||||
uses_region_group_mapping = self._uses_region_group_mapping
|
||||
assert len(region_group_ids) == self.num_regions
|
||||
if not region_group_ids and num_regions == 1:
|
||||
region_group_ids = [0]
|
||||
assert len(region_group_ids) == num_regions
|
||||
if uses_region_group_mapping is None:
|
||||
uses_region_group_mapping = len(set(region_group_ids)) > 1
|
||||
region_group_ids_array = np.asarray(region_group_ids, dtype=np.int32)
|
||||
@@ -397,8 +407,6 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
# Config.
|
||||
self.vllm_config = vllm_config
|
||||
# mypy will complain on re-assignment otherwise.
|
||||
self.block_size: int = cast(int, vllm_config.cache_config.block_size)
|
||||
|
||||
if vllm_config.kv_transfer_config is None:
|
||||
raise ValueError("kv_transfer_config must be set for NixlConnector")
|
||||
@@ -420,6 +428,17 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
|
||||
self.kv_cache_config = kv_cache_config
|
||||
transfer_block_sizes = [
|
||||
group.kv_cache_spec.block_size
|
||||
for group in kv_cache_config.transfer_groups
|
||||
if get_representative_spec_type(group.kv_cache_spec)
|
||||
not in (MambaSpec, CircularBufferSpec)
|
||||
]
|
||||
self.block_size = (
|
||||
math.lcm(*transfer_block_sizes)
|
||||
if transfer_block_sizes
|
||||
else cast(int, vllm_config.cache_config.block_size)
|
||||
)
|
||||
# Per-layer specs, unwrapping UniformTypeKVCacheSpecs group wrappers.
|
||||
self._layer_specs: dict[str, KVCacheSpec] = {}
|
||||
for group in kv_cache_config.transfer_groups:
|
||||
@@ -599,12 +618,15 @@ class NixlBaseConnectorWorker:
|
||||
# Number of NIXL regions. Currently one region per cache
|
||||
# (so 1 per layer for MLA, otherwise 2 per layer)
|
||||
self.num_regions = 0
|
||||
self.region_mem_types: list[str] = []
|
||||
self.region_group_ids: list[int] = []
|
||||
self._uses_region_group_mapping = False
|
||||
self.region_names: list[str] = []
|
||||
self.region_num_blocks: list[int] = []
|
||||
self.region_mem_types: list[str] = []
|
||||
self._mixed_mem_types = False
|
||||
self._desc_is_dram_by_block_size: dict[int, np.ndarray] = {}
|
||||
self._desc_pos_by_block_size: dict[int, np.ndarray] = {}
|
||||
self._dram_src_handles_by_block_size: dict[int, int] = {}
|
||||
|
||||
# PP>1 (push mode): this worker holds a contiguous layer slice and
|
||||
# transfers into the matching sub-range of a PP=1 remote's regions.
|
||||
@@ -644,9 +666,6 @@ class NixlBaseConnectorWorker:
|
||||
self.dst_region_group_ids: dict[EngineId, list[int]] = {}
|
||||
self.dst_uses_region_group_mapping: dict[EngineId, bool] = {}
|
||||
self.dst_region_mem_types: dict[EngineId, list[str]] = {}
|
||||
self._desc_is_dram_by_block_size: dict[int, np.ndarray] = {}
|
||||
self._desc_pos_by_block_size: dict[int, np.ndarray] = {}
|
||||
self._dram_src_handles_by_block_size: dict[int, int] = {}
|
||||
self._registered_descs: list[Any] = []
|
||||
|
||||
# In progress transfers.
|
||||
@@ -667,6 +686,7 @@ class NixlBaseConnectorWorker:
|
||||
# Uses Queue for thread-safe cross-thread coordination with the
|
||||
# background handshake thread, matching the _ready_requests pattern.
|
||||
self._failed_recv_reqs: queue.Queue[ReqId] = queue.Queue()
|
||||
self._failed_inflight_recvs: set[ReqId] = set()
|
||||
self._pending_recv_notifs: dict[ReqId, list[tuple[str, bytes]]] = {}
|
||||
|
||||
# Handshake metadata of this worker for NIXL transfers.
|
||||
@@ -690,7 +710,6 @@ class NixlBaseConnectorWorker:
|
||||
"engine_ttl", 3600.0
|
||||
)
|
||||
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
self.model_config = vllm_config.model_config
|
||||
|
||||
self.use_mla = self.model_config.use_mla
|
||||
@@ -1240,7 +1259,7 @@ class NixlBaseConnectorWorker:
|
||||
self.use_host_buffer,
|
||||
)
|
||||
|
||||
registration_ranges: dict[tuple[int, str], tuple[int, int, int, str]] = {}
|
||||
registration_ranges: dict[tuple[int, str], tuple[int, int, int]] = {}
|
||||
region_mem_types: list[str] = []
|
||||
seen_base_addresses: list[int] = []
|
||||
self._ssm_region_indices = []
|
||||
@@ -1280,10 +1299,16 @@ class NixlBaseConnectorWorker:
|
||||
# K and V are packed into the content dim, so each attention layer is a
|
||||
# single NIXL region whose block transfers as one unit. Mamba layers instead
|
||||
# register separate conv/ssm sub-regions (see `_build_mamba_local`).
|
||||
# P and D may allocate equivalent transferable layers in different
|
||||
# cache-group orders. Keep their region lists aligned without putting
|
||||
# layers.10 before layers.2, which would break PP region slicing.
|
||||
layer_names = (
|
||||
xfer_buffers
|
||||
if self._is_csa_linear
|
||||
else sorted(xfer_buffers, key=_region_sort_key)
|
||||
else sorted(
|
||||
xfer_buffers,
|
||||
key=_region_sort_key,
|
||||
)
|
||||
)
|
||||
for layer_name in layer_names:
|
||||
cache = xfer_buffers[layer_name]
|
||||
@@ -1299,6 +1324,7 @@ class NixlBaseConnectorWorker:
|
||||
layer_name,
|
||||
)
|
||||
continue
|
||||
group_index = self.kv_cache_config.transfer_group_index_by_layer[layer_name]
|
||||
# `layer_spec.page_size_bytes` only accounts for logical page_size, that is
|
||||
# the page_size assuming constant `self._logical_num_blocks`.
|
||||
physical_page_size = (
|
||||
@@ -1307,17 +1333,48 @@ class NixlBaseConnectorWorker:
|
||||
else layer_spec.page_size_bytes
|
||||
// self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
group_id = self.kv_cache_config.transfer_group_index_by_layer[layer_name]
|
||||
group = self.kv_cache_config.transfer_groups[group_index]
|
||||
if group.host_resident:
|
||||
logical_num_blocks = self.kv_cache_config.hisparse_host_num_blocks
|
||||
assert logical_num_blocks is not None
|
||||
else:
|
||||
logical_num_blocks = self.kv_cache_config.num_blocks
|
||||
group_id = group_index
|
||||
num_blocks = (
|
||||
self._logical_num_blocks
|
||||
logical_num_blocks
|
||||
if isinstance(layer_spec, MambaSpec)
|
||||
else self.num_blocks
|
||||
else logical_num_blocks * self._physical_blocks_per_logical_kv_block
|
||||
)
|
||||
base_addr = cache.data_ptr()
|
||||
is_mla_region = isinstance(
|
||||
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
|
||||
)
|
||||
logger.debug(
|
||||
"Registering layer %s with cache shape: %s", layer_name, cache.shape
|
||||
)
|
||||
storage = cache.untyped_storage()
|
||||
storage_addr = storage.data_ptr()
|
||||
is_host_resident = group.host_resident
|
||||
if cache.device.type == "cpu":
|
||||
mem_type = "DRAM"
|
||||
region_device_id = 0
|
||||
else:
|
||||
mem_type = self.nixl_memory_type
|
||||
region_device_id = max(cache.get_device(), 0)
|
||||
self.device_id = region_device_id
|
||||
if is_host_resident:
|
||||
registration_base = cache.data_ptr()
|
||||
registration_len = cache.nbytes
|
||||
else:
|
||||
registration_base = storage_addr
|
||||
registration_len = storage.nbytes()
|
||||
storage_key = (registration_base, mem_type)
|
||||
if storage_key not in registration_ranges:
|
||||
registration_ranges[storage_key] = (
|
||||
registration_base,
|
||||
registration_base + registration_len,
|
||||
region_device_id,
|
||||
)
|
||||
|
||||
if isinstance(layer_spec, KpoolTailSpec):
|
||||
compressed_owner = compressed_region_owners.get(cache.data_ptr())
|
||||
@@ -1345,22 +1402,6 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
continue
|
||||
|
||||
if cache.device.type == "cpu":
|
||||
mem_type = "DRAM"
|
||||
region_device_id = 0
|
||||
else:
|
||||
mem_type = self.nixl_memory_type
|
||||
region_device_id = max(cache.get_device(), 0)
|
||||
self.device_id = region_device_id
|
||||
registration_ranges.setdefault(
|
||||
(storage_addr, mem_type),
|
||||
(storage_addr, storage.nbytes(), region_device_id, ""),
|
||||
)
|
||||
|
||||
is_mla_region = isinstance(
|
||||
layer_spec, (MLAAttentionSpec, SlidingWindowMLASpec)
|
||||
)
|
||||
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
physical_ratio = self._physical_blocks_per_logical_kv_block
|
||||
block_len = physical_page_size // physical_ratio
|
||||
@@ -1381,14 +1422,13 @@ class NixlBaseConnectorWorker:
|
||||
block_stride = physical_page_size
|
||||
else:
|
||||
block_stride = cache.stride(0) * cache.element_size()
|
||||
storage_is_block_major = num_blocks * block_stride == storage.nbytes()
|
||||
# A layer whose [H, N, C] interior is dense addresses its own page
|
||||
# as one chunk. Otherwise the block's whole row is the only
|
||||
# contiguous transfer unit.
|
||||
hnc_contiguous = (
|
||||
cache.ndim == 4
|
||||
and cache.stride(2) == cache.shape[3]
|
||||
and cache.stride(1) == cache.shape[2] * cache.shape[3]
|
||||
storage_is_block_major = num_blocks * block_stride == registration_len
|
||||
# A view with a contiguous page can own a strided region even
|
||||
# when sibling layers share the same backing allocation.
|
||||
page_contiguous = (
|
||||
cache.ndim > 1
|
||||
and cache[0].is_contiguous()
|
||||
and cache[0].nbytes == physical_page_size
|
||||
)
|
||||
virtual_transfer_pages = _uses_dense_virtual_transfer_pages(
|
||||
layer_spec, cache, physical_page_size, num_blocks
|
||||
@@ -1399,13 +1439,14 @@ class NixlBaseConnectorWorker:
|
||||
(cache.data_ptr(), physical_page_size, physical_page_size)
|
||||
]
|
||||
elif storage_is_block_major and (
|
||||
(packed_storage and is_mla_region)
|
||||
or (not hnc_contiguous and not self._is_csa_linear)
|
||||
not page_contiguous
|
||||
and ((packed_storage and is_mla_region) or not self._is_csa_linear)
|
||||
):
|
||||
# Packed MLA layouts transfer the complete storage row.
|
||||
storage_block_len = storage.nbytes() // num_blocks
|
||||
# TODO(Lucas): handle TP slicing for packed_storage; for now
|
||||
# restrict to MLA (DSv4) where kv is replicated.
|
||||
storage_block_len = registration_len // num_blocks
|
||||
region_specs = [
|
||||
(storage_addr, storage_block_len, storage_block_len)
|
||||
(registration_base, storage_block_len, storage_block_len)
|
||||
]
|
||||
elif storage_is_block_major:
|
||||
region_specs = [
|
||||
@@ -1440,6 +1481,10 @@ class NixlBaseConnectorWorker:
|
||||
region_index = seen_base_addresses.index(base_addr)
|
||||
assert region_mem_types[region_index] == mem_type
|
||||
self._region_is_mla[region_index] |= is_mla_region
|
||||
if is_mla_region:
|
||||
self.block_len_per_layer[region_index] = block_len
|
||||
self.block_stride_per_layer[region_index] = block_stride
|
||||
self.region_num_blocks[region_index] = num_blocks
|
||||
if self.region_group_ids[region_index] != group_id:
|
||||
self.region_group_ids[region_index] = _SHARED_REGION_GROUP_ID
|
||||
else:
|
||||
@@ -1450,8 +1495,8 @@ class NixlBaseConnectorWorker:
|
||||
self.region_group_ids.append(group_id)
|
||||
self.region_names.append(layer_name)
|
||||
self.region_num_blocks.append(num_blocks)
|
||||
region_mem_types.append(mem_type)
|
||||
self._region_is_mla.append(is_mla_region)
|
||||
region_mem_types.append(mem_type)
|
||||
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
if layer_spec.tp_replicated:
|
||||
@@ -1496,7 +1541,6 @@ class NixlBaseConnectorWorker:
|
||||
== len(self.region_group_ids)
|
||||
== len(self.region_names)
|
||||
== len(self.region_num_blocks)
|
||||
== len(region_mem_types)
|
||||
)
|
||||
# Descriptor ids must be region-ordered, matching the remote side.
|
||||
self._scratch_region_indices.sort()
|
||||
@@ -1521,16 +1565,23 @@ class NixlBaseConnectorWorker:
|
||||
# Total local FA descriptors (boundary between FA and mamba descs).
|
||||
self.num_descs = sum(self.region_num_blocks)
|
||||
|
||||
self._mixed_mem_types = len(set(region_mem_types)) > 1
|
||||
if self._mixed_mem_types:
|
||||
assert self.use_mla and not self._has_mamba, (
|
||||
"Mixed-device KV registration is only supported for MLA "
|
||||
"models without Mamba layers."
|
||||
)
|
||||
for mem_type in sorted(set(region_mem_types)):
|
||||
ranges = [
|
||||
registration
|
||||
for (
|
||||
_,
|
||||
registration_mem_type,
|
||||
), registration in registration_ranges.items()
|
||||
if registration_mem_type == mem_type
|
||||
ranges_for_mem_type = [
|
||||
(start, end - start, device_id, "")
|
||||
for (_, cache_mem_type), (
|
||||
start,
|
||||
end,
|
||||
device_id,
|
||||
) in registration_ranges.items()
|
||||
if cache_mem_type == mem_type
|
||||
]
|
||||
descs = self.nixl_wrapper.get_reg_descs(ranges, mem_type)
|
||||
descs = self.nixl_wrapper.get_reg_descs(ranges_for_mem_type, mem_type)
|
||||
self.nixl_wrapper.register_memory(descs, backends=self.nixl_backends)
|
||||
self._registered_descs.append(descs)
|
||||
|
||||
@@ -1854,7 +1905,11 @@ class NixlBaseConnectorWorker:
|
||||
if self._mixed_mem_types:
|
||||
desc_is_dram = np.concatenate(
|
||||
[
|
||||
np.full(count * block_size_ratio, mem_type == "DRAM", dtype=bool)
|
||||
np.full(
|
||||
count * block_size_ratio,
|
||||
mem_type == "DRAM",
|
||||
dtype=bool,
|
||||
)
|
||||
for mem_type, count in zip(
|
||||
self.region_mem_types,
|
||||
self.region_num_blocks,
|
||||
@@ -1864,24 +1919,29 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
assert len(desc_is_dram) == len(blocks_data)
|
||||
desc_pos = np.empty(len(desc_is_dram), dtype=np.int64)
|
||||
dram_indices = np.flatnonzero(desc_is_dram)
|
||||
device_indices = np.flatnonzero(~desc_is_dram)
|
||||
desc_pos[dram_indices] = np.arange(len(dram_indices), dtype=np.int64)
|
||||
desc_pos[device_indices] = np.arange(len(device_indices), dtype=np.int64)
|
||||
dram_idx = np.where(desc_is_dram)[0]
|
||||
vram_idx = np.where(~desc_is_dram)[0]
|
||||
desc_pos[dram_idx] = np.arange(len(dram_idx), dtype=np.int64)
|
||||
desc_pos[vram_idx] = np.arange(len(vram_idx), dtype=np.int64)
|
||||
self._desc_is_dram_by_block_size[block_size] = desc_is_dram
|
||||
self._desc_pos_by_block_size[block_size] = desc_pos
|
||||
blocks_data[desc_is_dram, 2] = 0
|
||||
dram_descs = self.nixl_wrapper.get_xfer_descs(
|
||||
blocks_data[desc_is_dram], "DRAM"
|
||||
)
|
||||
|
||||
# DRAM descriptors are registered under CPU device 0.
|
||||
blocks_data = [
|
||||
(addr, length, 0) if is_dram else (addr, length, dev)
|
||||
for (addr, length, dev), is_dram in zip(
|
||||
blocks_data, desc_is_dram, strict=True
|
||||
)
|
||||
]
|
||||
dram_blocks = [blocks_data[i] for i in dram_idx]
|
||||
vram_blocks = [blocks_data[i] for i in vram_idx]
|
||||
dram_descs = self.nixl_wrapper.get_xfer_descs(dram_blocks, "DRAM")
|
||||
self._dram_src_handles_by_block_size[block_size] = (
|
||||
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", dram_descs)
|
||||
)
|
||||
device_descs = self.nixl_wrapper.get_xfer_descs(
|
||||
blocks_data[~desc_is_dram], self.nixl_memory_type
|
||||
)
|
||||
descs = self.nixl_wrapper.get_xfer_descs(vram_blocks, self.nixl_memory_type)
|
||||
return (
|
||||
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", device_descs),
|
||||
self.nixl_wrapper.prep_xfer_dlist("NIXL_INIT_AGENT", descs),
|
||||
blocks_data,
|
||||
)
|
||||
|
||||
@@ -1968,10 +2028,6 @@ class NixlBaseConnectorWorker:
|
||||
]
|
||||
nixl_agent_meta.block_lens = nixl_agent_meta.block_lens[start:end]
|
||||
nixl_agent_meta.block_strides = nixl_agent_meta.block_strides[start:end]
|
||||
if nixl_agent_meta.region_num_blocks is not None:
|
||||
nixl_agent_meta.region_num_blocks = nixl_agent_meta.region_num_blocks[
|
||||
start:end
|
||||
]
|
||||
if nixl_agent_meta.region_group_ids is not None:
|
||||
nixl_agent_meta.region_group_ids = nixl_agent_meta.region_group_ids[
|
||||
start:end
|
||||
@@ -1982,6 +2038,10 @@ class NixlBaseConnectorWorker:
|
||||
nixl_agent_meta.region_mem_types = nixl_agent_meta.region_mem_types[
|
||||
start:end
|
||||
]
|
||||
if nixl_agent_meta.region_num_blocks is not None:
|
||||
nixl_agent_meta.region_num_blocks = nixl_agent_meta.region_num_blocks[
|
||||
start:end
|
||||
]
|
||||
|
||||
### Register remote engine in TransferTopology (idempotent).
|
||||
assert self.transfer_topo is not None
|
||||
@@ -2026,11 +2086,12 @@ class NixlBaseConnectorWorker:
|
||||
nixl_agent_meta.region_num_blocks
|
||||
or [nixl_agent_meta.num_blocks] * num_remote_regions
|
||||
)
|
||||
self.dst_region_group_ids[engine_id] = nixl_agent_meta.region_group_ids or (
|
||||
remote_region_group_ids = nixl_agent_meta.region_group_ids or (
|
||||
self.region_group_ids
|
||||
if len(self.region_group_ids) == num_remote_regions
|
||||
else [0] * num_remote_regions
|
||||
)
|
||||
self.dst_region_group_ids[engine_id] = remote_region_group_ids
|
||||
self.dst_uses_region_group_mapping[engine_id] = (
|
||||
len(set(self.dst_region_group_ids[engine_id])) > 1
|
||||
)
|
||||
@@ -2138,8 +2199,6 @@ class NixlBaseConnectorWorker:
|
||||
mamba = self._build_mamba_remote(nixl_agent_meta, tp_ratio, transfer_info)
|
||||
blocks_data = np.concatenate([blocks_data, mamba])
|
||||
|
||||
# A mixed local destination is split into multiple READs below. The
|
||||
# producer remains a single descriptor list in this prerequisite.
|
||||
remote_mem_types = set(self.dst_region_mem_types[engine_id])
|
||||
if len(remote_mem_types) != 1:
|
||||
raise NotImplementedError(
|
||||
@@ -2540,9 +2599,10 @@ class NixlBaseConnectorWorker:
|
||||
indices=indices,
|
||||
)
|
||||
|
||||
def get_finished(self) -> tuple[set[str], set[str]]:
|
||||
def get_transfer_results(self) -> KVConnectorTransferResults:
|
||||
"""
|
||||
Get requests that are done sending or recving on this specific worker.
|
||||
Get transfers that completed on this specific worker.
|
||||
|
||||
The scheduler process (via the MultiprocExecutor) will use this output
|
||||
to track which workers are done.
|
||||
"""
|
||||
@@ -2564,7 +2624,6 @@ class NixlBaseConnectorWorker:
|
||||
# Add failed requests to done_recving for scheduler tracking
|
||||
# (blocks are already marked invalid, scheduler will handle recompute)
|
||||
done_recving.update(failed_recv_reqs)
|
||||
|
||||
if len(done_sending) > 0 or len(done_recving) > 0:
|
||||
logger.debug(
|
||||
"Rank %s, get_finished: %s requests done sending "
|
||||
@@ -2577,6 +2636,7 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
block_ids_for_blocksize_post_process = defaultdict(list)
|
||||
block_ids_for_heterogeneous_attn_post_process = list[list[int]]()
|
||||
direct_device_recving = set[str]()
|
||||
for req_id in done_recving:
|
||||
# clean up metadata for completed requests
|
||||
meta = self._recving_metadata.pop(req_id, None)
|
||||
@@ -2594,6 +2654,8 @@ class NixlBaseConnectorWorker:
|
||||
if self.use_host_buffer:
|
||||
self.sync_recved_kv_to_device(req_id, meta)
|
||||
|
||||
direct_device_recving.add(req_id)
|
||||
|
||||
# Post processing for heteroblocksize/layout, and for blocks the
|
||||
# transfer clipped. The latter happens either at remote-block
|
||||
# granularity (block_size_ratio > 1) or at kernel-block
|
||||
@@ -2642,7 +2704,7 @@ class NixlBaseConnectorWorker:
|
||||
for block_ids in block_ids_for_heterogeneous_attn_post_process:
|
||||
self.post_process_device_kv_on_receive_heterogeneous_attn(block_ids)
|
||||
|
||||
self._sync_device_after_mamba_recv(done_recving, failed_recv_reqs)
|
||||
self._sync_device_after_direct_recv(direct_device_recving)
|
||||
|
||||
# Handle timeout to avoid stranding blocks on remote.
|
||||
now = time.perf_counter()
|
||||
@@ -2664,20 +2726,21 @@ class NixlBaseConnectorWorker:
|
||||
del self._reqs_to_send[req_id]
|
||||
done_sending.add(req_id)
|
||||
|
||||
return done_sending, done_recving
|
||||
return KVConnectorTransferResults(
|
||||
finished_sending=done_sending,
|
||||
finished_recving=done_recving,
|
||||
failed_recving=failed_recv_reqs,
|
||||
)
|
||||
|
||||
def _sync_device_after_mamba_recv(
|
||||
self,
|
||||
done_recving: set[str],
|
||||
failed_recv_reqs: set[str],
|
||||
) -> None:
|
||||
"""Synchronize ROCm direct-GPU Mamba receives before model execution."""
|
||||
if (
|
||||
not current_platform.is_rocm()
|
||||
or not self._has_mamba
|
||||
or self.use_host_buffer
|
||||
or not (done_recving - failed_recv_reqs)
|
||||
):
|
||||
def get_finished(self) -> tuple[set[str], set[str]]:
|
||||
"""Compatibility wrapper for the legacy completion API."""
|
||||
results = self.get_transfer_results()
|
||||
return results.finished_sending, results.finished_recving
|
||||
|
||||
def _sync_device_after_direct_recv(self, done_recving: set[str]) -> None:
|
||||
"""Make direct NIXL writes visible before model execution."""
|
||||
requires_sync = current_platform.is_rocm() and self._has_mamba
|
||||
if self.use_host_buffer or not done_recving or not requires_sync:
|
||||
return
|
||||
|
||||
torch.accelerator.synchronize()
|
||||
@@ -2731,11 +2794,11 @@ class NixlBaseConnectorWorker:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
elif xfer_state == "PROC":
|
||||
in_progress.append(handle)
|
||||
continue
|
||||
else:
|
||||
self._log_failure(
|
||||
failure_type="transfer_failed",
|
||||
msg="Marking blocks as invalid",
|
||||
msg="Deferring request completion until its last "
|
||||
"xfer is terminal",
|
||||
req_id=req_id,
|
||||
xfer_state=xfer_state,
|
||||
)
|
||||
@@ -2743,21 +2806,20 @@ class NixlBaseConnectorWorker:
|
||||
except Exception as e:
|
||||
self._log_failure(
|
||||
failure_type="transfer_exception",
|
||||
msg="Marking blocks as invalid",
|
||||
msg="Handle is unpollable; treating it as terminal",
|
||||
req_id=req_id,
|
||||
error=e,
|
||||
)
|
||||
self._handle_failed_transfer(req_id, handle)
|
||||
|
||||
if not in_progress:
|
||||
# Only report request as completed when all transfers are done.
|
||||
# A request failed in an earlier poll was already reported via
|
||||
# _failed_recv_reqs and its metadata popped by get_finished();
|
||||
# don't report it again, just drop the remaining handles.
|
||||
if req_id in self._recving_metadata:
|
||||
done_req_ids.add(req_id)
|
||||
self._send_pending_recv_notifs(req_id)
|
||||
del transfers[req_id]
|
||||
done_req_ids.add(req_id)
|
||||
if req_id in self._failed_inflight_recvs:
|
||||
self._failed_inflight_recvs.remove(req_id)
|
||||
self._report_failed_recv(req_id)
|
||||
else:
|
||||
self._send_pending_recv_notifs(req_id)
|
||||
else:
|
||||
transfers[req_id] = in_progress
|
||||
return done_req_ids
|
||||
@@ -2771,22 +2833,25 @@ class NixlBaseConnectorWorker:
|
||||
req_id: The request ID.
|
||||
handle: The transfer handle.
|
||||
"""
|
||||
# (multi-read) One handle is created per remote rank, and they do not
|
||||
# all fail in the same _pop_done_transfers poll. The request is
|
||||
# reported failed on the first one, which pops its metadata in
|
||||
# get_finished(); on the later failures only the handle cleanup is left.
|
||||
# TODO (NickLucche) handle failed transfer for HMA.
|
||||
# A split READ's notification is sent only after every handle succeeds.
|
||||
# A sibling READ may still be writing these blocks. Retain metadata
|
||||
# and defer invalidation until every handle is terminal.
|
||||
self._pending_recv_notifs.pop(req_id, None)
|
||||
if (meta := self._recving_metadata.get(req_id)) is not None:
|
||||
if not self._is_hma_required:
|
||||
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
if req_id in self._recving_transfers:
|
||||
self._failed_inflight_recvs.add(req_id)
|
||||
else:
|
||||
self._report_failed_recv(req_id)
|
||||
if handle is not None:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self.xfer_stats.record_failed_transfer()
|
||||
|
||||
def _report_failed_recv(self, req_id: str) -> None:
|
||||
if (meta := self._recving_metadata.get(req_id)) is not None:
|
||||
if not self._is_hma_required:
|
||||
self._invalid_block_ids.put(set(meta.local_block_ids[0]))
|
||||
self._failed_recv_reqs.put(req_id)
|
||||
|
||||
def _send_pending_recv_notifs(self, req_id: str) -> None:
|
||||
"""Send notifications deferred by split DRAM/VRAM reads."""
|
||||
for agent_name, notif_id in self._pending_recv_notifs.pop(req_id, []):
|
||||
try:
|
||||
self.nixl_wrapper.send_notif(agent_name, notif_msg=notif_id)
|
||||
@@ -2909,10 +2974,10 @@ class NixlBaseConnectorWorker:
|
||||
return block_ids
|
||||
block_arange = np.arange(0, ratio).reshape(1, -1)
|
||||
# Mamba blocks have no logical<>physical discrepancy (block-size=1)
|
||||
group_specs = self.kv_cache_config.transfer_groups
|
||||
physical_block_ids = []
|
||||
for i, group in enumerate(block_ids):
|
||||
if _is_ssm_spec(get_representative_spec_type(group_specs[i].kv_cache_spec)):
|
||||
spec = self.kv_cache_config.transfer_groups[i].kv_cache_spec
|
||||
if _is_ssm_spec(get_representative_spec_type(spec)):
|
||||
physical_block_ids.append(group)
|
||||
else:
|
||||
physical_block_ids.append(
|
||||
@@ -3215,9 +3280,44 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
|
||||
def __del__(self):
|
||||
self.shutdown()
|
||||
with contextlib.suppress(Exception):
|
||||
self.shutdown()
|
||||
|
||||
def shutdown(self):
|
||||
def _finish_shutdown(self) -> None:
|
||||
self._recving_transfers.clear()
|
||||
try:
|
||||
for handle in self.src_xfer_handles_by_block_size.values():
|
||||
self.nixl_wrapper.release_dlist_handle(handle)
|
||||
for handles in self.src_xfer_handles_by_tp_ratio.values():
|
||||
for handle in handles:
|
||||
self.nixl_wrapper.release_dlist_handle(handle)
|
||||
for handles in self._dram_src_handles_by_tp_ratio.values():
|
||||
for handle in handles:
|
||||
self.nixl_wrapper.release_dlist_handle(handle)
|
||||
for handle in self._dram_src_handles_by_block_size.values():
|
||||
self.nixl_wrapper.release_dlist_handle(handle)
|
||||
except Exception:
|
||||
logger.exception("NIXL dlist-handle release failed at shutdown.")
|
||||
self.src_xfer_handles_by_block_size.clear()
|
||||
self.src_xfer_handles_by_tp_ratio.clear()
|
||||
self._dram_src_handles_by_tp_ratio.clear()
|
||||
self._dram_src_handles_by_block_size.clear()
|
||||
try:
|
||||
for engine_id in list(self._remote_agents):
|
||||
self._cleanup_remote_engine(engine_id, log_eviction=False)
|
||||
except Exception:
|
||||
logger.exception("NIXL remote-engine cleanup failed at shutdown.")
|
||||
try:
|
||||
for desc in self._registered_descs:
|
||||
self.nixl_wrapper.deregister_memory(desc)
|
||||
finally:
|
||||
self._registered_descs.clear()
|
||||
# Drop cache references before their owners release registered
|
||||
# host memory; handshake futures may outlive model-runner shutdown.
|
||||
self.device_kv_caches = {}
|
||||
self.host_xfer_buffers = {}
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Shutdown the connector worker."""
|
||||
if not hasattr(self, "_handshake_initiation_executor"):
|
||||
# error happens during init, no need to shutdown
|
||||
@@ -3227,22 +3327,4 @@ class NixlBaseConnectorWorker:
|
||||
for handle in handles:
|
||||
self.nixl_wrapper.release_xfer_handle(handle)
|
||||
self._recving_transfers.clear()
|
||||
for handle in self.src_xfer_handles_by_block_size.values():
|
||||
self.nixl_wrapper.release_dlist_handle(handle)
|
||||
self.src_xfer_handles_by_block_size.clear()
|
||||
for handles in self.src_xfer_handles_by_tp_ratio.values():
|
||||
for handle in handles:
|
||||
self.nixl_wrapper.release_dlist_handle(handle)
|
||||
self.src_xfer_handles_by_tp_ratio.clear()
|
||||
for handles in self._dram_src_handles_by_tp_ratio.values():
|
||||
for handle in handles:
|
||||
self.nixl_wrapper.release_dlist_handle(handle)
|
||||
self._dram_src_handles_by_tp_ratio.clear()
|
||||
for handle in self._dram_src_handles_by_block_size.values():
|
||||
self.nixl_wrapper.release_dlist_handle(handle)
|
||||
self._dram_src_handles_by_block_size.clear()
|
||||
for engine_id in list(self._remote_agents):
|
||||
self._cleanup_remote_engine(engine_id, log_eviction=False)
|
||||
for desc in self._registered_descs:
|
||||
self.nixl_wrapper.deregister_memory(desc)
|
||||
self._registered_descs.clear()
|
||||
self._finish_shutdown()
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorHandshakeMetadata,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorTransferResults,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
@@ -234,6 +235,18 @@ class NixlBaseConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.get_finished()
|
||||
|
||||
def get_transfer_results(
|
||||
self, finished_req_ids: set[str]
|
||||
) -> KVConnectorTransferResults:
|
||||
assert self.connector_worker is not None
|
||||
results = self.connector_worker.get_transfer_results()
|
||||
if (
|
||||
self.kv_transfer_config.kv_role == "kv_producer"
|
||||
and self.connector_worker.pcp_rank > 0
|
||||
):
|
||||
results.finished_sending.clear()
|
||||
return results
|
||||
|
||||
def get_block_ids_with_load_errors(self) -> set[int]:
|
||||
"""Get block IDs that failed to load via NIXL."""
|
||||
assert self.connector_worker is not None
|
||||
|
||||
@@ -305,7 +305,9 @@ class NixlConnectorMetadata(KVConnectorMetadata):
|
||||
local_num_computed_blocks: tuple[int, ...] = (),
|
||||
):
|
||||
req = self._add_new_req(
|
||||
local_block_ids, kv_transfer_params, local_num_computed_blocks
|
||||
local_block_ids,
|
||||
kv_transfer_params,
|
||||
local_num_computed_blocks,
|
||||
)
|
||||
req.remote = RemoteMeta(
|
||||
block_ids=kv_transfer_params["remote_block_ids"],
|
||||
|
||||
@@ -166,11 +166,10 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
|
||||
dcp_active = self.dcp_size > 1 or remote_info.remote_dcp_size > 1
|
||||
local_block_ids = meta.local_physical_block_ids
|
||||
local_region_groups = getattr(self, "region_group_ids", [])
|
||||
remote_region_groups = getattr(self, "dst_region_group_ids", {}).get(
|
||||
engine_id, local_region_groups
|
||||
)
|
||||
if local_region_groups != remote_region_groups:
|
||||
remote_region_groups = self.dst_region_group_ids[engine_id]
|
||||
local_region_groups = self.region_group_ids or remote_region_groups
|
||||
groups_differ = local_region_groups != remote_region_groups
|
||||
if groups_differ:
|
||||
if not self.use_mla or self._has_mamba:
|
||||
raise NotImplementedError(
|
||||
"Different NIXL cache-group layouts are only supported for "
|
||||
@@ -292,6 +291,9 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
spec.remote_rank
|
||||
]
|
||||
|
||||
# Once a read routes the request to failure reporting, the
|
||||
# scheduler may free and reuse its blocks, so no sibling READs
|
||||
# may be posted (and P must not be notified).
|
||||
if not self._read_blocks(
|
||||
read_spec=spec,
|
||||
request_id=req_id,
|
||||
@@ -329,6 +331,10 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
"""
|
||||
Post a READ point-to-point xfer request from a single local worker to
|
||||
a single remote worker.
|
||||
|
||||
Returns True when the read was posted (or was unnecessary), False
|
||||
when the request was routed to failure reporting — the caller must
|
||||
not post further transfers for it.
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
remote_rank = read_spec.remote_rank
|
||||
@@ -416,11 +422,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
dst_num_blocks=self.dst_num_blocks[dst_engine_id],
|
||||
block_size_ratio=None,
|
||||
physical_blocks_per_logical=remote_info.remote_physical_blocks_per_logical,
|
||||
region_num_blocks=self.dst_region_num_blocks[dst_engine_id],
|
||||
region_num_blocks=(self.dst_region_num_blocks.get(dst_engine_id) or None),
|
||||
region_group_ids=(
|
||||
list(range(self.num_regions))
|
||||
if read_spec.block_ids_by_region
|
||||
else self.dst_region_group_ids[dst_engine_id]
|
||||
else (self.dst_region_group_ids.get(dst_engine_id) or None)
|
||||
),
|
||||
uses_region_group_mapping=(
|
||||
self.num_regions > 1
|
||||
@@ -433,11 +439,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
dst_num_blocks=self.dst_num_blocks[self.engine_id],
|
||||
block_size_ratio=block_size_ratio,
|
||||
physical_blocks_per_logical=self._physical_blocks_per_logical_kv_block,
|
||||
region_num_blocks=self.dst_region_num_blocks[self.engine_id],
|
||||
region_num_blocks=(self.dst_region_num_blocks.get(self.engine_id) or None),
|
||||
region_group_ids=(
|
||||
list(range(self.num_regions))
|
||||
if read_spec.block_ids_by_region
|
||||
else self.region_group_ids
|
||||
else (self.region_group_ids or None)
|
||||
),
|
||||
uses_region_group_mapping=(
|
||||
self.num_regions > 1
|
||||
@@ -537,11 +543,11 @@ class NixlPullConnectorWorker(NixlBaseConnectorWorker):
|
||||
self._pending_recv_notifs.setdefault(request_id, []).append(
|
||||
(notif_agent, notif_id)
|
||||
)
|
||||
for index, handle in enumerate(handles):
|
||||
for i, handle in enumerate(handles):
|
||||
try:
|
||||
self.nixl_wrapper.transfer(handle)
|
||||
except Exception:
|
||||
for unstarted in handles[index:]:
|
||||
for unstarted in handles[i:]:
|
||||
self.nixl_wrapper.release_xfer_handle(unstarted)
|
||||
raise
|
||||
self._recving_transfers[request_id].append(handle)
|
||||
|
||||
@@ -41,6 +41,9 @@ from typing import TYPE_CHECKING, Any
|
||||
import msgspec
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import BlockIds
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorTransferResults,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.base_worker import (
|
||||
NixlBaseConnectorWorker,
|
||||
)
|
||||
@@ -145,7 +148,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
self._push_writer_thread.start()
|
||||
logger.info("nixl-push-writer thread started (rank=%d)", self.tp_rank)
|
||||
|
||||
def shutdown(self):
|
||||
def shutdown(self) -> None:
|
||||
self._push_writer_stop.set()
|
||||
# Unblock the writer if it's waiting in the no-active-state branch.
|
||||
self._push_writer_wake.set()
|
||||
@@ -522,7 +525,14 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
)
|
||||
remote_block_ids = meta.remote.block_ids
|
||||
local_block_ids = meta.local_physical_block_ids
|
||||
num_groups = len(local_block_ids)
|
||||
local_region_groups = self.region_group_ids
|
||||
remote_region_groups = self.dst_region_group_ids[engine_id]
|
||||
groups_differ = local_region_groups != remote_region_groups
|
||||
if groups_differ:
|
||||
raise NotImplementedError(
|
||||
"NixlPushConnector does not support different producer and "
|
||||
"consumer cache-group layouts"
|
||||
)
|
||||
|
||||
# MLA latent is replicated across D's TP ranks: the tp-mapping
|
||||
# collapses it to one rank (fine for reads), but push must WRITE every
|
||||
@@ -537,6 +547,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
else:
|
||||
write_ranks = list(plan.all_source_ranks)
|
||||
|
||||
num_groups = len(local_block_ids)
|
||||
|
||||
def group_ids(block_ids: BlockIds, rank: int) -> BlockIds:
|
||||
return [
|
||||
list(block_ids[g])
|
||||
@@ -777,13 +789,14 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
self._reqs_to_send.pop(req_id, None)
|
||||
return notified_req_ids
|
||||
|
||||
def get_finished(self) -> tuple[set[str], set[str]]:
|
||||
def get_transfer_results(self) -> KVConnectorTransferResults:
|
||||
# Engine main thread asking for completions: also wake the writer
|
||||
# so it gets a chance to drain NIXL notifs (heartbeats, completion
|
||||
# notifs, late PUSH_REGs) even if it had been parked.
|
||||
self._push_writer_wake.set()
|
||||
|
||||
done_sending, done_recving = super().get_finished()
|
||||
results = super().get_transfer_results()
|
||||
done_sending = results.finished_sending
|
||||
|
||||
# ``_pop_done_transfers`` mutates ``_sending_transfers``; the
|
||||
# writer thread also appends to it, so guard the pop.
|
||||
@@ -803,4 +816,4 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
if done_sending:
|
||||
self._push_writer_wake.set()
|
||||
|
||||
return done_sending, done_recving
|
||||
return results
|
||||
|
||||
@@ -12,6 +12,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupRole,
|
||||
KVCacheSpec,
|
||||
MLAAttentionSpec,
|
||||
SlidingWindowMLASpec,
|
||||
@@ -31,6 +32,16 @@ if TYPE_CHECKING:
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
|
||||
def get_offloading_group_ids(kv_cache_config: "KVCacheConfig") -> tuple[int, ...]:
|
||||
if kv_cache_config.hisparse_host_num_blocks is None:
|
||||
return tuple(range(len(kv_cache_config.kv_cache_groups)))
|
||||
return tuple(
|
||||
group_id
|
||||
for group_id, group in enumerate(kv_cache_config.kv_cache_groups)
|
||||
if group.role is KVCacheGroupRole.HISPARSE_INDEXER
|
||||
)
|
||||
|
||||
|
||||
def build_offloading_config(
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
@@ -43,15 +54,22 @@ def build_offloading_config(
|
||||
engine_id = kv_transfer_config.engine_id
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
selected_groups = tuple(
|
||||
(group_id, kv_cache_config.kv_cache_groups[group_id])
|
||||
for group_id in get_offloading_group_ids(kv_cache_config)
|
||||
)
|
||||
if not selected_groups:
|
||||
raise ValueError("KV offloading found no eligible cache groups.")
|
||||
groups = tuple(
|
||||
OffloadingGroupConfig(
|
||||
group_id=group_id,
|
||||
tokens_per_block=resolve_dcp_kv_block_size(
|
||||
group.kv_cache_spec,
|
||||
parallel_config.decode_context_parallel_size,
|
||||
),
|
||||
layer_names=tuple(group.layer_names),
|
||||
)
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
for group_id, group in selected_groups
|
||||
)
|
||||
|
||||
_, tokens_per_hash = resolve_kv_cache_block_sizes(kv_cache_config, vllm_config)
|
||||
@@ -103,11 +121,20 @@ def build_offloading_config(
|
||||
)
|
||||
|
||||
worker_kv_bytes_per_block = 0
|
||||
if kv_cache_config.num_blocks > 0 and kv_cache_config.kv_cache_tensors:
|
||||
all_groups_selected = len(selected_groups) == len(kv_cache_config.kv_cache_groups)
|
||||
if (
|
||||
all_groups_selected
|
||||
and kv_cache_config.num_blocks > 0
|
||||
and kv_cache_config.kv_cache_tensors
|
||||
):
|
||||
# Every KVCacheTensor describes placement within the same backing allocation,
|
||||
# so its size is the total, not a per-tensor share.
|
||||
total_gpu_kv_bytes = kv_cache_config.kv_cache_tensors[0].size
|
||||
worker_kv_bytes_per_block = total_gpu_kv_bytes // kv_cache_config.num_blocks
|
||||
elif kv_cache_config.num_blocks > 0:
|
||||
worker_kv_bytes_per_block = sum(
|
||||
group.kv_cache_spec.page_size_bytes for _, group in selected_groups
|
||||
)
|
||||
|
||||
single_group_spec = (
|
||||
kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
|
||||
@@ -152,8 +152,9 @@ def resolve_mamba_align_size(
|
||||
such groups agree on the same value.
|
||||
"""
|
||||
mamba_align_size: int | None = None
|
||||
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
|
||||
kv_spec = kv_cache_config.kv_cache_groups[idx].kv_cache_spec
|
||||
for group in spec.config.groups:
|
||||
tokens_per_block = group.tokens_per_block
|
||||
kv_spec = kv_cache_config.kv_cache_groups[group.group_id].kv_cache_spec
|
||||
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode in (
|
||||
"align",
|
||||
"all",
|
||||
@@ -188,8 +189,9 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
# each segment can never serve a load hit. Relevant for hybrid
|
||||
# architectures like DeepSeek V4 (MLA + SWA groups).
|
||||
full_attn_tokens_per_chunk: set[int] = set()
|
||||
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
|
||||
kv_spec = kv_cache_config.kv_cache_groups[idx].kv_cache_spec
|
||||
for group in spec.config.groups:
|
||||
tokens_per_block = group.tokens_per_block
|
||||
kv_spec = kv_cache_config.kv_cache_groups[group.group_id].kv_cache_spec
|
||||
sw = get_sliding_window_size_in_chunks(
|
||||
kv_spec, tokens_per_block * spec.blocks_per_chunk
|
||||
)
|
||||
@@ -214,9 +216,9 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
retention_interval = vllm_config.cache_config.prefix_cache_retention_interval
|
||||
|
||||
eagle_groups = {
|
||||
idx
|
||||
for idx, g in enumerate(kv_cache_config.kv_cache_groups)
|
||||
if g.is_eagle_group
|
||||
group.group_id
|
||||
for group in spec.config.groups
|
||||
if kv_cache_config.kv_cache_groups[group.group_id].is_eagle_group
|
||||
}
|
||||
|
||||
use_eagle_block_drop = (
|
||||
@@ -249,8 +251,10 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
)
|
||||
|
||||
kv_group_configs_list: list[GroupOffloadConfig] = []
|
||||
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
|
||||
kv_cache_group = kv_cache_config.kv_cache_groups[idx]
|
||||
for group in spec.config.groups:
|
||||
group_id = group.group_id
|
||||
tokens_per_block = group.tokens_per_block
|
||||
kv_cache_group = kv_cache_config.kv_cache_groups[group_id]
|
||||
kv_spec = kv_cache_group.kv_cache_spec
|
||||
sw = get_sliding_window_size_in_chunks(
|
||||
kv_spec, tokens_per_block * spec.blocks_per_chunk
|
||||
@@ -261,7 +265,7 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
)
|
||||
kv_group_configs_list.append(
|
||||
GroupOffloadConfig(
|
||||
group_idx=idx,
|
||||
group_idx=group_id,
|
||||
tokens_per_block=tokens_per_block,
|
||||
tokens_per_chunk=tokens_per_block * spec.blocks_per_chunk,
|
||||
hashes_per_chunk=(
|
||||
@@ -272,7 +276,7 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
kv_cache_spec=kv_spec,
|
||||
manager_cls=manager_cls,
|
||||
kv_event_group_spec=get_offloading_event_group_spec(kv_cache_group),
|
||||
is_eagle_group=idx in eagle_groups,
|
||||
is_eagle_group=group_id in eagle_groups,
|
||||
requires_cow_source=(
|
||||
isinstance(kv_spec, MambaSpec)
|
||||
and kv_spec.mamba_cache_mode == "align"
|
||||
@@ -408,9 +412,10 @@ class RequestOffloadState:
|
||||
if new_block_id_groups is None:
|
||||
return
|
||||
|
||||
assert len(new_block_id_groups) == len(self.group_states)
|
||||
for group_state, new_blocks in zip(self.group_states, new_block_id_groups):
|
||||
group_state.block_ids.extend(new_blocks)
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs, self.group_states
|
||||
):
|
||||
group_state.block_ids.extend(new_block_id_groups[group_config.group_idx])
|
||||
|
||||
def storable_chunks(
|
||||
self,
|
||||
@@ -541,11 +546,11 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
full_attention_groups: list[int] = []
|
||||
sliding_window_groups: list[int] = []
|
||||
for group_config in self.config.kv_group_configs:
|
||||
for config_idx, group_config in enumerate(self.config.kv_group_configs):
|
||||
if group_config.sliding_window_size_in_chunks is None:
|
||||
full_attention_groups.append(group_config.group_idx)
|
||||
full_attention_groups.append(config_idx)
|
||||
else:
|
||||
sliding_window_groups.append(group_config.group_idx)
|
||||
sliding_window_groups.append(config_idx)
|
||||
|
||||
# sort sliding window groups by window size in decreasing order
|
||||
def _sliding_window_sort_key(i: int) -> int:
|
||||
@@ -737,7 +742,11 @@ class OffloadingConnectorScheduler:
|
||||
req_status.req_context,
|
||||
)
|
||||
|
||||
def _lookup_complete_chunks(self, req_status: RequestOffloadState) -> int | None:
|
||||
def _lookup_complete_chunks(
|
||||
self,
|
||||
req_status: RequestOffloadState,
|
||||
max_num_new_tokens: int | None = None,
|
||||
) -> int | None:
|
||||
"""
|
||||
Find how many tokens beyond num_locally_computed_tokens can be loaded.
|
||||
|
||||
@@ -748,6 +757,10 @@ class OffloadingConnectorScheduler:
|
||||
"""
|
||||
num_computed_tokens = req_status.num_locally_computed_tokens
|
||||
max_hit_size_tokens: int = req_status.req.num_tokens
|
||||
if max_num_new_tokens is not None:
|
||||
max_hit_size_tokens = min(
|
||||
max_hit_size_tokens, num_computed_tokens + max_num_new_tokens
|
||||
)
|
||||
if self._sliding_window_groups:
|
||||
# the last prompt token has to be recomputed to get the logprobs
|
||||
# for sliding window attention, we must reduce by 1 to make sure
|
||||
@@ -929,8 +942,12 @@ class OffloadingConnectorScheduler:
|
||||
hash_idx = boundary_tokens // self.config.tokens_per_hash - 1
|
||||
return make_offload_key(request.block_hashes[hash_idx], group_idx)
|
||||
|
||||
def _lookup(self, req_status: RequestOffloadState) -> int | None:
|
||||
complete_hit = self._lookup_complete_chunks(req_status)
|
||||
def _lookup(
|
||||
self,
|
||||
req_status: RequestOffloadState,
|
||||
max_num_new_tokens: int | None = None,
|
||||
) -> int | None:
|
||||
complete_hit = self._lookup_complete_chunks(req_status, max_num_new_tokens)
|
||||
req_status.partial_tail_boundary = None
|
||||
if complete_hit is None or not self.config.supports_partial_tail:
|
||||
return complete_hit
|
||||
@@ -939,9 +956,10 @@ class OffloadingConnectorScheduler:
|
||||
complete_boundary = local_tokens + complete_hit
|
||||
tokens_per_hash = self.config.tokens_per_hash
|
||||
block_end = complete_boundary + self._partial_tail_block_size
|
||||
max_boundary = round_down(
|
||||
min(req_status.req.num_prompt_tokens - 1, block_end - 1), tokens_per_hash
|
||||
)
|
||||
max_boundary = min(req_status.req.num_prompt_tokens - 1, block_end - 1)
|
||||
if max_num_new_tokens is not None:
|
||||
max_boundary = min(max_boundary, local_tokens + max_num_new_tokens)
|
||||
max_boundary = round_down(max_boundary, tokens_per_hash)
|
||||
if max_boundary <= complete_boundary:
|
||||
return complete_hit
|
||||
|
||||
@@ -990,7 +1008,10 @@ class OffloadingConnectorScheduler:
|
||||
self._req_status[request.request_id] = req_status
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
self,
|
||||
request: Request,
|
||||
num_computed_tokens: int,
|
||||
max_num_new_tokens: int | None = None,
|
||||
) -> tuple[int | None, bool]:
|
||||
"""
|
||||
Get number of new tokens that can be loaded beyond the
|
||||
@@ -1030,7 +1051,7 @@ class OffloadingConnectorScheduler:
|
||||
num_hit_tokens = 0
|
||||
else:
|
||||
lookup_start = time.monotonic()
|
||||
num_hit_tokens = self._lookup(req_status)
|
||||
num_hit_tokens = self._lookup(req_status, max_num_new_tokens)
|
||||
self._connector_stats.observe_histogram(
|
||||
_ConnectorMetricName.LOOKUP_SYNC_DELAY,
|
||||
time.monotonic() - lookup_start,
|
||||
@@ -1065,11 +1086,11 @@ class OffloadingConnectorScheduler:
|
||||
# per group
|
||||
group_sizes: list[int] = []
|
||||
block_indices: list[int] = []
|
||||
for group_config, group_state, group_blocks in zip(
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs,
|
||||
req_status.group_states,
|
||||
blocks.blocks,
|
||||
):
|
||||
group_blocks = blocks.blocks[group_config.group_idx]
|
||||
self._current_batch_allocated_block_ids.update(
|
||||
block.block_id for block in group_blocks if block.block_id != 0
|
||||
)
|
||||
@@ -1184,7 +1205,8 @@ class OffloadingConnectorScheduler:
|
||||
for grp_idx in self._sliding_window_groups
|
||||
)
|
||||
req_status.update_block_id_groups(new_block_id_groups)
|
||||
for new_blocks in new_block_id_groups:
|
||||
for group_config in self.config.kv_group_configs:
|
||||
new_blocks = new_block_id_groups[group_config.group_idx]
|
||||
for bid in new_blocks:
|
||||
if bid != 0:
|
||||
self._current_batch_allocated_block_ids.add(bid)
|
||||
@@ -1218,6 +1240,10 @@ class OffloadingConnectorScheduler:
|
||||
) -> dict[int, TransferJob]:
|
||||
store_jobs: dict[int, TransferJob] = {}
|
||||
num_groups = len(self.config.kv_group_configs)
|
||||
config_idx_by_group = {
|
||||
config.group_idx: idx
|
||||
for idx, config in enumerate(self.config.kv_group_configs)
|
||||
}
|
||||
for req_id, entries in handoffs.items():
|
||||
req_status = self._req_status.get(req_id)
|
||||
if req_status is None:
|
||||
@@ -1225,7 +1251,10 @@ class OffloadingConnectorScheduler:
|
||||
req = req_status.req
|
||||
max_boundary = self._calc_num_offloadable_tokens(req_status, req.num_tokens)
|
||||
for group_idx, block_id, boundary in entries:
|
||||
group_config = self.config.kv_group_configs[group_idx]
|
||||
config_idx = config_idx_by_group.get(group_idx)
|
||||
if config_idx is None:
|
||||
continue
|
||||
group_config = self.config.kv_group_configs[config_idx]
|
||||
if (
|
||||
block_id == 0
|
||||
or boundary > max_boundary
|
||||
@@ -1254,9 +1283,11 @@ class OffloadingConnectorScheduler:
|
||||
fenced_block_ids=[block_id],
|
||||
)
|
||||
group_sizes = [0] * num_groups
|
||||
group_sizes[group_idx] = 1
|
||||
group_sizes[config_idx] = 1
|
||||
block_indices = [0] * num_groups
|
||||
block_indices[group_idx] = boundary // group_config.tokens_per_block - 1
|
||||
block_indices[config_idx] = (
|
||||
boundary // group_config.tokens_per_block - 1
|
||||
)
|
||||
store_jobs[job_id] = TransferJob(
|
||||
req_id=req_id,
|
||||
src_spec=GPULoadStoreSpec(
|
||||
@@ -1300,6 +1331,12 @@ class OffloadingConnectorScheduler:
|
||||
assert len(boundaries) == 1
|
||||
boundary = boundaries.pop()
|
||||
req = req_status.req
|
||||
group_states = {
|
||||
group.group_idx: state
|
||||
for group, state in zip(
|
||||
self.config.kv_group_configs, req_status.group_states
|
||||
)
|
||||
}
|
||||
max_boundary = min(
|
||||
req.num_prompt_tokens,
|
||||
req_status.max_offload_tokens or req.num_prompt_tokens,
|
||||
@@ -1315,7 +1352,7 @@ class OffloadingConnectorScheduler:
|
||||
block_idx = boundary // self._partial_tail_block_size
|
||||
if any(
|
||||
group.group_idx not in self._cow_source_groups
|
||||
and block_idx >= len(req_status.group_states[group.group_idx].block_ids)
|
||||
and block_idx >= len(group_states[group.group_idx].block_ids)
|
||||
for group in self.config.kv_group_configs
|
||||
):
|
||||
continue
|
||||
@@ -1326,7 +1363,7 @@ class OffloadingConnectorScheduler:
|
||||
block_ids = [
|
||||
cow_blocks[group.group_idx]
|
||||
if group.group_idx in self._cow_source_groups
|
||||
else req_status.group_states[group.group_idx].block_ids[block_idx]
|
||||
else group_states[group.group_idx].block_ids[block_idx]
|
||||
for group in self.config.kv_group_configs
|
||||
]
|
||||
assert all(block_id != 0 for block_id in block_ids)
|
||||
|
||||
@@ -14,6 +14,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingWorkerMetadata,
|
||||
ReqId,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
|
||||
get_offloading_group_ids,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
@@ -65,7 +68,10 @@ class OffloadingConnectorWorker:
|
||||
|
||||
def register_kv_caches(self, kv_caches: dict[str, torch.Tensor]):
|
||||
kv_cache_config = self.kv_cache_config
|
||||
num_blocks = kv_cache_config.num_blocks
|
||||
selected_group_ids = get_offloading_group_ids(kv_cache_config)
|
||||
selected_groups = tuple(
|
||||
kv_cache_config.kv_cache_groups[group_id] for group_id in selected_group_ids
|
||||
)
|
||||
mappings = derive_canonical_mappings(
|
||||
self.vllm_config, kv_cache_config, kv_caches
|
||||
)
|
||||
@@ -76,7 +82,9 @@ class OffloadingConnectorWorker:
|
||||
unpadded_page_size_bytes: dict[str, int] = {}
|
||||
# layer_name -> size of page in bytes
|
||||
page_size_bytes: dict[str, int] = {}
|
||||
for kv_cache_group in kv_cache_config.kv_cache_groups:
|
||||
for kv_cache_group in selected_groups:
|
||||
assert not kv_cache_group.host_resident
|
||||
num_blocks = kv_cache_config.num_blocks
|
||||
group_layer_names = kv_cache_group.layer_names
|
||||
group_kv_cache_spec = kv_cache_group.kv_cache_spec
|
||||
if isinstance(group_kv_cache_spec, UniformTypeKVCacheSpecs):
|
||||
@@ -131,8 +139,11 @@ class OffloadingConnectorWorker:
|
||||
),
|
||||
None,
|
||||
)
|
||||
if packed_layer_name is not None:
|
||||
if packed_layer_name is not None and len(selected_groups) == len(
|
||||
kv_cache_config.kv_cache_groups
|
||||
):
|
||||
(tensor,) = tensors_per_block[packed_layer_name]
|
||||
num_blocks = tensor.shape[0]
|
||||
block_stride = tensor.stride(0)
|
||||
packed_tensor = tensor.as_strided(
|
||||
(num_blocks, block_stride),
|
||||
@@ -142,10 +153,7 @@ class OffloadingConnectorWorker:
|
||||
self._init_worker(
|
||||
CanonicalKVCaches(
|
||||
[CanonicalKVCacheTensor(packed_tensor, block_stride)],
|
||||
[
|
||||
[CanonicalKVCacheRef(0, block_stride)]
|
||||
for _ in kv_cache_config.kv_cache_groups
|
||||
],
|
||||
[[CanonicalKVCacheRef(0, block_stride)] for _ in selected_groups],
|
||||
)
|
||||
)
|
||||
return
|
||||
@@ -192,7 +200,7 @@ class OffloadingConnectorWorker:
|
||||
)
|
||||
|
||||
group_data_refs: list[list[CanonicalKVCacheRef]] = []
|
||||
for kv_cache_group in kv_cache_config.kv_cache_groups:
|
||||
for kv_cache_group in selected_groups:
|
||||
group_refs: list[CanonicalKVCacheRef] = []
|
||||
for layer_name in kv_cache_group.layer_names:
|
||||
group_refs += block_data_refs[layer_name]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Iterable
|
||||
from functools import cached_property
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
@@ -25,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
|
||||
build_offloading_config,
|
||||
get_offloading_group_ids,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
@@ -38,6 +40,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.worker import (
|
||||
)
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.kv_cache_coordinator import HybridKVCacheCoordinator
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
@@ -47,6 +50,25 @@ from vllm.v1.request import Request
|
||||
|
||||
|
||||
class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
@cached_property
|
||||
def _bounding_group_ids(self) -> tuple[int, ...]:
|
||||
"""Prefix-cacheable groups this connector does not offload.
|
||||
|
||||
Never offer tokens for a range some group cannot cover: past such a
|
||||
group's own cached prefix its KV would be left unwritten.
|
||||
"""
|
||||
offloaded = set(get_offloading_group_ids(self._kv_cache_config))
|
||||
return tuple(
|
||||
group_id
|
||||
for group_id, group in enumerate(self._kv_cache_config.kv_cache_groups)
|
||||
if group.kv_cache_spec.prefix_cacheable and group_id not in offloaded
|
||||
)
|
||||
|
||||
@property
|
||||
def scheduler(self) -> OffloadingConnectorScheduler:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler
|
||||
|
||||
@property
|
||||
def requires_kv_delivery(self) -> bool:
|
||||
# Runs as kv_both, but is a best-effort cache: a dropped save is just a
|
||||
@@ -138,9 +160,30 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
) -> tuple[int | None, bool]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens
|
||||
request,
|
||||
num_computed_tokens,
|
||||
max_num_new_tokens=self._max_loadable_tokens(request, num_computed_tokens),
|
||||
)
|
||||
|
||||
def _max_loadable_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> int | None:
|
||||
"""How far past ``num_computed_tokens`` a load may reach, if bounded.
|
||||
|
||||
Bounded by the deepest prefix the groups this connector does not
|
||||
offload already hold, since nothing refills them beyond it.
|
||||
"""
|
||||
if not self._bounding_group_ids:
|
||||
return None
|
||||
assert self._kv_cache_manager is not None
|
||||
coordinator = self._kv_cache_manager.coordinator
|
||||
assert isinstance(coordinator, HybridKVCacheCoordinator)
|
||||
_, per_group_hits = coordinator.find_longest_cache_hit_per_group(
|
||||
request.block_hashes, request.num_tokens - 1
|
||||
)
|
||||
bound = min(per_group_hits[group_id] for group_id in self._bounding_group_ids)
|
||||
return max(0, bound - num_computed_tokens)
|
||||
|
||||
def update_state_after_alloc(
|
||||
self, request: "Request", blocks: "KVCacheBlocks", num_external_tokens: int
|
||||
):
|
||||
@@ -185,6 +228,8 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
|
||||
@classmethod
|
||||
def get_required_kvcache_layout(cls, vllm_config: VllmConfig) -> str | None:
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
return "BLHNC"
|
||||
return "LBHNC"
|
||||
|
||||
def reset_cache(self) -> bool | None:
|
||||
|
||||
@@ -78,13 +78,11 @@ def ensure_kv_transfer_initialized(
|
||||
|
||||
global _KV_CONNECTOR_AGENT
|
||||
|
||||
if vllm_config.kv_transfer_config is None:
|
||||
if _KV_CONNECTOR_AGENT is not None:
|
||||
return
|
||||
|
||||
if (
|
||||
vllm_config.kv_transfer_config.is_kv_transfer_instance
|
||||
and _KV_CONNECTOR_AGENT is None
|
||||
):
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
if kv_transfer_config is not None and kv_transfer_config.is_kv_transfer_instance:
|
||||
# NIXL P/D requires an interleave_size equal to block_size.
|
||||
vllm_config.adjust_dcp_kv_cache_interleave_size(kv_cache_config)
|
||||
_sync_engine_id_across_tp(vllm_config)
|
||||
|
||||
Reference in New Issue
Block a user