[3/N][Core][KV Connector] Support reliable partial-tail KV offload for sub-block prompts (#49502)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
544cb724c8
commit
d742856610
@@ -21,6 +21,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
@@ -83,6 +84,9 @@ class MooncakeStoreCoordinator:
|
||||
self.kv_cache_groups = kv_cache_groups
|
||||
self.hash_block_size = hash_block_size
|
||||
self.lcm_block_size = scheduler_block_size
|
||||
self.enable_partial_hash_hits = partial_hash_hits_enabled(
|
||||
kv_cache_groups, hash_block_size
|
||||
)
|
||||
self.use_eagle = use_eagle
|
||||
# Mirror vLLM core's KVCacheCoordinator.retention_interval.
|
||||
self.retention_interval = retention_interval
|
||||
@@ -94,7 +98,12 @@ class MooncakeStoreCoordinator:
|
||||
self._verify_and_split_kv_cache_groups()
|
||||
|
||||
def align_lookup_length(self, length: int) -> int:
|
||||
return length // self.lcm_block_size * self.lcm_block_size
|
||||
alignment = (
|
||||
self.hash_block_size
|
||||
if self.enable_partial_hash_hits
|
||||
else self.lcm_block_size
|
||||
)
|
||||
return length // alignment * alignment
|
||||
|
||||
def _verify_and_split_kv_cache_groups(self) -> None:
|
||||
"""Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but
|
||||
@@ -226,9 +235,14 @@ class MooncakeStoreCoordinator:
|
||||
retention_interval: int | None,
|
||||
num_prompt_tokens: int | None,
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
assert aligned_token_len % self.lcm_block_size == 0, (
|
||||
mask_alignment = (
|
||||
self.hash_block_size
|
||||
if self.enable_partial_hash_hits
|
||||
else self.lcm_block_size
|
||||
)
|
||||
assert aligned_token_len % mask_alignment == 0, (
|
||||
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
|
||||
f"lcm_block_size ({self.lcm_block_size})"
|
||||
f"{mask_alignment}"
|
||||
)
|
||||
masks: list[list[bool] | None] = []
|
||||
for g_idx, g in enumerate(self.kv_cache_groups):
|
||||
@@ -278,17 +292,21 @@ class MooncakeStoreCoordinator:
|
||||
one already removed by the lookup.
|
||||
"""
|
||||
eagle_indices = self.eagle_attn_group_indices if apply_eagle else set()
|
||||
alignment_tokens = (
|
||||
self.hash_block_size
|
||||
if self.enable_partial_hash_hits
|
||||
else self.lcm_block_size
|
||||
)
|
||||
if len(self.attention_groups) == 1:
|
||||
spec, group_ids, manager_cls = self.attention_groups[0]
|
||||
hashes = self.block_hashes_for_spec(block_hashes, spec)
|
||||
hit_blocks, hit_length = manager_cls.find_longest_cache_hit(
|
||||
block_hashes=hashes, # type: ignore[arg-type]
|
||||
block_hashes=block_hashes, # type: ignore[arg-type]
|
||||
max_length=max_length,
|
||||
kv_cache_group_ids=group_ids,
|
||||
block_pool=cast(BlockPool, cached_block_pool),
|
||||
kv_cache_spec=spec,
|
||||
drop_eagle_block=(0 in eagle_indices),
|
||||
alignment_tokens=spec.block_size,
|
||||
alignment_tokens=alignment_tokens,
|
||||
)
|
||||
num_groups = len(self.kv_cache_groups)
|
||||
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
|
||||
@@ -320,17 +338,23 @@ class MooncakeStoreCoordinator:
|
||||
|
||||
drop_eagle_block = idx in eagle_indices and idx not in eagle_verified
|
||||
_max_length = curr_hit_length
|
||||
if drop_eagle_block:
|
||||
_max_length = min(curr_hit_length + spec.block_size, max_length)
|
||||
hashes = self.block_hashes_for_spec(block_hashes, spec)
|
||||
if drop_eagle_block and not isinstance(spec, MambaSpec):
|
||||
eagle_margin = (
|
||||
self.hash_block_size
|
||||
if self.enable_partial_hash_hits
|
||||
and manager_cls.supports_fine_grained_hash_lookup
|
||||
and spec.block_size > self.hash_block_size
|
||||
else spec.block_size
|
||||
)
|
||||
_max_length = min(curr_hit_length + eagle_margin, max_length)
|
||||
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
|
||||
block_hashes=hashes, # type: ignore[arg-type]
|
||||
block_hashes=block_hashes, # type: ignore[arg-type]
|
||||
max_length=_max_length,
|
||||
kv_cache_group_ids=group_ids,
|
||||
block_pool=cast(BlockPool, cached_block_pool),
|
||||
kv_cache_spec=spec,
|
||||
drop_eagle_block=drop_eagle_block,
|
||||
alignment_tokens=self.lcm_block_size,
|
||||
alignment_tokens=alignment_tokens,
|
||||
)
|
||||
if drop_eagle_block:
|
||||
eagle_verified.add(idx)
|
||||
@@ -348,10 +372,11 @@ class MooncakeStoreCoordinator:
|
||||
break
|
||||
|
||||
# Truncate full-attention hit_blocks to final converged length;
|
||||
# other specs already trim themselves inside their hit logic.
|
||||
# other specs already trim themselves inside their hit logic. cdiv keeps
|
||||
# the partial tail block when hit_length is not block-aligned.
|
||||
spec0, group_ids0, _ = self.attention_groups[0]
|
||||
if isinstance(spec0, FullAttentionSpec):
|
||||
num_blocks = hit_length // spec0.block_size
|
||||
num_blocks = cdiv(hit_length, spec0.block_size)
|
||||
for gid in group_ids0:
|
||||
full_blks = hit_blocks_by_group[gid]
|
||||
assert full_blks is not None
|
||||
@@ -368,3 +393,18 @@ def _unwrap_spec(spec: KVCacheSpec) -> KVCacheSpec:
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
return next(iter(spec.kv_cache_specs.values()))
|
||||
return spec
|
||||
|
||||
|
||||
def partial_hash_hits_enabled(
|
||||
kv_cache_groups: list[KVCacheGroupSpec], hash_block_size: int
|
||||
) -> bool:
|
||||
"""Mirror of core's ``HybridKVCacheCoordinator.enable_partial_hash_hits``
|
||||
(its dcp == 1 clause holds: the connector rejects hybrid + DCP/PCP > 1).
|
||||
Single copy on purpose — scheduler and coordinator must not disagree.
|
||||
"""
|
||||
return any(
|
||||
isinstance(spec := _unwrap_spec(g.kv_cache_spec), MambaSpec)
|
||||
and spec.mamba_cache_mode == "align"
|
||||
and spec.block_size > hash_block_size
|
||||
for g in kv_cache_groups
|
||||
)
|
||||
|
||||
@@ -228,16 +228,28 @@ class ChunkedTokenDatabase:
|
||||
n = len(chunks)
|
||||
starts = np.fromiter((c[0] for c in chunks), dtype=np.int64, count=n)
|
||||
spans = np.fromiter((c[1] for c in chunks), dtype=np.int64, count=n) - starts
|
||||
assert not (spans % self.block_size).any()
|
||||
assert not (spans % self.hash_block_size).any()
|
||||
bids = np.fromiter(
|
||||
(block_ids[i] for i in (starts // self.block_size).tolist()),
|
||||
dtype=np.int64,
|
||||
count=n,
|
||||
)
|
||||
addrs = base[None, :] + bids[:, None] * blen[None, :]
|
||||
sizes = blen[None, :] * (spans // self.block_size)[:, None]
|
||||
block_counts = (spans + self.block_size - 1) // self.block_size
|
||||
sizes = blen[None, :] * block_counts[:, None]
|
||||
return addrs.tolist(), sizes.tolist(), bids.tolist()
|
||||
|
||||
def prepare_value_for_block(self, block_id: int) -> tuple[list[int], list[int]]:
|
||||
"""Return addresses and sizes for one physical block slot."""
|
||||
addr_list = []
|
||||
size_list = []
|
||||
length = len(self.block_len)
|
||||
for index, base_addr in enumerate(self.kv_caches_base_addr):
|
||||
addr = base_addr + block_id * self.block_len[index % length]
|
||||
addr_list.append(addr)
|
||||
size_list.append(self.block_len[index % length])
|
||||
return addr_list, size_list
|
||||
|
||||
def process_tokens(
|
||||
self,
|
||||
token_len: int,
|
||||
@@ -256,7 +268,8 @@ class ChunkedTokenDatabase:
|
||||
rank regardless of where the processed suffix begins.
|
||||
|
||||
Args:
|
||||
token_len: Total number of tokens.
|
||||
token_len: Total number of tokens. Must be hash-block aligned and
|
||||
covered by ``block_hashes`` when hashes are present.
|
||||
block_hashes: Block hashes computed at ``hash_block_size`` granularity.
|
||||
When ``block_size > hash_block_size`` each group's ``block_size`` chunk
|
||||
is keyed by its last sub-hash via ``chunk_hashes_for_block_size``.
|
||||
@@ -269,11 +282,10 @@ class ChunkedTokenDatabase:
|
||||
assert put_step > 0
|
||||
if not block_hashes:
|
||||
return
|
||||
chunk_hashes: Sequence[BlockHash] = chunk_hashes_for_block_size(
|
||||
block_hashes, self.hash_block_size, self.block_size
|
||||
)
|
||||
assert token_len % self.hash_block_size == 0
|
||||
assert token_len // self.hash_block_size <= len(block_hashes)
|
||||
start_chunk = max(0, cdiv(mask_num, self.block_size))
|
||||
max_chunks = min(len(chunk_hashes), cdiv(token_len, self.block_size))
|
||||
max_chunks = cdiv(token_len, self.block_size)
|
||||
if chunk_mask is not None:
|
||||
max_chunks = min(max_chunks, start_chunk + len(chunk_mask))
|
||||
for chunk_id in range(start_chunk, max_chunks):
|
||||
@@ -281,9 +293,9 @@ class ChunkedTokenDatabase:
|
||||
continue
|
||||
if chunk_id % put_step != put_step_rank:
|
||||
continue
|
||||
h = chunk_hashes[chunk_id]
|
||||
start_idx = chunk_id * self.block_size
|
||||
end_idx = min(start_idx + self.block_size, token_len)
|
||||
h = block_hashes[end_idx // self.hash_block_size - 1]
|
||||
yield start_idx, end_idx, h
|
||||
|
||||
|
||||
@@ -306,6 +318,7 @@ class RequestTracker:
|
||||
allocated_block_ids: tuple[list[int], ...]
|
||||
num_saved_tokens: int = 0
|
||||
token_ids: list[int] | None = None
|
||||
has_pending_offload: bool = False
|
||||
# Snapshot of the prefill range length at tracker creation time.
|
||||
# For a fresh request this is len(prompt). For a resumed-from-preemption
|
||||
# request it includes previously-generated tokens, which are re-prefilled.
|
||||
@@ -316,6 +329,7 @@ class RequestTracker:
|
||||
self.allocated_block_ids = ()
|
||||
self.num_saved_tokens = 0
|
||||
self.token_ids = None
|
||||
self.has_pending_offload = False
|
||||
self.prefill_end_tokens = 0
|
||||
|
||||
def update(
|
||||
@@ -352,6 +366,11 @@ class ReqMeta:
|
||||
|
||||
token_ids: list[int] | None = None
|
||||
num_prompt_tokens: int | None = None
|
||||
# Core-provided per-mamba-group
|
||||
# (group_id, cow_block_id, boundary_tokens) for this request's partial tail.
|
||||
# Present only on the producer's CoW step; drives the connector's offload
|
||||
# (the FA group's block is derived from block_ids and boundary_tokens).
|
||||
partial_tail_offloads: list[tuple[int, int, int]] | None = None
|
||||
|
||||
@staticmethod
|
||||
def from_request_tracker(
|
||||
|
||||
@@ -11,6 +11,9 @@ from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator import ( # noqa: E501
|
||||
partial_hash_hits_enabled,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
LoadSpec,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
@@ -63,6 +66,9 @@ class MooncakeStoreScheduler:
|
||||
self._block_size, self._hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
self.enable_partial_hash_hits = partial_hash_hits_enabled(
|
||||
kv_cache_config.kv_cache_groups, self._hash_block_size
|
||||
)
|
||||
|
||||
# Per-request state
|
||||
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
|
||||
@@ -80,7 +86,12 @@ class MooncakeStoreScheduler:
|
||||
Returns ``(None, False)`` when an async lookup is still in flight,
|
||||
signaling the scheduler to retry this request on a later step.
|
||||
"""
|
||||
if request.num_tokens < self._block_size:
|
||||
# Fine-grained hits may land on a hash boundary inside a block; without
|
||||
# partial hits, prefixes shorter than one physical block are skipped.
|
||||
align = (
|
||||
self._hash_block_size if self.enable_partial_hash_hits else self._block_size
|
||||
)
|
||||
if request.num_tokens < align:
|
||||
return 0, False
|
||||
|
||||
num_external_hit_tokens = self.client.lookup(
|
||||
@@ -338,6 +349,44 @@ class MooncakeStoreScheduler:
|
||||
if req_meta is not None:
|
||||
meta.add_request(req_meta)
|
||||
|
||||
# Flush partial-tail offloads in the step they arrive: the CoW copy is
|
||||
# enqueued before the connector event records, so this step's event
|
||||
# fences the cow block. Ride the request's save meta when present, else
|
||||
# emit an offload-only ReqMeta (token_len_chunk=0 skips the normal
|
||||
# save; can_save=True takes the normal enqueue path).
|
||||
step_partial_tails = getattr(scheduler_output, "partial_tail_offloads", None)
|
||||
if step_partial_tails and not force_skip_save:
|
||||
pending = dict(step_partial_tails)
|
||||
for req_meta in meta.requests:
|
||||
if req_meta.can_save:
|
||||
groups = pending.pop(req_meta.req_id, None)
|
||||
if groups:
|
||||
req_meta.partial_tail_offloads = groups
|
||||
tracker = self._request_trackers.get(req_meta.req_id)
|
||||
if tracker is not None:
|
||||
tracker.has_pending_offload = True
|
||||
for req_id, groups in pending.items():
|
||||
tracker = self._request_trackers.get(req_id)
|
||||
req_tuple = self._unfinished_requests.get(req_id)
|
||||
if tracker is None or req_tuple is None:
|
||||
# Request finished/preempted within this step; its blocks
|
||||
# are going away, so the offload is conservatively dropped.
|
||||
logger.debug("Dropping partial-tail offload for request %s", req_id)
|
||||
continue
|
||||
assert len({boundary for _, _, boundary in groups}) == 1
|
||||
tracker.has_pending_offload = True
|
||||
meta.add_request(
|
||||
ReqMeta(
|
||||
req_id=req_id,
|
||||
token_len_chunk=0,
|
||||
block_ids=tracker.allocated_block_ids,
|
||||
block_hashes=req_tuple[0].block_hashes,
|
||||
can_save=True,
|
||||
num_prompt_tokens=tracker.prefill_end_tokens,
|
||||
partial_tail_offloads=groups,
|
||||
)
|
||||
)
|
||||
|
||||
return meta
|
||||
|
||||
def request_finished(
|
||||
@@ -352,7 +401,9 @@ class MooncakeStoreScheduler:
|
||||
# Missing tracker can happen when the request is aborted before the
|
||||
# connector observes the normal finished lifecycle or is preempted
|
||||
# before finishing.
|
||||
if tracker is None or tracker.num_saved_tokens <= 0:
|
||||
if tracker is None or (
|
||||
tracker.num_saved_tokens <= 0 and not tracker.has_pending_offload
|
||||
):
|
||||
return False, None
|
||||
total_blocks = sum(len(g) for g in block_ids)
|
||||
delay_free_blocks = total_blocks > 0
|
||||
|
||||
@@ -61,6 +61,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.protocol import
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.utils.network_utils import get_ip, make_zmq_socket
|
||||
from vllm.v1.attention.backends.utils import NULL_BLOCK_ID
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
maybe_convert_block_hash,
|
||||
@@ -531,6 +532,164 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self._skip_store_requests.clear()
|
||||
return True
|
||||
|
||||
def _maybe_offload_partial_tail(self, req_meta: ReqMeta) -> bool:
|
||||
"""Offload the request's sub-block partial tail (its last prompt hash
|
||||
boundary) so a later request can hit the sub-block prefix.
|
||||
|
||||
Covers every block from the normal save's lcm floor to the boundary:
|
||||
the normal save floors to ``lcm_block_size``, so a smaller-block
|
||||
group's full blocks in that gap are never persisted elsewhere, and
|
||||
the consumer's lookup needs every group at every probed boundary.
|
||||
Full blocks are keyed by their block-end hash, the partial boundary
|
||||
block by the boundary sub-hash; the mamba "align" boundary block is
|
||||
the core-provided CoW block. All keys are deduped against the store.
|
||||
|
||||
Returns:
|
||||
True when no put is needed or every put succeeds, False otherwise.
|
||||
"""
|
||||
if not self.coord.enable_partial_hash_hits or not req_meta.block_hashes:
|
||||
return True
|
||||
partial_tail_offloads = req_meta.partial_tail_offloads
|
||||
if not partial_tail_offloads:
|
||||
return True
|
||||
hash_block_size = self.coord.hash_block_size
|
||||
boundaries = {boundary for _, _, boundary in partial_tail_offloads}
|
||||
if len(boundaries) != 1:
|
||||
raise ValueError(
|
||||
"Partial-tail offloads for one request must share a boundary"
|
||||
)
|
||||
boundary = boundaries.pop()
|
||||
if boundary == 0:
|
||||
return True
|
||||
if boundary // hash_block_size - 1 >= len(req_meta.block_hashes):
|
||||
return True
|
||||
mamba_offloads = {
|
||||
group_id: block_id for group_id, block_id, _ in partial_tail_offloads
|
||||
}
|
||||
|
||||
keys: list[str] = []
|
||||
addrs: list[list[int]] = []
|
||||
sizes: list[list[int]] = []
|
||||
saved = self._saved_offset.get(req_meta.req_id, 0)
|
||||
for g_idx, db in enumerate(self.token_databases):
|
||||
group_blocks = req_meta.block_ids[g_idx]
|
||||
# Distribute across ranks by the same rule as normal chunks.
|
||||
put_step = self.group_put_steps[g_idx]
|
||||
put_step_rank = (self.tp_rank + g_idx) % put_step
|
||||
# Always include the boundary block: its sub-hash key is written
|
||||
# only here, even if normal saves already advanced past it.
|
||||
last_block = cdiv(boundary, db.block_size) - 1
|
||||
for block_idx in range(
|
||||
min(saved // db.block_size, last_block), last_block + 1
|
||||
):
|
||||
if block_idx % put_step != put_step_rank:
|
||||
continue
|
||||
valid_end = min((block_idx + 1) * db.block_size, boundary)
|
||||
key_hash = req_meta.block_hashes[valid_end // hash_block_size - 1]
|
||||
if (
|
||||
g_idx in mamba_offloads
|
||||
and valid_end == boundary
|
||||
and boundary % db.block_size != 0
|
||||
):
|
||||
block_id = mamba_offloads[g_idx]
|
||||
else:
|
||||
if block_idx >= len(group_blocks):
|
||||
continue
|
||||
block_id = group_blocks[block_idx]
|
||||
if block_id == NULL_BLOCK_ID:
|
||||
logger.debug(
|
||||
"Skipping unavailable partial-tail source block "
|
||||
"(req=%s, group=%d, block=%d)",
|
||||
req_meta.req_id,
|
||||
g_idx,
|
||||
block_idx,
|
||||
)
|
||||
continue
|
||||
addr, size = db.prepare_value_for_block(block_id)
|
||||
keys.append(db.key_for(key_hash))
|
||||
addrs.append(addr)
|
||||
sizes.append(size)
|
||||
|
||||
if not keys:
|
||||
return True
|
||||
exists_start = time.perf_counter()
|
||||
try:
|
||||
exists = self.store.batch_is_exist(keys)
|
||||
except Exception as e:
|
||||
self._record_operation(
|
||||
"save_exists",
|
||||
exists_start,
|
||||
len(keys),
|
||||
status="error",
|
||||
num_failed_keys=len(keys),
|
||||
)
|
||||
logger.error(
|
||||
"Failed to check partial-tail keys for request %s: %s",
|
||||
req_meta.req_id,
|
||||
e,
|
||||
)
|
||||
return False
|
||||
self._record_operation("save_exists", exists_start, len(keys))
|
||||
missing = [i for i, e in enumerate(exists) if e != 1]
|
||||
if not missing:
|
||||
return True
|
||||
keys = [keys[i] for i in missing]
|
||||
addrs = [addrs[i] for i in missing]
|
||||
sizes = [sizes[i] for i in missing]
|
||||
if req_meta.current_event is not None:
|
||||
# Fence the CoW block copy enqueued earlier this step.
|
||||
req_meta.current_event.synchronize()
|
||||
batch_bytes = _sum_batch_bytes(sizes)
|
||||
put_start = time.perf_counter()
|
||||
try:
|
||||
res = self.store.batch_put_from_multi_buffers(
|
||||
keys, addrs, sizes, self.replicate_config
|
||||
)
|
||||
except Exception as e:
|
||||
self._record_operation(
|
||||
"save_put",
|
||||
put_start,
|
||||
len(keys),
|
||||
num_bytes=batch_bytes,
|
||||
status="error",
|
||||
num_failed_keys=len(keys),
|
||||
)
|
||||
logger.error(
|
||||
"Failed to put partial-tail keys for request %s: %s",
|
||||
req_meta.req_id,
|
||||
e,
|
||||
)
|
||||
return False
|
||||
|
||||
failed = [i for i, value in enumerate(res) if value < 0]
|
||||
self._record_operation(
|
||||
"save_put",
|
||||
put_start,
|
||||
len(keys),
|
||||
num_bytes=batch_bytes,
|
||||
status="partial_failure" if failed else "ok",
|
||||
num_failed_keys=len(failed),
|
||||
)
|
||||
if failed:
|
||||
failed_codes = {res[i] for i in failed}
|
||||
logger.warning(
|
||||
"Partial-tail put failed for request %s: %d/%d keys failed (codes=%s)",
|
||||
req_meta.req_id,
|
||||
len(failed),
|
||||
len(keys),
|
||||
failed_codes,
|
||||
)
|
||||
if MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes:
|
||||
self._mark_request_skipped_for_pressure(req_meta.req_id)
|
||||
return False
|
||||
|
||||
if self._clear_store_pressure():
|
||||
logger.info(
|
||||
"Mooncake CPU/disk offloading pressure cleared after a "
|
||||
"successful partial-tail batch"
|
||||
)
|
||||
return True
|
||||
|
||||
def _handle_request(self, req_meta: ReqMeta):
|
||||
# Cache hits are always a multiple of ``lcm_block_size`` tokens, which
|
||||
# is also ``store_mask``'s precondition.
|
||||
@@ -548,9 +707,6 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
# so the scheduler can release the GPU blocks it pinned for this
|
||||
# request (via `delay_free_blocks`) even when the store path raises.
|
||||
try:
|
||||
if token_len == 0:
|
||||
return
|
||||
|
||||
if self._should_skip_request(req_id):
|
||||
logger.debug(
|
||||
"Skipping Mooncake store for request %s while CPU/disk "
|
||||
@@ -559,6 +715,16 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
)
|
||||
return
|
||||
|
||||
# Offload the sub-block partial tail (independent of the normal
|
||||
# block-aligned save, which may be skipped this step).
|
||||
if req_meta.partial_tail_offloads is not None and not (
|
||||
self._maybe_offload_partial_tail(req_meta)
|
||||
):
|
||||
return
|
||||
|
||||
if token_len == 0:
|
||||
return
|
||||
|
||||
# Resume from where this rank left off; only the new suffix is saved.
|
||||
save_start = self._saved_offset.get(req_id, 0)
|
||||
|
||||
@@ -1385,7 +1551,7 @@ class MooncakeStoreWorker:
|
||||
self.recv_request_queue.put(request)
|
||||
|
||||
assert self.load_async, "load_async must be True for better performance."
|
||||
# Issue stores with CUDA event synchronization
|
||||
# Issue stores with CUDA event synchronization.
|
||||
if self.kv_role in ["kv_producer", "kv_both"]:
|
||||
current_event = None
|
||||
for request in meta.requests:
|
||||
@@ -1501,21 +1667,32 @@ class MooncakeStoreWorker:
|
||||
# candidate_meta stores the (group, hash_bytes) for key slice.
|
||||
candidate_keys: list[str] = []
|
||||
candidate_meta: list[tuple[int, bytes]] = []
|
||||
lookup_masks = self.coord.lookup_mask(token_len)
|
||||
fine_grained = self.coord.enable_partial_hash_hits
|
||||
lookup_masks = None if fine_grained else self.coord.lookup_mask(token_len)
|
||||
for g_idx, db in enumerate(self.token_dbs):
|
||||
spec_block_size = db.block_size
|
||||
lookup_mask = lookup_masks[g_idx]
|
||||
key_prefixes = self._lookup_key_prefixes[g_idx]
|
||||
group_hashes = self.coord.block_hashes_for_spec(
|
||||
block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec
|
||||
)
|
||||
max_chunks = min(len(group_hashes), cdiv(token_len, spec_block_size))
|
||||
mask_limit = (
|
||||
max_chunks if lookup_mask is None else min(max_chunks, len(lookup_mask))
|
||||
)
|
||||
for chunk_id in range(mask_limit):
|
||||
if lookup_mask is not None and not lookup_mask[chunk_id]:
|
||||
continue
|
||||
if fine_grained:
|
||||
max_units = min(len(block_hashes), token_len // self.hash_block_size)
|
||||
unit_ids: range | list[int] = range(max_units)
|
||||
group_hashes: Sequence[BlockHash] = block_hashes
|
||||
else:
|
||||
lookup_mask = lookup_masks[g_idx] # type: ignore[index]
|
||||
group_hashes = self.coord.block_hashes_for_spec(
|
||||
block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec
|
||||
)
|
||||
max_chunks = min(len(group_hashes), cdiv(token_len, spec_block_size))
|
||||
mask_limit = (
|
||||
max_chunks
|
||||
if lookup_mask is None
|
||||
else min(max_chunks, len(lookup_mask))
|
||||
)
|
||||
unit_ids = [
|
||||
chunk_id
|
||||
for chunk_id in range(mask_limit)
|
||||
if lookup_mask is None or lookup_mask[chunk_id]
|
||||
]
|
||||
for chunk_id in unit_ids:
|
||||
h = group_hashes[chunk_id]
|
||||
hash_hex = h.hex()
|
||||
for key_prefix in key_prefixes:
|
||||
|
||||
Reference in New Issue
Block a user