[KV Connector] Keep MooncakeStore full hits block-aligned (#43494)

Signed-off-by: Dao Le <daole@inferact.ai>
Signed-off-by: Dao Le <Dao007forever@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Dao007forever
2026-05-23 23:15:03 -07:00
committed by GitHub
co-authored by Claude
parent 33d7cbe02c
commit 0902d8e62f
4 changed files with 74 additions and 39 deletions
@@ -68,12 +68,6 @@ class MooncakeStoreScheduler:
kv_cache_config, vllm_config
)
self._discard_partial_chunks = (
vllm_config.kv_transfer_config.get_from_extra_config(
"discard_partial_chunks", True
)
)
# Per-request state
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
@@ -88,18 +82,19 @@ class MooncakeStoreScheduler:
) -> tuple[int, bool]:
"""Check for external KV cache hit."""
# Look up against the full prefill range, not just the prompt.
if self._discard_partial_chunks:
token_len = request.num_tokens // self._block_size * self._block_size
else:
token_len = request.num_tokens
token_len = request.num_tokens // self._block_size * self._block_size
if token_len < self._block_size:
return 0, False
num_external_hit_tokens = self.client.lookup(token_len, request.block_hashes)
if num_external_hit_tokens == request.num_tokens:
num_external_hit_tokens -= 1
# Leave a sub-block tail uncomputed for sampling, on a block
# boundary so the recv-side load mask covers every yielded chunk.
num_external_hit_tokens = max(
0,
(request.num_tokens - 1) // self._block_size * self._block_size,
)
if num_external_hit_tokens < num_computed_tokens:
need_to_allocate = 0
@@ -214,9 +209,7 @@ class MooncakeStoreScheduler:
self._request_trackers[request.req_id] = request_tracker
last_chunk_tokens_num = (
(len(prefill_tokens) // self._block_size * self._block_size)
if self._discard_partial_chunks
else len(prefill_tokens)
len(prefill_tokens) // self._block_size * self._block_size
)
req_meta = ReqMeta.from_request_tracker(
@@ -226,7 +219,6 @@ class MooncakeStoreScheduler:
skip_save=force_skip_save,
block_hashes=request_real.block_hashes,
is_last_chunk=(request_tracker.token_len >= last_chunk_tokens_num),
discard_partial_chunks=self._discard_partial_chunks,
original_block_size=self.original_block_size,
)
if req_meta is not None:
@@ -269,9 +261,7 @@ class MooncakeStoreScheduler:
self._request_trackers[req_id] = request_tracker
last_chunk_tokens_num = (
(len(prefill_tokens) // self._block_size * self._block_size)
if self._discard_partial_chunks
else len(prefill_tokens)
len(prefill_tokens) // self._block_size * self._block_size
)
req_meta = ReqMeta.from_request_tracker(
request_tracker,
@@ -282,7 +272,6 @@ class MooncakeStoreScheduler:
is_last_chunk=(
request_tracker.token_len >= last_chunk_tokens_num
),
discard_partial_chunks=self._discard_partial_chunks,
original_block_size=self.original_block_size,
)
else:
@@ -310,9 +299,7 @@ class MooncakeStoreScheduler:
request_tracker.update(new_block_ids)
last_chunk_tokens_num = (
(prefill_end // self._block_size * self._block_size)
if self._discard_partial_chunks
else prefill_end
prefill_end // self._block_size * self._block_size
)
req_meta = ReqMeta.from_request_tracker(
request_tracker,
@@ -323,7 +310,6 @@ class MooncakeStoreScheduler:
is_last_chunk=(
request_tracker.token_len >= last_chunk_tokens_num
),
discard_partial_chunks=self._discard_partial_chunks,
original_block_size=self.original_block_size,
)
@@ -341,10 +327,6 @@ class MooncakeStoreScheduler:
if not load_spec:
continue
num_tokens_to_compute = load_spec.kvpool_cached_tokens
if (num_tokens_to_compute % self._block_size != 0) and (
num_tokens_to_compute == unfinished_req.num_tokens - 1
):
num_tokens_to_compute = num_tokens_to_compute + 1
request_tracker = RequestTracker(
req_id=request_id,
token_len=num_tokens_to_compute,
@@ -358,7 +340,6 @@ class MooncakeStoreScheduler:
load_spec=load_spec,
skip_save=None,
block_hashes=unfinished_req.block_hashes,
discard_partial_chunks=self._discard_partial_chunks,
)
if req_meta is not None:
meta.add_request(req_meta)
@@ -1195,14 +1195,7 @@ class MooncakeStoreWorker:
if load_spec is None or not load_spec.can_load:
continue
token_len = request.token_len_chunk
if (load_spec.kvpool_cached_tokens % self.block_size != 0) and (
load_spec.kvpool_cached_tokens == token_len - 1
):
token_len = load_spec.kvpool_cached_tokens + 1
else:
token_len = load_spec.kvpool_cached_tokens
load_spec.token_len = token_len
load_spec.token_len = load_spec.kvpool_cached_tokens
assert self.kv_recv_thread is not None
self.kv_recv_thread.add_request(request)