[KV Connector] Keep MooncakeStore full hits block-aligned (#43494)
Signed-off-by: Dao Le <daole@inferact.ai> Signed-off-by: Dao Le <Dao007forever@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -68,12 +68,6 @@ class MooncakeStoreScheduler:
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
|
||||
self._discard_partial_chunks = (
|
||||
vllm_config.kv_transfer_config.get_from_extra_config(
|
||||
"discard_partial_chunks", True
|
||||
)
|
||||
)
|
||||
|
||||
# Per-request state
|
||||
self.load_specs: dict[str, LoadSpec] = {} # to be loaded
|
||||
self._request_trackers: dict[str, RequestTracker] = {} # scheduled new requests
|
||||
@@ -88,18 +82,19 @@ class MooncakeStoreScheduler:
|
||||
) -> tuple[int, bool]:
|
||||
"""Check for external KV cache hit."""
|
||||
# Look up against the full prefill range, not just the prompt.
|
||||
if self._discard_partial_chunks:
|
||||
token_len = request.num_tokens // self._block_size * self._block_size
|
||||
else:
|
||||
token_len = request.num_tokens
|
||||
|
||||
token_len = request.num_tokens // self._block_size * self._block_size
|
||||
if token_len < self._block_size:
|
||||
return 0, False
|
||||
|
||||
num_external_hit_tokens = self.client.lookup(token_len, request.block_hashes)
|
||||
|
||||
if num_external_hit_tokens == request.num_tokens:
|
||||
num_external_hit_tokens -= 1
|
||||
# Leave a sub-block tail uncomputed for sampling, on a block
|
||||
# boundary so the recv-side load mask covers every yielded chunk.
|
||||
num_external_hit_tokens = max(
|
||||
0,
|
||||
(request.num_tokens - 1) // self._block_size * self._block_size,
|
||||
)
|
||||
|
||||
if num_external_hit_tokens < num_computed_tokens:
|
||||
need_to_allocate = 0
|
||||
@@ -214,9 +209,7 @@ class MooncakeStoreScheduler:
|
||||
self._request_trackers[request.req_id] = request_tracker
|
||||
|
||||
last_chunk_tokens_num = (
|
||||
(len(prefill_tokens) // self._block_size * self._block_size)
|
||||
if self._discard_partial_chunks
|
||||
else len(prefill_tokens)
|
||||
len(prefill_tokens) // self._block_size * self._block_size
|
||||
)
|
||||
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
@@ -226,7 +219,6 @@ class MooncakeStoreScheduler:
|
||||
skip_save=force_skip_save,
|
||||
block_hashes=request_real.block_hashes,
|
||||
is_last_chunk=(request_tracker.token_len >= last_chunk_tokens_num),
|
||||
discard_partial_chunks=self._discard_partial_chunks,
|
||||
original_block_size=self.original_block_size,
|
||||
)
|
||||
if req_meta is not None:
|
||||
@@ -269,9 +261,7 @@ class MooncakeStoreScheduler:
|
||||
self._request_trackers[req_id] = request_tracker
|
||||
|
||||
last_chunk_tokens_num = (
|
||||
(len(prefill_tokens) // self._block_size * self._block_size)
|
||||
if self._discard_partial_chunks
|
||||
else len(prefill_tokens)
|
||||
len(prefill_tokens) // self._block_size * self._block_size
|
||||
)
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
@@ -282,7 +272,6 @@ class MooncakeStoreScheduler:
|
||||
is_last_chunk=(
|
||||
request_tracker.token_len >= last_chunk_tokens_num
|
||||
),
|
||||
discard_partial_chunks=self._discard_partial_chunks,
|
||||
original_block_size=self.original_block_size,
|
||||
)
|
||||
else:
|
||||
@@ -310,9 +299,7 @@ class MooncakeStoreScheduler:
|
||||
request_tracker.update(new_block_ids)
|
||||
|
||||
last_chunk_tokens_num = (
|
||||
(prefill_end // self._block_size * self._block_size)
|
||||
if self._discard_partial_chunks
|
||||
else prefill_end
|
||||
prefill_end // self._block_size * self._block_size
|
||||
)
|
||||
req_meta = ReqMeta.from_request_tracker(
|
||||
request_tracker,
|
||||
@@ -323,7 +310,6 @@ class MooncakeStoreScheduler:
|
||||
is_last_chunk=(
|
||||
request_tracker.token_len >= last_chunk_tokens_num
|
||||
),
|
||||
discard_partial_chunks=self._discard_partial_chunks,
|
||||
original_block_size=self.original_block_size,
|
||||
)
|
||||
|
||||
@@ -341,10 +327,6 @@ class MooncakeStoreScheduler:
|
||||
if not load_spec:
|
||||
continue
|
||||
num_tokens_to_compute = load_spec.kvpool_cached_tokens
|
||||
if (num_tokens_to_compute % self._block_size != 0) and (
|
||||
num_tokens_to_compute == unfinished_req.num_tokens - 1
|
||||
):
|
||||
num_tokens_to_compute = num_tokens_to_compute + 1
|
||||
request_tracker = RequestTracker(
|
||||
req_id=request_id,
|
||||
token_len=num_tokens_to_compute,
|
||||
@@ -358,7 +340,6 @@ class MooncakeStoreScheduler:
|
||||
load_spec=load_spec,
|
||||
skip_save=None,
|
||||
block_hashes=unfinished_req.block_hashes,
|
||||
discard_partial_chunks=self._discard_partial_chunks,
|
||||
)
|
||||
if req_meta is not None:
|
||||
meta.add_request(req_meta)
|
||||
|
||||
@@ -1195,14 +1195,7 @@ class MooncakeStoreWorker:
|
||||
if load_spec is None or not load_spec.can_load:
|
||||
continue
|
||||
|
||||
token_len = request.token_len_chunk
|
||||
if (load_spec.kvpool_cached_tokens % self.block_size != 0) and (
|
||||
load_spec.kvpool_cached_tokens == token_len - 1
|
||||
):
|
||||
token_len = load_spec.kvpool_cached_tokens + 1
|
||||
else:
|
||||
token_len = load_spec.kvpool_cached_tokens
|
||||
load_spec.token_len = token_len
|
||||
load_spec.token_len = load_spec.kvpool_cached_tokens
|
||||
|
||||
assert self.kv_recv_thread is not None
|
||||
self.kv_recv_thread.add_request(request)
|
||||
|
||||
Reference in New Issue
Block a user