[Bugfix][Mooncake] Offload producer partial tails on request finish (#52832)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Co-authored-by: Roger Wang <hey@rogerw.io>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Dao007forever
2026-09-01 13:23:35 -07:00
committed by GitHub
co-authored by Roger Wang OpenAI Codex
parent 55178f2d09
commit 7cb9a88f50
10 changed files with 375 additions and 10 deletions
@@ -12,6 +12,7 @@ import pytest
import torch
from tests.v1.core.test_prefix_caching import make_kv_cache_manager, make_request
from vllm.distributed.kv_transfer.kv_connector.v1.base import SupportsHMA
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_utils import (
KVCacheBlockCopy,
@@ -735,11 +736,121 @@ def test_boundary_state_offloads_returns_cow_target():
assert block_id != source_block_id
# Draining clears it.
assert manager.take_boundary_state_offloads() == {}
# CoW consumed the producer marker, so finishing cannot emit the same
# boundary a second time.
assert manager.finalize_partial_tail_offloads(req0) == []
manager.block_pool.free_blocks(retained)
manager.free(req0)
def test_finished_partial_tail_uses_table_source_once():
"""A finish without another allocation hands off the unchanged table block."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
source_block_id = manager.get_blocks("0").get_block_ids()[1][1]
req0.num_computed_tokens = 6
offloads = manager.finalize_partial_tail_offloads(req0)
assert offloads == [(1, source_block_id, 6)]
assert manager.finalize_partial_tail_offloads(req0) == []
assert manager.take_boundary_state_offloads() == {}
# A table state that advanced beyond the boundary is unsafe and is
# consumed without producing a handoff.
advanced_manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req1 = make_request("1", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed, _ = advanced_manager.get_computed_blocks(req1)
assert (
advanced_manager.allocate_slots(req1, 6, num_computed, computed_blocks)
is not None
)
req1.num_computed_tokens = 7
assert advanced_manager.finalize_partial_tail_offloads(req1) == []
assert advanced_manager.finalize_partial_tail_offloads(req1) == []
def test_connector_finish_registers_partial_tail_before_cleanup():
class _Connector(SupportsHMA):
def register_finished_partial_tail(self, request, block_ids, offloads):
self.registered = (request, block_ids, offloads)
return True
def request_finished_all_groups(self, request, block_ids):
return False, None
scheduler = object.__new__(Scheduler)
scheduler.connector = _Connector()
scheduler.vllm_config = SimpleNamespace(
kv_transfer_config=SimpleNamespace(is_kv_producer=True)
)
scheduler.kv_cache_manager = MagicMock()
scheduler.kv_cache_manager.finalize_partial_tail_offloads.return_value = [
(1, 9, 12)
]
scheduler.kv_cache_manager.get_block_ids_for_computed_tokens.return_value = (
[3],
[9],
)
request = SimpleNamespace(
request_id="0",
num_computed_tokens=12,
num_in_flight_tokens=0,
num_prompt_tokens=12,
)
delay_free, kv_xfer_params = scheduler._connector_finished(request)
assert delay_free is True
assert kv_xfer_params is None
assert scheduler.connector.registered == (
request,
([3], [9]),
[(1, 9, 12)],
)
scheduler.kv_cache_manager.remove_skipped_blocks.assert_called_once()
def test_block_pool_touch_pins_released_cow_target():
"""The connector can rescue an offered CoW target after its step-scoped
retention is released by using the bound BlockPool's touch method."""