[Bugfix][Mooncake] Offload producer partial tails on request finish (#52832)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Co-authored-by: Roger Wang <hey@rogerw.io> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Roger Wang
OpenAI Codex
parent
55178f2d09
commit
7cb9a88f50
@@ -12,6 +12,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.core.test_prefix_caching import make_kv_cache_manager, make_request
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import SupportsHMA
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
KVCacheBlockCopy,
|
||||
@@ -735,11 +736,121 @@ def test_boundary_state_offloads_returns_cow_target():
|
||||
assert block_id != source_block_id
|
||||
# Draining clears it.
|
||||
assert manager.take_boundary_state_offloads() == {}
|
||||
# CoW consumed the producer marker, so finishing cannot emit the same
|
||||
# boundary a second time.
|
||||
assert manager.finalize_partial_tail_offloads(req0) == []
|
||||
|
||||
manager.block_pool.free_blocks(retained)
|
||||
manager.free(req0)
|
||||
|
||||
|
||||
def test_finished_partial_tail_uses_table_source_once():
|
||||
"""A finish without another allocation hands off the unchanged table block."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
source_block_id = manager.get_blocks("0").get_block_ids()[1][1]
|
||||
|
||||
req0.num_computed_tokens = 6
|
||||
offloads = manager.finalize_partial_tail_offloads(req0)
|
||||
|
||||
assert offloads == [(1, source_block_id, 6)]
|
||||
assert manager.finalize_partial_tail_offloads(req0) == []
|
||||
assert manager.take_boundary_state_offloads() == {}
|
||||
|
||||
# A table state that advanced beyond the boundary is unsafe and is
|
||||
# consumed without producing a handoff.
|
||||
advanced_manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = advanced_manager.get_computed_blocks(req1)
|
||||
assert (
|
||||
advanced_manager.allocate_slots(req1, 6, num_computed, computed_blocks)
|
||||
is not None
|
||||
)
|
||||
req1.num_computed_tokens = 7
|
||||
assert advanced_manager.finalize_partial_tail_offloads(req1) == []
|
||||
assert advanced_manager.finalize_partial_tail_offloads(req1) == []
|
||||
|
||||
|
||||
def test_connector_finish_registers_partial_tail_before_cleanup():
|
||||
class _Connector(SupportsHMA):
|
||||
def register_finished_partial_tail(self, request, block_ids, offloads):
|
||||
self.registered = (request, block_ids, offloads)
|
||||
return True
|
||||
|
||||
def request_finished_all_groups(self, request, block_ids):
|
||||
return False, None
|
||||
|
||||
scheduler = object.__new__(Scheduler)
|
||||
scheduler.connector = _Connector()
|
||||
scheduler.vllm_config = SimpleNamespace(
|
||||
kv_transfer_config=SimpleNamespace(is_kv_producer=True)
|
||||
)
|
||||
scheduler.kv_cache_manager = MagicMock()
|
||||
scheduler.kv_cache_manager.finalize_partial_tail_offloads.return_value = [
|
||||
(1, 9, 12)
|
||||
]
|
||||
scheduler.kv_cache_manager.get_block_ids_for_computed_tokens.return_value = (
|
||||
[3],
|
||||
[9],
|
||||
)
|
||||
request = SimpleNamespace(
|
||||
request_id="0",
|
||||
num_computed_tokens=12,
|
||||
num_in_flight_tokens=0,
|
||||
num_prompt_tokens=12,
|
||||
)
|
||||
|
||||
delay_free, kv_xfer_params = scheduler._connector_finished(request)
|
||||
|
||||
assert delay_free is True
|
||||
assert kv_xfer_params is None
|
||||
assert scheduler.connector.registered == (
|
||||
request,
|
||||
([3], [9]),
|
||||
[(1, 9, 12)],
|
||||
)
|
||||
scheduler.kv_cache_manager.remove_skipped_blocks.assert_called_once()
|
||||
|
||||
|
||||
def test_block_pool_touch_pins_released_cow_target():
|
||||
"""The connector can rescue an offered CoW target after its step-scoped
|
||||
retention is released by using the bound BlockPool's touch method."""
|
||||
|
||||
@@ -42,6 +42,7 @@ def _make_bare_scheduler(
|
||||
scheduler._unfinished_request_ids = {"req-0"}
|
||||
scheduler._unfinished_requests = {}
|
||||
scheduler._request_trackers = {}
|
||||
scheduler._finished_partial_tail_metas = {}
|
||||
scheduler._gpu_block_pool = BlockPool(
|
||||
num_gpu_blocks=64, enable_caching=True, hash_block_size=hash_block_size
|
||||
)
|
||||
@@ -971,6 +972,65 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
assert tracker.has_pending_offload is True
|
||||
|
||||
|
||||
def test_finished_partial_tail_is_pre_pinned_as_store_job():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
scheduler.client = SimpleNamespace(discard=lambda *_: None)
|
||||
request = SimpleNamespace(
|
||||
request_id="req-0",
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
)
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=12,
|
||||
allocated_block_ids=([3], [9]),
|
||||
token_ids=list(range(12)),
|
||||
prefill_end_tokens=12,
|
||||
)
|
||||
block_ids = ([3], [9])
|
||||
|
||||
delay_free = scheduler.register_finished_partial_tail(
|
||||
request,
|
||||
block_ids,
|
||||
[(1, 9, 12)],
|
||||
)
|
||||
|
||||
# The exact source is pinned immediately, so the request can be freed
|
||||
# before the next connector metadata build.
|
||||
assert delay_free is False
|
||||
assert scheduler._gpu_block_pool.blocks[9].ref_cnt == 1
|
||||
assert scheduler._gpu_block_pool.blocks[3].ref_cnt == 0
|
||||
|
||||
out = SimpleNamespace(
|
||||
finished_req_ids={"req-0"},
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
kv_connector_block_state=_make_connector_block_state(),
|
||||
)
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert len(meta.requests) == 1
|
||||
req_meta = meta.requests[0]
|
||||
assert req_meta.req_id == "req-0"
|
||||
assert req_meta.token_len_chunk == 0
|
||||
assert req_meta.block_ids == block_ids
|
||||
assert req_meta.block_hashes == request.block_hashes
|
||||
assert req_meta.boundary_state_offloads == [(1, 9, 12)]
|
||||
assert scheduler._pinned_saves[req_meta.store_job_id][0] == [9]
|
||||
assert scheduler._gpu_block_pool.blocks[9].ref_cnt == 1
|
||||
assert scheduler._finished_partial_tail_metas == {}
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({req_meta.store_job_id: 1}))
|
||||
assert scheduler._gpu_block_pool.blocks[9].ref_cnt == 0
|
||||
|
||||
|
||||
def test_decode_boundary_state_offload_dropped_unclaimed():
|
||||
# A hand-off past the prefill end can never complete a joint hybrid hit
|
||||
# (every other group stops saving there), so it is neither transferred nor
|
||||
|
||||
@@ -148,6 +148,26 @@ KVConnectorFactory.register_connector(
|
||||
)
|
||||
|
||||
|
||||
def test_register_finished_partial_tail_notifies_every_connector():
|
||||
connector = object.__new__(MultiConnector)
|
||||
first = MagicMock(spec_set=KVConnectorBase_V1)
|
||||
second = MagicMock(spec_set=KVConnectorBase_V1)
|
||||
first.register_finished_partial_tail.return_value = True
|
||||
second.register_finished_partial_tail.return_value = False
|
||||
connector._connectors = [first, second]
|
||||
request = MagicMock()
|
||||
block_ids = ([1], [2])
|
||||
offloads = [(1, 2, 12)]
|
||||
|
||||
assert connector.register_finished_partial_tail(request, block_ids, offloads)
|
||||
first.register_finished_partial_tail.assert_called_once_with(
|
||||
request, block_ids, offloads
|
||||
)
|
||||
second.register_finished_partial_tail.assert_called_once_with(
|
||||
request, block_ids, offloads
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mc() -> MultiConnector:
|
||||
"""MultiConnector using two mocked connectors"""
|
||||
|
||||
Reference in New Issue
Block a user