[Metrics] Expose cached prompt tokens by cache tier (#56318)
Signed-off-by: Cam Quilici <cjquilici@gmail.com> Signed-off-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Cam Quilici
Nick Hill
Yifan Qiao
parent
ad0f67a7ec
commit
e37e51dd24
@@ -3,7 +3,7 @@
|
||||
import dataclasses
|
||||
from concurrent.futures import Future
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
@@ -60,6 +60,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import (
|
||||
DraftTokenIds,
|
||||
ECConnectorOutput,
|
||||
@@ -2613,6 +2614,65 @@ def test_kv_connector_honors_skip_reading_prefix_cache():
|
||||
assert output.num_scheduled_tokens[scoring.request_id] == BLOCK_SIZE * 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
def test_kv_connector_records_external_cache_hit_sources(monkeypatch, is_async):
|
||||
block_size = 16
|
||||
num_matched_tokens = 2 * block_size
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
use_kv_connector=mock_kv(
|
||||
matched_tokens=num_matched_tokens,
|
||||
is_async=is_async,
|
||||
),
|
||||
block_size=block_size,
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=2 * num_matched_tokens,
|
||||
block_size=block_size,
|
||||
)[0]
|
||||
assert scheduler.connector is not None
|
||||
get_sources = Mock(
|
||||
return_value={CacheHitSource.P2P: block_size, CacheHitSource.HOST: block_size}
|
||||
)
|
||||
update_state = Mock(wraps=scheduler.connector.update_state_after_alloc)
|
||||
calls = Mock()
|
||||
calls.attach_mock(update_state, "allocated")
|
||||
calls.attach_mock(get_sources, "sources")
|
||||
monkeypatch.setattr(scheduler.connector, "update_state_after_alloc", update_state)
|
||||
monkeypatch.setattr(
|
||||
scheduler.connector, "get_external_cache_hit_sources", get_sources
|
||||
)
|
||||
|
||||
scheduler.add_request(request)
|
||||
# A failed allocation is not an admission: nothing is attributed.
|
||||
with patch.object(scheduler.kv_cache_manager, "allocate_slots", return_value=None):
|
||||
scheduler.schedule()
|
||||
get_sources.assert_not_called()
|
||||
update_state.assert_not_called()
|
||||
|
||||
scheduler.schedule()
|
||||
# Attribution runs once, after the connector has built its load plan.
|
||||
get_sources.assert_called_once_with(request, num_matched_tokens)
|
||||
assert [c[0] for c in calls.mock_calls] == ["allocated", "sources"]
|
||||
connector_stats = scheduler.connector_prefix_cache_stats
|
||||
assert connector_stats is not None
|
||||
assert connector_stats.hits == num_matched_tokens
|
||||
assert connector_stats.hits_by_source == {
|
||||
CacheHitSource.P2P: block_size,
|
||||
CacheHitSource.HOST: block_size,
|
||||
}
|
||||
|
||||
if is_async:
|
||||
# Re-admission after the async load completes is not a new hit.
|
||||
scheduler.make_stats()
|
||||
_step_until_kv_transfer_finished(scheduler, [request.request_id])
|
||||
get_sources.assert_called_once()
|
||||
connector_stats = scheduler.connector_prefix_cache_stats
|
||||
assert connector_stats is not None
|
||||
assert connector_stats.hits_by_source == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
def test_kv_connector_basic(is_async: bool):
|
||||
"""Test whether Scheduler with KVConnector schedules tokens, allocates
|
||||
|
||||
@@ -77,6 +77,7 @@ from vllm.v1.kv_offload.base import (
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
@@ -197,14 +198,18 @@ def test_swa_offload_window_covers_unaligned_hit(
|
||||
manager.complete_load(job.keys, state.req_context)
|
||||
|
||||
|
||||
def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler:
|
||||
def _make_partial_tail_scheduler(
|
||||
recurrent_only: bool = False,
|
||||
) -> OffloadingConnectorScheduler:
|
||||
vllm_config = _make_vllm_config(extra_config={"self_describing_kv_events": True})
|
||||
vllm_config.cache_config.prefix_match_unit = 4
|
||||
vllm_config.cache_config.prefix_match_unit = None if recurrent_only else 4
|
||||
vllm_config.speculative_config = None
|
||||
vllm_config.kv_events_config = KVEventsConfig(
|
||||
enable_kv_cache_events=True, publisher="null"
|
||||
)
|
||||
kv_cache_config = _make_mamba_hybrid_kv_cache_config()
|
||||
if recurrent_only:
|
||||
kv_cache_config.kv_cache_groups = kv_cache_config.kv_cache_groups[1:]
|
||||
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
|
||||
return OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
|
||||
|
||||
@@ -576,6 +581,284 @@ def test_recurrent_group_unhashed_block_does_not_truncate_load_boundary():
|
||||
assert 42 in loaded
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("recurrent_source", "expected"),
|
||||
[
|
||||
("host", {CacheHitSource.P2P: 16, CacheHitSource.HOST: 12}),
|
||||
("p2p", {CacheHitSource.P2P: 28}),
|
||||
(None, {CacheHitSource.P2P: 16, CacheHitSource.HOST: 12}),
|
||||
],
|
||||
)
|
||||
def test_external_cache_hit_sources_preserve_partial_tail_origin(
|
||||
recurrent_source, expected
|
||||
):
|
||||
"""Recurrent state covers the whole prefix; the outermost tier wins."""
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(scheduler)
|
||||
req_status = scheduler._req_status["req"]
|
||||
req_status.update_offload_keys()
|
||||
|
||||
scheduler.manager.lookup.return_value = LookupResult.HIT
|
||||
assert scheduler._lookup(req_status) == 28
|
||||
|
||||
def get_load_source(key, req_context):
|
||||
if get_offload_group_idx(key) == 1:
|
||||
assert recurrent_source is not None
|
||||
return recurrent_source
|
||||
block_hash = get_offload_block_hash(key)
|
||||
return "p2p" if block_hash == b"h3" else "host"
|
||||
|
||||
scheduler.manager.get_load_source.side_effect = get_load_source
|
||||
|
||||
recurrent_block = KVCacheBlock(41)
|
||||
if recurrent_source is None:
|
||||
recurrent_block.set_block_hash(
|
||||
make_block_hash_with_group_id(BlockHash(b"cached"), 1), 12
|
||||
)
|
||||
scheduler.update_state_after_alloc(
|
||||
request,
|
||||
KVCacheBlocks(
|
||||
(
|
||||
[KVCacheBlock(31), KVCacheBlock(32)],
|
||||
[KVCacheBlock(0, is_null=True), recurrent_block],
|
||||
)
|
||||
),
|
||||
num_external_tokens=28,
|
||||
)
|
||||
assert req_status.partial_tail_boundary is None
|
||||
assert scheduler.get_external_cache_hit_sources(request, 28) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("second_source", "expected"),
|
||||
[
|
||||
("host", {CacheHitSource.HOST: 4, CacheHitSource.DISK: 4}),
|
||||
("p2p", {CacheHitSource.P2P: 4, CacheHitSource.DISK: 4}),
|
||||
],
|
||||
)
|
||||
def test_external_cache_hit_sources_reconcile_different_group_chunk_sizes(
|
||||
request_runner,
|
||||
second_source,
|
||||
expected,
|
||||
):
|
||||
"""Groups with different chunk sizes overlap; the outermost tier wins."""
|
||||
block_size = 4
|
||||
groups = [
|
||||
KVCacheGroupSpec(
|
||||
[f"layer{group_idx}"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size * (group_idx + 1),
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
for group_idx in range(2)
|
||||
]
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=False,
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
scheduler = runner.connector_scheduler
|
||||
request = MagicMock(request_id="mixed")
|
||||
req_status = RequestOffloadState(
|
||||
config=scheduler.config,
|
||||
req=request,
|
||||
req_context=ReqContext(req_id=request.request_id),
|
||||
offloading_context=RequestOffloadingContext(policy=OffloadPolicy.CHUNK_LEVEL),
|
||||
)
|
||||
for group_idx, group_state in enumerate(req_status.group_states):
|
||||
group_state.offload_keys = [
|
||||
make_offload_key(f"h{chunk_idx}".encode(), group_idx)
|
||||
for chunk_idx in range(2 // (group_idx + 1))
|
||||
]
|
||||
scheduler._req_status[request.request_id] = req_status
|
||||
runner.manager.get_load_source.side_effect = lambda key, req_context: (
|
||||
("host" if get_offload_block_hash(key) == b"h0" else "disk")
|
||||
if get_offload_group_idx(key) == 0
|
||||
else second_source
|
||||
)
|
||||
scheduler.update_state_after_alloc(
|
||||
request,
|
||||
KVCacheBlocks(([KVCacheBlock(1), KVCacheBlock(2)], [KVCacheBlock(3)])),
|
||||
2 * block_size,
|
||||
)
|
||||
|
||||
assert scheduler.get_external_cache_hit_sources(request, 2 * block_size) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("ranges", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
[(8, 12, "host"), (0, 8, "host"), (4, 10, "host")],
|
||||
{CacheHitSource.HOST: 8, CacheHitSource.EXTERNAL_UNSPECIFIED: 4},
|
||||
id="overlapping-same-source-and-adjacent-endpoints",
|
||||
),
|
||||
pytest.param(
|
||||
[(10, 20, "p2p"), (6, 10, "disk"), (0, 8, "host"), (4, 12, "host")],
|
||||
{CacheHitSource.HOST: 2, CacheHitSource.DISK: 4, CacheHitSource.P2P: 6},
|
||||
id="nested-tiers-and-clipped-ranges",
|
||||
),
|
||||
pytest.param(
|
||||
[(0, 4, "disk"), (16, 20, "disk"), (8, 8, "disk")],
|
||||
{CacheHitSource.EXTERNAL_UNSPECIFIED: 12},
|
||||
id="no-covering-ranges",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_external_cache_hit_sources_range_boundaries(ranges, expected):
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(scheduler)
|
||||
state = scheduler._req_status[request.request_id]
|
||||
state.num_locally_computed_tokens = 4
|
||||
for idx, (lo, hi, source) in enumerate(ranges):
|
||||
key = make_offload_key(str(idx).encode(), 0)
|
||||
state.load_key_ranges.append((lo, hi, key))
|
||||
scheduler.manager.get_load_source.side_effect = lambda key, ctx: ranges[
|
||||
int(get_offload_block_hash(key))
|
||||
][2]
|
||||
|
||||
result = scheduler.get_external_cache_hit_sources(request, 12)
|
||||
assert result == expected
|
||||
assert sum(result.values()) == 12
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", ["host", "disk", "p2p", "external_unspecified"])
|
||||
def test_external_cache_hit_sources_recurrent_only_state(source):
|
||||
scheduler = _make_partial_tail_scheduler(recurrent_only=True)
|
||||
request = _make_partial_tail_request(scheduler)
|
||||
request.num_prompt_tokens = request.num_tokens = 64
|
||||
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(4)]
|
||||
request.all_token_ids = list(range(64))
|
||||
state = scheduler._req_status[request.request_id]
|
||||
state.update_offload_keys()
|
||||
scheduler.manager.lookup.return_value = LookupResult.HIT
|
||||
assert scheduler._lookup(state) == 48
|
||||
scheduler.manager.get_load_source.return_value = source
|
||||
scheduler.update_state_after_alloc(
|
||||
request,
|
||||
KVCacheBlocks(
|
||||
(
|
||||
[
|
||||
KVCacheBlock(0, is_null=True),
|
||||
KVCacheBlock(0, is_null=True),
|
||||
KVCacheBlock(41),
|
||||
],
|
||||
)
|
||||
),
|
||||
48,
|
||||
)
|
||||
assert scheduler.get_external_cache_hit_sources(request, 48) == {
|
||||
CacheHitSource(source): 48
|
||||
}
|
||||
scheduler.manager.get_load_source.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sparse_kind", ["sliding", "chunked_local"])
|
||||
@pytest.mark.parametrize("blocks_per_chunk", [1, 2])
|
||||
@pytest.mark.parametrize("local_tokens", [0, 4])
|
||||
@pytest.mark.parametrize("full_attention", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"sparse_source", ["host", "disk", "p2p", "external_unspecified", "mixed"]
|
||||
)
|
||||
def test_external_cache_hit_sources_use_loaded_attention_ranges(
|
||||
sparse_kind,
|
||||
blocks_per_chunk,
|
||||
local_tokens,
|
||||
full_attention,
|
||||
sparse_source,
|
||||
):
|
||||
"""SWA/local attention affects only loaded positions, not omitted history."""
|
||||
block_size = 4
|
||||
chunk_size = block_size * blocks_per_chunk
|
||||
spec_kwargs = dict(
|
||||
block_size=block_size, num_kv_heads=1, head_size=1, dtype=torch.float32
|
||||
)
|
||||
sparse_spec = (
|
||||
SlidingWindowSpec(**spec_kwargs, sliding_window=2 * chunk_size)
|
||||
if sparse_kind == "sliding"
|
||||
else ChunkedLocalAttentionSpec(
|
||||
**spec_kwargs, attention_chunk_size=2 * chunk_size
|
||||
)
|
||||
)
|
||||
groups = [KVCacheGroupSpec(["sparse"], sparse_spec)]
|
||||
if full_attention:
|
||||
groups.insert(0, KVCacheGroupSpec(["full"], FullAttentionSpec(**spec_kwargs)))
|
||||
vllm_config = _make_vllm_config(extra_config={"block_size": chunk_size})
|
||||
vllm_config.cache_config.block_size = block_size
|
||||
vllm_config.speculative_config = None
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
spec = MockOffloadingSpec(build_offloading_config(vllm_config, kv_cache_config))
|
||||
scheduler = OffloadingConnectorScheduler(spec, vllm_config, kv_cache_config)
|
||||
request = MagicMock(request_id="sparse")
|
||||
req_status = RequestOffloadState(
|
||||
config=scheduler.config,
|
||||
req=request,
|
||||
req_context=ReqContext(req_id=request.request_id),
|
||||
offloading_context=RequestOffloadingContext(policy=OffloadPolicy.CHUNK_LEVEL),
|
||||
num_locally_computed_tokens=local_tokens,
|
||||
)
|
||||
scheduler._req_status[request.request_id] = req_status
|
||||
block_groups = []
|
||||
for group_idx, state in enumerate(req_status.group_states):
|
||||
state.offload_keys = [
|
||||
make_offload_key(str(i).encode(), group_idx) for i in range(4)
|
||||
]
|
||||
block_groups.append(
|
||||
[
|
||||
KVCacheBlock(
|
||||
0
|
||||
if group_idx == len(groups) - 1 and i < 2 * blocks_per_chunk
|
||||
else 1 + group_idx * 100 + i,
|
||||
is_null=group_idx == len(groups) - 1 and i < 2 * blocks_per_chunk,
|
||||
)
|
||||
for i in range(4 * blocks_per_chunk)
|
||||
]
|
||||
)
|
||||
|
||||
def get_load_source(key, req_context):
|
||||
chunk_idx = int(get_offload_block_hash(key))
|
||||
if get_offload_group_idx(key) == len(groups) - 1:
|
||||
assert chunk_idx >= 2 # Earlier sparse keys were not loaded.
|
||||
if sparse_source == "mixed":
|
||||
return "host" if chunk_idx == 2 else "disk"
|
||||
return sparse_source
|
||||
return "host"
|
||||
|
||||
scheduler.manager.get_load_source.side_effect = get_load_source
|
||||
count = 4 * chunk_size - local_tokens
|
||||
scheduler.update_state_after_alloc(
|
||||
request, KVCacheBlocks(tuple(block_groups)), count
|
||||
)
|
||||
expected: dict[CacheHitSource, int] = {}
|
||||
|
||||
def expect(source: CacheHitSource, num_tokens: int) -> None:
|
||||
expected[source] = expected.get(source, 0) + num_tokens
|
||||
|
||||
expect(
|
||||
CacheHitSource.HOST if full_attention else CacheHitSource.EXTERNAL_UNSPECIFIED,
|
||||
2 * chunk_size - local_tokens,
|
||||
)
|
||||
for chunk_idx in range(2, 4):
|
||||
source = (
|
||||
("host" if chunk_idx == 2 else "disk")
|
||||
if sparse_source == "mixed"
|
||||
else sparse_source
|
||||
)
|
||||
expect(CacheHitSource(source), chunk_size)
|
||||
result = scheduler.get_external_cache_hit_sources(request, count)
|
||||
assert result == expected
|
||||
assert sum(result.values()) == count
|
||||
assert scheduler.get_external_cache_hit_sources(request, 0) == {}
|
||||
|
||||
|
||||
def test_partial_lookup_requires_every_cache_group():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
_make_partial_tail_request(scheduler)
|
||||
@@ -1400,7 +1683,6 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
|
||||
# are within the window → loads blocks 1,2
|
||||
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
|
||||
)
|
||||
|
||||
runner.manager.touch.assert_not_called()
|
||||
|
||||
# 3 blocks are hit on GPU [0, 1, 2]
|
||||
|
||||
@@ -57,6 +57,7 @@ from vllm.v1.kv_offload.base import (
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.config import OffloadingConfig
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
|
||||
@@ -129,6 +130,7 @@ class MockOffloadingSpec(OffloadingSpec):
|
||||
self.manager = MagicMock(spec=OffloadingManager)
|
||||
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
|
||||
self.manager.lookup.return_value = LookupResult.MISS
|
||||
self.manager.get_load_source.return_value = CacheHitSource.EXTERNAL_UNSPECIFIED
|
||||
self.manager.get_stats.return_value = None
|
||||
self.manager.on_new_request.return_value = RequestOffloadingContext()
|
||||
self.handler = MockOffloadingWorker()
|
||||
|
||||
@@ -8,7 +8,6 @@ prefix cache hash table so future requests cannot match and reuse corrupted data
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -16,6 +15,7 @@ from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
from .utils import (
|
||||
create_mock_connector,
|
||||
create_model_runner_output,
|
||||
create_request,
|
||||
create_scheduler,
|
||||
@@ -73,7 +73,7 @@ def test_invalid_blocks_evicted_prevents_cache_pollution(
|
||||
}
|
||||
|
||||
# mock connector indicating sync load
|
||||
fail_scheduler.connector = Mock()
|
||||
fail_scheduler.connector = create_mock_connector()
|
||||
fail_scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, False)
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -10,6 +9,7 @@ from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.request import FinishReason, Request, RequestStatus
|
||||
|
||||
from .utils import (
|
||||
create_mock_connector,
|
||||
create_model_runner_output,
|
||||
create_request,
|
||||
create_scheduler,
|
||||
@@ -56,7 +56,7 @@ def test_error_propagation_sync_load(fail_scheduler: Scheduler):
|
||||
request.request_id: num_external_computed_tokens,
|
||||
}
|
||||
|
||||
fail_scheduler.connector = Mock()
|
||||
fail_scheduler.connector = create_mock_connector()
|
||||
fail_scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, False)
|
||||
)
|
||||
@@ -110,7 +110,7 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler):
|
||||
request.request_id: num_external_computed_tokens,
|
||||
}
|
||||
|
||||
fail_scheduler.connector = Mock()
|
||||
fail_scheduler.connector = create_mock_connector()
|
||||
fail_scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
fail_scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, True)
|
||||
|
||||
@@ -11,7 +11,6 @@ These tests verify correct behavior in three scenarios:
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -19,6 +18,7 @@ from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.request import FinishReason, Request, RequestStatus
|
||||
|
||||
from .utils import (
|
||||
create_mock_connector,
|
||||
create_model_runner_output,
|
||||
create_request,
|
||||
create_scheduler,
|
||||
@@ -83,7 +83,7 @@ def test_sync_recompute_blocks_not_freed_for_running_requests(
|
||||
}
|
||||
|
||||
# mock connector indicating sync load
|
||||
recompute_scheduler.connector = Mock()
|
||||
recompute_scheduler.connector = create_mock_connector()
|
||||
recompute_scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, False)
|
||||
)
|
||||
@@ -206,7 +206,7 @@ def test_sync_fail_invalid_blocks_evicted(fail_scheduler: Scheduler):
|
||||
}
|
||||
|
||||
# mock connector indicating sync load
|
||||
fail_scheduler.connector = Mock()
|
||||
fail_scheduler.connector = create_mock_connector()
|
||||
fail_scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, False)
|
||||
)
|
||||
@@ -323,7 +323,7 @@ def test_async_recompute_blocks_not_cached_when_invalid(
|
||||
}
|
||||
|
||||
# mock connector indicating async load
|
||||
recompute_scheduler.connector = Mock()
|
||||
recompute_scheduler.connector = create_mock_connector()
|
||||
recompute_scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, True)
|
||||
)
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -10,6 +9,7 @@ from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.request import Request, RequestStatus
|
||||
|
||||
from .utils import (
|
||||
create_mock_connector,
|
||||
create_model_runner_output,
|
||||
create_request,
|
||||
create_scheduler,
|
||||
@@ -41,7 +41,7 @@ def test_failed_receive_completion_honors_failure_policy(policy):
|
||||
scheduler = create_scheduler(create_vllm_config(kv_load_failure_policy=policy))
|
||||
request = create_request(num_tokens=3 * scheduler.block_size)
|
||||
scheduler.add_request(request)
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector = create_mock_connector()
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = [
|
||||
(2 * scheduler.block_size, True),
|
||||
@@ -103,7 +103,7 @@ def test_async_load_failure(
|
||||
request3.request_id: num_external_computed_tokens,
|
||||
}
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector = create_mock_connector()
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True)
|
||||
@@ -181,7 +181,7 @@ def test_sync_load_failure(
|
||||
request3.request_id: num_external_computed_tokens,
|
||||
}
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector = create_mock_connector()
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=False)
|
||||
)
|
||||
@@ -262,7 +262,7 @@ def test_sync_load_failure_with_shared_blocks(
|
||||
request1.request_id: num_external_computed_tokens,
|
||||
}
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector = create_mock_connector()
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=False)
|
||||
)
|
||||
@@ -333,7 +333,7 @@ def test_async_progressive_load_failure(
|
||||
request.request_id: num_external_computed_tokens,
|
||||
}
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector = create_mock_connector()
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True)
|
||||
|
||||
@@ -38,6 +38,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheTensor,
|
||||
MambaSpec,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
|
||||
from .utils import create_vllm_config
|
||||
@@ -208,6 +209,29 @@ def test_scheduler_role_initializes_store_scheduler_only():
|
||||
mock_scheduler.return_value.bind_gpu_block_pool.assert_called_once_with(block_pool)
|
||||
|
||||
|
||||
def test_scheduler_reports_mooncake_cache_source():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
with (
|
||||
set_current_vllm_config(vllm_config),
|
||||
patch(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store."
|
||||
"connector.MooncakeStoreScheduler"
|
||||
),
|
||||
):
|
||||
connector = mooncake_store_connector.MooncakeStoreConnector(
|
||||
vllm_config, KVConnectorRole.SCHEDULER, kv_cache_config
|
||||
)
|
||||
|
||||
# Mooncake Store is an external cache service. Its internal memory and
|
||||
# local-storage policy is intentionally hidden behind that stable label.
|
||||
assert connector.get_external_cache_hit_sources(
|
||||
None, # type: ignore[arg-type]
|
||||
32,
|
||||
) == {CacheHitSource.EXTERNAL_UNSPECIFIED: 32}
|
||||
|
||||
|
||||
def test_worker_methods_delegate_to_store_worker():
|
||||
vllm_config = _make_vllm_config()
|
||||
kv_cache_config = _make_kv_cache_config()
|
||||
|
||||
@@ -34,6 +34,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
|
||||
NixlKVConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput, KVConnectorWorkerMetadata
|
||||
|
||||
MODEL_NAME = "meta-llama/Llama-3.2-1B-Instruct"
|
||||
@@ -246,6 +247,18 @@ def test_multi_connector_rejects_multiple_mem_pool_contexts(mc: MultiConnector):
|
||||
mc.get_mem_pool_context()
|
||||
|
||||
|
||||
def test_cache_hit_sources_delegate_to_selected_connector(mc: MultiConnector):
|
||||
request = MagicMock(request_id="request")
|
||||
mc._requests_to_connector[request.request_id] = 1
|
||||
sources = {CacheHitSource.DISK: 32}
|
||||
mc._connectors[1].get_external_cache_hit_sources.return_value = sources
|
||||
|
||||
assert mc.get_external_cache_hit_sources(request, 32) is sources
|
||||
mc._connectors[1].get_external_cache_hit_sources.assert_called_once_with(
|
||||
request, 32
|
||||
)
|
||||
|
||||
|
||||
# Helper function to compare directories recursively
|
||||
def _compare_directories(dir1: Path, dir2: Path) -> bool:
|
||||
"""Compares two directories recursively for identical content."""
|
||||
|
||||
@@ -36,6 +36,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheTensor,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.simple_kv_offload.metadata import (
|
||||
@@ -232,6 +233,9 @@ def test_nixl_wins_load_over_cpu_offload():
|
||||
assert not sched_out.has_sync_kv_loads
|
||||
|
||||
assert mc._requests_to_connector[request.request_id] == 0
|
||||
assert mc.get_external_cache_hit_sources(request, 2 * BLOCK_SIZE) == {
|
||||
CacheHitSource.P2P: 2 * BLOCK_SIZE
|
||||
}
|
||||
|
||||
meta = sched_out.kv_connector_metadata
|
||||
assert isinstance(meta, MultiKVConnectorMetadata)
|
||||
@@ -302,6 +306,9 @@ def test_cpu_offload_wins_when_nixl_has_no_match():
|
||||
assert hit_tokens is not None and hit_tokens > 0
|
||||
assert mc._requests_to_connector[req2.request_id] == 1
|
||||
assert is_async is True
|
||||
assert mc.get_external_cache_hit_sources(req2, hit_tokens) == {
|
||||
CacheHitSource.HOST: hit_tokens
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("swa_enabled", [False, True], ids=["fa_only", "fa_sw"])
|
||||
|
||||
@@ -6,6 +6,7 @@ from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain, count
|
||||
from typing import Any, Literal
|
||||
from unittest.mock import Mock
|
||||
|
||||
import torch
|
||||
|
||||
@@ -42,6 +43,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.structured_output import StructuredOutputManager
|
||||
@@ -49,6 +51,14 @@ from vllm.v1.structured_output import StructuredOutputManager
|
||||
EOS_TOKEN_ID = 50256
|
||||
|
||||
|
||||
def create_mock_connector() -> Mock:
|
||||
connector = Mock()
|
||||
connector.get_external_cache_hit_sources.side_effect = lambda request, num_tokens: (
|
||||
{CacheHitSource.EXTERNAL_UNSPECIFIED: num_tokens} if num_tokens else {}
|
||||
)
|
||||
return connector
|
||||
|
||||
|
||||
def assert_scheduler_empty(scheduler: Scheduler):
|
||||
"""Confirm the scheduler is "empty" - i.e. no leaks."""
|
||||
# Scheduler Metadata.
|
||||
|
||||
@@ -17,6 +17,7 @@ import pytest
|
||||
from vllm.v1.kv_offload.tiering.base import SecondaryTierManager
|
||||
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
|
||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
@@ -43,12 +44,22 @@ def _make_mock_args():
|
||||
|
||||
def test_pre_registered_tiers_can_be_imported():
|
||||
"""CI sentinel: registered tiers import and yield SecondaryTierManager."""
|
||||
# Derived from each tier's ``medium``; P2P has none and sets its own.
|
||||
expected_sources = {
|
||||
"example": CacheHitSource.HOST,
|
||||
"fs": CacheHitSource.DISK,
|
||||
"obj": CacheHitSource.DISK,
|
||||
"p2p": CacheHitSource.P2P,
|
||||
}
|
||||
for tier_type in SecondaryTierFactory._registry:
|
||||
# KVCR is an optional external dependency and may not be installed.
|
||||
if tier_type == "kvcr" and find_spec("kvcr") is None:
|
||||
continue
|
||||
cls = SecondaryTierFactory._registry[tier_type]()
|
||||
assert issubclass(cls, SecondaryTierManager)
|
||||
assert cls.cache_hit_source == expected_sources.get(
|
||||
tier_type, CacheHitSource.EXTERNAL_UNSPECIFIED
|
||||
)
|
||||
|
||||
|
||||
def test_example_tier_registered():
|
||||
@@ -95,7 +106,8 @@ def test_create_multiple_tiers():
|
||||
assert all(isinstance(tier, ExampleSecondaryTierManager) for tier in tiers)
|
||||
|
||||
|
||||
def test_register_new_tier_type():
|
||||
@pytest.mark.parametrize("tier_type", ["custom_tier", "cpu", "nvme"])
|
||||
def test_register_new_tier_type(tier_type):
|
||||
"""Verify that new tier types can be registered and created.
|
||||
|
||||
This is how external projects add custom secondary tiers
|
||||
@@ -103,20 +115,22 @@ def test_register_new_tier_type():
|
||||
"""
|
||||
# Register a new tier type (reuse example manager for simplicity)
|
||||
SecondaryTierFactory.register_tier(
|
||||
"custom_tier",
|
||||
tier_type,
|
||||
"vllm.v1.kv_offload.tiering.example.manager",
|
||||
"ExampleSecondaryTierManager",
|
||||
)
|
||||
|
||||
primary_kv_view, offloading_spec = _make_mock_args()
|
||||
tier = SecondaryTierFactory.create_secondary_tier(
|
||||
{"type": "custom_tier", "custom_param": 99},
|
||||
{"type": tier_type, "custom_param": 99},
|
||||
primary_kv_view,
|
||||
offloading_spec,
|
||||
)
|
||||
|
||||
assert tier.tier_type == "custom_tier"
|
||||
assert tier.tier_type == tier_type
|
||||
assert isinstance(tier, ExampleSecondaryTierManager)
|
||||
# Attribution follows the tier's medium, never its registration name.
|
||||
assert tier.cache_hit_source is CacheHitSource.HOST
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -53,6 +53,7 @@ from vllm.v1.kv_offload.tiering.manager import (
|
||||
TieringOffloadingManager,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.spec import TieringOffloadingSpec
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
|
||||
_CTX = ReqContext(req_id="test")
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock()
|
||||
@@ -500,8 +501,13 @@ class TestTieringOffloadingManager:
|
||||
# Lookup should find all chunks in primary
|
||||
assert count_hits(self.manager, chunks) == 3
|
||||
|
||||
def test_promotion_from_secondary(self, manager_setup):
|
||||
@pytest.mark.parametrize(
|
||||
"source",
|
||||
[CacheHitSource.EXTERNAL_UNSPECIFIED, CacheHitSource.DISK, CacheHitSource.P2P],
|
||||
)
|
||||
def test_promotion_from_secondary(self, manager_setup, monkeypatch, source):
|
||||
"""Test promotion of chunks from secondary to primary tier."""
|
||||
monkeypatch.setattr(ExampleSecondaryTierManager, "cache_hit_source", source)
|
||||
chunks = to_keys(range(3))
|
||||
|
||||
# Manually add chunks to secondary tier (simulate previous cascade)
|
||||
@@ -525,6 +531,27 @@ class TestTieringOffloadingManager:
|
||||
# Next lookup should succeed
|
||||
assert count_hits(self.manager, chunks) == 3
|
||||
|
||||
# The request that caused the promotion retains the secondary origin.
|
||||
assert all(
|
||||
self.manager.get_load_source(chunk, _CTX) == source for chunk in chunks
|
||||
)
|
||||
|
||||
# A later request sees the chunks as ordinary host-memory primary hits.
|
||||
later_context = ReqContext(req_id="later")
|
||||
self.manager.on_new_request(later_context)
|
||||
assert all(
|
||||
self.manager.lookup(chunk, later_context) is LookupResult.HIT
|
||||
for chunk in chunks
|
||||
)
|
||||
assert all(
|
||||
self.manager.get_load_source(chunk, later_context) == "host"
|
||||
for chunk in chunks
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source",
|
||||
[CacheHitSource.EXTERNAL_UNSPECIFIED, CacheHitSource.DISK, CacheHitSource.P2P],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("successful_indices", "expected_results"),
|
||||
[
|
||||
@@ -540,8 +567,9 @@ class TestTieringOffloadingManager:
|
||||
ids=["partial", "legacy-full-failure"],
|
||||
)
|
||||
def test_failed_promotion_keeps_only_successful_chunks(
|
||||
self, manager_setup, successful_indices, expected_results
|
||||
self, manager_setup, monkeypatch, source, successful_indices, expected_results
|
||||
):
|
||||
monkeypatch.setattr(ExampleSecondaryTierManager, "cache_hit_source", source)
|
||||
chunks = to_keys(range(3))
|
||||
for chunk in chunks:
|
||||
self.secondary_tier1.chunks[chunk] = True
|
||||
@@ -571,6 +599,14 @@ class TestTieringOffloadingManager:
|
||||
assert [
|
||||
self.primary_tier.lookup(chunk, _CTX) for chunk in chunks
|
||||
] == expected_results
|
||||
expected_source_keys = (
|
||||
set()
|
||||
if successful_indices is None
|
||||
else {chunks[i] for i in successful_indices}
|
||||
)
|
||||
assert set(self.manager._request_load_sources[_CTX.req_id]) == (
|
||||
expected_source_keys
|
||||
)
|
||||
|
||||
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
|
||||
"""Resolved lookups report one sync delay sample per tier and chunk."""
|
||||
@@ -1315,6 +1351,7 @@ class TestTieringOffloadingManager:
|
||||
# Orchestrator state cleared.
|
||||
assert self.manager._jobs == {}
|
||||
assert self.manager._pending_load_submissions == {}
|
||||
assert self.manager._request_load_sources == {}
|
||||
assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id}
|
||||
assert self.manager._processed_jobs_this_step is False
|
||||
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm.v1.core.sched.output import ScheduledEncoderInputStats, SchedulerOutput
|
||||
from vllm.v1.engine import EngineCoreOutputs, FinishReason
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.metrics.stats import (
|
||||
IterationStats,
|
||||
PrefillStats,
|
||||
PrefixCacheStats,
|
||||
PromptTokenStats,
|
||||
RequestStateStats,
|
||||
SchedulerIterationDetails,
|
||||
@@ -45,6 +48,96 @@ def test_scheduler_iteration_details_serialization():
|
||||
assert decoded.scheduler_stats.iteration_details == iteration_details
|
||||
|
||||
|
||||
def test_prefix_cache_stats_hits_by_source_accumulate():
|
||||
stats = PrefixCacheStats()
|
||||
stats.record(
|
||||
num_tokens=8,
|
||||
num_hits=5,
|
||||
preempted=False,
|
||||
hits_by_source={CacheHitSource.HOST: 3, CacheHitSource.DISK: 2},
|
||||
)
|
||||
stats.record(
|
||||
num_tokens=8,
|
||||
num_hits=6,
|
||||
preempted=False,
|
||||
hits_by_source={CacheHitSource.P2P: 5, CacheHitSource.HOST: 1},
|
||||
)
|
||||
# Zero counts never create an entry.
|
||||
stats.record(
|
||||
num_tokens=8,
|
||||
num_hits=4,
|
||||
preempted=False,
|
||||
hits_by_source={CacheHitSource.DISK: 4, CacheHitSource.P2P: 0},
|
||||
)
|
||||
|
||||
assert stats.hits_by_source == {
|
||||
CacheHitSource.HOST: 4,
|
||||
CacheHitSource.DISK: 6,
|
||||
CacheHitSource.P2P: 5,
|
||||
}
|
||||
assert sum(stats.hits_by_source.values()) == stats.hits == 15
|
||||
|
||||
|
||||
def test_prefix_cache_stats_record_hits_by_source():
|
||||
stats = PrefixCacheStats()
|
||||
stats.record(num_tokens=32, num_hits=16, preempted=False)
|
||||
stats.record(
|
||||
num_tokens=32,
|
||||
num_hits=24,
|
||||
preempted=False,
|
||||
hits_by_source={CacheHitSource.HOST: 16, CacheHitSource.DISK: 8},
|
||||
)
|
||||
# Preempted re-admissions never contribute to hits or the split.
|
||||
stats.record(
|
||||
num_tokens=32,
|
||||
num_hits=32,
|
||||
preempted=True,
|
||||
hits_by_source={CacheHitSource.P2P: 32},
|
||||
)
|
||||
# A connector miscount is reported as unspecified, not dropped or raised.
|
||||
stats.record(
|
||||
num_tokens=8,
|
||||
num_hits=8,
|
||||
preempted=False,
|
||||
hits_by_source={CacheHitSource.HOST: 4},
|
||||
)
|
||||
|
||||
assert stats.requests == 3
|
||||
assert stats.hits == 48
|
||||
assert stats.preempted_hits == 32
|
||||
assert stats.hits_by_source == {
|
||||
CacheHitSource.HOST: 16,
|
||||
CacheHitSource.DISK: 8,
|
||||
CacheHitSource.EXTERNAL_UNSPECIFIED: 8,
|
||||
}
|
||||
assert sum(stats.hits_by_source.values()) == 48 - 16
|
||||
|
||||
|
||||
def test_prefix_cache_stats_hits_by_source_serialization():
|
||||
connector_stats = PrefixCacheStats()
|
||||
connector_stats.record(
|
||||
num_tokens=16,
|
||||
num_hits=8,
|
||||
preempted=False,
|
||||
hits_by_source={CacheHitSource.P2P: 4, CacheHitSource.HOST: 4},
|
||||
)
|
||||
outputs = EngineCoreOutputs(
|
||||
scheduler_stats=SchedulerStats(
|
||||
connector_prefix_cache_stats=connector_stats,
|
||||
)
|
||||
)
|
||||
|
||||
encoded = MsgpackEncoder().encode(outputs)
|
||||
decoded = MsgpackDecoder(EngineCoreOutputs).decode(encoded)
|
||||
|
||||
assert decoded.scheduler_stats is not None
|
||||
assert decoded.scheduler_stats.connector_prefix_cache_stats is not None
|
||||
assert decoded.scheduler_stats.connector_prefix_cache_stats.hits_by_source == {
|
||||
CacheHitSource.P2P: 4,
|
||||
CacheHitSource.HOST: 4,
|
||||
}
|
||||
|
||||
|
||||
def test_compute_iteration_details_includes_encoder_stats():
|
||||
scheduler_output = SchedulerOutput.make_empty()
|
||||
scheduler_output.scheduled_encoder_input_stats = ScheduledEncoderInputStats(
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.config import (
|
||||
)
|
||||
from vllm.config.cache import MambaCacheMode
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
|
||||
SimpleCPUOffloadConnector,
|
||||
)
|
||||
@@ -53,6 +54,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.simple_kv_offload.manager import SimpleCPUOffloadScheduler
|
||||
@@ -213,6 +215,37 @@ class SchedulerFixture:
|
||||
num_groups: int = 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "source"),
|
||||
[("cpu", CacheHitSource.HOST), ("disk", CacheHitSource.DISK)],
|
||||
)
|
||||
def test_connector_reports_configured_cache_source(
|
||||
tmp_path, backend: str, source: CacheHitSource
|
||||
):
|
||||
kv_cache_config = _make_kv_cache_config(num_blocks=4)
|
||||
vllm_config = _make_vllm_config()
|
||||
extra_config = {
|
||||
"cpu_bytes_to_use_per_rank": _BYTES_PER_BLOCK * 4,
|
||||
"kv_offload_backend": backend,
|
||||
}
|
||||
if backend == "disk":
|
||||
extra_config.update(
|
||||
disk_path=str(tmp_path / "kv-offload.bin"),
|
||||
disk_capacity_bytes=_BYTES_PER_BLOCK * 4,
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config = extra_config
|
||||
|
||||
connector = SimpleCPUOffloadConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.SCHEDULER,
|
||||
kv_cache_config,
|
||||
)
|
||||
|
||||
assert connector.get_external_cache_hit_sources(None, 32) == {source: 32} # type: ignore[arg-type]
|
||||
assert connector.get_external_cache_hit_sources(None, 0) == {} # type: ignore[arg-type]
|
||||
|
||||
|
||||
def make_scheduler(
|
||||
num_cpu_blocks: int = 8,
|
||||
num_gpu_blocks: int = 16,
|
||||
|
||||
@@ -114,6 +114,7 @@ def test_torch_profiler_records_each_profile_round(tmp_path):
|
||||
profiler="torch",
|
||||
torch_profiler_dir=str(tmp_path),
|
||||
torch_profiler_dump_cuda_time_total=False,
|
||||
torch_profiler_with_stack=False,
|
||||
),
|
||||
worker_name="worker",
|
||||
local_rank=1,
|
||||
|
||||
Reference in New Issue
Block a user