[Metrics] Expose cached prompt tokens by cache tier (#56318)
Signed-off-by: Cam Quilici <cjquilici@gmail.com> Signed-off-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Cam Quilici <cameron@semianalysis.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Cam Quilici
Nick Hill
Yifan Qiao
parent
ad0f67a7ec
commit
e37e51dd24
@@ -23,6 +23,7 @@ from vllm.config import (
|
||||
)
|
||||
from vllm.config.cache import MambaCacheMode
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1 import KVConnectorRole
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.simple_cpu_offload_connector import (
|
||||
SimpleCPUOffloadConnector,
|
||||
)
|
||||
@@ -53,6 +54,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
from vllm.v1.simple_kv_offload.manager import SimpleCPUOffloadScheduler
|
||||
@@ -213,6 +215,37 @@ class SchedulerFixture:
|
||||
num_groups: int = 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("backend", "source"),
|
||||
[("cpu", CacheHitSource.HOST), ("disk", CacheHitSource.DISK)],
|
||||
)
|
||||
def test_connector_reports_configured_cache_source(
|
||||
tmp_path, backend: str, source: CacheHitSource
|
||||
):
|
||||
kv_cache_config = _make_kv_cache_config(num_blocks=4)
|
||||
vllm_config = _make_vllm_config()
|
||||
extra_config = {
|
||||
"cpu_bytes_to_use_per_rank": _BYTES_PER_BLOCK * 4,
|
||||
"kv_offload_backend": backend,
|
||||
}
|
||||
if backend == "disk":
|
||||
extra_config.update(
|
||||
disk_path=str(tmp_path / "kv-offload.bin"),
|
||||
disk_capacity_bytes=_BYTES_PER_BLOCK * 4,
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config = extra_config
|
||||
|
||||
connector = SimpleCPUOffloadConnector(
|
||||
vllm_config,
|
||||
KVConnectorRole.SCHEDULER,
|
||||
kv_cache_config,
|
||||
)
|
||||
|
||||
assert connector.get_external_cache_hit_sources(None, 32) == {source: 32} # type: ignore[arg-type]
|
||||
assert connector.get_external_cache_hit_sources(None, 0) == {} # type: ignore[arg-type]
|
||||
|
||||
|
||||
def make_scheduler(
|
||||
num_cpu_blocks: int = 8,
|
||||
num_gpu_blocks: int = 16,
|
||||
|
||||
Reference in New Issue
Block a user