Prefix Cache Aware Scheduling [1/n] (#10128)

Signed-off-by: rickyx <rickyx@anyscale.com>
This commit is contained in:
Ricky Xu
2024-11-22 21:15:55 -08:00
committed by GitHub
parent 7c25fe45a6
commit 4634a89d18
13 changed files with 962 additions and 236 deletions
+4
View File
@@ -121,3 +121,7 @@ class BlockSpaceManager(ABC):
def get_prefix_cache_hit_rate(self, device: Device) -> float:
"""Prefix cache hit rate. -1 means not supported or disabled."""
pass
@abstractmethod
def get_num_cached_tokens(self, seq: Sequence) -> int:
pass