[Core][Bugfix]Refactor block manager for better testability (#3492)
This commit is contained in:
@@ -324,6 +324,8 @@ class CacheConfig:
|
||||
vLLM execution.
|
||||
swap_space: Size of the CPU swap space per GPU (in GiB).
|
||||
cache_dtype: Data type for kv cache storage.
|
||||
forced_num_gpu_blocks: Number of GPU blocks to use. This overrides the
|
||||
profiled num_gpu_blocks if specified. Does nothing if None.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -332,12 +334,14 @@ class CacheConfig:
|
||||
gpu_memory_utilization: float,
|
||||
swap_space: int,
|
||||
cache_dtype: str,
|
||||
forced_num_gpu_blocks: Optional[int] = None,
|
||||
sliding_window: Optional[int] = None,
|
||||
enable_prefix_caching: bool = False,
|
||||
) -> None:
|
||||
self.block_size = block_size
|
||||
self.gpu_memory_utilization = gpu_memory_utilization
|
||||
self.swap_space_bytes = swap_space * _GB
|
||||
self.forced_num_gpu_blocks = forced_num_gpu_blocks
|
||||
self.cache_dtype = cache_dtype
|
||||
self.sliding_window = sliding_window
|
||||
self.enable_prefix_caching = enable_prefix_caching
|
||||
@@ -528,6 +532,7 @@ class SchedulerConfig:
|
||||
and generated text).
|
||||
delay_factor: Apply a delay (of delay factor multiplied by previous
|
||||
prompt latency) before scheduling next prompt.
|
||||
use_v2_block_manager: Whether to use the BlockSpaceManagerV2 or not.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -535,6 +540,7 @@ class SchedulerConfig:
|
||||
max_num_batched_tokens: Optional[int],
|
||||
max_num_seqs: int,
|
||||
max_model_len: int,
|
||||
use_v2_block_manager: bool = False,
|
||||
delay_factor: float = 0.0,
|
||||
) -> None:
|
||||
if max_num_batched_tokens is not None:
|
||||
@@ -546,6 +552,7 @@ class SchedulerConfig:
|
||||
self.max_num_seqs = max_num_seqs
|
||||
self.max_model_len = max_model_len
|
||||
self.delay_factor = delay_factor
|
||||
self.use_v2_block_manager = use_v2_block_manager
|
||||
self._verify_args()
|
||||
|
||||
def _verify_args(self) -> None:
|
||||
|
||||
Reference in New Issue
Block a user