[Bugfix] fix automatic prefix args and add log info (#3608)

This commit is contained in:
TianYu GUO
2024-03-25 05:35:22 -07:00
committed by GitHub
parent 925f3332ca
commit e67c295b0c
2 changed files with 7 additions and 1 deletions
+5
View File
@@ -9,6 +9,9 @@ from vllm.block import BlockTable, PhysicalTokenBlock
from vllm.sequence import Sequence, SequenceGroup, SequenceStatus
from vllm.utils import Device
from vllm.core.evictor import Evictor, EvictionPolicy, make_evictor
from vllm.logger import init_logger
logger = init_logger(__name__)
class BlockAllocatorBase(ABC):
@@ -241,11 +244,13 @@ class BlockSpaceManager:
self.watermark_blocks = int(watermark * num_gpu_blocks)
if self.enable_caching:
logger.info("enable automatic prefix caching")
self.gpu_allocator = CachedBlockAllocator(Device.GPU, block_size,
num_gpu_blocks)
self.cpu_allocator = CachedBlockAllocator(Device.CPU, block_size,
num_cpu_blocks)
else:
logger.info("disable automatic prefix caching")
self.gpu_allocator = UncachedBlockAllocator(
Device.GPU, block_size, num_gpu_blocks)
self.cpu_allocator = UncachedBlockAllocator(