[Bugfix][KV Cache] Prevent negative external block allocation (#52707)
Signed-off-by: haic0 <149741444+haic0@users.noreply.github.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -14,11 +14,13 @@ from vllm.v1.core.kv_cache_utils import (
|
|||||||
)
|
)
|
||||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||||
ChunkedLocalAttentionManager,
|
ChunkedLocalAttentionManager,
|
||||||
|
FullAttentionManager,
|
||||||
RSWAManager,
|
RSWAManager,
|
||||||
SlidingWindowManager,
|
SlidingWindowManager,
|
||||||
)
|
)
|
||||||
from vllm.v1.kv_cache_interface import (
|
from vllm.v1.kv_cache_interface import (
|
||||||
ChunkedLocalAttentionSpec,
|
ChunkedLocalAttentionSpec,
|
||||||
|
FullAttentionSpec,
|
||||||
RSWASpec,
|
RSWASpec,
|
||||||
SlidingWindowSpec,
|
SlidingWindowSpec,
|
||||||
)
|
)
|
||||||
@@ -26,6 +28,47 @@ from vllm.v1.kv_cache_interface import (
|
|||||||
pytestmark = pytest.mark.cpu_test
|
pytestmark = pytest.mark.cpu_test
|
||||||
|
|
||||||
|
|
||||||
|
def test_external_computed_blocks_do_not_corrupt_free_pool():
|
||||||
|
block_size = 4
|
||||||
|
spec = FullAttentionSpec(
|
||||||
|
block_size=block_size,
|
||||||
|
num_kv_heads=1,
|
||||||
|
head_size=1,
|
||||||
|
dtype=torch.float32,
|
||||||
|
)
|
||||||
|
block_pool = BlockPool(
|
||||||
|
num_gpu_blocks=10,
|
||||||
|
enable_caching=False,
|
||||||
|
hash_block_size=block_size,
|
||||||
|
)
|
||||||
|
manager = FullAttentionManager(
|
||||||
|
spec,
|
||||||
|
block_pool=block_pool,
|
||||||
|
enable_caching=False,
|
||||||
|
kv_cache_group_id=0,
|
||||||
|
scheduler_block_size=block_size,
|
||||||
|
)
|
||||||
|
request_id = "request"
|
||||||
|
manager.allocate_new_blocks(
|
||||||
|
request_id,
|
||||||
|
num_tokens=3 * block_size,
|
||||||
|
num_tokens_main_model=3 * block_size,
|
||||||
|
)
|
||||||
|
num_free_blocks = block_pool.get_num_free_blocks()
|
||||||
|
|
||||||
|
# Speculative allocations can exceed the blocks implied by the eventual
|
||||||
|
# external computed-token count. This must not request a negative number
|
||||||
|
# of blocks, which inflates the free-queue counter.
|
||||||
|
manager.allocate_external_computed_blocks(
|
||||||
|
request_id,
|
||||||
|
num_local_computed_tokens=0,
|
||||||
|
num_external_computed_tokens=block_size,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert block_pool.get_num_free_blocks() == num_free_blocks
|
||||||
|
assert len(manager.req_to_blocks[request_id]) == 3
|
||||||
|
|
||||||
|
|
||||||
def get_sliding_window_manager(
|
def get_sliding_window_manager(
|
||||||
sliding_window_spec,
|
sliding_window_spec,
|
||||||
block_pool,
|
block_pool,
|
||||||
|
|||||||
@@ -317,9 +317,10 @@ class SingleTypeKVCacheManager(ABC):
|
|||||||
return
|
return
|
||||||
|
|
||||||
req_blocks = self.req_to_blocks[request_id]
|
req_blocks = self.req_to_blocks[request_id]
|
||||||
allocated_blocks = self.block_pool.get_new_blocks(
|
num_new_blocks = max(
|
||||||
cdiv(num_total_computed_tokens, self.block_size) - len(req_blocks)
|
0, cdiv(num_total_computed_tokens, self.block_size) - len(req_blocks)
|
||||||
)
|
)
|
||||||
|
allocated_blocks = self.block_pool.get_new_blocks(num_new_blocks)
|
||||||
req_blocks.extend(allocated_blocks)
|
req_blocks.extend(allocated_blocks)
|
||||||
if self._record_new_block_ids:
|
if self._record_new_block_ids:
|
||||||
self.new_block_ids.extend(b.block_id for b in allocated_blocks)
|
self.new_block_ids.extend(b.block_id for b in allocated_blocks)
|
||||||
|
|||||||
Reference in New Issue
Block a user