Tier offload followup (#42529)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for TieringOffloadingManager and ExampleSecondaryTier.
|
||||
Unit tests for TieringOffloadingManager and ExampleSecondaryTierManager.
|
||||
|
||||
These tests verify:
|
||||
1. Basic tiered offloading operations (store, load, lookup)
|
||||
@@ -14,7 +14,6 @@ These tests verify:
|
||||
from collections.abc import Iterable
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -23,8 +22,7 @@ from vllm.v1.kv_offload.base import (
|
||||
ReqContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import JobMetadata
|
||||
from vllm.v1.kv_offload.tiering.example import ExampleSecondaryTier
|
||||
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
|
||||
from vllm.v1.kv_offload.tiering.manager import (
|
||||
CPUPrimaryTierOffloadingManager,
|
||||
TieringOffloadingManager,
|
||||
@@ -63,14 +61,17 @@ def count_hits(manager, keys: list[OffloadKey]) -> int | None:
|
||||
return count
|
||||
|
||||
|
||||
class TestExampleSecondaryTier:
|
||||
"""Tests for ExampleSecondaryTier implementation."""
|
||||
class TestExampleSecondaryTierManager:
|
||||
"""Tests for ExampleSecondaryTierManager implementation."""
|
||||
|
||||
def test_basic_store_and_lookup(self):
|
||||
"""Test basic store and lookup operations."""
|
||||
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
|
||||
tier = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
|
||||
tier = ExampleSecondaryTierManager(
|
||||
vllm_config=_MOCK_VLLM_CONFIG,
|
||||
primary_kv_view=mock_view,
|
||||
tier_type="example",
|
||||
custom_param=67,
|
||||
)
|
||||
|
||||
# Initially empty
|
||||
@@ -88,81 +89,6 @@ class TestExampleSecondaryTier:
|
||||
# Third block not present
|
||||
assert tier.lookup(blocks[2], _CTX) is False
|
||||
|
||||
def test_lru_eviction(self):
|
||||
"""Test LRU eviction policy."""
|
||||
mock_view = memoryview(torch.zeros((4, 16), dtype=torch.int8).numpy())
|
||||
tier = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=3
|
||||
)
|
||||
|
||||
# Fill tier to capacity
|
||||
blocks = to_keys(range(3))
|
||||
for block in blocks:
|
||||
tier.blocks[block] = True
|
||||
|
||||
assert tier.get_num_blocks() == 3
|
||||
|
||||
# Touch first block (make it most recently used)
|
||||
tier.touch([blocks[0]], _CTX)
|
||||
|
||||
# Store new block should evict blocks[1] (least recently used)
|
||||
new_block = to_keys([3])[0]
|
||||
|
||||
tier.submit_store(
|
||||
JobMetadata(
|
||||
job_id=1,
|
||||
keys=[new_block],
|
||||
block_ids=np.array([0], dtype=np.int64),
|
||||
is_promotion=False,
|
||||
req_context=_CTX,
|
||||
)
|
||||
)
|
||||
|
||||
# Complete the job
|
||||
tier.get_finished()
|
||||
|
||||
# Verify new block is stored and blocks[1] was evicted (LRU)
|
||||
assert new_block in tier.blocks
|
||||
assert blocks[1] not in tier.blocks
|
||||
# blocks[0] and blocks[2] should still be present
|
||||
assert blocks[0] in tier.blocks
|
||||
assert blocks[2] in tier.blocks
|
||||
|
||||
def test_async_simulation(self):
|
||||
"""Test simulated async behavior."""
|
||||
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
|
||||
tier = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG,
|
||||
primary_kv_view=mock_view,
|
||||
max_blocks=10,
|
||||
simulate_async=True,
|
||||
)
|
||||
|
||||
blocks = to_keys(range(2))
|
||||
|
||||
# Submit store job
|
||||
tier.submit_store(
|
||||
JobMetadata(
|
||||
job_id=1,
|
||||
keys=blocks,
|
||||
block_ids=np.array([0, 1], dtype=np.int64),
|
||||
is_promotion=False,
|
||||
req_context=_CTX,
|
||||
)
|
||||
)
|
||||
|
||||
# Blocks should not yet be stored (pending async completion)
|
||||
assert tier.get_num_blocks() == 0
|
||||
|
||||
# First get_finished() should complete the job
|
||||
completed = list(tier.get_finished())
|
||||
assert len(completed) == 1
|
||||
assert completed[0].job_id == 1
|
||||
assert completed[0].success is True
|
||||
|
||||
# Blocks should now be stored
|
||||
assert tier.get_num_blocks() == 2
|
||||
|
||||
|
||||
class TestTieringOffloadingManager:
|
||||
"""Tests for TieringOffloadingManager."""
|
||||
@@ -178,11 +104,15 @@ class TestTieringOffloadingManager:
|
||||
mock_view = mock_region.create_kv_memoryview()
|
||||
|
||||
# Create secondary tiers with the primary view
|
||||
self.secondary_tier1 = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
|
||||
self.secondary_tier1 = ExampleSecondaryTierManager(
|
||||
vllm_config=_MOCK_VLLM_CONFIG,
|
||||
primary_kv_view=mock_view,
|
||||
tier_type="example",
|
||||
)
|
||||
self.secondary_tier2 = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
|
||||
self.secondary_tier2 = ExampleSecondaryTierManager(
|
||||
vllm_config=_MOCK_VLLM_CONFIG,
|
||||
primary_kv_view=mock_view,
|
||||
tier_type="example",
|
||||
)
|
||||
|
||||
# Create tiered manager
|
||||
@@ -359,20 +289,19 @@ class TestTieringOffloadingManager:
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
list(self.manager.take_events())
|
||||
|
||||
self.secondary_tier1.touch = MagicMock(wraps=self.secondary_tier1.touch)
|
||||
self.secondary_tier2.touch = MagicMock(wraps=self.secondary_tier2.touch)
|
||||
|
||||
# Touch blocks
|
||||
self.manager.touch(blocks, _CTX)
|
||||
|
||||
# Verify touch was called on primary tier (check LRU order)
|
||||
# In LRU, touched blocks should be at the end
|
||||
primary_keys = list(self.primary_tier._policy.blocks.keys())
|
||||
assert primary_keys[-3:] == list(reversed(blocks))
|
||||
|
||||
# Verify touch was called on all secondary tiers
|
||||
secondary1_keys = list(self.secondary_tier1.blocks.keys())
|
||||
assert secondary1_keys[-3:] == list(reversed(blocks))
|
||||
|
||||
secondary2_keys = list(self.secondary_tier2.blocks.keys())
|
||||
assert secondary2_keys[-3:] == list(reversed(blocks))
|
||||
# Verify touch was propagated to all secondary tiers
|
||||
self.secondary_tier1.touch.assert_called_once_with(blocks, _CTX)
|
||||
self.secondary_tier2.touch.assert_called_once_with(blocks, _CTX)
|
||||
|
||||
def test_failed_store_no_cascade(self, manager_setup):
|
||||
"""Test that failed GPU→primary store doesn't cascade."""
|
||||
@@ -396,59 +325,6 @@ class TestTieringOffloadingManager:
|
||||
self.secondary_tier1.submit_store.assert_not_called()
|
||||
self.secondary_tier2.submit_store.assert_not_called()
|
||||
|
||||
def test_multiple_secondary_tiers_independent_eviction(self):
|
||||
"""Test that secondary tiers manage their own evictions."""
|
||||
mock_region = _mock_mmap_region(10)
|
||||
mock_view = mock_region.create_kv_memoryview()
|
||||
|
||||
# Create tier with small capacity
|
||||
small_tier = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG,
|
||||
primary_kv_view=mock_view,
|
||||
max_blocks=5,
|
||||
simulate_async=False,
|
||||
)
|
||||
large_tier = ExampleSecondaryTier(
|
||||
vllm_config=_MOCK_VLLM_CONFIG,
|
||||
primary_kv_view=mock_view,
|
||||
max_blocks=10,
|
||||
simulate_async=False,
|
||||
)
|
||||
|
||||
# Create a fresh primary tier for this test
|
||||
primary_tier = CPUPrimaryTierOffloadingManager(
|
||||
num_blocks=10, mmap_region=mock_region
|
||||
)
|
||||
|
||||
manager = TieringOffloadingManager(
|
||||
primary_tier=primary_tier,
|
||||
secondary_tiers=[small_tier, large_tier],
|
||||
)
|
||||
|
||||
# First, store 5 blocks to fill the small tier
|
||||
blocks1 = to_keys(range(5))
|
||||
result = manager.prepare_store(blocks1, _CTX)
|
||||
assert result is not None
|
||||
manager.complete_store(blocks1, _CTX, success=True)
|
||||
list(manager.take_events())
|
||||
|
||||
# Both tiers should have 5 blocks
|
||||
assert small_tier.get_num_blocks() == 5
|
||||
assert large_tier.get_num_blocks() == 5
|
||||
|
||||
# Now store 3 more blocks - small tier should evict 3 blocks
|
||||
blocks2 = to_keys(range(5, 8))
|
||||
result = manager.prepare_store(blocks2, _CTX)
|
||||
assert result is not None
|
||||
manager.complete_store(blocks2, _CTX, success=True)
|
||||
list(manager.take_events())
|
||||
|
||||
# Small tier should still have 5 blocks (evicted 3, added 3)
|
||||
assert small_tier.get_num_blocks() == 5
|
||||
|
||||
# Large tier should have all 8 blocks
|
||||
assert large_tier.get_num_blocks() == 8
|
||||
|
||||
def test_lookup_batches_submit_load_per_request(self, manager_setup):
|
||||
"""lookup() defers submit_load until take_events(), one call per request.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user