Tier offload followup (#42529)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-05-18 19:41:58 +00:00
committed by GitHub
parent 8fc1c284b9
commit 84747489de
8 changed files with 226 additions and 448 deletions
+23 -147
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for TieringOffloadingManager and ExampleSecondaryTier.
Unit tests for TieringOffloadingManager and ExampleSecondaryTierManager.
These tests verify:
1. Basic tiered offloading operations (store, load, lookup)
@@ -14,7 +14,6 @@ These tests verify:
from collections.abc import Iterable
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
@@ -23,8 +22,7 @@ from vllm.v1.kv_offload.base import (
ReqContext,
make_offload_key,
)
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.example import ExampleSecondaryTier
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
from vllm.v1.kv_offload.tiering.manager import (
CPUPrimaryTierOffloadingManager,
TieringOffloadingManager,
@@ -63,14 +61,17 @@ def count_hits(manager, keys: list[OffloadKey]) -> int | None:
return count
class TestExampleSecondaryTier:
"""Tests for ExampleSecondaryTier implementation."""
class TestExampleSecondaryTierManager:
"""Tests for ExampleSecondaryTierManager implementation."""
def test_basic_store_and_lookup(self):
"""Test basic store and lookup operations."""
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
tier = ExampleSecondaryTierManager(
vllm_config=_MOCK_VLLM_CONFIG,
primary_kv_view=mock_view,
tier_type="example",
custom_param=67,
)
# Initially empty
@@ -88,81 +89,6 @@ class TestExampleSecondaryTier:
# Third block not present
assert tier.lookup(blocks[2], _CTX) is False
def test_lru_eviction(self):
"""Test LRU eviction policy."""
mock_view = memoryview(torch.zeros((4, 16), dtype=torch.int8).numpy())
tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=3
)
# Fill tier to capacity
blocks = to_keys(range(3))
for block in blocks:
tier.blocks[block] = True
assert tier.get_num_blocks() == 3
# Touch first block (make it most recently used)
tier.touch([blocks[0]], _CTX)
# Store new block should evict blocks[1] (least recently used)
new_block = to_keys([3])[0]
tier.submit_store(
JobMetadata(
job_id=1,
keys=[new_block],
block_ids=np.array([0], dtype=np.int64),
is_promotion=False,
req_context=_CTX,
)
)
# Complete the job
tier.get_finished()
# Verify new block is stored and blocks[1] was evicted (LRU)
assert new_block in tier.blocks
assert blocks[1] not in tier.blocks
# blocks[0] and blocks[2] should still be present
assert blocks[0] in tier.blocks
assert blocks[2] in tier.blocks
def test_async_simulation(self):
"""Test simulated async behavior."""
mock_view = memoryview(torch.zeros((10, 16), dtype=torch.int8).numpy())
tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG,
primary_kv_view=mock_view,
max_blocks=10,
simulate_async=True,
)
blocks = to_keys(range(2))
# Submit store job
tier.submit_store(
JobMetadata(
job_id=1,
keys=blocks,
block_ids=np.array([0, 1], dtype=np.int64),
is_promotion=False,
req_context=_CTX,
)
)
# Blocks should not yet be stored (pending async completion)
assert tier.get_num_blocks() == 0
# First get_finished() should complete the job
completed = list(tier.get_finished())
assert len(completed) == 1
assert completed[0].job_id == 1
assert completed[0].success is True
# Blocks should now be stored
assert tier.get_num_blocks() == 2
class TestTieringOffloadingManager:
"""Tests for TieringOffloadingManager."""
@@ -178,11 +104,15 @@ class TestTieringOffloadingManager:
mock_view = mock_region.create_kv_memoryview()
# Create secondary tiers with the primary view
self.secondary_tier1 = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
self.secondary_tier1 = ExampleSecondaryTierManager(
vllm_config=_MOCK_VLLM_CONFIG,
primary_kv_view=mock_view,
tier_type="example",
)
self.secondary_tier2 = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG, primary_kv_view=mock_view, max_blocks=10
self.secondary_tier2 = ExampleSecondaryTierManager(
vllm_config=_MOCK_VLLM_CONFIG,
primary_kv_view=mock_view,
tier_type="example",
)
# Create tiered manager
@@ -359,20 +289,19 @@ class TestTieringOffloadingManager:
self.manager.complete_store(blocks, _CTX, success=True)
list(self.manager.take_events())
self.secondary_tier1.touch = MagicMock(wraps=self.secondary_tier1.touch)
self.secondary_tier2.touch = MagicMock(wraps=self.secondary_tier2.touch)
# Touch blocks
self.manager.touch(blocks, _CTX)
# Verify touch was called on primary tier (check LRU order)
# In LRU, touched blocks should be at the end
primary_keys = list(self.primary_tier._policy.blocks.keys())
assert primary_keys[-3:] == list(reversed(blocks))
# Verify touch was called on all secondary tiers
secondary1_keys = list(self.secondary_tier1.blocks.keys())
assert secondary1_keys[-3:] == list(reversed(blocks))
secondary2_keys = list(self.secondary_tier2.blocks.keys())
assert secondary2_keys[-3:] == list(reversed(blocks))
# Verify touch was propagated to all secondary tiers
self.secondary_tier1.touch.assert_called_once_with(blocks, _CTX)
self.secondary_tier2.touch.assert_called_once_with(blocks, _CTX)
def test_failed_store_no_cascade(self, manager_setup):
"""Test that failed GPU→primary store doesn't cascade."""
@@ -396,59 +325,6 @@ class TestTieringOffloadingManager:
self.secondary_tier1.submit_store.assert_not_called()
self.secondary_tier2.submit_store.assert_not_called()
def test_multiple_secondary_tiers_independent_eviction(self):
"""Test that secondary tiers manage their own evictions."""
mock_region = _mock_mmap_region(10)
mock_view = mock_region.create_kv_memoryview()
# Create tier with small capacity
small_tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG,
primary_kv_view=mock_view,
max_blocks=5,
simulate_async=False,
)
large_tier = ExampleSecondaryTier(
vllm_config=_MOCK_VLLM_CONFIG,
primary_kv_view=mock_view,
max_blocks=10,
simulate_async=False,
)
# Create a fresh primary tier for this test
primary_tier = CPUPrimaryTierOffloadingManager(
num_blocks=10, mmap_region=mock_region
)
manager = TieringOffloadingManager(
primary_tier=primary_tier,
secondary_tiers=[small_tier, large_tier],
)
# First, store 5 blocks to fill the small tier
blocks1 = to_keys(range(5))
result = manager.prepare_store(blocks1, _CTX)
assert result is not None
manager.complete_store(blocks1, _CTX, success=True)
list(manager.take_events())
# Both tiers should have 5 blocks
assert small_tier.get_num_blocks() == 5
assert large_tier.get_num_blocks() == 5
# Now store 3 more blocks - small tier should evict 3 blocks
blocks2 = to_keys(range(5, 8))
result = manager.prepare_store(blocks2, _CTX)
assert result is not None
manager.complete_store(blocks2, _CTX, success=True)
list(manager.take_events())
# Small tier should still have 5 blocks (evicted 3, added 3)
assert small_tier.get_num_blocks() == 5
# Large tier should have all 8 blocks
assert large_tier.get_num_blocks() == 8
def test_lookup_batches_submit_load_per_request(self, manager_setup):
"""lookup() defers submit_load until take_events(), one call per request.