File system secondary tier implemented in python (#41735)

Signed-off-by: Rotem Shavitt <rshavitt@gmail.com>
Signed-off-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Rotem Shavitt
2026-05-24 18:14:44 +00:00
committed by GitHub
co-authored by Or Ozeri
parent d56285c747
commit d0a100c87a
9 changed files with 980 additions and 4 deletions
@@ -107,7 +107,7 @@ class MockSubscriber:
self.sub.close()
def _wait_for_prefix_cache_reset(llm: LLM) -> None:
def _wait_for_prefix_cache_reset(llm: LLM, reset_connector: bool = False) -> None:
"""Wait for async offload transfers to finish so prefix cache can reset.
The GPU-to-CPU offload runs on a CUDA stream asynchronously. While blocks
@@ -115,10 +115,14 @@ def _wait_for_prefix_cache_reset(llm: LLM) -> None:
``False``. Between retries we send a dummy single-token prefill to force
the engine to step, which polls the worker for completed transfers and
frees GPU blocks.
Args:
llm: The LLM instance to reset.
reset_connector: If True, also reset the KV connector state.
"""
_dummy_params = SamplingParams(max_tokens=1)
deadline = time.monotonic() + _RESET_CACHE_TIMEOUT
while not llm.reset_prefix_cache():
while not llm.reset_prefix_cache(reset_connector=reset_connector):
if time.monotonic() > deadline:
raise TimeoutError(
"reset_prefix_cache did not succeed within "
@@ -133,7 +137,9 @@ def _wait_for_prefix_cache_reset(llm: LLM) -> None:
)
def _latency_test(llm: LLM, subscriber: MockSubscriber | None):
def _latency_test(
llm: LLM, subscriber: MockSubscriber | None, reset_connector: bool = False
):
sampling_params = SamplingParams(max_tokens=1)
num_times_cpu_better_than_cold = 0
@@ -163,7 +169,7 @@ def _latency_test(llm: LLM, subscriber: MockSubscriber | None):
# Wait for the async CPU offload to finish, then reset prefix cache
# so the next generate() must reload from CPU rather than GPU.
_wait_for_prefix_cache_reset(llm)
_wait_for_prefix_cache_reset(llm, reset_connector=reset_connector)
# Verify CPU stored events arrived (offload is done before we
# attempt to load from CPU).
@@ -337,3 +343,49 @@ def test_tiering_offloading() -> None:
finally:
subscriber.close()
del llm
def test_fs_tiering_offloading(tmp_path) -> None:
"""Tests OffloadingConnector with TieringOffloadingSpec
+ fs_python secondary tier."""
extra_config: dict = {
"cpu_bytes_to_use": 1 << 30,
"block_size": 48,
"spec_name": "TieringOffloadingSpec",
"secondary_tiers": [{"type": "fs_python", "root_dir": str(tmp_path)}],
}
kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
kv_connector_extra_config=extra_config,
)
port: int
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("0.0.0.0", 0))
port = s.getsockname()[1]
events_endpoint = f"tcp://*:{port}"
kv_events_config = KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
endpoint=events_endpoint,
topic="test",
)
llm = LLM(
model="meta-llama/Llama-3.2-1B-Instruct",
max_model_len=512,
gpu_memory_utilization=0.5,
kv_events_config=kv_events_config,
kv_transfer_config=kv_transfer_config,
)
subscriber = MockSubscriber(
events_endpoint.replace("*", "127.0.0.1"),
topic=kv_events_config.topic,
)
try:
_latency_test(llm, subscriber, reset_connector=True)
_accuracy_test(llm, subscriber)
finally:
subscriber.close()
del llm
+127
View File
@@ -0,0 +1,127 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for FileMapper."""
from unittest.mock import MagicMock
from vllm.v1.kv_offload.base import (
OffloadingSpec,
make_offload_key,
)
from vllm.v1.kv_offload.file_mapper import FileMapper
# ---------------------------------------------------------------------------
# Shared mocks (mirrors test_fs_tier.py pattern)
# ---------------------------------------------------------------------------
_MOCK_VLLM_CONFIG = MagicMock()
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
_MOCK_KV_CACHE_CONFIG = MagicMock()
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
_MOCK_OFFLOADING_SPEC = MagicMock(spec=OffloadingSpec)
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
# ---------------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------------
def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
"""Helper to create FileMapper with customizable mock config."""
# Create a copy of the mock config to avoid modifying the global one
mock_vllm_config = MagicMock()
mock_vllm_config.model_config.model = kwargs.get("model_name", "test-model")
mock_vllm_config.cache_config.block_size = kwargs.get("hash_block_size", 16)
mock_vllm_config.cache_config.cache_dtype = (
f"torch.{kwargs.get('dtype', 'float16')}"
)
mock_vllm_config.parallel_config.tensor_parallel_size = kwargs.get("tp_size", 1)
mock_vllm_config.parallel_config.pipeline_parallel_size = kwargs.get("pp_size", 1)
mock_vllm_config.parallel_config.prefill_context_parallel_size = kwargs.get(
"pcp_size", 1
)
mock_vllm_config.parallel_config.decode_context_parallel_size = kwargs.get(
"dcp_size", 1
)
mock_vllm_config.parallel_config.rank = kwargs.get("rank", 0)
mock_kv_cache_config = MagicMock()
mock_kv_cache_config.kv_cache_groups = []
mock_offloading_spec = MagicMock(spec=OffloadingSpec)
mock_offloading_spec.vllm_config = mock_vllm_config
mock_offloading_spec.kv_cache_config = mock_kv_cache_config
mock_offloading_spec.block_size_factor = kwargs.get("block_size_factor", 1)
return FileMapper.from_offloading_spec(
root_dir=kwargs.get("root_dir", "/tmp/cache"),
offloading_spec=mock_offloading_spec,
gpu_blocks_per_file=mock_offloading_spec.block_size_factor,
)
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
def test_get_file_name_full_structure():
"""
Path must match: <base_path>_r<rank>/<hhh>/<hh>_g<group_idx>/<hash_hex>.bin
Concretely:
- The segment immediately after base_path must end with `_r0`
- The next segment is the first 3 hex chars of the block hash
- The next segment is <2 hex chars>_g<group_idx>
- The final segment is <full hash hex>.bin
"""
rank = 3
group_idx = 2
block_hash = bytes(range(8)) # deterministic, non-zero bytes
fm = make_mapper_from_offloading_spec(rank=rank)
key = make_offload_key(block_hash, group_idx)
path = fm.get_file_name(key)
expected_path = (
"/tmp/cache/test-model_588656ebcc66_r3/000/10_g2/0001020304050607.bin"
)
assert path == expected_path
def test_get_run_config_fields():
fm = make_mapper_from_offloading_spec(
model_name="my-model",
dtype="bfloat16",
tp_size=2,
)
cfg = fm.get_run_config()
assert cfg == {
"model_name": "my-model",
"hash_block_size": 16,
"gpu_blocks_per_file": 1,
"tp_size": 2,
"pp_size": 1,
"pcp_size": 1,
"dcp_size": 1,
"dtype": "bfloat16",
"kv_cache_groups": [],
"inference_engine": "vllm",
}
def test_get_config_file_path():
fm = make_mapper_from_offloading_spec()
config_path = fm.get_config_file_path()
assert config_path == f"{fm.base_path}/config.json"
+256
View File
@@ -0,0 +1,256 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for FileSystemTierManager.
These tests use real disk I/O to verify the Python filesystem tier implementation.
The tier manager writes KV cache blocks to disk and reads them back, verifying
data integrity throughout the process.
"""
import os
import time
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, make_offload_key
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.fs.manager import (
FileSystemTierManager,
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
_BLOCK_ELEMENTS = 512 * 1024 # 2 MB per block (float32 × 512K = 2MB)
_DTYPE = torch.float32
_CTX = ReqContext(req_id="test")
_MOCK_VLLM_CONFIG = MagicMock()
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
_MOCK_KV_CACHE_CONFIG = MagicMock()
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
_MOCK_OFFLOADING_SPEC = MagicMock()
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
def key(n: int) -> OffloadKey:
return make_offload_key(n.to_bytes(8, "big"), 0)
def make_job(
job_id: int,
keys: list[OffloadKey],
block_ids: list[int] | None = None,
is_promotion: bool = False,
) -> JobMetadata:
if block_ids is None:
block_ids = list(range(len(keys)))
return JobMetadata(
job_id=job_id,
keys=keys,
block_ids=np.array(block_ids, dtype=np.int64),
is_promotion=is_promotion,
req_context=_CTX,
)
def drain(tier: FileSystemTierManager, max_rounds: int = 40) -> list:
"""
Call get_finished() repeatedly until no new results arrive for 5
consecutive rounds or max_rounds is reached.
"""
results = []
idle = 0
for _ in range(max_rounds):
time.sleep(0.01)
new = list(tier.get_finished())
results.extend(new)
if new:
idle = 0
else:
idle += 1
if idle >= 5:
break
return results
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture
def fs_tier(tmp_path):
tensor = torch.zeros((4, _BLOCK_ELEMENTS), dtype=_DTYPE)
mock_view = memoryview(tensor.numpy())
tier = FileSystemTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=mock_view,
tier_type="fs_python",
root_dir=str(tmp_path),
n_read_threads=4,
n_write_threads=4,
)
yield tier, tensor
tier.shutdown()
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
def test_lookup_empty_tier(fs_tier):
tier, _ = fs_tier
assert tier.lookup(key(1), _CTX) is False
assert tier.lookup(key(2), _CTX) is False
def test_store_creates_file_and_lookup_succeeds(fs_tier):
tier, _ = fs_tier
job = make_job(1, [key(1)], [0])
tier.submit_store(job)
results = drain(tier)
assert len(results) == 1
assert results[0].success
assert tier.lookup(key(1), _CTX) is True
dest = tier.file_mapper.get_file_name(key(1))
assert os.path.exists(dest), f"Expected file at {dest}"
def test_store_then_load_roundtrip(fs_tier):
tier, _ = fs_tier
job_s = make_job(1, [key(1), key(2)], [0, 1])
tier.submit_store(job_s)
store_results = drain(tier)
assert all(r.success for r in store_results)
assert tier.lookup(key(1), _CTX) is True
assert tier.lookup(key(2), _CTX) is True
job_l = make_job(2, [key(1), key(2)], [2, 3], is_promotion=True)
tier.submit_load(job_l)
load_results = drain(tier)
assert all(r.success for r in load_results)
# Blocks stay on disk after load
assert tier.lookup(key(1), _CTX) is True
assert tier.lookup(key(2), _CTX) is True
def test_invalid_path_raises_at_construction():
"""Construction must fail immediately when the config file cannot be written."""
tensor = torch.zeros((32, _BLOCK_ELEMENTS), dtype=_DTYPE)
mock_view = memoryview(tensor.numpy())
with pytest.raises(OSError):
FileSystemTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=mock_view,
tier_type="fs_python",
root_dir="/dev/null/invalid_path",
)
def test_failed_load_missing_file(fs_tier):
"""Test that loading a block whose file does not exist results in a failed job."""
tier, _ = fs_tier
job = make_job(1, [key(99)], [0], is_promotion=True)
tier.submit_load(job)
results = drain(tier)
assert len(results) == 1
assert not results[0].success
def test_multiple_jobs_tracked_independently(fs_tier):
tier, _ = fs_tier
job1 = make_job(1, [key(1)], [0])
job2 = make_job(2, [key(2)], [1])
tier.submit_store(job1)
tier.submit_store(job2)
results = drain(tier)
job_ids = {r.job_id for r in results}
assert job_ids == {1, 2}
assert tier.lookup(key(1), _CTX) is True
assert tier.lookup(key(2), _CTX) is True
def test_multi_block_job_partial_failure(fs_tier):
"""A load job where one block file is missing yields a single failed JobResult."""
tier, _ = fs_tier
# Store two of three keys
tier.submit_store(make_job(1, [key(10), key(11)], [0, 1]))
assert all(r.success for r in drain(tier))
# Load all three — key(99) was never stored
tier.submit_load(
make_job(2, [key(10), key(11), key(99)], [0, 1, 2], is_promotion=True)
)
results = drain(tier)
assert len(results) == 1
assert results[0].job_id == 2
assert not results[0].success
def test_shutdown_discards_pending_tasks(fs_tier):
"""Shutdown clears both queues and stops all worker threads without draining."""
tier, _ = fs_tier
# Submit many tasks to ensure some remain pending
for i in range(10):
tier.submit_store(make_job(i, [key(i)], [i % 4]))
# Shutdown immediately without draining
tier.shutdown()
# Verify queues are cleared and threads stopped
assert len(tier._pool._load_q) == 0
assert len(tier._pool._store_q) == 0
assert all(not t.is_alive() for t in tier._pool._threads)
def test_store_load_data_integrity(fs_tier):
"""Data written by store must be exactly recovered by load."""
tier, tensor = fs_tier
# Populate tensor with random data
tensor[:] = torch.rand((4, _BLOCK_ELEMENTS), dtype=_DTYPE)
# Store first 2 blocks
num_store = 2
expected = tensor[:num_store].clone()
store_ids = list(range(num_store))
keys = [key(i) for i in range(num_store)]
tier.submit_store(make_job(1, keys, store_ids))
results = drain(tier)
assert all(r.success for r in results)
# Overwrite source blocks to prove data is read from disk
tensor[:num_store] = 0.0
# Load into last 2 blocks
load_ids = [2, 3]
tier.submit_load(make_job(2, keys, load_ids, is_promotion=True))
results = drain(tier)
assert all(r.success for r in results)
for i, bid in enumerate(load_ids):
assert torch.allclose(tensor[bid], expected[i]), (
f"Block {bid} data mismatch after store+load"
)