File system secondary tier implemented in python (#41735)
Signed-off-by: Rotem Shavitt <rshavitt@gmail.com> Signed-off-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Or Ozeri
parent
d56285c747
commit
d0a100c87a
@@ -107,7 +107,7 @@ class MockSubscriber:
|
||||
self.sub.close()
|
||||
|
||||
|
||||
def _wait_for_prefix_cache_reset(llm: LLM) -> None:
|
||||
def _wait_for_prefix_cache_reset(llm: LLM, reset_connector: bool = False) -> None:
|
||||
"""Wait for async offload transfers to finish so prefix cache can reset.
|
||||
|
||||
The GPU-to-CPU offload runs on a CUDA stream asynchronously. While blocks
|
||||
@@ -115,10 +115,14 @@ def _wait_for_prefix_cache_reset(llm: LLM) -> None:
|
||||
``False``. Between retries we send a dummy single-token prefill to force
|
||||
the engine to step, which polls the worker for completed transfers and
|
||||
frees GPU blocks.
|
||||
|
||||
Args:
|
||||
llm: The LLM instance to reset.
|
||||
reset_connector: If True, also reset the KV connector state.
|
||||
"""
|
||||
_dummy_params = SamplingParams(max_tokens=1)
|
||||
deadline = time.monotonic() + _RESET_CACHE_TIMEOUT
|
||||
while not llm.reset_prefix_cache():
|
||||
while not llm.reset_prefix_cache(reset_connector=reset_connector):
|
||||
if time.monotonic() > deadline:
|
||||
raise TimeoutError(
|
||||
"reset_prefix_cache did not succeed within "
|
||||
@@ -133,7 +137,9 @@ def _wait_for_prefix_cache_reset(llm: LLM) -> None:
|
||||
)
|
||||
|
||||
|
||||
def _latency_test(llm: LLM, subscriber: MockSubscriber | None):
|
||||
def _latency_test(
|
||||
llm: LLM, subscriber: MockSubscriber | None, reset_connector: bool = False
|
||||
):
|
||||
sampling_params = SamplingParams(max_tokens=1)
|
||||
|
||||
num_times_cpu_better_than_cold = 0
|
||||
@@ -163,7 +169,7 @@ def _latency_test(llm: LLM, subscriber: MockSubscriber | None):
|
||||
|
||||
# Wait for the async CPU offload to finish, then reset prefix cache
|
||||
# so the next generate() must reload from CPU rather than GPU.
|
||||
_wait_for_prefix_cache_reset(llm)
|
||||
_wait_for_prefix_cache_reset(llm, reset_connector=reset_connector)
|
||||
|
||||
# Verify CPU stored events arrived (offload is done before we
|
||||
# attempt to load from CPU).
|
||||
@@ -337,3 +343,49 @@ def test_tiering_offloading() -> None:
|
||||
finally:
|
||||
subscriber.close()
|
||||
del llm
|
||||
|
||||
|
||||
def test_fs_tiering_offloading(tmp_path) -> None:
|
||||
"""Tests OffloadingConnector with TieringOffloadingSpec
|
||||
+ fs_python secondary tier."""
|
||||
extra_config: dict = {
|
||||
"cpu_bytes_to_use": 1 << 30,
|
||||
"block_size": 48,
|
||||
"spec_name": "TieringOffloadingSpec",
|
||||
"secondary_tiers": [{"type": "fs_python", "root_dir": str(tmp_path)}],
|
||||
}
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="OffloadingConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config=extra_config,
|
||||
)
|
||||
|
||||
port: int
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("0.0.0.0", 0))
|
||||
port = s.getsockname()[1]
|
||||
events_endpoint = f"tcp://*:{port}"
|
||||
kv_events_config = KVEventsConfig(
|
||||
enable_kv_cache_events=True,
|
||||
publisher="zmq",
|
||||
endpoint=events_endpoint,
|
||||
topic="test",
|
||||
)
|
||||
|
||||
llm = LLM(
|
||||
model="meta-llama/Llama-3.2-1B-Instruct",
|
||||
max_model_len=512,
|
||||
gpu_memory_utilization=0.5,
|
||||
kv_events_config=kv_events_config,
|
||||
kv_transfer_config=kv_transfer_config,
|
||||
)
|
||||
subscriber = MockSubscriber(
|
||||
events_endpoint.replace("*", "127.0.0.1"),
|
||||
topic=kv_events_config.topic,
|
||||
)
|
||||
try:
|
||||
_latency_test(llm, subscriber, reset_connector=True)
|
||||
_accuracy_test(llm, subscriber)
|
||||
finally:
|
||||
subscriber.close()
|
||||
del llm
|
||||
|
||||
@@ -0,0 +1,127 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for FileMapper."""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
OffloadingSpec,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Shared mocks (mirrors test_fs_tier.py pattern)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_MOCK_VLLM_CONFIG = MagicMock()
|
||||
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
|
||||
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
|
||||
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
|
||||
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
|
||||
|
||||
_MOCK_KV_CACHE_CONFIG = MagicMock()
|
||||
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
|
||||
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock(spec=OffloadingSpec)
|
||||
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
|
||||
"""Helper to create FileMapper with customizable mock config."""
|
||||
# Create a copy of the mock config to avoid modifying the global one
|
||||
mock_vllm_config = MagicMock()
|
||||
mock_vllm_config.model_config.model = kwargs.get("model_name", "test-model")
|
||||
mock_vllm_config.cache_config.block_size = kwargs.get("hash_block_size", 16)
|
||||
mock_vllm_config.cache_config.cache_dtype = (
|
||||
f"torch.{kwargs.get('dtype', 'float16')}"
|
||||
)
|
||||
mock_vllm_config.parallel_config.tensor_parallel_size = kwargs.get("tp_size", 1)
|
||||
mock_vllm_config.parallel_config.pipeline_parallel_size = kwargs.get("pp_size", 1)
|
||||
mock_vllm_config.parallel_config.prefill_context_parallel_size = kwargs.get(
|
||||
"pcp_size", 1
|
||||
)
|
||||
mock_vllm_config.parallel_config.decode_context_parallel_size = kwargs.get(
|
||||
"dcp_size", 1
|
||||
)
|
||||
mock_vllm_config.parallel_config.rank = kwargs.get("rank", 0)
|
||||
|
||||
mock_kv_cache_config = MagicMock()
|
||||
mock_kv_cache_config.kv_cache_groups = []
|
||||
|
||||
mock_offloading_spec = MagicMock(spec=OffloadingSpec)
|
||||
mock_offloading_spec.vllm_config = mock_vllm_config
|
||||
mock_offloading_spec.kv_cache_config = mock_kv_cache_config
|
||||
mock_offloading_spec.block_size_factor = kwargs.get("block_size_factor", 1)
|
||||
|
||||
return FileMapper.from_offloading_spec(
|
||||
root_dir=kwargs.get("root_dir", "/tmp/cache"),
|
||||
offloading_spec=mock_offloading_spec,
|
||||
gpu_blocks_per_file=mock_offloading_spec.block_size_factor,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_get_file_name_full_structure():
|
||||
"""
|
||||
Path must match: <base_path>_r<rank>/<hhh>/<hh>_g<group_idx>/<hash_hex>.bin
|
||||
|
||||
Concretely:
|
||||
- The segment immediately after base_path must end with `_r0`
|
||||
- The next segment is the first 3 hex chars of the block hash
|
||||
- The next segment is <2 hex chars>_g<group_idx>
|
||||
- The final segment is <full hash hex>.bin
|
||||
"""
|
||||
rank = 3
|
||||
group_idx = 2
|
||||
block_hash = bytes(range(8)) # deterministic, non-zero bytes
|
||||
fm = make_mapper_from_offloading_spec(rank=rank)
|
||||
key = make_offload_key(block_hash, group_idx)
|
||||
path = fm.get_file_name(key)
|
||||
|
||||
expected_path = (
|
||||
"/tmp/cache/test-model_588656ebcc66_r3/000/10_g2/0001020304050607.bin"
|
||||
)
|
||||
assert path == expected_path
|
||||
|
||||
|
||||
def test_get_run_config_fields():
|
||||
fm = make_mapper_from_offloading_spec(
|
||||
model_name="my-model",
|
||||
dtype="bfloat16",
|
||||
tp_size=2,
|
||||
)
|
||||
cfg = fm.get_run_config()
|
||||
assert cfg == {
|
||||
"model_name": "my-model",
|
||||
"hash_block_size": 16,
|
||||
"gpu_blocks_per_file": 1,
|
||||
"tp_size": 2,
|
||||
"pp_size": 1,
|
||||
"pcp_size": 1,
|
||||
"dcp_size": 1,
|
||||
"dtype": "bfloat16",
|
||||
"kv_cache_groups": [],
|
||||
"inference_engine": "vllm",
|
||||
}
|
||||
|
||||
|
||||
def test_get_config_file_path():
|
||||
fm = make_mapper_from_offloading_spec()
|
||||
config_path = fm.get_config_file_path()
|
||||
assert config_path == f"{fm.base_path}/config.json"
|
||||
@@ -0,0 +1,256 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Unit tests for FileSystemTierManager.
|
||||
|
||||
These tests use real disk I/O to verify the Python filesystem tier implementation.
|
||||
The tier manager writes KV cache blocks to disk and reads them back, verifying
|
||||
data integrity throughout the process.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, make_offload_key
|
||||
from vllm.v1.kv_offload.tiering.base import JobMetadata
|
||||
from vllm.v1.kv_offload.tiering.fs.manager import (
|
||||
FileSystemTierManager,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_BLOCK_ELEMENTS = 512 * 1024 # 2 MB per block (float32 × 512K = 2MB)
|
||||
_DTYPE = torch.float32
|
||||
_CTX = ReqContext(req_id="test")
|
||||
|
||||
_MOCK_VLLM_CONFIG = MagicMock()
|
||||
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
|
||||
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
|
||||
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
|
||||
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
|
||||
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
|
||||
|
||||
_MOCK_KV_CACHE_CONFIG = MagicMock()
|
||||
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
|
||||
|
||||
_MOCK_OFFLOADING_SPEC = MagicMock()
|
||||
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
|
||||
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
|
||||
|
||||
|
||||
def key(n: int) -> OffloadKey:
|
||||
return make_offload_key(n.to_bytes(8, "big"), 0)
|
||||
|
||||
|
||||
def make_job(
|
||||
job_id: int,
|
||||
keys: list[OffloadKey],
|
||||
block_ids: list[int] | None = None,
|
||||
is_promotion: bool = False,
|
||||
) -> JobMetadata:
|
||||
if block_ids is None:
|
||||
block_ids = list(range(len(keys)))
|
||||
return JobMetadata(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=np.array(block_ids, dtype=np.int64),
|
||||
is_promotion=is_promotion,
|
||||
req_context=_CTX,
|
||||
)
|
||||
|
||||
|
||||
def drain(tier: FileSystemTierManager, max_rounds: int = 40) -> list:
|
||||
"""
|
||||
Call get_finished() repeatedly until no new results arrive for 5
|
||||
consecutive rounds or max_rounds is reached.
|
||||
"""
|
||||
results = []
|
||||
idle = 0
|
||||
for _ in range(max_rounds):
|
||||
time.sleep(0.01)
|
||||
new = list(tier.get_finished())
|
||||
results.extend(new)
|
||||
if new:
|
||||
idle = 0
|
||||
else:
|
||||
idle += 1
|
||||
if idle >= 5:
|
||||
break
|
||||
return results
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fs_tier(tmp_path):
|
||||
tensor = torch.zeros((4, _BLOCK_ELEMENTS), dtype=_DTYPE)
|
||||
mock_view = memoryview(tensor.numpy())
|
||||
tier = FileSystemTierManager(
|
||||
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
||||
primary_kv_view=mock_view,
|
||||
tier_type="fs_python",
|
||||
root_dir=str(tmp_path),
|
||||
n_read_threads=4,
|
||||
n_write_threads=4,
|
||||
)
|
||||
yield tier, tensor
|
||||
tier.shutdown()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_lookup_empty_tier(fs_tier):
|
||||
tier, _ = fs_tier
|
||||
assert tier.lookup(key(1), _CTX) is False
|
||||
assert tier.lookup(key(2), _CTX) is False
|
||||
|
||||
|
||||
def test_store_creates_file_and_lookup_succeeds(fs_tier):
|
||||
tier, _ = fs_tier
|
||||
job = make_job(1, [key(1)], [0])
|
||||
tier.submit_store(job)
|
||||
results = drain(tier)
|
||||
assert len(results) == 1
|
||||
assert results[0].success
|
||||
assert tier.lookup(key(1), _CTX) is True
|
||||
dest = tier.file_mapper.get_file_name(key(1))
|
||||
assert os.path.exists(dest), f"Expected file at {dest}"
|
||||
|
||||
|
||||
def test_store_then_load_roundtrip(fs_tier):
|
||||
tier, _ = fs_tier
|
||||
job_s = make_job(1, [key(1), key(2)], [0, 1])
|
||||
tier.submit_store(job_s)
|
||||
store_results = drain(tier)
|
||||
assert all(r.success for r in store_results)
|
||||
|
||||
assert tier.lookup(key(1), _CTX) is True
|
||||
assert tier.lookup(key(2), _CTX) is True
|
||||
|
||||
job_l = make_job(2, [key(1), key(2)], [2, 3], is_promotion=True)
|
||||
tier.submit_load(job_l)
|
||||
load_results = drain(tier)
|
||||
assert all(r.success for r in load_results)
|
||||
# Blocks stay on disk after load
|
||||
assert tier.lookup(key(1), _CTX) is True
|
||||
assert tier.lookup(key(2), _CTX) is True
|
||||
|
||||
|
||||
def test_invalid_path_raises_at_construction():
|
||||
"""Construction must fail immediately when the config file cannot be written."""
|
||||
tensor = torch.zeros((32, _BLOCK_ELEMENTS), dtype=_DTYPE)
|
||||
mock_view = memoryview(tensor.numpy())
|
||||
|
||||
with pytest.raises(OSError):
|
||||
FileSystemTierManager(
|
||||
offloading_spec=_MOCK_OFFLOADING_SPEC,
|
||||
primary_kv_view=mock_view,
|
||||
tier_type="fs_python",
|
||||
root_dir="/dev/null/invalid_path",
|
||||
)
|
||||
|
||||
|
||||
def test_failed_load_missing_file(fs_tier):
|
||||
"""Test that loading a block whose file does not exist results in a failed job."""
|
||||
tier, _ = fs_tier
|
||||
job = make_job(1, [key(99)], [0], is_promotion=True)
|
||||
tier.submit_load(job)
|
||||
results = drain(tier)
|
||||
assert len(results) == 1
|
||||
assert not results[0].success
|
||||
|
||||
|
||||
def test_multiple_jobs_tracked_independently(fs_tier):
|
||||
tier, _ = fs_tier
|
||||
job1 = make_job(1, [key(1)], [0])
|
||||
job2 = make_job(2, [key(2)], [1])
|
||||
tier.submit_store(job1)
|
||||
tier.submit_store(job2)
|
||||
results = drain(tier)
|
||||
job_ids = {r.job_id for r in results}
|
||||
assert job_ids == {1, 2}
|
||||
assert tier.lookup(key(1), _CTX) is True
|
||||
assert tier.lookup(key(2), _CTX) is True
|
||||
|
||||
|
||||
def test_multi_block_job_partial_failure(fs_tier):
|
||||
"""A load job where one block file is missing yields a single failed JobResult."""
|
||||
tier, _ = fs_tier
|
||||
# Store two of three keys
|
||||
tier.submit_store(make_job(1, [key(10), key(11)], [0, 1]))
|
||||
assert all(r.success for r in drain(tier))
|
||||
|
||||
# Load all three — key(99) was never stored
|
||||
tier.submit_load(
|
||||
make_job(2, [key(10), key(11), key(99)], [0, 1, 2], is_promotion=True)
|
||||
)
|
||||
results = drain(tier)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].job_id == 2
|
||||
assert not results[0].success
|
||||
|
||||
|
||||
def test_shutdown_discards_pending_tasks(fs_tier):
|
||||
"""Shutdown clears both queues and stops all worker threads without draining."""
|
||||
tier, _ = fs_tier
|
||||
# Submit many tasks to ensure some remain pending
|
||||
for i in range(10):
|
||||
tier.submit_store(make_job(i, [key(i)], [i % 4]))
|
||||
|
||||
# Shutdown immediately without draining
|
||||
tier.shutdown()
|
||||
|
||||
# Verify queues are cleared and threads stopped
|
||||
assert len(tier._pool._load_q) == 0
|
||||
assert len(tier._pool._store_q) == 0
|
||||
assert all(not t.is_alive() for t in tier._pool._threads)
|
||||
|
||||
|
||||
def test_store_load_data_integrity(fs_tier):
|
||||
"""Data written by store must be exactly recovered by load."""
|
||||
tier, tensor = fs_tier
|
||||
# Populate tensor with random data
|
||||
tensor[:] = torch.rand((4, _BLOCK_ELEMENTS), dtype=_DTYPE)
|
||||
|
||||
# Store first 2 blocks
|
||||
num_store = 2
|
||||
expected = tensor[:num_store].clone()
|
||||
|
||||
store_ids = list(range(num_store))
|
||||
keys = [key(i) for i in range(num_store)]
|
||||
|
||||
tier.submit_store(make_job(1, keys, store_ids))
|
||||
results = drain(tier)
|
||||
assert all(r.success for r in results)
|
||||
|
||||
# Overwrite source blocks to prove data is read from disk
|
||||
tensor[:num_store] = 0.0
|
||||
|
||||
# Load into last 2 blocks
|
||||
load_ids = [2, 3]
|
||||
tier.submit_load(make_job(2, keys, load_ids, is_promotion=True))
|
||||
results = drain(tier)
|
||||
assert all(r.success for r in results)
|
||||
|
||||
for i, bid in enumerate(load_ids):
|
||||
assert torch.allclose(tensor[bid], expected[i]), (
|
||||
f"Block {bid} data mismatch after store+load"
|
||||
)
|
||||
Reference in New Issue
Block a user