File system secondary tier implemented in python (#41735)

Signed-off-by: Rotem Shavitt <rshavitt@gmail.com>
Signed-off-by: Or Ozeri <oro@il.ibm.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Rotem Shavitt
2026-05-24 18:14:44 +00:00
committed by GitHub
co-authored by Or Ozeri
parent d56285c747
commit d0a100c87a
9 changed files with 980 additions and 4 deletions
@@ -107,7 +107,7 @@ class MockSubscriber:
self.sub.close()
def _wait_for_prefix_cache_reset(llm: LLM) -> None:
def _wait_for_prefix_cache_reset(llm: LLM, reset_connector: bool = False) -> None:
"""Wait for async offload transfers to finish so prefix cache can reset.
The GPU-to-CPU offload runs on a CUDA stream asynchronously. While blocks
@@ -115,10 +115,14 @@ def _wait_for_prefix_cache_reset(llm: LLM) -> None:
``False``. Between retries we send a dummy single-token prefill to force
the engine to step, which polls the worker for completed transfers and
frees GPU blocks.
Args:
llm: The LLM instance to reset.
reset_connector: If True, also reset the KV connector state.
"""
_dummy_params = SamplingParams(max_tokens=1)
deadline = time.monotonic() + _RESET_CACHE_TIMEOUT
while not llm.reset_prefix_cache():
while not llm.reset_prefix_cache(reset_connector=reset_connector):
if time.monotonic() > deadline:
raise TimeoutError(
"reset_prefix_cache did not succeed within "
@@ -133,7 +137,9 @@ def _wait_for_prefix_cache_reset(llm: LLM) -> None:
)
def _latency_test(llm: LLM, subscriber: MockSubscriber | None):
def _latency_test(
llm: LLM, subscriber: MockSubscriber | None, reset_connector: bool = False
):
sampling_params = SamplingParams(max_tokens=1)
num_times_cpu_better_than_cold = 0
@@ -163,7 +169,7 @@ def _latency_test(llm: LLM, subscriber: MockSubscriber | None):
# Wait for the async CPU offload to finish, then reset prefix cache
# so the next generate() must reload from CPU rather than GPU.
_wait_for_prefix_cache_reset(llm)
_wait_for_prefix_cache_reset(llm, reset_connector=reset_connector)
# Verify CPU stored events arrived (offload is done before we
# attempt to load from CPU).
@@ -337,3 +343,49 @@ def test_tiering_offloading() -> None:
finally:
subscriber.close()
del llm
def test_fs_tiering_offloading(tmp_path) -> None:
"""Tests OffloadingConnector with TieringOffloadingSpec
+ fs_python secondary tier."""
extra_config: dict = {
"cpu_bytes_to_use": 1 << 30,
"block_size": 48,
"spec_name": "TieringOffloadingSpec",
"secondary_tiers": [{"type": "fs_python", "root_dir": str(tmp_path)}],
}
kv_transfer_config = KVTransferConfig(
kv_connector="OffloadingConnector",
kv_role="kv_both",
kv_connector_extra_config=extra_config,
)
port: int
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("0.0.0.0", 0))
port = s.getsockname()[1]
events_endpoint = f"tcp://*:{port}"
kv_events_config = KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
endpoint=events_endpoint,
topic="test",
)
llm = LLM(
model="meta-llama/Llama-3.2-1B-Instruct",
max_model_len=512,
gpu_memory_utilization=0.5,
kv_events_config=kv_events_config,
kv_transfer_config=kv_transfer_config,
)
subscriber = MockSubscriber(
events_endpoint.replace("*", "127.0.0.1"),
topic=kv_events_config.topic,
)
try:
_latency_test(llm, subscriber, reset_connector=True)
_accuracy_test(llm, subscriber)
finally:
subscriber.close()
del llm
+127
View File
@@ -0,0 +1,127 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for FileMapper."""
from unittest.mock import MagicMock
from vllm.v1.kv_offload.base import (
OffloadingSpec,
make_offload_key,
)
from vllm.v1.kv_offload.file_mapper import FileMapper
# ---------------------------------------------------------------------------
# Shared mocks (mirrors test_fs_tier.py pattern)
# ---------------------------------------------------------------------------
_MOCK_VLLM_CONFIG = MagicMock()
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
_MOCK_KV_CACHE_CONFIG = MagicMock()
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
_MOCK_OFFLOADING_SPEC = MagicMock(spec=OffloadingSpec)
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
# ---------------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------------
def make_mapper_from_offloading_spec(**kwargs) -> FileMapper:
"""Helper to create FileMapper with customizable mock config."""
# Create a copy of the mock config to avoid modifying the global one
mock_vllm_config = MagicMock()
mock_vllm_config.model_config.model = kwargs.get("model_name", "test-model")
mock_vllm_config.cache_config.block_size = kwargs.get("hash_block_size", 16)
mock_vllm_config.cache_config.cache_dtype = (
f"torch.{kwargs.get('dtype', 'float16')}"
)
mock_vllm_config.parallel_config.tensor_parallel_size = kwargs.get("tp_size", 1)
mock_vllm_config.parallel_config.pipeline_parallel_size = kwargs.get("pp_size", 1)
mock_vllm_config.parallel_config.prefill_context_parallel_size = kwargs.get(
"pcp_size", 1
)
mock_vllm_config.parallel_config.decode_context_parallel_size = kwargs.get(
"dcp_size", 1
)
mock_vllm_config.parallel_config.rank = kwargs.get("rank", 0)
mock_kv_cache_config = MagicMock()
mock_kv_cache_config.kv_cache_groups = []
mock_offloading_spec = MagicMock(spec=OffloadingSpec)
mock_offloading_spec.vllm_config = mock_vllm_config
mock_offloading_spec.kv_cache_config = mock_kv_cache_config
mock_offloading_spec.block_size_factor = kwargs.get("block_size_factor", 1)
return FileMapper.from_offloading_spec(
root_dir=kwargs.get("root_dir", "/tmp/cache"),
offloading_spec=mock_offloading_spec,
gpu_blocks_per_file=mock_offloading_spec.block_size_factor,
)
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
def test_get_file_name_full_structure():
"""
Path must match: <base_path>_r<rank>/<hhh>/<hh>_g<group_idx>/<hash_hex>.bin
Concretely:
- The segment immediately after base_path must end with `_r0`
- The next segment is the first 3 hex chars of the block hash
- The next segment is <2 hex chars>_g<group_idx>
- The final segment is <full hash hex>.bin
"""
rank = 3
group_idx = 2
block_hash = bytes(range(8)) # deterministic, non-zero bytes
fm = make_mapper_from_offloading_spec(rank=rank)
key = make_offload_key(block_hash, group_idx)
path = fm.get_file_name(key)
expected_path = (
"/tmp/cache/test-model_588656ebcc66_r3/000/10_g2/0001020304050607.bin"
)
assert path == expected_path
def test_get_run_config_fields():
fm = make_mapper_from_offloading_spec(
model_name="my-model",
dtype="bfloat16",
tp_size=2,
)
cfg = fm.get_run_config()
assert cfg == {
"model_name": "my-model",
"hash_block_size": 16,
"gpu_blocks_per_file": 1,
"tp_size": 2,
"pp_size": 1,
"pcp_size": 1,
"dcp_size": 1,
"dtype": "bfloat16",
"kv_cache_groups": [],
"inference_engine": "vllm",
}
def test_get_config_file_path():
fm = make_mapper_from_offloading_spec()
config_path = fm.get_config_file_path()
assert config_path == f"{fm.base_path}/config.json"
+256
View File
@@ -0,0 +1,256 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Unit tests for FileSystemTierManager.
These tests use real disk I/O to verify the Python filesystem tier implementation.
The tier manager writes KV cache blocks to disk and reads them back, verifying
data integrity throughout the process.
"""
import os
import time
from unittest.mock import MagicMock
import numpy as np
import pytest
import torch
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, make_offload_key
from vllm.v1.kv_offload.tiering.base import JobMetadata
from vllm.v1.kv_offload.tiering.fs.manager import (
FileSystemTierManager,
)
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
_BLOCK_ELEMENTS = 512 * 1024 # 2 MB per block (float32 × 512K = 2MB)
_DTYPE = torch.float32
_CTX = ReqContext(req_id="test")
_MOCK_VLLM_CONFIG = MagicMock()
_MOCK_VLLM_CONFIG.model_config.model = "test-model"
_MOCK_VLLM_CONFIG.cache_config.block_size = 16
_MOCK_VLLM_CONFIG.cache_config.cache_dtype = "torch.float32"
_MOCK_VLLM_CONFIG.parallel_config.tensor_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.pipeline_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.prefill_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.decode_context_parallel_size = 1
_MOCK_VLLM_CONFIG.parallel_config.rank = 0
_MOCK_KV_CACHE_CONFIG = MagicMock()
_MOCK_KV_CACHE_CONFIG.kv_cache_groups = []
_MOCK_OFFLOADING_SPEC = MagicMock()
_MOCK_OFFLOADING_SPEC.vllm_config = _MOCK_VLLM_CONFIG
_MOCK_OFFLOADING_SPEC.kv_cache_config = _MOCK_KV_CACHE_CONFIG
_MOCK_OFFLOADING_SPEC.block_size_factor = 1
def key(n: int) -> OffloadKey:
return make_offload_key(n.to_bytes(8, "big"), 0)
def make_job(
job_id: int,
keys: list[OffloadKey],
block_ids: list[int] | None = None,
is_promotion: bool = False,
) -> JobMetadata:
if block_ids is None:
block_ids = list(range(len(keys)))
return JobMetadata(
job_id=job_id,
keys=keys,
block_ids=np.array(block_ids, dtype=np.int64),
is_promotion=is_promotion,
req_context=_CTX,
)
def drain(tier: FileSystemTierManager, max_rounds: int = 40) -> list:
"""
Call get_finished() repeatedly until no new results arrive for 5
consecutive rounds or max_rounds is reached.
"""
results = []
idle = 0
for _ in range(max_rounds):
time.sleep(0.01)
new = list(tier.get_finished())
results.extend(new)
if new:
idle = 0
else:
idle += 1
if idle >= 5:
break
return results
# ---------------------------------------------------------------------------
# Fixtures
# ---------------------------------------------------------------------------
@pytest.fixture
def fs_tier(tmp_path):
tensor = torch.zeros((4, _BLOCK_ELEMENTS), dtype=_DTYPE)
mock_view = memoryview(tensor.numpy())
tier = FileSystemTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=mock_view,
tier_type="fs_python",
root_dir=str(tmp_path),
n_read_threads=4,
n_write_threads=4,
)
yield tier, tensor
tier.shutdown()
# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------
def test_lookup_empty_tier(fs_tier):
tier, _ = fs_tier
assert tier.lookup(key(1), _CTX) is False
assert tier.lookup(key(2), _CTX) is False
def test_store_creates_file_and_lookup_succeeds(fs_tier):
tier, _ = fs_tier
job = make_job(1, [key(1)], [0])
tier.submit_store(job)
results = drain(tier)
assert len(results) == 1
assert results[0].success
assert tier.lookup(key(1), _CTX) is True
dest = tier.file_mapper.get_file_name(key(1))
assert os.path.exists(dest), f"Expected file at {dest}"
def test_store_then_load_roundtrip(fs_tier):
tier, _ = fs_tier
job_s = make_job(1, [key(1), key(2)], [0, 1])
tier.submit_store(job_s)
store_results = drain(tier)
assert all(r.success for r in store_results)
assert tier.lookup(key(1), _CTX) is True
assert tier.lookup(key(2), _CTX) is True
job_l = make_job(2, [key(1), key(2)], [2, 3], is_promotion=True)
tier.submit_load(job_l)
load_results = drain(tier)
assert all(r.success for r in load_results)
# Blocks stay on disk after load
assert tier.lookup(key(1), _CTX) is True
assert tier.lookup(key(2), _CTX) is True
def test_invalid_path_raises_at_construction():
"""Construction must fail immediately when the config file cannot be written."""
tensor = torch.zeros((32, _BLOCK_ELEMENTS), dtype=_DTYPE)
mock_view = memoryview(tensor.numpy())
with pytest.raises(OSError):
FileSystemTierManager(
offloading_spec=_MOCK_OFFLOADING_SPEC,
primary_kv_view=mock_view,
tier_type="fs_python",
root_dir="/dev/null/invalid_path",
)
def test_failed_load_missing_file(fs_tier):
"""Test that loading a block whose file does not exist results in a failed job."""
tier, _ = fs_tier
job = make_job(1, [key(99)], [0], is_promotion=True)
tier.submit_load(job)
results = drain(tier)
assert len(results) == 1
assert not results[0].success
def test_multiple_jobs_tracked_independently(fs_tier):
tier, _ = fs_tier
job1 = make_job(1, [key(1)], [0])
job2 = make_job(2, [key(2)], [1])
tier.submit_store(job1)
tier.submit_store(job2)
results = drain(tier)
job_ids = {r.job_id for r in results}
assert job_ids == {1, 2}
assert tier.lookup(key(1), _CTX) is True
assert tier.lookup(key(2), _CTX) is True
def test_multi_block_job_partial_failure(fs_tier):
"""A load job where one block file is missing yields a single failed JobResult."""
tier, _ = fs_tier
# Store two of three keys
tier.submit_store(make_job(1, [key(10), key(11)], [0, 1]))
assert all(r.success for r in drain(tier))
# Load all three — key(99) was never stored
tier.submit_load(
make_job(2, [key(10), key(11), key(99)], [0, 1, 2], is_promotion=True)
)
results = drain(tier)
assert len(results) == 1
assert results[0].job_id == 2
assert not results[0].success
def test_shutdown_discards_pending_tasks(fs_tier):
"""Shutdown clears both queues and stops all worker threads without draining."""
tier, _ = fs_tier
# Submit many tasks to ensure some remain pending
for i in range(10):
tier.submit_store(make_job(i, [key(i)], [i % 4]))
# Shutdown immediately without draining
tier.shutdown()
# Verify queues are cleared and threads stopped
assert len(tier._pool._load_q) == 0
assert len(tier._pool._store_q) == 0
assert all(not t.is_alive() for t in tier._pool._threads)
def test_store_load_data_integrity(fs_tier):
"""Data written by store must be exactly recovered by load."""
tier, tensor = fs_tier
# Populate tensor with random data
tensor[:] = torch.rand((4, _BLOCK_ELEMENTS), dtype=_DTYPE)
# Store first 2 blocks
num_store = 2
expected = tensor[:num_store].clone()
store_ids = list(range(num_store))
keys = [key(i) for i in range(num_store)]
tier.submit_store(make_job(1, keys, store_ids))
results = drain(tier)
assert all(r.success for r in results)
# Overwrite source blocks to prove data is read from disk
tensor[:num_store] = 0.0
# Load into last 2 blocks
load_ids = [2, 3]
tier.submit_load(make_job(2, keys, load_ids, is_promotion=True))
results = drain(tier)
assert all(r.success for r in results)
for i, bid in enumerate(load_ids):
assert torch.allclose(tensor[bid], expected[i]), (
f"Block {bid} data mismatch after store+load"
)
+126
View File
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import hashlib
import json
from vllm.v1.kv_offload.base import (
OffloadingSpec,
OffloadKey,
get_offload_block_hash,
get_offload_group_idx,
)
_BASE_PATH_HASH_LEN = 12
_CONFIG_FILENAME = "config.json"
class FileMapper:
"""
FileMapper maps KV blocks (given by their hash) to file names.
"""
def __init__(
self,
root_dir: str,
model_name: str,
hash_block_size: int,
gpu_blocks_per_file: int,
tp_size: int,
pp_size: int,
pcp_size: int,
dcp_size: int,
rank: int,
dtype: str,
kv_cache_groups: list[dict] | None = None,
inference_engine: str = "vllm",
parallel_agnostic: bool = False,
):
"""
Initialize the file mapper. Each worker constructs its own, but
`config.json` is shared across workers since rank lives outside the hash.
When `parallel_agnostic=True`, tp/pp/pcp/dcp are forced to 1 and rank
to 0 so multiple parallelism layouts collapse into the same folder.
"""
if parallel_agnostic:
tp_size = pp_size = pcp_size = dcp_size = 1
rank = 0
self.rank: int = rank
self.fields: dict = {
"model_name": model_name,
"hash_block_size": hash_block_size,
"gpu_blocks_per_file": gpu_blocks_per_file,
"tp_size": tp_size,
"pp_size": pp_size,
"pcp_size": pcp_size,
"dcp_size": dcp_size,
"dtype": str(dtype),
"kv_cache_groups": kv_cache_groups or [],
"inference_engine": inference_engine,
}
self.base_path: str = self._compute_base_path(root_dir, self.fields)
@classmethod
def from_offloading_spec(
cls,
root_dir: str,
offloading_spec: OffloadingSpec,
gpu_blocks_per_file: int = 1,
parallel_agnostic: bool = False,
) -> "FileMapper":
"""Build a FileMapper from an OffloadingSpec."""
vllm_config = offloading_spec.vllm_config
kv_cache_config = offloading_spec.kv_cache_config
parallel_config = vllm_config.parallel_config
dtype = str(vllm_config.cache_config.cache_dtype).replace("torch.", "")
kv_cache_groups = [
{
"block_size": group.kv_cache_spec.block_size,
"layer_names": list(group.layer_names),
}
for group in kv_cache_config.kv_cache_groups
]
return cls(
root_dir=root_dir,
model_name=vllm_config.model_config.model,
hash_block_size=vllm_config.cache_config.block_size,
gpu_blocks_per_file=gpu_blocks_per_file,
tp_size=parallel_config.tensor_parallel_size,
pp_size=parallel_config.pipeline_parallel_size,
pcp_size=parallel_config.prefill_context_parallel_size,
dcp_size=parallel_config.decode_context_parallel_size,
rank=parallel_config.rank,
dtype=dtype,
kv_cache_groups=kv_cache_groups,
parallel_agnostic=parallel_agnostic,
)
def get_file_name(self, key: OffloadKey) -> str:
"""Map an OffloadKey to <base>_r<rank>/<hhh>/<hh>_g<group_idx>/<hash>.bin."""
hash_hex = get_offload_block_hash(key).hex()
group_idx = get_offload_group_idx(key)
subfolder1, subfolder2 = hash_hex[:3], hash_hex[3:5]
return (
f"{self.base_path}_r{self.rank}"
f"/{subfolder1}/{subfolder2}_g{group_idx}/{hash_hex}.bin"
)
def get_run_config(self) -> dict:
return dict(self.fields)
def get_config_file_path(self) -> str:
return f"{self.base_path}/{_CONFIG_FILENAME}"
@staticmethod
def _compute_base_path(root_dir: str, fields: dict) -> str:
"""
Layout: <root_dir>/<safe_model_name>_<sha256-prefix>/.
safe_model_name replaces '/' with '_' so HuggingFace IDs don't nest.
"""
canonical = json.dumps(fields, sort_keys=True, separators=(",", ":"))
digest = hashlib.sha256(canonical.encode("utf-8")).hexdigest()[
:_BASE_PATH_HASH_LEN
]
safe_model_name = fields["model_name"].replace("/", "_")
return f"{root_dir}/{safe_model_name}_{digest}"
+6
View File
@@ -57,3 +57,9 @@ SecondaryTierFactory.register_tier(
"vllm.v1.kv_offload.tiering.example.manager",
"ExampleSecondaryTierManager",
)
SecondaryTierFactory.register_tier(
"fs_python",
"vllm.v1.kv_offload.tiering.fs.manager",
"FileSystemTierManager",
)
+101
View File
@@ -0,0 +1,101 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import os
import random
import threading
logger = logging.getLogger(__name__)
# O_DIRECT is Linux-specific and not available on macOS
O_DIRECT = getattr(os, "O_DIRECT", 0)
# Thread-local storage for unique temporary file suffixes
_thread_local = threading.local()
def _get_tmp_suffix() -> str:
"""Generate a thread-local unique suffix for temporary files."""
try:
return _thread_local.tmp_suffix
except AttributeError:
_thread_local.tmp_suffix = f"_{random.randint(0, 2**63 - 1)}.tmp"
return _thread_local.tmp_suffix
def _ensure_dirs(path: str) -> None:
"""Create parent directories of *path* if they don't exist."""
os.makedirs(os.path.dirname(path), exist_ok=True)
def store_block(
dest_path: str,
buffer: memoryview,
offset: int,
block_size: int,
) -> None:
"""
Store callback: Writes to a temp file then atomically replaces the destination.
"""
# Check if block already exists to avoid redundant writes
if os.path.exists(dest_path):
return
tmp_path = dest_path + _get_tmp_suffix()
# Ensure parent directories exist
_ensure_dirs(dest_path)
# Write block atomically. Cast to a flat byte view so the slice uses byte
# indices; the raw memoryview may be multi-dimensional with itemsize > 1.
view_slice = buffer.cast("B")[offset : offset + block_size]
try:
fd = os.open(
tmp_path,
os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC | O_DIRECT,
0o644,
)
try:
written = os.write(fd, view_slice)
if written < len(view_slice):
raise OSError(
f"Short write: expected {len(view_slice)} bytes, wrote {written}"
)
finally:
os.close(fd)
os.replace(tmp_path, dest_path)
except Exception:
try:
os.remove(tmp_path)
except OSError as cleanup_exc:
logger.warning("Failed to remove temp file %s: %s", tmp_path, cleanup_exc)
raise
def load_block(
source_path: str,
view: memoryview,
offset: int,
block_size: int,
) -> None:
"""
Load callback: read one KV block from disk. Remove the file on failure.
"""
fd: int | None = None
view_slice = view.cast("B")[offset : offset + block_size]
try:
fd = os.open(source_path, os.O_RDONLY | O_DIRECT)
bytes_read = os.readv(fd, [view_slice])
if bytes_read < block_size:
raise OSError(f"Short read: expected {block_size} bytes, read {bytes_read}")
except Exception:
try:
os.remove(source_path)
except OSError as cleanup_exc:
logger.warning(
"Failed to remove unreadable file %s: %s", source_path, cleanup_exc
)
raise
finally:
if fd is not None:
os.close(fd)
+150
View File
@@ -0,0 +1,150 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
FileSystemTierManager: Pure-Python file system secondary tier for KV cache offloading.
Store path:
Data is written to a temp file (<dest_path.tmp>) via os.write,
then os.replace'd to the final path (without .tmp).
Load path:
Data is read from the block file directly via os.readv into the
provided memoryview slice.
File naming: <base_path>_r<rank>/<hhh>/<hh>_g<group_idx>/<hash_hex>.bin
(hash-based subdirectories to limit directory fan-out)
"""
import functools
import json
import os
from collections.abc import Iterable
from typing import TYPE_CHECKING
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.base import (
JobMetadata,
JobResult,
SecondaryTierManager,
)
from vllm.v1.kv_offload.tiering.fs.io import load_block, store_block
from vllm.v1.kv_offload.tiering.fs.thread_pool import DualQueueThreadPool
if TYPE_CHECKING:
from vllm.v1.kv_offload.base import OffloadingSpec
logger = init_logger(__name__)
class FileSystemTierManager(SecondaryTierManager):
"""
Pure-Python disk-backed secondary tier.
Read-priority threads service load jobs preferentially; write-priority
threads service store jobs preferentially. Both groups can drain either
queue, so neither starves.
submit_store / submit_load are non-blocking: they enqueue tasks and return.
get_finished() polls job completion and returns completed JobResults.
"""
def __init__(
self,
offloading_spec: "OffloadingSpec",
primary_kv_view: memoryview,
tier_type: str,
root_dir: str,
n_read_threads: int = 16,
n_write_threads: int = 16,
):
"""
Args:
offloading_spec: contains the vllm_config, kv_cache_config
and block_size_factor.
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
tier_type: Tier type identifier, set by SecondaryTierFactory.
root_dir: Root directory for block files.
n_read_threads: Number of read-priority I/O threads.
n_write_threads: Number of write-priority I/O threads.
"""
super().__init__(offloading_spec, primary_kv_view, tier_type)
# Extract block size from primary view
assert primary_kv_view.strides is not None, (
"primary_kv_view.strides cannot be None"
)
self._block_size: int = primary_kv_view.strides[0]
# Create file mapper
self.file_mapper = FileMapper.from_offloading_spec(
root_dir=root_dir,
offloading_spec=offloading_spec,
gpu_blocks_per_file=offloading_spec.block_size_factor,
)
# Write config file
config_path = self.file_mapper.get_config_file_path()
os.makedirs(os.path.dirname(config_path), exist_ok=True)
if not os.path.exists(config_path):
with open(config_path, "w") as f:
json.dump(
self.file_mapper.get_run_config(), f, indent=2, sort_keys=True
)
self._pool = DualQueueThreadPool(
n_read_threads,
n_write_threads,
thread_name_prefix="vllm_kv_py_fs",
)
def lookup(
self, key: OffloadKey, req_context: ReqContext | None = None
) -> bool | None:
return os.path.exists(self.file_mapper.get_file_name(key))
def submit_store(self, job_metadata: JobMetadata) -> None:
tasks = (
functools.partial(
store_block,
self.file_mapper.get_file_name(key),
self._primary_kv_view,
int(bid) * self._block_size,
self._block_size,
)
for key, bid in zip(job_metadata.keys, job_metadata.block_ids)
)
self._pool.enqueue_store(job_metadata.job_id, len(job_metadata.keys), tasks)
def submit_load(self, job_metadata: JobMetadata) -> None:
tasks = (
functools.partial(
load_block,
self.file_mapper.get_file_name(key),
self._primary_kv_view,
int(bid) * self._block_size,
self._block_size,
)
for key, bid in zip(job_metadata.keys, job_metadata.block_ids)
)
self._pool.enqueue_load(job_metadata.job_id, len(job_metadata.keys), tasks)
def get_finished(self) -> Iterable[JobResult]:
"""
Collect completed jobs from the finished-jobs queue.
"""
return (
JobResult(job_id=job_id, success=success)
for job_id, success in self._pool.get_finished()
)
def shutdown(self) -> None:
"""
Release resources held by this tier.
Shuts down the thread pool, clearing pending tasks and waiting for
active threads to complete.
"""
self._pool.shutdown(wait=True)
@@ -0,0 +1,158 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Thread pool:
Two queues (load, store) and two sets of threads:
- Load-priority threads: drain the load queue first, then the store queue.
- Store-priority threads: drain the store queue first, then the load queue.
Load jobs are enqueued to the load queue; store jobs to the store queue.
"""
import threading
from collections import deque
from collections.abc import Callable, Iterable
from vllm.logger import init_logger
from vllm.v1.kv_offload.tiering.base import JobId
logger = init_logger(__name__)
class JobState:
"""
Thread-safe completion tracker for a set of per-block I/O tasks.
Each task calls task_done(success) when it finishes.
"""
__slots__ = ("_job_id", "_n_tasks", "_completed", "_success", "_lock")
def __init__(self, job_id: JobId, n_tasks: int) -> None:
self._job_id: JobId = job_id
self._n_tasks = n_tasks
self._completed = 0
self._success = True
self._lock = threading.Lock()
@property
def job_id(self) -> JobId:
return self._job_id
def task_done(self, success: bool) -> tuple[bool, bool]:
"""Returns if job completed and success flag"""
with self._lock:
self._completed += 1
if not success:
self._success = False
return self._completed == self._n_tasks, self._success
class DualQueueThreadPool:
"""
Thread pool with two task queues (load and store) and two thread groups.
Load-priority threads drain the load queue first, then fall back to the
store queue. Store-priority threads do the reverse. Both queues share
a single condition variable.
"""
def __init__(
self,
n_read_threads: int,
n_write_threads: int,
thread_name_prefix: str = "fs_secondary_tier",
) -> None:
self._load_q: deque = deque()
self._store_q: deque = deque()
self._condition = threading.Condition(threading.Lock())
self._stop = False
self._threads: list[threading.Thread] = []
self._finished_q: deque[tuple[JobId, bool]] = deque()
for i in range(n_read_threads):
t = threading.Thread(
target=self._worker,
args=(True,),
name=f"{thread_name_prefix}_l{i}",
daemon=True,
)
t.start()
self._threads.append(t)
for i in range(n_write_threads):
t = threading.Thread(
target=self._worker,
args=(False,),
name=f"{thread_name_prefix}_s{i}",
daemon=True,
)
t.start()
self._threads.append(t)
def enqueue_load(
self,
job_id: JobId,
n_tasks: int,
tasks: Iterable[Callable],
) -> None:
"""Enqueue load tasks for a job (high-priority for load-priority threads)."""
state = JobState(job_id, n_tasks)
with self._condition:
for fn in tasks:
self._load_q.append((fn, state))
self._condition.notify(n_tasks)
def enqueue_store(
self,
job_id: JobId,
n_tasks: int,
tasks: Iterable[Callable],
) -> None:
"""Enqueue store tasks for a job (high-priority for store-priority threads)."""
state = JobState(job_id, n_tasks)
with self._condition:
for fn in tasks:
self._store_q.append((fn, state))
self._condition.notify(n_tasks)
def get_finished(self) -> list[tuple[JobId, bool]]:
jobs = []
while self._finished_q:
jobs.append(self._finished_q.popleft())
return jobs
def shutdown(self, wait: bool = True) -> None:
with self._condition:
self._stop = True
self._load_q.clear()
self._store_q.clear()
self._condition.notify_all()
if wait:
for t in self._threads:
t.join()
def _worker(self, load_priority: bool) -> None:
# Wait for tasks, process from primary queue first, fall back to secondary.
while True:
with self._condition:
self._condition.wait_for(
lambda: self._stop or self._load_q or self._store_q
)
if self._stop:
return
primary = self._load_q if load_priority else self._store_q
secondary = self._store_q if load_priority else self._load_q
task, state = primary.popleft() if primary else secondary.popleft()
try:
task()
job_finished, success = state.task_done(True)
except Exception as exc:
logger.error(
"FileSystemTierManagerPython: job %s block I/O failed: %s",
state.job_id,
exc,
)
job_finished, success = state.task_done(False)
if job_finished:
self._finished_q.append((state.job_id, success))