[KV Offload] Define clean backend configuration boundary (#48150)

Signed-off-by: Change72 <changg@nvidia.com>
Signed-off-by: Chang Guo <cguo51@asu.edu>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Chang Guo
2026-07-16 13:27:05 +03:00
committed by GitHub
co-authored by Codex Claude Fable 5 Or Ozeri OpenAI Codex
parent 8c3393f373
commit a9531edfa6
27 changed files with 1287 additions and 855 deletions
+9 -56
View File
@@ -14,14 +14,13 @@ import numpy as np
import torch
from vllm.logger import init_logger
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
)
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.kv_offload.config import OffloadingConfig
# `OffloadKey` identifies an offloaded block. It combines a block hash with
# its KV cache group index, encoded as raw bytes to avoid tuple GC overhead.
@@ -482,22 +481,15 @@ class OffloadingSpec(ABC):
"""Return Prometheus metric definitions emitted by this spec."""
return {}
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
def __init__(self, config: OffloadingConfig):
logger.warning(
"Initializing OffloadingSpec. This API is experimental and "
"subject to change in the future as we iterate the design."
)
self.vllm_config = vllm_config
self.kv_cache_config = kv_cache_config
kv_transfer_config = vllm_config.kv_transfer_config
assert kv_transfer_config is not None
self.extra_config = kv_transfer_config.kv_connector_extra_config
kv_events_config = vllm_config.kv_events_config
self.config = config
self.extra_config = config.extra_config
self.kv_events_config = OffloadingKVEventsConfig(
enable_kv_cache_events=(
kv_events_config is not None and kv_events_config.enable_kv_cache_events
),
enable_kv_cache_events=config.enable_kv_cache_events,
self_describing_kv_events=bool(
self.extra_config.get("self_describing_kv_events", False)
),
@@ -511,48 +503,9 @@ class OffloadingSpec(ABC):
self.extra_config.get("offload_prompt_only", True)
)
parallel_config = vllm_config.parallel_config
context_parallel_factor = (
parallel_config.decode_context_parallel_size
* parallel_config.prefill_context_parallel_size
)
# gpu block size per group
self.gpu_block_size: tuple[int, ...] = tuple(
kv_cache_group.kv_cache_spec.block_size * context_parallel_factor
for kv_cache_group in kv_cache_config.kv_cache_groups
)
# hash_block_size must match what the scheduler uses for
# Request.block_hashes (resolved via resolve_kv_cache_block_sizes).
_, self.hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
)
for block_size in self.gpu_block_size:
assert block_size % self.hash_block_size == 0, (
f"gpu_block_size={block_size} not divisible by "
f"hash_block_size={self.hash_block_size}. "
f"Hybrid models (e.g. Mamba+Attention) need "
f"--enable-prefix-caching to align block sizes."
)
# offloaded_block_size / gpu_block_size
self.block_size_factor: int = 1
offloaded_block_size = self.extra_config.get("block_size")
if offloaded_block_size is not None:
offloaded_block_size_int = int(offloaded_block_size)
gpu_block_sizes = set(self.gpu_block_size)
assert len(gpu_block_sizes) == 1, (
"If 'block_size' is specified in kv_connector_extra_config, "
"there must be at least one KV cache group, "
"and all groups must have the same block size."
)
gpu_block_size = gpu_block_sizes.pop()
assert offloaded_block_size_int % gpu_block_size == 0
self.block_size_factor = offloaded_block_size_int // gpu_block_size
self.tokens_per_block = tuple(group.tokens_per_block for group in config.groups)
self.tokens_per_hash = config.cache.tokens_per_hash
self.blocks_per_chunk = config.cache.blocks_per_chunk
@abstractmethod
def get_manager(self) -> OffloadingManager: