[KV Offload] Define clean backend configuration boundary (#48150)
Signed-off-by: Change72 <changg@nvidia.com> Signed-off-by: Chang Guo <cguo51@asu.edu> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
Claude Fable 5
Or Ozeri
OpenAI Codex
parent
8c3393f373
commit
a9531edfa6
@@ -14,14 +14,13 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
|
||||
from vllm.v1.kv_offload.config import OffloadingConfig
|
||||
|
||||
# `OffloadKey` identifies an offloaded block. It combines a block hash with
|
||||
# its KV cache group index, encoded as raw bytes to avoid tuple GC overhead.
|
||||
@@ -482,22 +481,15 @@ class OffloadingSpec(ABC):
|
||||
"""Return Prometheus metric definitions emitted by this spec."""
|
||||
return {}
|
||||
|
||||
def __init__(self, vllm_config: "VllmConfig", kv_cache_config: "KVCacheConfig"):
|
||||
def __init__(self, config: OffloadingConfig):
|
||||
logger.warning(
|
||||
"Initializing OffloadingSpec. This API is experimental and "
|
||||
"subject to change in the future as we iterate the design."
|
||||
)
|
||||
self.vllm_config = vllm_config
|
||||
self.kv_cache_config = kv_cache_config
|
||||
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
assert kv_transfer_config is not None
|
||||
self.extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
kv_events_config = vllm_config.kv_events_config
|
||||
self.config = config
|
||||
self.extra_config = config.extra_config
|
||||
self.kv_events_config = OffloadingKVEventsConfig(
|
||||
enable_kv_cache_events=(
|
||||
kv_events_config is not None and kv_events_config.enable_kv_cache_events
|
||||
),
|
||||
enable_kv_cache_events=config.enable_kv_cache_events,
|
||||
self_describing_kv_events=bool(
|
||||
self.extra_config.get("self_describing_kv_events", False)
|
||||
),
|
||||
@@ -511,48 +503,9 @@ class OffloadingSpec(ABC):
|
||||
self.extra_config.get("offload_prompt_only", True)
|
||||
)
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
context_parallel_factor = (
|
||||
parallel_config.decode_context_parallel_size
|
||||
* parallel_config.prefill_context_parallel_size
|
||||
)
|
||||
|
||||
# gpu block size per group
|
||||
self.gpu_block_size: tuple[int, ...] = tuple(
|
||||
kv_cache_group.kv_cache_spec.block_size * context_parallel_factor
|
||||
for kv_cache_group in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
|
||||
# hash_block_size must match what the scheduler uses for
|
||||
# Request.block_hashes (resolved via resolve_kv_cache_block_sizes).
|
||||
_, self.hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
)
|
||||
|
||||
for block_size in self.gpu_block_size:
|
||||
assert block_size % self.hash_block_size == 0, (
|
||||
f"gpu_block_size={block_size} not divisible by "
|
||||
f"hash_block_size={self.hash_block_size}. "
|
||||
f"Hybrid models (e.g. Mamba+Attention) need "
|
||||
f"--enable-prefix-caching to align block sizes."
|
||||
)
|
||||
|
||||
# offloaded_block_size / gpu_block_size
|
||||
self.block_size_factor: int = 1
|
||||
|
||||
offloaded_block_size = self.extra_config.get("block_size")
|
||||
if offloaded_block_size is not None:
|
||||
offloaded_block_size_int = int(offloaded_block_size)
|
||||
gpu_block_sizes = set(self.gpu_block_size)
|
||||
assert len(gpu_block_sizes) == 1, (
|
||||
"If 'block_size' is specified in kv_connector_extra_config, "
|
||||
"there must be at least one KV cache group, "
|
||||
"and all groups must have the same block size."
|
||||
)
|
||||
gpu_block_size = gpu_block_sizes.pop()
|
||||
|
||||
assert offloaded_block_size_int % gpu_block_size == 0
|
||||
self.block_size_factor = offloaded_block_size_int // gpu_block_size
|
||||
self.tokens_per_block = tuple(group.tokens_per_block for group in config.groups)
|
||||
self.tokens_per_hash = config.cache.tokens_per_hash
|
||||
self.blocks_per_chunk = config.cache.blocks_per_chunk
|
||||
|
||||
@abstractmethod
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
|
||||
Reference in New Issue
Block a user