[KV Offload] Define clean backend configuration boundary (#48150)

Signed-off-by: Change72 <changg@nvidia.com>
Signed-off-by: Chang Guo <cguo51@asu.edu>
Co-authored-by: Codex <codex@openai.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Chang Guo
2026-07-16 13:27:05 +03:00
committed by GitHub
co-authored by Codex Claude Fable 5 Or Ozeri OpenAI Codex
parent 8c3393f373
commit a9531edfa6
27 changed files with 1287 additions and 855 deletions
@@ -0,0 +1,131 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Translate vLLM KV cache metadata for native offloading backends."""
from typing import TYPE_CHECKING
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
from vllm.v1.kv_cache_interface import FullAttentionSpec, MLAAttentionSpec
from vllm.v1.kv_offload.config import (
OffloadingCacheConfig,
OffloadingConfig,
OffloadingGroupConfig,
OffloadingModelConfig,
OffloadingParallelConfig,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheTensor
def is_kv_cache_tensor_packed(kv_cache_tensor: "KVCacheTensor") -> bool:
"""Return whether a KV cache tensor uses a packed block stride."""
return bool(kv_cache_tensor.block_stride)
def build_offloading_config(
vllm_config: "VllmConfig",
kv_cache_config: "KVCacheConfig",
) -> OffloadingConfig:
"""Translate vLLM configuration into the native offloading boundary."""
kv_transfer_config = vllm_config.kv_transfer_config
assert kv_transfer_config is not None
extra_config = kv_transfer_config.kv_connector_extra_config
assert kv_transfer_config.engine_id is not None
engine_id = kv_transfer_config.engine_id
parallel_config = vllm_config.parallel_config
context_parallel_factor = (
parallel_config.decode_context_parallel_size
* parallel_config.prefill_context_parallel_size
)
groups = tuple(
OffloadingGroupConfig(
tokens_per_block=(group.kv_cache_spec.block_size * context_parallel_factor),
layer_names=tuple(group.layer_names),
)
for group in kv_cache_config.kv_cache_groups
)
_, tokens_per_hash = resolve_kv_cache_block_sizes(kv_cache_config, vllm_config)
for group in groups:
assert group.tokens_per_block % tokens_per_hash == 0, (
f"tokens_per_block={group.tokens_per_block} not divisible by "
f"tokens_per_hash={tokens_per_hash}. "
f"Hybrid models (e.g. Mamba+Attention) need "
f"--enable-prefix-caching to align block sizes."
)
blocks_per_chunk = 1
tokens_per_chunk = extra_config.get("block_size")
if tokens_per_chunk is not None:
tokens_per_chunk_int = int(tokens_per_chunk)
unique_tokens_per_block = {group.tokens_per_block for group in groups}
assert len(unique_tokens_per_block) == 1, (
"If 'block_size' is specified in kv_connector_extra_config, "
"there must be at least one KV cache group, "
"and all groups must have the same block size."
)
tokens_per_block = unique_tokens_per_block.pop()
assert tokens_per_chunk_int % tokens_per_block == 0
blocks_per_chunk = tokens_per_chunk_int // tokens_per_block
worker_kv_bytes_per_block = 0
if kv_cache_config.num_blocks > 0:
packed_tensors = tuple(
is_kv_cache_tensor_packed(tensor)
for tensor in kv_cache_config.kv_cache_tensors
)
is_packed = any(packed_tensors)
assert not is_packed or all(packed_tensors)
total_gpu_kv_bytes = (
kv_cache_config.kv_cache_tensors[0].size
if is_packed
else sum(tensor.size for tensor in kv_cache_config.kv_cache_tensors)
)
worker_kv_bytes_per_block = total_gpu_kv_bytes // kv_cache_config.num_blocks
# Only a single non-MLA full-attention group is parallelism-invariant:
# MLA latent KV is replicated per rank (never head-sharded), and the V2
# model runner's KV layout is not known to be parallelism-invariant.
single_group = (
kv_cache_config.kv_cache_groups[0].kv_cache_spec
if len(kv_cache_config.kv_cache_groups) == 1
else None
)
is_parallelism_agnostic = (
not vllm_config.use_v2_model_runner
and single_group is not None
and isinstance(single_group, FullAttentionSpec)
and not isinstance(single_group, MLAAttentionSpec)
)
kv_events_config = vllm_config.kv_events_config
return OffloadingConfig(
groups=groups,
worker_kv_bytes_per_block=worker_kv_bytes_per_block,
enable_kv_cache_events=(
kv_events_config is not None and kv_events_config.enable_kv_cache_events
),
extra_config=extra_config,
engine_id=engine_id,
model=OffloadingModelConfig(
name=vllm_config.model_config.model,
dtype=str(vllm_config.cache_config.cache_dtype).replace("torch.", ""),
),
cache=OffloadingCacheConfig(
tokens_per_hash=tokens_per_hash,
blocks_per_chunk=blocks_per_chunk,
),
parallel=OffloadingParallelConfig(
rank=parallel_config.rank,
world_size=parallel_config.world_size,
tp_size=parallel_config.tensor_parallel_size,
pp_size=parallel_config.pipeline_parallel_size,
pcp_size=parallel_config.prefill_context_parallel_size,
dcp_size=parallel_config.decode_context_parallel_size,
data_parallel_index=parallel_config.data_parallel_index,
is_parallelism_agnostic=is_parallelism_agnostic,
),
)
@@ -101,7 +101,7 @@ class OffloadingEventsTracker:
self,
req: Request,
group_config: "GroupOffloadConfig",
offload_block_idx: int,
chunk_idx: int,
offload_key: OffloadKey,
) -> None:
"""Snapshot the KV cache event payload for one offloaded chunk.
@@ -111,9 +111,9 @@ class OffloadingEventsTracker:
"""
if not self.self_describing_enabled:
return
if group_config.sliding_window_size_in_blocks is not None:
if group_config.sliding_window_size_in_chunks is not None:
return
meta = self._build_event_metadata(req, group_config, offload_block_idx)
meta = self._build_event_metadata(req, group_config, chunk_idx)
self._pending_event_metadata[offload_key] = meta
def take_events(self, events: Iterable[OffloadingEvent]) -> Iterable[KVCacheEvent]:
@@ -142,19 +142,19 @@ class OffloadingEventsTracker:
self,
req: Request,
group_config: "GroupOffloadConfig",
offload_block_idx: int,
chunk_idx: int,
) -> _OffloadEventMetadata:
"""Build the payload snapshot for one offloaded chunk: its
constituent per-block hashes, the whole chunk's tokens, and the
per-block ``block_size``."""
hbf = group_config.hash_block_size_factor
hbf = group_config.hashes_per_chunk
assert hbf > 0
assert offload_block_idx >= 0
assert chunk_idx >= 0
# per-block token count (= the GPU/hash block size)
sub_block_size = group_config.offloaded_block_size // hbf
tokens_per_hash = group_config.tokens_per_chunk // hbf
# chunk c covers hash-blocks [c*hbf, (c+1)*hbf); its tail block's hash
# is the chunk's OffloadKey.
first_hash_idx = offload_block_idx * hbf
first_hash_idx = chunk_idx * hbf
last_hash_idx = first_hash_idx + hbf
assert first_hash_idx >= 0
assert last_hash_idx <= len(req.block_hashes)
@@ -164,7 +164,7 @@ class OffloadingEventsTracker:
chunk_hashes.append(block_hash)
assert len(chunk_hashes) == hbf
if group_config.sliding_window_size_in_blocks is not None:
if group_config.sliding_window_size_in_chunks is not None:
# record_store filters these out before calling this helper.
raise AssertionError("self-describing events only support full attention")
@@ -175,8 +175,8 @@ class OffloadingEventsTracker:
parent_block_hash = req.block_hashes[first_hash_idx - 1]
assert parent_block_hash is not None
tok_start = offload_block_idx * group_config.offloaded_block_size
tok_end = tok_start + group_config.offloaded_block_size
tok_start = chunk_idx * group_config.tokens_per_chunk
tok_end = tok_start + group_config.tokens_per_chunk
assert tok_end <= len(req.all_token_ids)
token_ids = tuple(req.all_token_ids[tok_start:tok_end])
@@ -190,7 +190,7 @@ class OffloadingEventsTracker:
block_hashes=tuple(chunk_hashes),
parent_block_hash=parent_block_hash,
token_ids=token_ids,
block_size=sub_block_size,
block_size=tokens_per_hash,
lora_id=lora_id,
lora_name=lora_name,
extra_keys=None,
@@ -321,10 +321,10 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
self.histogram_transfer_size: dict[tuple[int, str], PromMetricT] = {}
self.counter_kv_bytes: dict[tuple[int, str], PromMetricT] = {}
self.counter_kv_transfer_time: dict[tuple[int, str], PromMetricT] = {}
spec_cls = OffloadingSpecFactory.get_spec_cls(vllm_config)
kv_transfer_config = vllm_config.kv_transfer_config
assert kv_transfer_config is not None
extra_config = kv_transfer_config.kv_connector_extra_config
spec_cls = OffloadingSpecFactory.get_spec_cls(extra_config)
self._offloading_metric_metadata: dict[str, OffloadingMetricMetadata] = {
**spec_cls.build_metric_definitions(extra_config),
**get_connector_metric_definitions(),
@@ -6,6 +6,7 @@ from dataclasses import dataclass, field
from itertools import islice
from typing import Any, NamedTuple
from vllm.config import VllmConfig
from vllm.distributed.kv_events import KVCacheEvent
from vllm.distributed.kv_transfer.kv_connector.utils import yield_req_data
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
@@ -31,6 +32,7 @@ from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheSpec,
MambaSpec,
SlidingWindowSpec,
@@ -74,32 +76,32 @@ class TransferJobStatus:
class GroupOffloadConfig(NamedTuple):
group_idx: int
gpu_block_size: int
offloaded_block_size: int
hash_block_size_factor: int
tokens_per_block: int
tokens_per_chunk: int
hashes_per_chunk: int
# KV cache spec metadata propagated onto emitted BlockStored events so
# KV-aware consumers can classify and filter the group.
kv_event_group_spec: OffloadingEventGroupSpec
# None below means full attention
sliding_window_size_in_blocks: int | None
# Number of this group's offloaded blocks per full-attention alignment
# segment. Used to skip storing SWA blocks that can never serve a load
sliding_window_size_in_chunks: int | None
# Number of this group's offloaded chunks per full-attention alignment
# segment. Used to skip storing SWA chunks that can never serve a load
# hit (e.g. DeepSeek V4 where SWA groups have much smaller block sizes
# than the MLA full-attention group).
# None for full-attention groups or when the optimization doesn't apply.
alignment_block_count: int | None = None
# True for EAGLE/MTP draft-model attention groups. The trailing block
alignment_chunk_count: int | None = None
# True for EAGLE/MTP draft-model attention groups. The trailing chunk
# of these groups is volatile and lacks a stable hash, so it must
# be excluded from store and load scheduling.
is_eagle_group: bool = False
def get_sliding_window_size_in_blocks(
kv_cache_spec: KVCacheSpec, offloaded_block_size: int
def get_sliding_window_size_in_chunks(
kv_cache_spec: KVCacheSpec, tokens_per_chunk: int
) -> int | None:
if isinstance(kv_cache_spec, SlidingWindowSpec):
assert kv_cache_spec.sliding_window > 0
return cdiv(kv_cache_spec.sliding_window, offloaded_block_size)
return cdiv(kv_cache_spec.sliding_window, tokens_per_chunk)
if isinstance(kv_cache_spec, MambaSpec):
# Mamba depends on a single state
@@ -109,116 +111,121 @@ def get_sliding_window_size_in_blocks(
return None
def resolve_mamba_align_size(spec: "OffloadingSpec") -> int | None:
def resolve_mamba_align_size(
spec: "OffloadingSpec", kv_cache_config: KVCacheConfig
) -> int | None:
"""Scan all KV cache groups in *spec* and return the single mamba alignment
size, or None if no group requires mamba alignment.
For MambaSpec groups in "align" cache mode the hit window must be rounded
down to a multiple of the offloaded block size. Asserts that all such
down to a multiple of the offloaded chunk size. Asserts that all such
groups agree on the same value.
"""
mamba_align_size: int | None = None
for idx, gpu_block_size in enumerate(spec.gpu_block_size):
kv_spec = spec.kv_cache_config.kv_cache_groups[idx].kv_cache_spec
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
kv_spec = kv_cache_config.kv_cache_groups[idx].kv_cache_spec
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode == "align":
offload_block_size = gpu_block_size * spec.block_size_factor
assert mamba_align_size is None or mamba_align_size == offload_block_size
mamba_align_size = offload_block_size
tokens_per_chunk = tokens_per_block * spec.blocks_per_chunk
assert mamba_align_size is None or mamba_align_size == tokens_per_chunk
mamba_align_size = tokens_per_chunk
return mamba_align_size
class SchedulerOffloadConfig(NamedTuple):
kv_group_configs: tuple[GroupOffloadConfig, ...]
block_size_factor: int
blocks_per_chunk: int
num_workers: int
offload_prompt_only: bool
@classmethod
def from_spec(cls, spec: OffloadingSpec) -> "SchedulerOffloadConfig":
def from_spec(
cls,
spec: OffloadingSpec,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
) -> "SchedulerOffloadConfig":
# Determine the alignment token count from the full-attention group(s).
# This is the offloaded_block_size of the full-attention group; load
# This is the tokens_per_chunk of the full-attention group; load
# hits are always aligned to this boundary, so SWA blocks earlier in
# each segment can never serve a load hit. Relevant for hybrid
# architectures like DeepSeek V4 (MLA + SWA groups).
full_attn_offloaded_block_sizes: set[int] = set()
for idx, gpu_block_size in enumerate(spec.gpu_block_size):
kv_spec = spec.kv_cache_config.kv_cache_groups[idx].kv_cache_spec
sw = get_sliding_window_size_in_blocks(
kv_spec, gpu_block_size * spec.block_size_factor
full_attn_tokens_per_chunk: set[int] = set()
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
kv_spec = kv_cache_config.kv_cache_groups[idx].kv_cache_spec
sw = get_sliding_window_size_in_chunks(
kv_spec, tokens_per_block * spec.blocks_per_chunk
)
if sw is None:
full_attn_offloaded_block_sizes.add(
gpu_block_size * spec.block_size_factor
)
full_attn_tokens_per_chunk.add(tokens_per_block * spec.blocks_per_chunk)
# Only apply the optimization if there's a single consistent
# full-attention alignment size.
alignment_tokens: int | None = None
if len(full_attn_offloaded_block_sizes) == 1:
alignment_tokens = full_attn_offloaded_block_sizes.pop()
if len(full_attn_tokens_per_chunk) == 1:
alignment_tokens = full_attn_tokens_per_chunk.pop()
def _alignment_block_count(
offloaded_block_size: int,
sliding_window_size_in_blocks: int | None,
def _alignment_chunk_count(
tokens_per_chunk: int,
sliding_window_size_in_chunks: int | None,
) -> int | None:
if alignment_tokens is None or sliding_window_size_in_blocks is None:
if alignment_tokens is None or sliding_window_size_in_chunks is None:
return None
if alignment_tokens <= offloaded_block_size:
if alignment_tokens <= tokens_per_chunk:
return None
per_segment = alignment_tokens // offloaded_block_size
if sliding_window_size_in_blocks >= per_segment:
per_segment = alignment_tokens // tokens_per_chunk
if sliding_window_size_in_chunks >= per_segment:
return None
return per_segment
eagle_groups = {
idx
for idx, g in enumerate(spec.kv_cache_config.kv_cache_groups)
for idx, g in enumerate(kv_cache_config.kv_cache_groups)
if g.is_eagle_group
}
use_eagle = (
spec.vllm_config.speculative_config is not None
and spec.vllm_config.speculative_config.use_eagle()
vllm_config.speculative_config is not None
and vllm_config.speculative_config.use_eagle()
)
if use_eagle and not eagle_groups:
eagle_groups = set(range(len(spec.kv_cache_config.kv_cache_groups)))
eagle_groups = set(range(len(kv_cache_config.kv_cache_groups)))
if eagle_groups:
logger.info(
"KV offloading: EAGLE/MTP draft attention groups %s "
"detected. The trailing block of these groups will be "
"detected. The trailing chunk of these groups will be "
"excluded from offloading due to volatility.",
sorted(eagle_groups),
)
return cls(
num_workers=spec.vllm_config.parallel_config.world_size,
num_workers=vllm_config.parallel_config.world_size,
kv_group_configs=tuple(
GroupOffloadConfig(
group_idx=idx,
gpu_block_size=gpu_block_size,
offloaded_block_size=gpu_block_size * spec.block_size_factor,
hash_block_size_factor=(
(gpu_block_size * spec.block_size_factor)
// spec.hash_block_size
tokens_per_block=tokens_per_block,
tokens_per_chunk=tokens_per_block * spec.blocks_per_chunk,
hashes_per_chunk=(
(tokens_per_block * spec.blocks_per_chunk)
// spec.tokens_per_hash
),
sliding_window_size_in_blocks=(
sw := get_sliding_window_size_in_blocks(
spec.kv_cache_config.kv_cache_groups[idx].kv_cache_spec,
gpu_block_size * spec.block_size_factor,
sliding_window_size_in_chunks=(
sw := get_sliding_window_size_in_chunks(
kv_cache_config.kv_cache_groups[idx].kv_cache_spec,
tokens_per_block * spec.blocks_per_chunk,
)
),
alignment_block_count=_alignment_block_count(
gpu_block_size * spec.block_size_factor, sw
alignment_chunk_count=_alignment_chunk_count(
tokens_per_block * spec.blocks_per_chunk, sw
),
kv_event_group_spec=get_offloading_event_group_spec(
spec.kv_cache_config.kv_cache_groups[idx]
kv_cache_config.kv_cache_groups[idx]
),
is_eagle_group=idx in eagle_groups,
)
for idx, gpu_block_size in enumerate(spec.gpu_block_size)
for idx, tokens_per_block in enumerate(spec.tokens_per_block)
),
block_size_factor=spec.block_size_factor,
blocks_per_chunk=spec.blocks_per_chunk,
offload_prompt_only=spec.offload_prompt_only,
)
@@ -227,11 +234,11 @@ class SchedulerOffloadConfig(NamedTuple):
class RequestGroupState:
offload_keys: list[OffloadKey] = field(default_factory=list)
block_ids: list[int] = field(default_factory=list)
# index of next block (of size offloaded_block_size) to offload
next_stored_block_idx: int = 0
# number of offloaded blocks hit (including GPU prefix cache)
# Index of the next chunk to offload.
next_stored_chunk_idx: int = 0
# Number of offloaded chunks hit (including GPU prefix cache)
# when the request first started
num_hit_blocks: int = 0
num_hit_chunks: int = 0
@dataclass(slots=True)
@@ -278,11 +285,11 @@ class RequestOffloadState:
):
for req_block_hash in islice(
self.req.block_hashes,
group_config.hash_block_size_factor * len(group_state.offload_keys)
+ group_config.hash_block_size_factor
group_config.hashes_per_chunk * len(group_state.offload_keys)
+ group_config.hashes_per_chunk
- 1,
None,
group_config.hash_block_size_factor,
group_config.hashes_per_chunk,
):
group_state.offload_keys.append(
make_offload_key(req_block_hash, group_config.group_idx)
@@ -298,46 +305,46 @@ class RequestOffloadState:
for group_state, new_blocks in zip(self.group_states, new_block_id_groups):
group_state.block_ids.extend(new_blocks)
def storable_blocks(
def storable_chunks(
self, group_config: "GroupOffloadConfig", num_offloadable_tokens: int
) -> int:
"""Number of leading offloaded blocks eligible for store.
"""Number of leading offloaded chunks eligible for store.
For eagle/MTP groups the volatile trailing block of the offloadable
For eagle/MTP groups the volatile trailing chunk of the offloadable
range is excluded while decoding: the draft-layer KV of the last
accepted position may be rewritten after spec-token rejection. During
prefill the trailing block is stable (the draft input for a chunk's
prefill the trailing chunk is stable (the draft input for a chunk's
last position is the next prompt token), so it is stored immediately.
The exclusion must be applied consistently everywhere
``next_stored_block_idx`` is derived: otherwise the trailing block of
``next_stored_chunk_idx`` is derived: otherwise the trailing chunk of
each step is skipped on collection but jumped over by
``next_stored_block_idx``, so it is never re-considered and a
``next_stored_chunk_idx``, so it is never re-considered and a
permanent hole breaks prefix-reuse lookup.
"""
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
num_chunks = num_offloadable_tokens // group_config.tokens_per_chunk
is_decoding = num_offloadable_tokens > self.req.num_prompt_tokens
if group_config.is_eagle_group and is_decoding:
num_blocks = max(0, num_blocks - 1)
return num_blocks
num_chunks = max(0, num_chunks - 1)
return num_chunks
def advance_stored_idx(self, num_offloadable_tokens: int) -> None:
# max(): at the prefill->decode transition of a block-aligned prompt,
# storable_blocks drops by one (the eagle exclusion kicks in), and the
# index must not move backwards past already-stored blocks.
# max(): at the prefill->decode transition of a chunk-aligned prompt,
# storable_chunks drops by one (the eagle exclusion kicks in), and the
# index must not move backwards past already-stored chunks.
for group_config, group_state in zip(
self.config.kv_group_configs, self.group_states
):
group_state.next_stored_block_idx = max(
group_state.next_stored_block_idx,
self.storable_blocks(group_config, num_offloadable_tokens),
group_state.next_stored_chunk_idx = max(
group_state.next_stored_chunk_idx,
self.storable_chunks(group_config, num_offloadable_tokens),
)
def update_num_hit_blocks(self, num_cached_tokens: int) -> None:
def update_num_hit_chunks(self, num_cached_tokens: int) -> None:
for group_config, group_state in zip(
self.config.kv_group_configs, self.group_states
):
group_state.num_hit_blocks = (
num_cached_tokens // group_config.offloaded_block_size
group_state.num_hit_chunks = (
num_cached_tokens // group_config.tokens_per_chunk
)
@@ -354,22 +361,26 @@ class OffloadingConnectorScheduler:
def __init__(
self,
spec: OffloadingSpec,
vllm_config: VllmConfig,
kv_cache_config: KVCacheConfig,
):
self.config = SchedulerOffloadConfig.from_spec(spec)
self.config = SchedulerOffloadConfig.from_spec(
spec, vllm_config, kv_cache_config
)
self.manager: OffloadingManager = spec.get_manager()
self._connector_stats = OffloadingConnectorStats()
full_attention_groups: list[int] = []
sliding_window_groups: list[int] = []
for group_config in self.config.kv_group_configs:
if group_config.sliding_window_size_in_blocks is None:
if group_config.sliding_window_size_in_chunks is None:
full_attention_groups.append(group_config.group_idx)
else:
sliding_window_groups.append(group_config.group_idx)
# sort sliding window groups by window size in decreasing order
def _sliding_window_sort_key(i: int) -> int:
val = self.config.kv_group_configs[i].sliding_window_size_in_blocks
val = self.config.kv_group_configs[i].sliding_window_size_in_chunks
assert val is not None
return val
@@ -378,7 +389,9 @@ class OffloadingConnectorScheduler:
# used by _lookup
self._sliding_window_groups: tuple[int, ...] = tuple(sliding_window_groups)
self._lookup_groups = tuple(full_attention_groups) + self._sliding_window_groups
self._mamba_align_size: int | None = resolve_mamba_align_size(spec)
self._mamba_align_size: int | None = resolve_mamba_align_size(
spec, kv_cache_config
)
self._req_status: dict[ReqId, RequestOffloadState] = {}
self._current_batch_load_jobs: dict[int, TransferJob] = {}
@@ -386,9 +399,9 @@ class OffloadingConnectorScheduler:
# GPU block IDs allocated in the current engine step
self._current_batch_allocated_block_ids: set[int] = set()
# if GPU prefix caching is enabled,
# track loaded blocks to avoid redundant loads
self._blocks_being_loaded: set[OffloadKey] | None = (
set() if spec.vllm_config.cache_config.enable_prefix_caching else None
# Track loaded chunks to avoid redundant loads.
self._chunks_being_loaded: set[OffloadKey] | None = (
set() if vllm_config.cache_config.enable_prefix_caching else None
)
# Job ID counter shared by loads and stores.
@@ -434,7 +447,7 @@ class OffloadingConnectorScheduler:
def _maximal_prefix_lookup(
self, keys: Iterable[OffloadKey], req_context: ReqContext
) -> int | None:
"""Return the number of consecutive offloaded blocks from the start,
"""Return the number of consecutive offloaded chunks from the start,
or None if the backend deferred a lookup."""
hit_count = 0
defer_lookup = False
@@ -490,18 +503,18 @@ class OffloadingConnectorScheduler:
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
):
if group_config.sliding_window_size_in_blocks is None:
if group_config.sliding_window_size_in_chunks is None:
self.manager.touch(group_state.offload_keys, req_status.req_context)
else:
# we aim to keep just blocks that are necessary to hit
# the original request (+ decoded blocks)
blocks_to_skip = max(
# Keep only chunks needed to hit the original request, plus
# decoded chunks.
chunks_to_skip = max(
0,
group_state.num_hit_blocks
- group_config.sliding_window_size_in_blocks,
group_state.num_hit_chunks
- group_config.sliding_window_size_in_chunks,
)
self.manager.touch(
group_state.offload_keys[blocks_to_skip:],
group_state.offload_keys[chunks_to_skip:],
req_status.req_context,
)
@@ -531,7 +544,7 @@ class OffloadingConnectorScheduler:
defer_lookup = False
lookup_groups = self._lookup_groups
# Tracks which eagle groups have already popped their volatile trailing block
# Tracks which eagle groups have already popped their volatile trailing chunk
# in the current convergence iteration. Reset when a non-eagle group
# tightens the hit boundary, requiring a fresh pop.
eagle_verified: set[int] = set()
@@ -544,79 +557,76 @@ class OffloadingConnectorScheduler:
group_idx
]
group_state: RequestGroupState = req_status.group_states[group_idx]
offloaded_block_size = group_config.offloaded_block_size
tokens_per_chunk = group_config.tokens_per_chunk
offload_keys = group_state.offload_keys
assert (
len(offload_keys)
>= req_status.req.num_tokens // offloaded_block_size
len(offload_keys) >= req_status.req.num_tokens // tokens_per_chunk
)
is_eagle_unverified = (
group_config.is_eagle_group and group_idx not in eagle_verified
)
# Constrain to block-aligned boundary for this group
# Constrain to a chunk-aligned boundary for this group.
max_hit_size_tokens = min(
max_hit_size_tokens, len(offload_keys) * offloaded_block_size
max_hit_size_tokens, len(offload_keys) * tokens_per_chunk
)
if max_hit_size_tokens - num_computed_tokens < offloaded_block_size:
# we can only load less than a block, better skip
if max_hit_size_tokens - num_computed_tokens < tokens_per_chunk:
# We can only load less than a chunk, so skip.
return 0
sliding_window_size_in_blocks = (
group_config.sliding_window_size_in_blocks
sliding_window_size_in_chunks = (
group_config.sliding_window_size_in_chunks
)
# For eagle groups, query one extra block that will be popped.
# For eagle groups, query one extra chunk that will be popped.
# We only need to increase the query size for sliding window groups.
query_max = max_hit_size_tokens
if is_eagle_unverified and sliding_window_size_in_blocks is not None:
if is_eagle_unverified and sliding_window_size_in_chunks is not None:
query_max = min(
max_hit_size_tokens + offloaded_block_size,
len(offload_keys) * offloaded_block_size,
max_hit_size_tokens + tokens_per_chunk,
len(offload_keys) * tokens_per_chunk,
)
num_blocks = min(
cdiv(query_max, offloaded_block_size), len(offload_keys)
)
start_block_idx = num_computed_tokens // offloaded_block_size
offload_keys = offload_keys[start_block_idx:num_blocks]
num_chunks = min(cdiv(query_max, tokens_per_chunk), len(offload_keys))
start_chunk_idx = num_computed_tokens // tokens_per_chunk
offload_keys = offload_keys[start_chunk_idx:num_chunks]
# end index (in the sliced offload_keys) up to which we
# have backend-confirmed hits
num_hit_blocks: int | None
if sliding_window_size_in_blocks is None:
num_hit_blocks = self._maximal_prefix_lookup(
num_hit_chunks: int | None
if sliding_window_size_in_chunks is None:
num_hit_chunks = self._maximal_prefix_lookup(
offload_keys, req_status.req_context
)
else:
required_window = sliding_window_size_in_blocks
required_window = sliding_window_size_in_chunks
if is_eagle_unverified:
required_window += 1
num_hit_blocks = self._sliding_window_lookup(
num_hit_chunks = self._sliding_window_lookup(
offload_keys,
required_window,
req_status.req_context,
)
if num_hit_blocks == 0:
if num_hit_chunks == 0:
return 0
if num_hit_blocks is None:
if num_hit_chunks is None:
defer_lookup = True
else:
if is_eagle_unverified:
num_hit_blocks -= 1
num_hit_chunks -= 1
eagle_verified.add(group_idx)
max_hit_size_tokens = min(
max_hit_size_tokens,
offloaded_block_size * (start_block_idx + num_hit_blocks),
tokens_per_chunk * (start_chunk_idx + num_hit_chunks),
)
new_num_hit_tokens = max_hit_size_tokens - num_computed_tokens
if new_num_hit_tokens < offloaded_block_size:
# we can only load less than a block, better skip
if new_num_hit_tokens < tokens_per_chunk:
# We can only load less than a chunk, so skip.
return 0
if new_num_hit_tokens < num_hit_tokens:
@@ -632,7 +642,7 @@ class OffloadingConnectorScheduler:
# sliding window works with the new_num_hit_tokens
lookup_groups = self._sliding_window_groups
looked_up_sliding_window |= sliding_window_size_in_blocks is not None
looked_up_sliding_window |= sliding_window_size_in_chunks is not None
num_hit_tokens = new_num_hit_tokens
if defer_lookup:
@@ -642,28 +652,28 @@ class OffloadingConnectorScheduler:
)
return None
# possibly delay request if any of the hit blocks is already being loaded
if self._blocks_being_loaded:
# Possibly delay the request if any hit chunk is already being loaded.
if self._chunks_being_loaded:
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
):
offloaded_block_size = group_config.offloaded_block_size
sliding_window_size_in_blocks = (
group_config.sliding_window_size_in_blocks
tokens_per_chunk = group_config.tokens_per_chunk
sliding_window_size_in_chunks = (
group_config.sliding_window_size_in_chunks
)
offload_keys = group_state.offload_keys
num_blocks = cdiv(
num_computed_tokens + num_hit_tokens, offloaded_block_size
num_chunks = cdiv(
num_computed_tokens + num_hit_tokens, tokens_per_chunk
)
start_block_idx = num_computed_tokens // offloaded_block_size
offload_keys = offload_keys[start_block_idx:num_blocks]
if sliding_window_size_in_blocks is not None:
offload_keys = offload_keys[-sliding_window_size_in_blocks:]
if any(key in self._blocks_being_loaded for key in offload_keys):
# hit blocks are being loaded, delay request
start_chunk_idx = num_computed_tokens // tokens_per_chunk
offload_keys = offload_keys[start_chunk_idx:num_chunks]
if sliding_window_size_in_chunks is not None:
offload_keys = offload_keys[-sliding_window_size_in_chunks:]
if any(key in self._chunks_being_loaded for key in offload_keys):
# Hit chunks are being loaded, so delay the request.
logger.debug(
"Delaying request %s since some of its"
" blocks are already being loaded",
" chunks are already being loaded",
req_status.req.request_id,
)
return None
@@ -740,7 +750,7 @@ class OffloadingConnectorScheduler:
req_status.deferred_lookup_start_time = lookup_start
else:
self._maybe_observe_lookup_async_delay(req_status)
req_status.update_num_hit_blocks(num_computed_tokens + (num_hit_tokens or 0))
req_status.update_num_hit_chunks(num_computed_tokens + (num_hit_tokens or 0))
self._touch(req_status)
@@ -771,10 +781,10 @@ class OffloadingConnectorScheduler:
block.block_id for block in group_blocks if block.block_id != 0
)
gpu_block_size = group_config.gpu_block_size
offloaded_block_size = group_config.offloaded_block_size
tokens_per_block = group_config.tokens_per_block
tokens_per_chunk = group_config.tokens_per_chunk
offload_keys = group_state.offload_keys
num_gpu_blocks = cdiv(num_cached_tokens, gpu_block_size)
num_gpu_blocks = cdiv(num_cached_tokens, tokens_per_block)
assert len(group_blocks) >= num_gpu_blocks
num_locally_computed_gpu_blocks = num_gpu_blocks
@@ -786,24 +796,24 @@ class OffloadingConnectorScheduler:
assert (
num_locally_computed_tokens
<= num_locally_computed_gpu_blocks * gpu_block_size
<= num_locally_computed_gpu_blocks * tokens_per_block
)
num_pending_gpu_blocks = num_gpu_blocks - num_locally_computed_gpu_blocks
if group_config.sliding_window_size_in_blocks is not None:
if group_config.sliding_window_size_in_chunks is not None:
assert (
num_pending_gpu_blocks
<= group_config.sliding_window_size_in_blocks
* self.config.block_size_factor
<= group_config.sliding_window_size_in_chunks
* self.config.blocks_per_chunk
)
num_blocks = cdiv(num_cached_tokens, offloaded_block_size)
assert len(offload_keys) >= num_blocks
num_chunks = cdiv(num_cached_tokens, tokens_per_chunk)
assert len(offload_keys) >= num_chunks
if num_pending_gpu_blocks:
start_block_idx = (
num_locally_computed_gpu_blocks // self.config.block_size_factor
start_chunk_idx = (
num_locally_computed_gpu_blocks // self.config.blocks_per_chunk
)
keys_to_load.extend(offload_keys[start_block_idx:num_blocks])
keys_to_load.extend(offload_keys[start_chunk_idx:num_chunks])
dst_block_ids.extend(
block.block_id
@@ -814,11 +824,11 @@ class OffloadingConnectorScheduler:
group_sizes.append(num_pending_gpu_blocks)
block_indices.append(num_locally_computed_gpu_blocks)
# Skip prefix-hit blocks for block-level policy; for
# request-level, next_stored_block_idx stays at 0 so all
# blocks (including hits) are offloaded.
# Skip prefix-hit chunks for block-level policy; for
# request-level, next_stored_chunk_idx stays at 0 so all
# chunks (including hits) are offloaded.
if req_status.offloading_context.policy == OffloadPolicy.BLOCK_LEVEL:
group_state.next_stored_block_idx = num_blocks
group_state.next_stored_chunk_idx = num_chunks
src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context)
dst_spec = GPULoadStoreSpec(
@@ -841,8 +851,8 @@ class OffloadingConnectorScheduler:
is_store=False,
)
if self._blocks_being_loaded is not None:
self._blocks_being_loaded.update(keys_to_load)
if self._chunks_being_loaded is not None:
self._chunks_being_loaded.update(keys_to_load)
def _update_req_states(self, scheduler_output: SchedulerOutput) -> None:
"""
@@ -877,16 +887,16 @@ class OffloadingConnectorScheduler:
# Zero out stale block_ids in sliding window groups' pending-store
# positions. Only sliding window groups can have stale entries (blocks
# freed by remove_skipped_blocks then reallocated). Only positions in
# [next_stored_block_idx * bsf, end) need checking where end is the
# [next_stored_chunk_idx * bsf, end) need checking where end is the
# pre-extend length: earlier positions were already offloaded, later
# ones are fresh allocations from this step.
if self._sliding_window_groups and self._current_batch_allocated_block_ids:
block_size_factor = self.config.block_size_factor
blocks_per_chunk = self.config.blocks_per_chunk
for req_id, req_status in self._req_status.items():
ends = new_block_ids_end.get(req_id)
for i, grp_idx in enumerate(self._sliding_window_groups):
group_state = req_status.group_states[grp_idx]
start = group_state.next_stored_block_idx * block_size_factor
start = group_state.next_stored_chunk_idx * blocks_per_chunk
end = ends[i] if ends is not None else len(group_state.block_ids)
for j in range(start, end):
if (
@@ -899,7 +909,7 @@ class OffloadingConnectorScheduler:
self,
scheduler_output: SchedulerOutput,
) -> dict[int, TransferJob]:
block_size_factor = self.config.block_size_factor
blocks_per_chunk = self.config.blocks_per_chunk
store_jobs: dict[int, TransferJob] = {}
for req_id in scheduler_output.num_scheduled_tokens:
req_status = self._req_status.get(req_id)
@@ -915,59 +925,59 @@ class OffloadingConnectorScheduler:
if max_offload_tokens is not None:
num_offloadable_tokens = min(num_offloadable_tokens, max_offload_tokens)
# Skip decode-phase blocks: clamp to the prompt length so only
# prefill (prompt) blocks become eligible for store. next_stored_idx
# never advances past this boundary, so decode blocks are never
# Skip decode-phase chunks: clamp to the prompt length so only
# prefill chunks become eligible for store. next_stored_chunk_idx
# never advances past this boundary, so decode chunks are never
# queued in this or any later step.
if self.config.offload_prompt_only:
num_offloadable_tokens = min(
num_offloadable_tokens, req.num_prompt_tokens
)
# Filter out blocks skipped due to sliding window attention / SSM
# Filter out chunks skipped due to sliding window attention / SSM
# or unreachable by the load path's alignment constraints.
new_offload_keys: list[OffloadKey] = []
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
):
num_blocks = req_status.storable_blocks(
num_chunks = req_status.storable_chunks(
group_config, num_offloadable_tokens
)
start_block_idx = group_state.next_stored_block_idx
if num_blocks <= start_block_idx:
start_chunk_idx = group_state.next_stored_chunk_idx
if num_chunks <= start_chunk_idx:
continue
offload_keys = group_state.offload_keys[start_block_idx:num_blocks]
# For each block to offload, take the last corresponding GPU block.
# e.g. if block size factor is 3 and GPU block IDs are
# 1 5 6 7 2 4 9 3 8 then we'll take blocks 6 4 8.
offload_keys = group_state.offload_keys[start_chunk_idx:num_chunks]
# For each chunk, take the last corresponding GPU block. For
# blocks_per_chunk=3 and GPU block IDs 1 5 6 7 2 4 9 3 8,
# this selects GPU blocks 6 4 8.
# A block_id of 0 means either a sliding window / SSM skip
# or a stale entry that was zeroed out — skip it either way.
offload_block_ids = group_state.block_ids[
start_block_idx * block_size_factor
+ block_size_factor
- 1 : num_blocks * block_size_factor : block_size_factor
start_chunk_idx * blocks_per_chunk
+ blocks_per_chunk
- 1 : num_chunks * blocks_per_chunk : blocks_per_chunk
]
assert len(offload_keys) == len(offload_block_ids)
alignment_block_count = group_config.alignment_block_count
tail = group_config.sliding_window_size_in_blocks
alignment_chunk_count = group_config.alignment_chunk_count
tail = group_config.sliding_window_size_in_chunks
for key_idx, (offload_key, block_id) in enumerate(
zip(offload_keys, offload_block_ids)
):
if block_id == 0:
continue
# Skip SWA blocks that can never serve a load hit:
# Skip SWA chunks that can never serve a load hit:
# within each full-attention alignment segment, only the
# trailing `tail` blocks are reachable by
# trailing `tail` chunks are reachable by
# _sliding_window_lookup. For DeepSeek V4 with 100K
# tokens this reduces SWA stores by ~78%.
if alignment_block_count is not None:
if alignment_chunk_count is not None:
assert tail is not None
abs_block_idx = start_block_idx + key_idx
pos_in_segment = abs_block_idx % alignment_block_count
if pos_in_segment < alignment_block_count - tail:
abs_chunk_idx = start_chunk_idx + key_idx
pos_in_segment = abs_chunk_idx % alignment_chunk_count
if pos_in_segment < alignment_chunk_count - tail:
continue
new_offload_keys.append(offload_key)
@@ -982,7 +992,7 @@ class OffloadingConnectorScheduler:
self._connector_stats.increase_counter(
_ConnectorMetricName.ALLOCATION_FAILURE
)
logger.warning("Request %s: cannot store blocks", req_id)
logger.warning("Request %s: cannot store chunks", req_id)
continue
if not store_output.keys_to_store:
@@ -1002,29 +1012,29 @@ class OffloadingConnectorScheduler:
self.config.kv_group_configs, req_status.group_states
):
is_sliding_window = (
group_config.sliding_window_size_in_blocks is not None
group_config.sliding_window_size_in_chunks is not None
)
num_blocks = req_status.storable_blocks(
num_chunks = req_status.storable_chunks(
group_config, num_offloadable_tokens
)
start_block_idx = group_state.next_stored_block_idx
start_chunk_idx = group_state.next_stored_chunk_idx
block_ids = group_state.block_ids
num_group_blocks = 0
start_gpu_block_idx: int | None = None
for idx, offload_key in enumerate(
group_state.offload_keys[start_block_idx:num_blocks]
group_state.offload_keys[start_chunk_idx:num_chunks]
):
if offload_key not in keys_to_store:
continue
offloaded_block_idx = start_block_idx + idx
chunk_idx = start_chunk_idx + idx
self._events_tracker.record_store(
req, group_config, offloaded_block_idx, offload_key
req, group_config, chunk_idx, offload_key
)
gpu_block_idx = offloaded_block_idx * block_size_factor
for i in range(block_size_factor):
gpu_block_idx = chunk_idx * blocks_per_chunk
for i in range(blocks_per_chunk):
block_id = block_ids[gpu_block_idx + i]
if block_id == 0:
continue
@@ -1039,8 +1049,8 @@ class OffloadingConnectorScheduler:
group_sizes.append(num_group_blocks)
block_indices.append(start_gpu_block_idx or 0)
group_state.next_stored_block_idx = max(
group_state.next_stored_block_idx, num_blocks
group_state.next_stored_chunk_idx = max(
group_state.next_stored_chunk_idx, num_chunks
)
src_spec = GPULoadStoreSpec(
@@ -1076,7 +1086,7 @@ class OffloadingConnectorScheduler:
)
logger.debug(
"Request %s offloading %s blocks upto %d tokens (job %d)",
"Request %s offloading %s chunks upto %d tokens (job %d)",
req_id,
len(keys_to_store),
num_offloadable_tokens,
@@ -1198,8 +1208,8 @@ class OffloadingConnectorScheduler:
self.manager.complete_store(job_status.keys, req_status.req_context)
else:
self.manager.complete_load(job_status.keys, req_status.req_context)
if self._blocks_being_loaded:
self._blocks_being_loaded.difference_update(job_status.keys)
if self._chunks_being_loaded:
self._chunks_being_loaded.difference_update(job_status.keys)
if self._block_id_to_pending_jobs:
# Sliding window blocks are tracked from store creation
# and must be cleaned up unconditionally.
@@ -1289,7 +1299,7 @@ class OffloadingConnectorScheduler:
yield from self._events_tracker.take_events(self.manager.take_events())
def reset_cache(self) -> None:
"""Reset the offloading manager cache, evicting all stored blocks."""
"""Reset the offloading manager cache, evicting all stored chunks."""
# reset_cache cannot be called in the middle of a schedule step
assert not self._current_batch_load_jobs
@@ -1306,10 +1316,10 @@ class OffloadingConnectorScheduler:
# Reset offloading manager cache
self.manager.reset_cache()
# Reset store progress so active requests re-offload from block 0
# Reset store progress so active requests re-offload from chunk 0.
for status in self._req_status.values():
for group_state in status.group_states:
group_state.next_stored_block_idx = 0
group_state.next_stored_chunk_idx = 0
status.transfer_jobs.clear()
# Discard jobs and save job_counter to be able to discard worker responses
@@ -1323,8 +1333,8 @@ class OffloadingConnectorScheduler:
# Note: _current_batch_jobs_to_flush is intentionally NOT cleared.
# The load flush IDs collected above must be delivered to workers.
if self._blocks_being_loaded is not None:
self._blocks_being_loaded.clear()
if self._chunks_being_loaded is not None:
self._chunks_being_loaded.clear()
def shutdown(self) -> None:
self.manager.shutdown()
@@ -10,10 +10,14 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingWorkerMetadata,
ReqId,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
is_kv_cache_tensor_packed,
)
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionBackend
from vllm.v1.kv_cache_interface import (
AttentionSpec,
KVCacheConfig,
MambaSpec,
UniformTypeKVCacheSpecs,
)
@@ -33,8 +37,13 @@ logger = init_logger(__name__)
class OffloadingConnectorWorker:
"""Implementation of Worker side methods"""
def __init__(self, spec: OffloadingSpec):
def __init__(
self,
spec: OffloadingSpec,
kv_cache_config: KVCacheConfig,
):
self.spec = spec
self.kv_cache_config = kv_cache_config
self.worker: OffloadingWorker | None = None
# job_id -> req_id for in-flight loads.
@@ -50,7 +59,7 @@ class OffloadingConnectorWorker:
def register_kv_caches(
self, kv_caches: dict[str, torch.Tensor | list[torch.Tensor]]
):
kv_cache_config = self.spec.kv_cache_config
kv_cache_config = self.kv_cache_config
num_blocks = kv_cache_config.num_blocks
# Packed layouts (e.g. DSv4) set block_stride > 0; their tensors use
@@ -58,7 +67,7 @@ class OffloadingConnectorWorker:
# General (non-packed) layouts size the tensor at page_size_bytes per
# manager block, so page_size_bytes is the correct offloading stride.
layer_is_packed: dict[str, bool] = {
ln: bool(kv_tensor.block_stride)
ln: is_kv_cache_tensor_packed(kv_tensor)
for kv_tensor in kv_cache_config.kv_cache_tensors
for ln in kv_tensor.shared_by
}
@@ -142,7 +151,7 @@ class OffloadingConnectorWorker:
(
t
for t in kv_cache_config.kv_cache_tensors
if t.block_stride and t.shared_by
if is_kv_cache_tensor_packed(t) and t.shared_by
),
None,
)
@@ -237,7 +246,7 @@ class OffloadingConnectorWorker:
num_blocks_physical_dim = physical_to_logical.index(num_blocks_logical_dim)
assert num_blocks_physical_dim == 0
kv_cache_groups = self.spec.kv_cache_config.kv_cache_groups
kv_cache_groups = self.kv_cache_config.kv_cache_groups
assert len(kv_cache_groups) == 1
kv_cache_spec = kv_cache_groups[0].kv_cache_spec
num_layers = len(kv_cache_groups[0].layer_names)
@@ -23,6 +23,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
OffloadingConnectorMetadata,
OffloadingWorkerMetadata,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
build_offloading_config,
)
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
OffloadPromMetrics,
@@ -56,14 +59,17 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
):
super().__init__(vllm_config, role, kv_cache_config)
spec = OffloadingSpecFactory.create_spec(vllm_config, kv_cache_config)
offloading_config = build_offloading_config(vllm_config, kv_cache_config)
spec = OffloadingSpecFactory.create_spec(offloading_config)
self.connector_scheduler: OffloadingConnectorScheduler | None = None
self.connector_worker: OffloadingConnectorWorker | None = None
if role == KVConnectorRole.SCHEDULER:
self.connector_scheduler = OffloadingConnectorScheduler(spec)
self.connector_scheduler = OffloadingConnectorScheduler(
spec, vllm_config, kv_cache_config
)
elif role == KVConnectorRole.WORKER:
self.connector_worker = OffloadingConnectorWorker(spec)
self.connector_worker = OffloadingConnectorWorker(spec, kv_cache_config)
def shutdown(self) -> None:
if self.connector_worker is not None: