[KV Offload] Define clean backend configuration boundary (#48150)
Signed-off-by: Change72 <changg@nvidia.com> Signed-off-by: Chang Guo <cguo51@asu.edu> Co-authored-by: Codex <codex@openai.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Codex
Claude Fable 5
Or Ozeri
OpenAI Codex
parent
8c3393f373
commit
a9531edfa6
@@ -0,0 +1,131 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Translate vLLM KV cache metadata for native offloading backends."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, MLAAttentionSpec
|
||||
from vllm.v1.kv_offload.config import (
|
||||
OffloadingCacheConfig,
|
||||
OffloadingConfig,
|
||||
OffloadingGroupConfig,
|
||||
OffloadingModelConfig,
|
||||
OffloadingParallelConfig,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheTensor
|
||||
|
||||
|
||||
def is_kv_cache_tensor_packed(kv_cache_tensor: "KVCacheTensor") -> bool:
|
||||
"""Return whether a KV cache tensor uses a packed block stride."""
|
||||
return bool(kv_cache_tensor.block_stride)
|
||||
|
||||
|
||||
def build_offloading_config(
|
||||
vllm_config: "VllmConfig",
|
||||
kv_cache_config: "KVCacheConfig",
|
||||
) -> OffloadingConfig:
|
||||
"""Translate vLLM configuration into the native offloading boundary."""
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
assert kv_transfer_config is not None
|
||||
extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
assert kv_transfer_config.engine_id is not None
|
||||
engine_id = kv_transfer_config.engine_id
|
||||
|
||||
parallel_config = vllm_config.parallel_config
|
||||
context_parallel_factor = (
|
||||
parallel_config.decode_context_parallel_size
|
||||
* parallel_config.prefill_context_parallel_size
|
||||
)
|
||||
groups = tuple(
|
||||
OffloadingGroupConfig(
|
||||
tokens_per_block=(group.kv_cache_spec.block_size * context_parallel_factor),
|
||||
layer_names=tuple(group.layer_names),
|
||||
)
|
||||
for group in kv_cache_config.kv_cache_groups
|
||||
)
|
||||
|
||||
_, tokens_per_hash = resolve_kv_cache_block_sizes(kv_cache_config, vllm_config)
|
||||
for group in groups:
|
||||
assert group.tokens_per_block % tokens_per_hash == 0, (
|
||||
f"tokens_per_block={group.tokens_per_block} not divisible by "
|
||||
f"tokens_per_hash={tokens_per_hash}. "
|
||||
f"Hybrid models (e.g. Mamba+Attention) need "
|
||||
f"--enable-prefix-caching to align block sizes."
|
||||
)
|
||||
|
||||
blocks_per_chunk = 1
|
||||
tokens_per_chunk = extra_config.get("block_size")
|
||||
if tokens_per_chunk is not None:
|
||||
tokens_per_chunk_int = int(tokens_per_chunk)
|
||||
unique_tokens_per_block = {group.tokens_per_block for group in groups}
|
||||
assert len(unique_tokens_per_block) == 1, (
|
||||
"If 'block_size' is specified in kv_connector_extra_config, "
|
||||
"there must be at least one KV cache group, "
|
||||
"and all groups must have the same block size."
|
||||
)
|
||||
tokens_per_block = unique_tokens_per_block.pop()
|
||||
assert tokens_per_chunk_int % tokens_per_block == 0
|
||||
blocks_per_chunk = tokens_per_chunk_int // tokens_per_block
|
||||
|
||||
worker_kv_bytes_per_block = 0
|
||||
if kv_cache_config.num_blocks > 0:
|
||||
packed_tensors = tuple(
|
||||
is_kv_cache_tensor_packed(tensor)
|
||||
for tensor in kv_cache_config.kv_cache_tensors
|
||||
)
|
||||
is_packed = any(packed_tensors)
|
||||
assert not is_packed or all(packed_tensors)
|
||||
total_gpu_kv_bytes = (
|
||||
kv_cache_config.kv_cache_tensors[0].size
|
||||
if is_packed
|
||||
else sum(tensor.size for tensor in kv_cache_config.kv_cache_tensors)
|
||||
)
|
||||
worker_kv_bytes_per_block = total_gpu_kv_bytes // kv_cache_config.num_blocks
|
||||
|
||||
# Only a single non-MLA full-attention group is parallelism-invariant:
|
||||
# MLA latent KV is replicated per rank (never head-sharded), and the V2
|
||||
# model runner's KV layout is not known to be parallelism-invariant.
|
||||
single_group = (
|
||||
kv_cache_config.kv_cache_groups[0].kv_cache_spec
|
||||
if len(kv_cache_config.kv_cache_groups) == 1
|
||||
else None
|
||||
)
|
||||
is_parallelism_agnostic = (
|
||||
not vllm_config.use_v2_model_runner
|
||||
and single_group is not None
|
||||
and isinstance(single_group, FullAttentionSpec)
|
||||
and not isinstance(single_group, MLAAttentionSpec)
|
||||
)
|
||||
|
||||
kv_events_config = vllm_config.kv_events_config
|
||||
return OffloadingConfig(
|
||||
groups=groups,
|
||||
worker_kv_bytes_per_block=worker_kv_bytes_per_block,
|
||||
enable_kv_cache_events=(
|
||||
kv_events_config is not None and kv_events_config.enable_kv_cache_events
|
||||
),
|
||||
extra_config=extra_config,
|
||||
engine_id=engine_id,
|
||||
model=OffloadingModelConfig(
|
||||
name=vllm_config.model_config.model,
|
||||
dtype=str(vllm_config.cache_config.cache_dtype).replace("torch.", ""),
|
||||
),
|
||||
cache=OffloadingCacheConfig(
|
||||
tokens_per_hash=tokens_per_hash,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
),
|
||||
parallel=OffloadingParallelConfig(
|
||||
rank=parallel_config.rank,
|
||||
world_size=parallel_config.world_size,
|
||||
tp_size=parallel_config.tensor_parallel_size,
|
||||
pp_size=parallel_config.pipeline_parallel_size,
|
||||
pcp_size=parallel_config.prefill_context_parallel_size,
|
||||
dcp_size=parallel_config.decode_context_parallel_size,
|
||||
data_parallel_index=parallel_config.data_parallel_index,
|
||||
is_parallelism_agnostic=is_parallelism_agnostic,
|
||||
),
|
||||
)
|
||||
@@ -101,7 +101,7 @@ class OffloadingEventsTracker:
|
||||
self,
|
||||
req: Request,
|
||||
group_config: "GroupOffloadConfig",
|
||||
offload_block_idx: int,
|
||||
chunk_idx: int,
|
||||
offload_key: OffloadKey,
|
||||
) -> None:
|
||||
"""Snapshot the KV cache event payload for one offloaded chunk.
|
||||
@@ -111,9 +111,9 @@ class OffloadingEventsTracker:
|
||||
"""
|
||||
if not self.self_describing_enabled:
|
||||
return
|
||||
if group_config.sliding_window_size_in_blocks is not None:
|
||||
if group_config.sliding_window_size_in_chunks is not None:
|
||||
return
|
||||
meta = self._build_event_metadata(req, group_config, offload_block_idx)
|
||||
meta = self._build_event_metadata(req, group_config, chunk_idx)
|
||||
self._pending_event_metadata[offload_key] = meta
|
||||
|
||||
def take_events(self, events: Iterable[OffloadingEvent]) -> Iterable[KVCacheEvent]:
|
||||
@@ -142,19 +142,19 @@ class OffloadingEventsTracker:
|
||||
self,
|
||||
req: Request,
|
||||
group_config: "GroupOffloadConfig",
|
||||
offload_block_idx: int,
|
||||
chunk_idx: int,
|
||||
) -> _OffloadEventMetadata:
|
||||
"""Build the payload snapshot for one offloaded chunk: its
|
||||
constituent per-block hashes, the whole chunk's tokens, and the
|
||||
per-block ``block_size``."""
|
||||
hbf = group_config.hash_block_size_factor
|
||||
hbf = group_config.hashes_per_chunk
|
||||
assert hbf > 0
|
||||
assert offload_block_idx >= 0
|
||||
assert chunk_idx >= 0
|
||||
# per-block token count (= the GPU/hash block size)
|
||||
sub_block_size = group_config.offloaded_block_size // hbf
|
||||
tokens_per_hash = group_config.tokens_per_chunk // hbf
|
||||
# chunk c covers hash-blocks [c*hbf, (c+1)*hbf); its tail block's hash
|
||||
# is the chunk's OffloadKey.
|
||||
first_hash_idx = offload_block_idx * hbf
|
||||
first_hash_idx = chunk_idx * hbf
|
||||
last_hash_idx = first_hash_idx + hbf
|
||||
assert first_hash_idx >= 0
|
||||
assert last_hash_idx <= len(req.block_hashes)
|
||||
@@ -164,7 +164,7 @@ class OffloadingEventsTracker:
|
||||
chunk_hashes.append(block_hash)
|
||||
assert len(chunk_hashes) == hbf
|
||||
|
||||
if group_config.sliding_window_size_in_blocks is not None:
|
||||
if group_config.sliding_window_size_in_chunks is not None:
|
||||
# record_store filters these out before calling this helper.
|
||||
raise AssertionError("self-describing events only support full attention")
|
||||
|
||||
@@ -175,8 +175,8 @@ class OffloadingEventsTracker:
|
||||
parent_block_hash = req.block_hashes[first_hash_idx - 1]
|
||||
assert parent_block_hash is not None
|
||||
|
||||
tok_start = offload_block_idx * group_config.offloaded_block_size
|
||||
tok_end = tok_start + group_config.offloaded_block_size
|
||||
tok_start = chunk_idx * group_config.tokens_per_chunk
|
||||
tok_end = tok_start + group_config.tokens_per_chunk
|
||||
assert tok_end <= len(req.all_token_ids)
|
||||
token_ids = tuple(req.all_token_ids[tok_start:tok_end])
|
||||
|
||||
@@ -190,7 +190,7 @@ class OffloadingEventsTracker:
|
||||
block_hashes=tuple(chunk_hashes),
|
||||
parent_block_hash=parent_block_hash,
|
||||
token_ids=token_ids,
|
||||
block_size=sub_block_size,
|
||||
block_size=tokens_per_hash,
|
||||
lora_id=lora_id,
|
||||
lora_name=lora_name,
|
||||
extra_keys=None,
|
||||
|
||||
@@ -321,10 +321,10 @@ class OffloadPromMetrics(KVConnectorPromMetrics):
|
||||
self.histogram_transfer_size: dict[tuple[int, str], PromMetricT] = {}
|
||||
self.counter_kv_bytes: dict[tuple[int, str], PromMetricT] = {}
|
||||
self.counter_kv_transfer_time: dict[tuple[int, str], PromMetricT] = {}
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(vllm_config)
|
||||
kv_transfer_config = vllm_config.kv_transfer_config
|
||||
assert kv_transfer_config is not None
|
||||
extra_config = kv_transfer_config.kv_connector_extra_config
|
||||
spec_cls = OffloadingSpecFactory.get_spec_cls(extra_config)
|
||||
self._offloading_metric_metadata: dict[str, OffloadingMetricMetadata] = {
|
||||
**spec_cls.build_metric_definitions(extra_config),
|
||||
**get_connector_metric_definitions(),
|
||||
|
||||
@@ -6,6 +6,7 @@ from dataclasses import dataclass, field
|
||||
from itertools import islice
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_events import KVCacheEvent
|
||||
from vllm.distributed.kv_transfer.kv_connector.utils import yield_req_data
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import KVConnectorMetadata
|
||||
@@ -31,6 +32,7 @@ from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
SlidingWindowSpec,
|
||||
@@ -74,32 +76,32 @@ class TransferJobStatus:
|
||||
|
||||
class GroupOffloadConfig(NamedTuple):
|
||||
group_idx: int
|
||||
gpu_block_size: int
|
||||
offloaded_block_size: int
|
||||
hash_block_size_factor: int
|
||||
tokens_per_block: int
|
||||
tokens_per_chunk: int
|
||||
hashes_per_chunk: int
|
||||
# KV cache spec metadata propagated onto emitted BlockStored events so
|
||||
# KV-aware consumers can classify and filter the group.
|
||||
kv_event_group_spec: OffloadingEventGroupSpec
|
||||
# None below means full attention
|
||||
sliding_window_size_in_blocks: int | None
|
||||
# Number of this group's offloaded blocks per full-attention alignment
|
||||
# segment. Used to skip storing SWA blocks that can never serve a load
|
||||
sliding_window_size_in_chunks: int | None
|
||||
# Number of this group's offloaded chunks per full-attention alignment
|
||||
# segment. Used to skip storing SWA chunks that can never serve a load
|
||||
# hit (e.g. DeepSeek V4 where SWA groups have much smaller block sizes
|
||||
# than the MLA full-attention group).
|
||||
# None for full-attention groups or when the optimization doesn't apply.
|
||||
alignment_block_count: int | None = None
|
||||
# True for EAGLE/MTP draft-model attention groups. The trailing block
|
||||
alignment_chunk_count: int | None = None
|
||||
# True for EAGLE/MTP draft-model attention groups. The trailing chunk
|
||||
# of these groups is volatile and lacks a stable hash, so it must
|
||||
# be excluded from store and load scheduling.
|
||||
is_eagle_group: bool = False
|
||||
|
||||
|
||||
def get_sliding_window_size_in_blocks(
|
||||
kv_cache_spec: KVCacheSpec, offloaded_block_size: int
|
||||
def get_sliding_window_size_in_chunks(
|
||||
kv_cache_spec: KVCacheSpec, tokens_per_chunk: int
|
||||
) -> int | None:
|
||||
if isinstance(kv_cache_spec, SlidingWindowSpec):
|
||||
assert kv_cache_spec.sliding_window > 0
|
||||
return cdiv(kv_cache_spec.sliding_window, offloaded_block_size)
|
||||
return cdiv(kv_cache_spec.sliding_window, tokens_per_chunk)
|
||||
|
||||
if isinstance(kv_cache_spec, MambaSpec):
|
||||
# Mamba depends on a single state
|
||||
@@ -109,116 +111,121 @@ def get_sliding_window_size_in_blocks(
|
||||
return None
|
||||
|
||||
|
||||
def resolve_mamba_align_size(spec: "OffloadingSpec") -> int | None:
|
||||
def resolve_mamba_align_size(
|
||||
spec: "OffloadingSpec", kv_cache_config: KVCacheConfig
|
||||
) -> int | None:
|
||||
"""Scan all KV cache groups in *spec* and return the single mamba alignment
|
||||
size, or None if no group requires mamba alignment.
|
||||
|
||||
For MambaSpec groups in "align" cache mode the hit window must be rounded
|
||||
down to a multiple of the offloaded block size. Asserts that all such
|
||||
down to a multiple of the offloaded chunk size. Asserts that all such
|
||||
groups agree on the same value.
|
||||
"""
|
||||
mamba_align_size: int | None = None
|
||||
for idx, gpu_block_size in enumerate(spec.gpu_block_size):
|
||||
kv_spec = spec.kv_cache_config.kv_cache_groups[idx].kv_cache_spec
|
||||
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
|
||||
kv_spec = kv_cache_config.kv_cache_groups[idx].kv_cache_spec
|
||||
if isinstance(kv_spec, MambaSpec) and kv_spec.mamba_cache_mode == "align":
|
||||
offload_block_size = gpu_block_size * spec.block_size_factor
|
||||
assert mamba_align_size is None or mamba_align_size == offload_block_size
|
||||
mamba_align_size = offload_block_size
|
||||
tokens_per_chunk = tokens_per_block * spec.blocks_per_chunk
|
||||
assert mamba_align_size is None or mamba_align_size == tokens_per_chunk
|
||||
mamba_align_size = tokens_per_chunk
|
||||
return mamba_align_size
|
||||
|
||||
|
||||
class SchedulerOffloadConfig(NamedTuple):
|
||||
kv_group_configs: tuple[GroupOffloadConfig, ...]
|
||||
block_size_factor: int
|
||||
blocks_per_chunk: int
|
||||
num_workers: int
|
||||
offload_prompt_only: bool
|
||||
|
||||
@classmethod
|
||||
def from_spec(cls, spec: OffloadingSpec) -> "SchedulerOffloadConfig":
|
||||
def from_spec(
|
||||
cls,
|
||||
spec: OffloadingSpec,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
) -> "SchedulerOffloadConfig":
|
||||
# Determine the alignment token count from the full-attention group(s).
|
||||
# This is the offloaded_block_size of the full-attention group; load
|
||||
# This is the tokens_per_chunk of the full-attention group; load
|
||||
# hits are always aligned to this boundary, so SWA blocks earlier in
|
||||
# each segment can never serve a load hit. Relevant for hybrid
|
||||
# architectures like DeepSeek V4 (MLA + SWA groups).
|
||||
full_attn_offloaded_block_sizes: set[int] = set()
|
||||
for idx, gpu_block_size in enumerate(spec.gpu_block_size):
|
||||
kv_spec = spec.kv_cache_config.kv_cache_groups[idx].kv_cache_spec
|
||||
sw = get_sliding_window_size_in_blocks(
|
||||
kv_spec, gpu_block_size * spec.block_size_factor
|
||||
full_attn_tokens_per_chunk: set[int] = set()
|
||||
for idx, tokens_per_block in enumerate(spec.tokens_per_block):
|
||||
kv_spec = kv_cache_config.kv_cache_groups[idx].kv_cache_spec
|
||||
sw = get_sliding_window_size_in_chunks(
|
||||
kv_spec, tokens_per_block * spec.blocks_per_chunk
|
||||
)
|
||||
if sw is None:
|
||||
full_attn_offloaded_block_sizes.add(
|
||||
gpu_block_size * spec.block_size_factor
|
||||
)
|
||||
full_attn_tokens_per_chunk.add(tokens_per_block * spec.blocks_per_chunk)
|
||||
|
||||
# Only apply the optimization if there's a single consistent
|
||||
# full-attention alignment size.
|
||||
alignment_tokens: int | None = None
|
||||
if len(full_attn_offloaded_block_sizes) == 1:
|
||||
alignment_tokens = full_attn_offloaded_block_sizes.pop()
|
||||
if len(full_attn_tokens_per_chunk) == 1:
|
||||
alignment_tokens = full_attn_tokens_per_chunk.pop()
|
||||
|
||||
def _alignment_block_count(
|
||||
offloaded_block_size: int,
|
||||
sliding_window_size_in_blocks: int | None,
|
||||
def _alignment_chunk_count(
|
||||
tokens_per_chunk: int,
|
||||
sliding_window_size_in_chunks: int | None,
|
||||
) -> int | None:
|
||||
if alignment_tokens is None or sliding_window_size_in_blocks is None:
|
||||
if alignment_tokens is None or sliding_window_size_in_chunks is None:
|
||||
return None
|
||||
if alignment_tokens <= offloaded_block_size:
|
||||
if alignment_tokens <= tokens_per_chunk:
|
||||
return None
|
||||
per_segment = alignment_tokens // offloaded_block_size
|
||||
if sliding_window_size_in_blocks >= per_segment:
|
||||
per_segment = alignment_tokens // tokens_per_chunk
|
||||
if sliding_window_size_in_chunks >= per_segment:
|
||||
return None
|
||||
return per_segment
|
||||
|
||||
eagle_groups = {
|
||||
idx
|
||||
for idx, g in enumerate(spec.kv_cache_config.kv_cache_groups)
|
||||
for idx, g in enumerate(kv_cache_config.kv_cache_groups)
|
||||
if g.is_eagle_group
|
||||
}
|
||||
|
||||
use_eagle = (
|
||||
spec.vllm_config.speculative_config is not None
|
||||
and spec.vllm_config.speculative_config.use_eagle()
|
||||
vllm_config.speculative_config is not None
|
||||
and vllm_config.speculative_config.use_eagle()
|
||||
)
|
||||
if use_eagle and not eagle_groups:
|
||||
eagle_groups = set(range(len(spec.kv_cache_config.kv_cache_groups)))
|
||||
eagle_groups = set(range(len(kv_cache_config.kv_cache_groups)))
|
||||
|
||||
if eagle_groups:
|
||||
logger.info(
|
||||
"KV offloading: EAGLE/MTP draft attention groups %s "
|
||||
"detected. The trailing block of these groups will be "
|
||||
"detected. The trailing chunk of these groups will be "
|
||||
"excluded from offloading due to volatility.",
|
||||
sorted(eagle_groups),
|
||||
)
|
||||
|
||||
return cls(
|
||||
num_workers=spec.vllm_config.parallel_config.world_size,
|
||||
num_workers=vllm_config.parallel_config.world_size,
|
||||
kv_group_configs=tuple(
|
||||
GroupOffloadConfig(
|
||||
group_idx=idx,
|
||||
gpu_block_size=gpu_block_size,
|
||||
offloaded_block_size=gpu_block_size * spec.block_size_factor,
|
||||
hash_block_size_factor=(
|
||||
(gpu_block_size * spec.block_size_factor)
|
||||
// spec.hash_block_size
|
||||
tokens_per_block=tokens_per_block,
|
||||
tokens_per_chunk=tokens_per_block * spec.blocks_per_chunk,
|
||||
hashes_per_chunk=(
|
||||
(tokens_per_block * spec.blocks_per_chunk)
|
||||
// spec.tokens_per_hash
|
||||
),
|
||||
sliding_window_size_in_blocks=(
|
||||
sw := get_sliding_window_size_in_blocks(
|
||||
spec.kv_cache_config.kv_cache_groups[idx].kv_cache_spec,
|
||||
gpu_block_size * spec.block_size_factor,
|
||||
sliding_window_size_in_chunks=(
|
||||
sw := get_sliding_window_size_in_chunks(
|
||||
kv_cache_config.kv_cache_groups[idx].kv_cache_spec,
|
||||
tokens_per_block * spec.blocks_per_chunk,
|
||||
)
|
||||
),
|
||||
alignment_block_count=_alignment_block_count(
|
||||
gpu_block_size * spec.block_size_factor, sw
|
||||
alignment_chunk_count=_alignment_chunk_count(
|
||||
tokens_per_block * spec.blocks_per_chunk, sw
|
||||
),
|
||||
kv_event_group_spec=get_offloading_event_group_spec(
|
||||
spec.kv_cache_config.kv_cache_groups[idx]
|
||||
kv_cache_config.kv_cache_groups[idx]
|
||||
),
|
||||
is_eagle_group=idx in eagle_groups,
|
||||
)
|
||||
for idx, gpu_block_size in enumerate(spec.gpu_block_size)
|
||||
for idx, tokens_per_block in enumerate(spec.tokens_per_block)
|
||||
),
|
||||
block_size_factor=spec.block_size_factor,
|
||||
blocks_per_chunk=spec.blocks_per_chunk,
|
||||
offload_prompt_only=spec.offload_prompt_only,
|
||||
)
|
||||
|
||||
@@ -227,11 +234,11 @@ class SchedulerOffloadConfig(NamedTuple):
|
||||
class RequestGroupState:
|
||||
offload_keys: list[OffloadKey] = field(default_factory=list)
|
||||
block_ids: list[int] = field(default_factory=list)
|
||||
# index of next block (of size offloaded_block_size) to offload
|
||||
next_stored_block_idx: int = 0
|
||||
# number of offloaded blocks hit (including GPU prefix cache)
|
||||
# Index of the next chunk to offload.
|
||||
next_stored_chunk_idx: int = 0
|
||||
# Number of offloaded chunks hit (including GPU prefix cache)
|
||||
# when the request first started
|
||||
num_hit_blocks: int = 0
|
||||
num_hit_chunks: int = 0
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
@@ -278,11 +285,11 @@ class RequestOffloadState:
|
||||
):
|
||||
for req_block_hash in islice(
|
||||
self.req.block_hashes,
|
||||
group_config.hash_block_size_factor * len(group_state.offload_keys)
|
||||
+ group_config.hash_block_size_factor
|
||||
group_config.hashes_per_chunk * len(group_state.offload_keys)
|
||||
+ group_config.hashes_per_chunk
|
||||
- 1,
|
||||
None,
|
||||
group_config.hash_block_size_factor,
|
||||
group_config.hashes_per_chunk,
|
||||
):
|
||||
group_state.offload_keys.append(
|
||||
make_offload_key(req_block_hash, group_config.group_idx)
|
||||
@@ -298,46 +305,46 @@ class RequestOffloadState:
|
||||
for group_state, new_blocks in zip(self.group_states, new_block_id_groups):
|
||||
group_state.block_ids.extend(new_blocks)
|
||||
|
||||
def storable_blocks(
|
||||
def storable_chunks(
|
||||
self, group_config: "GroupOffloadConfig", num_offloadable_tokens: int
|
||||
) -> int:
|
||||
"""Number of leading offloaded blocks eligible for store.
|
||||
"""Number of leading offloaded chunks eligible for store.
|
||||
|
||||
For eagle/MTP groups the volatile trailing block of the offloadable
|
||||
For eagle/MTP groups the volatile trailing chunk of the offloadable
|
||||
range is excluded while decoding: the draft-layer KV of the last
|
||||
accepted position may be rewritten after spec-token rejection. During
|
||||
prefill the trailing block is stable (the draft input for a chunk's
|
||||
prefill the trailing chunk is stable (the draft input for a chunk's
|
||||
last position is the next prompt token), so it is stored immediately.
|
||||
The exclusion must be applied consistently everywhere
|
||||
``next_stored_block_idx`` is derived: otherwise the trailing block of
|
||||
``next_stored_chunk_idx`` is derived: otherwise the trailing chunk of
|
||||
each step is skipped on collection but jumped over by
|
||||
``next_stored_block_idx``, so it is never re-considered and a
|
||||
``next_stored_chunk_idx``, so it is never re-considered and a
|
||||
permanent hole breaks prefix-reuse lookup.
|
||||
"""
|
||||
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
|
||||
num_chunks = num_offloadable_tokens // group_config.tokens_per_chunk
|
||||
is_decoding = num_offloadable_tokens > self.req.num_prompt_tokens
|
||||
if group_config.is_eagle_group and is_decoding:
|
||||
num_blocks = max(0, num_blocks - 1)
|
||||
return num_blocks
|
||||
num_chunks = max(0, num_chunks - 1)
|
||||
return num_chunks
|
||||
|
||||
def advance_stored_idx(self, num_offloadable_tokens: int) -> None:
|
||||
# max(): at the prefill->decode transition of a block-aligned prompt,
|
||||
# storable_blocks drops by one (the eagle exclusion kicks in), and the
|
||||
# index must not move backwards past already-stored blocks.
|
||||
# max(): at the prefill->decode transition of a chunk-aligned prompt,
|
||||
# storable_chunks drops by one (the eagle exclusion kicks in), and the
|
||||
# index must not move backwards past already-stored chunks.
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs, self.group_states
|
||||
):
|
||||
group_state.next_stored_block_idx = max(
|
||||
group_state.next_stored_block_idx,
|
||||
self.storable_blocks(group_config, num_offloadable_tokens),
|
||||
group_state.next_stored_chunk_idx = max(
|
||||
group_state.next_stored_chunk_idx,
|
||||
self.storable_chunks(group_config, num_offloadable_tokens),
|
||||
)
|
||||
|
||||
def update_num_hit_blocks(self, num_cached_tokens: int) -> None:
|
||||
def update_num_hit_chunks(self, num_cached_tokens: int) -> None:
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs, self.group_states
|
||||
):
|
||||
group_state.num_hit_blocks = (
|
||||
num_cached_tokens // group_config.offloaded_block_size
|
||||
group_state.num_hit_chunks = (
|
||||
num_cached_tokens // group_config.tokens_per_chunk
|
||||
)
|
||||
|
||||
|
||||
@@ -354,22 +361,26 @@ class OffloadingConnectorScheduler:
|
||||
def __init__(
|
||||
self,
|
||||
spec: OffloadingSpec,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
self.config = SchedulerOffloadConfig.from_spec(spec)
|
||||
self.config = SchedulerOffloadConfig.from_spec(
|
||||
spec, vllm_config, kv_cache_config
|
||||
)
|
||||
self.manager: OffloadingManager = spec.get_manager()
|
||||
self._connector_stats = OffloadingConnectorStats()
|
||||
|
||||
full_attention_groups: list[int] = []
|
||||
sliding_window_groups: list[int] = []
|
||||
for group_config in self.config.kv_group_configs:
|
||||
if group_config.sliding_window_size_in_blocks is None:
|
||||
if group_config.sliding_window_size_in_chunks is None:
|
||||
full_attention_groups.append(group_config.group_idx)
|
||||
else:
|
||||
sliding_window_groups.append(group_config.group_idx)
|
||||
|
||||
# sort sliding window groups by window size in decreasing order
|
||||
def _sliding_window_sort_key(i: int) -> int:
|
||||
val = self.config.kv_group_configs[i].sliding_window_size_in_blocks
|
||||
val = self.config.kv_group_configs[i].sliding_window_size_in_chunks
|
||||
assert val is not None
|
||||
return val
|
||||
|
||||
@@ -378,7 +389,9 @@ class OffloadingConnectorScheduler:
|
||||
# used by _lookup
|
||||
self._sliding_window_groups: tuple[int, ...] = tuple(sliding_window_groups)
|
||||
self._lookup_groups = tuple(full_attention_groups) + self._sliding_window_groups
|
||||
self._mamba_align_size: int | None = resolve_mamba_align_size(spec)
|
||||
self._mamba_align_size: int | None = resolve_mamba_align_size(
|
||||
spec, kv_cache_config
|
||||
)
|
||||
|
||||
self._req_status: dict[ReqId, RequestOffloadState] = {}
|
||||
self._current_batch_load_jobs: dict[int, TransferJob] = {}
|
||||
@@ -386,9 +399,9 @@ class OffloadingConnectorScheduler:
|
||||
# GPU block IDs allocated in the current engine step
|
||||
self._current_batch_allocated_block_ids: set[int] = set()
|
||||
# if GPU prefix caching is enabled,
|
||||
# track loaded blocks to avoid redundant loads
|
||||
self._blocks_being_loaded: set[OffloadKey] | None = (
|
||||
set() if spec.vllm_config.cache_config.enable_prefix_caching else None
|
||||
# Track loaded chunks to avoid redundant loads.
|
||||
self._chunks_being_loaded: set[OffloadKey] | None = (
|
||||
set() if vllm_config.cache_config.enable_prefix_caching else None
|
||||
)
|
||||
|
||||
# Job ID counter shared by loads and stores.
|
||||
@@ -434,7 +447,7 @@ class OffloadingConnectorScheduler:
|
||||
def _maximal_prefix_lookup(
|
||||
self, keys: Iterable[OffloadKey], req_context: ReqContext
|
||||
) -> int | None:
|
||||
"""Return the number of consecutive offloaded blocks from the start,
|
||||
"""Return the number of consecutive offloaded chunks from the start,
|
||||
or None if the backend deferred a lookup."""
|
||||
hit_count = 0
|
||||
defer_lookup = False
|
||||
@@ -490,18 +503,18 @@ class OffloadingConnectorScheduler:
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs, req_status.group_states
|
||||
):
|
||||
if group_config.sliding_window_size_in_blocks is None:
|
||||
if group_config.sliding_window_size_in_chunks is None:
|
||||
self.manager.touch(group_state.offload_keys, req_status.req_context)
|
||||
else:
|
||||
# we aim to keep just blocks that are necessary to hit
|
||||
# the original request (+ decoded blocks)
|
||||
blocks_to_skip = max(
|
||||
# Keep only chunks needed to hit the original request, plus
|
||||
# decoded chunks.
|
||||
chunks_to_skip = max(
|
||||
0,
|
||||
group_state.num_hit_blocks
|
||||
- group_config.sliding_window_size_in_blocks,
|
||||
group_state.num_hit_chunks
|
||||
- group_config.sliding_window_size_in_chunks,
|
||||
)
|
||||
self.manager.touch(
|
||||
group_state.offload_keys[blocks_to_skip:],
|
||||
group_state.offload_keys[chunks_to_skip:],
|
||||
req_status.req_context,
|
||||
)
|
||||
|
||||
@@ -531,7 +544,7 @@ class OffloadingConnectorScheduler:
|
||||
defer_lookup = False
|
||||
lookup_groups = self._lookup_groups
|
||||
|
||||
# Tracks which eagle groups have already popped their volatile trailing block
|
||||
# Tracks which eagle groups have already popped their volatile trailing chunk
|
||||
# in the current convergence iteration. Reset when a non-eagle group
|
||||
# tightens the hit boundary, requiring a fresh pop.
|
||||
eagle_verified: set[int] = set()
|
||||
@@ -544,79 +557,76 @@ class OffloadingConnectorScheduler:
|
||||
group_idx
|
||||
]
|
||||
group_state: RequestGroupState = req_status.group_states[group_idx]
|
||||
offloaded_block_size = group_config.offloaded_block_size
|
||||
tokens_per_chunk = group_config.tokens_per_chunk
|
||||
offload_keys = group_state.offload_keys
|
||||
|
||||
assert (
|
||||
len(offload_keys)
|
||||
>= req_status.req.num_tokens // offloaded_block_size
|
||||
len(offload_keys) >= req_status.req.num_tokens // tokens_per_chunk
|
||||
)
|
||||
|
||||
is_eagle_unverified = (
|
||||
group_config.is_eagle_group and group_idx not in eagle_verified
|
||||
)
|
||||
|
||||
# Constrain to block-aligned boundary for this group
|
||||
# Constrain to a chunk-aligned boundary for this group.
|
||||
max_hit_size_tokens = min(
|
||||
max_hit_size_tokens, len(offload_keys) * offloaded_block_size
|
||||
max_hit_size_tokens, len(offload_keys) * tokens_per_chunk
|
||||
)
|
||||
if max_hit_size_tokens - num_computed_tokens < offloaded_block_size:
|
||||
# we can only load less than a block, better skip
|
||||
if max_hit_size_tokens - num_computed_tokens < tokens_per_chunk:
|
||||
# We can only load less than a chunk, so skip.
|
||||
return 0
|
||||
|
||||
sliding_window_size_in_blocks = (
|
||||
group_config.sliding_window_size_in_blocks
|
||||
sliding_window_size_in_chunks = (
|
||||
group_config.sliding_window_size_in_chunks
|
||||
)
|
||||
|
||||
# For eagle groups, query one extra block that will be popped.
|
||||
# For eagle groups, query one extra chunk that will be popped.
|
||||
# We only need to increase the query size for sliding window groups.
|
||||
query_max = max_hit_size_tokens
|
||||
if is_eagle_unverified and sliding_window_size_in_blocks is not None:
|
||||
if is_eagle_unverified and sliding_window_size_in_chunks is not None:
|
||||
query_max = min(
|
||||
max_hit_size_tokens + offloaded_block_size,
|
||||
len(offload_keys) * offloaded_block_size,
|
||||
max_hit_size_tokens + tokens_per_chunk,
|
||||
len(offload_keys) * tokens_per_chunk,
|
||||
)
|
||||
|
||||
num_blocks = min(
|
||||
cdiv(query_max, offloaded_block_size), len(offload_keys)
|
||||
)
|
||||
start_block_idx = num_computed_tokens // offloaded_block_size
|
||||
offload_keys = offload_keys[start_block_idx:num_blocks]
|
||||
num_chunks = min(cdiv(query_max, tokens_per_chunk), len(offload_keys))
|
||||
start_chunk_idx = num_computed_tokens // tokens_per_chunk
|
||||
offload_keys = offload_keys[start_chunk_idx:num_chunks]
|
||||
|
||||
# end index (in the sliced offload_keys) up to which we
|
||||
# have backend-confirmed hits
|
||||
num_hit_blocks: int | None
|
||||
if sliding_window_size_in_blocks is None:
|
||||
num_hit_blocks = self._maximal_prefix_lookup(
|
||||
num_hit_chunks: int | None
|
||||
if sliding_window_size_in_chunks is None:
|
||||
num_hit_chunks = self._maximal_prefix_lookup(
|
||||
offload_keys, req_status.req_context
|
||||
)
|
||||
else:
|
||||
required_window = sliding_window_size_in_blocks
|
||||
required_window = sliding_window_size_in_chunks
|
||||
if is_eagle_unverified:
|
||||
required_window += 1
|
||||
num_hit_blocks = self._sliding_window_lookup(
|
||||
num_hit_chunks = self._sliding_window_lookup(
|
||||
offload_keys,
|
||||
required_window,
|
||||
req_status.req_context,
|
||||
)
|
||||
if num_hit_blocks == 0:
|
||||
if num_hit_chunks == 0:
|
||||
return 0
|
||||
|
||||
if num_hit_blocks is None:
|
||||
if num_hit_chunks is None:
|
||||
defer_lookup = True
|
||||
else:
|
||||
if is_eagle_unverified:
|
||||
num_hit_blocks -= 1
|
||||
num_hit_chunks -= 1
|
||||
eagle_verified.add(group_idx)
|
||||
|
||||
max_hit_size_tokens = min(
|
||||
max_hit_size_tokens,
|
||||
offloaded_block_size * (start_block_idx + num_hit_blocks),
|
||||
tokens_per_chunk * (start_chunk_idx + num_hit_chunks),
|
||||
)
|
||||
|
||||
new_num_hit_tokens = max_hit_size_tokens - num_computed_tokens
|
||||
if new_num_hit_tokens < offloaded_block_size:
|
||||
# we can only load less than a block, better skip
|
||||
if new_num_hit_tokens < tokens_per_chunk:
|
||||
# We can only load less than a chunk, so skip.
|
||||
return 0
|
||||
|
||||
if new_num_hit_tokens < num_hit_tokens:
|
||||
@@ -632,7 +642,7 @@ class OffloadingConnectorScheduler:
|
||||
# sliding window works with the new_num_hit_tokens
|
||||
lookup_groups = self._sliding_window_groups
|
||||
|
||||
looked_up_sliding_window |= sliding_window_size_in_blocks is not None
|
||||
looked_up_sliding_window |= sliding_window_size_in_chunks is not None
|
||||
num_hit_tokens = new_num_hit_tokens
|
||||
|
||||
if defer_lookup:
|
||||
@@ -642,28 +652,28 @@ class OffloadingConnectorScheduler:
|
||||
)
|
||||
return None
|
||||
|
||||
# possibly delay request if any of the hit blocks is already being loaded
|
||||
if self._blocks_being_loaded:
|
||||
# Possibly delay the request if any hit chunk is already being loaded.
|
||||
if self._chunks_being_loaded:
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs, req_status.group_states
|
||||
):
|
||||
offloaded_block_size = group_config.offloaded_block_size
|
||||
sliding_window_size_in_blocks = (
|
||||
group_config.sliding_window_size_in_blocks
|
||||
tokens_per_chunk = group_config.tokens_per_chunk
|
||||
sliding_window_size_in_chunks = (
|
||||
group_config.sliding_window_size_in_chunks
|
||||
)
|
||||
offload_keys = group_state.offload_keys
|
||||
num_blocks = cdiv(
|
||||
num_computed_tokens + num_hit_tokens, offloaded_block_size
|
||||
num_chunks = cdiv(
|
||||
num_computed_tokens + num_hit_tokens, tokens_per_chunk
|
||||
)
|
||||
start_block_idx = num_computed_tokens // offloaded_block_size
|
||||
offload_keys = offload_keys[start_block_idx:num_blocks]
|
||||
if sliding_window_size_in_blocks is not None:
|
||||
offload_keys = offload_keys[-sliding_window_size_in_blocks:]
|
||||
if any(key in self._blocks_being_loaded for key in offload_keys):
|
||||
# hit blocks are being loaded, delay request
|
||||
start_chunk_idx = num_computed_tokens // tokens_per_chunk
|
||||
offload_keys = offload_keys[start_chunk_idx:num_chunks]
|
||||
if sliding_window_size_in_chunks is not None:
|
||||
offload_keys = offload_keys[-sliding_window_size_in_chunks:]
|
||||
if any(key in self._chunks_being_loaded for key in offload_keys):
|
||||
# Hit chunks are being loaded, so delay the request.
|
||||
logger.debug(
|
||||
"Delaying request %s since some of its"
|
||||
" blocks are already being loaded",
|
||||
" chunks are already being loaded",
|
||||
req_status.req.request_id,
|
||||
)
|
||||
return None
|
||||
@@ -740,7 +750,7 @@ class OffloadingConnectorScheduler:
|
||||
req_status.deferred_lookup_start_time = lookup_start
|
||||
else:
|
||||
self._maybe_observe_lookup_async_delay(req_status)
|
||||
req_status.update_num_hit_blocks(num_computed_tokens + (num_hit_tokens or 0))
|
||||
req_status.update_num_hit_chunks(num_computed_tokens + (num_hit_tokens or 0))
|
||||
|
||||
self._touch(req_status)
|
||||
|
||||
@@ -771,10 +781,10 @@ class OffloadingConnectorScheduler:
|
||||
block.block_id for block in group_blocks if block.block_id != 0
|
||||
)
|
||||
|
||||
gpu_block_size = group_config.gpu_block_size
|
||||
offloaded_block_size = group_config.offloaded_block_size
|
||||
tokens_per_block = group_config.tokens_per_block
|
||||
tokens_per_chunk = group_config.tokens_per_chunk
|
||||
offload_keys = group_state.offload_keys
|
||||
num_gpu_blocks = cdiv(num_cached_tokens, gpu_block_size)
|
||||
num_gpu_blocks = cdiv(num_cached_tokens, tokens_per_block)
|
||||
|
||||
assert len(group_blocks) >= num_gpu_blocks
|
||||
num_locally_computed_gpu_blocks = num_gpu_blocks
|
||||
@@ -786,24 +796,24 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
assert (
|
||||
num_locally_computed_tokens
|
||||
<= num_locally_computed_gpu_blocks * gpu_block_size
|
||||
<= num_locally_computed_gpu_blocks * tokens_per_block
|
||||
)
|
||||
num_pending_gpu_blocks = num_gpu_blocks - num_locally_computed_gpu_blocks
|
||||
|
||||
if group_config.sliding_window_size_in_blocks is not None:
|
||||
if group_config.sliding_window_size_in_chunks is not None:
|
||||
assert (
|
||||
num_pending_gpu_blocks
|
||||
<= group_config.sliding_window_size_in_blocks
|
||||
* self.config.block_size_factor
|
||||
<= group_config.sliding_window_size_in_chunks
|
||||
* self.config.blocks_per_chunk
|
||||
)
|
||||
|
||||
num_blocks = cdiv(num_cached_tokens, offloaded_block_size)
|
||||
assert len(offload_keys) >= num_blocks
|
||||
num_chunks = cdiv(num_cached_tokens, tokens_per_chunk)
|
||||
assert len(offload_keys) >= num_chunks
|
||||
if num_pending_gpu_blocks:
|
||||
start_block_idx = (
|
||||
num_locally_computed_gpu_blocks // self.config.block_size_factor
|
||||
start_chunk_idx = (
|
||||
num_locally_computed_gpu_blocks // self.config.blocks_per_chunk
|
||||
)
|
||||
keys_to_load.extend(offload_keys[start_block_idx:num_blocks])
|
||||
keys_to_load.extend(offload_keys[start_chunk_idx:num_chunks])
|
||||
|
||||
dst_block_ids.extend(
|
||||
block.block_id
|
||||
@@ -814,11 +824,11 @@ class OffloadingConnectorScheduler:
|
||||
group_sizes.append(num_pending_gpu_blocks)
|
||||
block_indices.append(num_locally_computed_gpu_blocks)
|
||||
|
||||
# Skip prefix-hit blocks for block-level policy; for
|
||||
# request-level, next_stored_block_idx stays at 0 so all
|
||||
# blocks (including hits) are offloaded.
|
||||
# Skip prefix-hit chunks for block-level policy; for
|
||||
# request-level, next_stored_chunk_idx stays at 0 so all
|
||||
# chunks (including hits) are offloaded.
|
||||
if req_status.offloading_context.policy == OffloadPolicy.BLOCK_LEVEL:
|
||||
group_state.next_stored_block_idx = num_blocks
|
||||
group_state.next_stored_chunk_idx = num_chunks
|
||||
|
||||
src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context)
|
||||
dst_spec = GPULoadStoreSpec(
|
||||
@@ -841,8 +851,8 @@ class OffloadingConnectorScheduler:
|
||||
is_store=False,
|
||||
)
|
||||
|
||||
if self._blocks_being_loaded is not None:
|
||||
self._blocks_being_loaded.update(keys_to_load)
|
||||
if self._chunks_being_loaded is not None:
|
||||
self._chunks_being_loaded.update(keys_to_load)
|
||||
|
||||
def _update_req_states(self, scheduler_output: SchedulerOutput) -> None:
|
||||
"""
|
||||
@@ -877,16 +887,16 @@ class OffloadingConnectorScheduler:
|
||||
# Zero out stale block_ids in sliding window groups' pending-store
|
||||
# positions. Only sliding window groups can have stale entries (blocks
|
||||
# freed by remove_skipped_blocks then reallocated). Only positions in
|
||||
# [next_stored_block_idx * bsf, end) need checking where end is the
|
||||
# [next_stored_chunk_idx * bsf, end) need checking where end is the
|
||||
# pre-extend length: earlier positions were already offloaded, later
|
||||
# ones are fresh allocations from this step.
|
||||
if self._sliding_window_groups and self._current_batch_allocated_block_ids:
|
||||
block_size_factor = self.config.block_size_factor
|
||||
blocks_per_chunk = self.config.blocks_per_chunk
|
||||
for req_id, req_status in self._req_status.items():
|
||||
ends = new_block_ids_end.get(req_id)
|
||||
for i, grp_idx in enumerate(self._sliding_window_groups):
|
||||
group_state = req_status.group_states[grp_idx]
|
||||
start = group_state.next_stored_block_idx * block_size_factor
|
||||
start = group_state.next_stored_chunk_idx * blocks_per_chunk
|
||||
end = ends[i] if ends is not None else len(group_state.block_ids)
|
||||
for j in range(start, end):
|
||||
if (
|
||||
@@ -899,7 +909,7 @@ class OffloadingConnectorScheduler:
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> dict[int, TransferJob]:
|
||||
block_size_factor = self.config.block_size_factor
|
||||
blocks_per_chunk = self.config.blocks_per_chunk
|
||||
store_jobs: dict[int, TransferJob] = {}
|
||||
for req_id in scheduler_output.num_scheduled_tokens:
|
||||
req_status = self._req_status.get(req_id)
|
||||
@@ -915,59 +925,59 @@ class OffloadingConnectorScheduler:
|
||||
if max_offload_tokens is not None:
|
||||
num_offloadable_tokens = min(num_offloadable_tokens, max_offload_tokens)
|
||||
|
||||
# Skip decode-phase blocks: clamp to the prompt length so only
|
||||
# prefill (prompt) blocks become eligible for store. next_stored_idx
|
||||
# never advances past this boundary, so decode blocks are never
|
||||
# Skip decode-phase chunks: clamp to the prompt length so only
|
||||
# prefill chunks become eligible for store. next_stored_chunk_idx
|
||||
# never advances past this boundary, so decode chunks are never
|
||||
# queued in this or any later step.
|
||||
if self.config.offload_prompt_only:
|
||||
num_offloadable_tokens = min(
|
||||
num_offloadable_tokens, req.num_prompt_tokens
|
||||
)
|
||||
|
||||
# Filter out blocks skipped due to sliding window attention / SSM
|
||||
# Filter out chunks skipped due to sliding window attention / SSM
|
||||
# or unreachable by the load path's alignment constraints.
|
||||
new_offload_keys: list[OffloadKey] = []
|
||||
for group_config, group_state in zip(
|
||||
self.config.kv_group_configs, req_status.group_states
|
||||
):
|
||||
num_blocks = req_status.storable_blocks(
|
||||
num_chunks = req_status.storable_chunks(
|
||||
group_config, num_offloadable_tokens
|
||||
)
|
||||
|
||||
start_block_idx = group_state.next_stored_block_idx
|
||||
if num_blocks <= start_block_idx:
|
||||
start_chunk_idx = group_state.next_stored_chunk_idx
|
||||
if num_chunks <= start_chunk_idx:
|
||||
continue
|
||||
offload_keys = group_state.offload_keys[start_block_idx:num_blocks]
|
||||
# For each block to offload, take the last corresponding GPU block.
|
||||
# e.g. if block size factor is 3 and GPU block IDs are
|
||||
# 1 5 6 7 2 4 9 3 8 then we'll take blocks 6 4 8.
|
||||
offload_keys = group_state.offload_keys[start_chunk_idx:num_chunks]
|
||||
# For each chunk, take the last corresponding GPU block. For
|
||||
# blocks_per_chunk=3 and GPU block IDs 1 5 6 7 2 4 9 3 8,
|
||||
# this selects GPU blocks 6 4 8.
|
||||
# A block_id of 0 means either a sliding window / SSM skip
|
||||
# or a stale entry that was zeroed out — skip it either way.
|
||||
offload_block_ids = group_state.block_ids[
|
||||
start_block_idx * block_size_factor
|
||||
+ block_size_factor
|
||||
- 1 : num_blocks * block_size_factor : block_size_factor
|
||||
start_chunk_idx * blocks_per_chunk
|
||||
+ blocks_per_chunk
|
||||
- 1 : num_chunks * blocks_per_chunk : blocks_per_chunk
|
||||
]
|
||||
assert len(offload_keys) == len(offload_block_ids)
|
||||
|
||||
alignment_block_count = group_config.alignment_block_count
|
||||
tail = group_config.sliding_window_size_in_blocks
|
||||
alignment_chunk_count = group_config.alignment_chunk_count
|
||||
tail = group_config.sliding_window_size_in_chunks
|
||||
|
||||
for key_idx, (offload_key, block_id) in enumerate(
|
||||
zip(offload_keys, offload_block_ids)
|
||||
):
|
||||
if block_id == 0:
|
||||
continue
|
||||
# Skip SWA blocks that can never serve a load hit:
|
||||
# Skip SWA chunks that can never serve a load hit:
|
||||
# within each full-attention alignment segment, only the
|
||||
# trailing `tail` blocks are reachable by
|
||||
# trailing `tail` chunks are reachable by
|
||||
# _sliding_window_lookup. For DeepSeek V4 with 100K
|
||||
# tokens this reduces SWA stores by ~78%.
|
||||
if alignment_block_count is not None:
|
||||
if alignment_chunk_count is not None:
|
||||
assert tail is not None
|
||||
abs_block_idx = start_block_idx + key_idx
|
||||
pos_in_segment = abs_block_idx % alignment_block_count
|
||||
if pos_in_segment < alignment_block_count - tail:
|
||||
abs_chunk_idx = start_chunk_idx + key_idx
|
||||
pos_in_segment = abs_chunk_idx % alignment_chunk_count
|
||||
if pos_in_segment < alignment_chunk_count - tail:
|
||||
continue
|
||||
new_offload_keys.append(offload_key)
|
||||
|
||||
@@ -982,7 +992,7 @@ class OffloadingConnectorScheduler:
|
||||
self._connector_stats.increase_counter(
|
||||
_ConnectorMetricName.ALLOCATION_FAILURE
|
||||
)
|
||||
logger.warning("Request %s: cannot store blocks", req_id)
|
||||
logger.warning("Request %s: cannot store chunks", req_id)
|
||||
continue
|
||||
|
||||
if not store_output.keys_to_store:
|
||||
@@ -1002,29 +1012,29 @@ class OffloadingConnectorScheduler:
|
||||
self.config.kv_group_configs, req_status.group_states
|
||||
):
|
||||
is_sliding_window = (
|
||||
group_config.sliding_window_size_in_blocks is not None
|
||||
group_config.sliding_window_size_in_chunks is not None
|
||||
)
|
||||
num_blocks = req_status.storable_blocks(
|
||||
num_chunks = req_status.storable_chunks(
|
||||
group_config, num_offloadable_tokens
|
||||
)
|
||||
start_block_idx = group_state.next_stored_block_idx
|
||||
start_chunk_idx = group_state.next_stored_chunk_idx
|
||||
block_ids = group_state.block_ids
|
||||
num_group_blocks = 0
|
||||
start_gpu_block_idx: int | None = None
|
||||
for idx, offload_key in enumerate(
|
||||
group_state.offload_keys[start_block_idx:num_blocks]
|
||||
group_state.offload_keys[start_chunk_idx:num_chunks]
|
||||
):
|
||||
if offload_key not in keys_to_store:
|
||||
continue
|
||||
|
||||
offloaded_block_idx = start_block_idx + idx
|
||||
chunk_idx = start_chunk_idx + idx
|
||||
|
||||
self._events_tracker.record_store(
|
||||
req, group_config, offloaded_block_idx, offload_key
|
||||
req, group_config, chunk_idx, offload_key
|
||||
)
|
||||
|
||||
gpu_block_idx = offloaded_block_idx * block_size_factor
|
||||
for i in range(block_size_factor):
|
||||
gpu_block_idx = chunk_idx * blocks_per_chunk
|
||||
for i in range(blocks_per_chunk):
|
||||
block_id = block_ids[gpu_block_idx + i]
|
||||
if block_id == 0:
|
||||
continue
|
||||
@@ -1039,8 +1049,8 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
group_sizes.append(num_group_blocks)
|
||||
block_indices.append(start_gpu_block_idx or 0)
|
||||
group_state.next_stored_block_idx = max(
|
||||
group_state.next_stored_block_idx, num_blocks
|
||||
group_state.next_stored_chunk_idx = max(
|
||||
group_state.next_stored_chunk_idx, num_chunks
|
||||
)
|
||||
|
||||
src_spec = GPULoadStoreSpec(
|
||||
@@ -1076,7 +1086,7 @@ class OffloadingConnectorScheduler:
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"Request %s offloading %s blocks upto %d tokens (job %d)",
|
||||
"Request %s offloading %s chunks upto %d tokens (job %d)",
|
||||
req_id,
|
||||
len(keys_to_store),
|
||||
num_offloadable_tokens,
|
||||
@@ -1198,8 +1208,8 @@ class OffloadingConnectorScheduler:
|
||||
self.manager.complete_store(job_status.keys, req_status.req_context)
|
||||
else:
|
||||
self.manager.complete_load(job_status.keys, req_status.req_context)
|
||||
if self._blocks_being_loaded:
|
||||
self._blocks_being_loaded.difference_update(job_status.keys)
|
||||
if self._chunks_being_loaded:
|
||||
self._chunks_being_loaded.difference_update(job_status.keys)
|
||||
if self._block_id_to_pending_jobs:
|
||||
# Sliding window blocks are tracked from store creation
|
||||
# and must be cleaned up unconditionally.
|
||||
@@ -1289,7 +1299,7 @@ class OffloadingConnectorScheduler:
|
||||
yield from self._events_tracker.take_events(self.manager.take_events())
|
||||
|
||||
def reset_cache(self) -> None:
|
||||
"""Reset the offloading manager cache, evicting all stored blocks."""
|
||||
"""Reset the offloading manager cache, evicting all stored chunks."""
|
||||
|
||||
# reset_cache cannot be called in the middle of a schedule step
|
||||
assert not self._current_batch_load_jobs
|
||||
@@ -1306,10 +1316,10 @@ class OffloadingConnectorScheduler:
|
||||
# Reset offloading manager cache
|
||||
self.manager.reset_cache()
|
||||
|
||||
# Reset store progress so active requests re-offload from block 0
|
||||
# Reset store progress so active requests re-offload from chunk 0.
|
||||
for status in self._req_status.values():
|
||||
for group_state in status.group_states:
|
||||
group_state.next_stored_block_idx = 0
|
||||
group_state.next_stored_chunk_idx = 0
|
||||
status.transfer_jobs.clear()
|
||||
|
||||
# Discard jobs and save job_counter to be able to discard worker responses
|
||||
@@ -1323,8 +1333,8 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
# Note: _current_batch_jobs_to_flush is intentionally NOT cleared.
|
||||
# The load flush IDs collected above must be delivered to workers.
|
||||
if self._blocks_being_loaded is not None:
|
||||
self._blocks_being_loaded.clear()
|
||||
if self._chunks_being_loaded is not None:
|
||||
self._chunks_being_loaded.clear()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self.manager.shutdown()
|
||||
|
||||
@@ -10,10 +10,14 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingWorkerMetadata,
|
||||
ReqId,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
|
||||
is_kv_cache_tensor_packed,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
KVCacheConfig,
|
||||
MambaSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
)
|
||||
@@ -33,8 +37,13 @@ logger = init_logger(__name__)
|
||||
class OffloadingConnectorWorker:
|
||||
"""Implementation of Worker side methods"""
|
||||
|
||||
def __init__(self, spec: OffloadingSpec):
|
||||
def __init__(
|
||||
self,
|
||||
spec: OffloadingSpec,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
):
|
||||
self.spec = spec
|
||||
self.kv_cache_config = kv_cache_config
|
||||
self.worker: OffloadingWorker | None = None
|
||||
|
||||
# job_id -> req_id for in-flight loads.
|
||||
@@ -50,7 +59,7 @@ class OffloadingConnectorWorker:
|
||||
def register_kv_caches(
|
||||
self, kv_caches: dict[str, torch.Tensor | list[torch.Tensor]]
|
||||
):
|
||||
kv_cache_config = self.spec.kv_cache_config
|
||||
kv_cache_config = self.kv_cache_config
|
||||
num_blocks = kv_cache_config.num_blocks
|
||||
|
||||
# Packed layouts (e.g. DSv4) set block_stride > 0; their tensors use
|
||||
@@ -58,7 +67,7 @@ class OffloadingConnectorWorker:
|
||||
# General (non-packed) layouts size the tensor at page_size_bytes per
|
||||
# manager block, so page_size_bytes is the correct offloading stride.
|
||||
layer_is_packed: dict[str, bool] = {
|
||||
ln: bool(kv_tensor.block_stride)
|
||||
ln: is_kv_cache_tensor_packed(kv_tensor)
|
||||
for kv_tensor in kv_cache_config.kv_cache_tensors
|
||||
for ln in kv_tensor.shared_by
|
||||
}
|
||||
@@ -142,7 +151,7 @@ class OffloadingConnectorWorker:
|
||||
(
|
||||
t
|
||||
for t in kv_cache_config.kv_cache_tensors
|
||||
if t.block_stride and t.shared_by
|
||||
if is_kv_cache_tensor_packed(t) and t.shared_by
|
||||
),
|
||||
None,
|
||||
)
|
||||
@@ -237,7 +246,7 @@ class OffloadingConnectorWorker:
|
||||
num_blocks_physical_dim = physical_to_logical.index(num_blocks_logical_dim)
|
||||
assert num_blocks_physical_dim == 0
|
||||
|
||||
kv_cache_groups = self.spec.kv_cache_config.kv_cache_groups
|
||||
kv_cache_groups = self.kv_cache_config.kv_cache_groups
|
||||
assert len(kv_cache_groups) == 1
|
||||
kv_cache_spec = kv_cache_groups[0].kv_cache_spec
|
||||
num_layers = len(kv_cache_groups[0].layer_names)
|
||||
|
||||
@@ -23,6 +23,9 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.common import (
|
||||
OffloadingConnectorMetadata,
|
||||
OffloadingWorkerMetadata,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.config import (
|
||||
build_offloading_config,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
OffloadPromMetrics,
|
||||
@@ -56,14 +59,17 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
):
|
||||
super().__init__(vllm_config, role, kv_cache_config)
|
||||
|
||||
spec = OffloadingSpecFactory.create_spec(vllm_config, kv_cache_config)
|
||||
offloading_config = build_offloading_config(vllm_config, kv_cache_config)
|
||||
spec = OffloadingSpecFactory.create_spec(offloading_config)
|
||||
|
||||
self.connector_scheduler: OffloadingConnectorScheduler | None = None
|
||||
self.connector_worker: OffloadingConnectorWorker | None = None
|
||||
if role == KVConnectorRole.SCHEDULER:
|
||||
self.connector_scheduler = OffloadingConnectorScheduler(spec)
|
||||
self.connector_scheduler = OffloadingConnectorScheduler(
|
||||
spec, vllm_config, kv_cache_config
|
||||
)
|
||||
elif role == KVConnectorRole.WORKER:
|
||||
self.connector_worker = OffloadingConnectorWorker(spec)
|
||||
self.connector_worker = OffloadingConnectorWorker(spec, kv_cache_config)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
if self.connector_worker is not None:
|
||||
|
||||
Reference in New Issue
Block a user