[kv_offload]: Add DSv4 support (#43142)

Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Or Ozeri
2026-05-24 11:10:12 +03:00
committed by GitHub
parent 0902d8e62f
commit 357fddf614
4 changed files with 214 additions and 14 deletions
@@ -65,6 +65,12 @@ class GroupOffloadConfig(NamedTuple):
hash_block_size_factor: int
# None below means full attention
sliding_window_size_in_blocks: int | None
# Number of this group's offloaded blocks per full-attention alignment
# segment. Used to skip storing SWA blocks that can never serve a load
# hit (e.g. DeepSeek V4 where SWA groups have much smaller block sizes
# than the MLA full-attention group).
# None for full-attention groups or when the optimization doesn't apply.
alignment_block_count: int | None = None
def get_sliding_window_size_in_blocks(
@@ -89,6 +95,41 @@ class SchedulerOffloadConfig(NamedTuple):
@classmethod
def from_spec(cls, spec: OffloadingSpec) -> "SchedulerOffloadConfig":
# Determine the alignment token count from the full-attention group(s).
# This is the offloaded_block_size of the full-attention group; load
# hits are always aligned to this boundary, so SWA blocks earlier in
# each segment can never serve a load hit. Relevant for hybrid
# architectures like DeepSeek V4 (MLA + SWA groups).
full_attn_offloaded_block_sizes: set[int] = set()
for idx, gpu_block_size in enumerate(spec.gpu_block_size):
kv_spec = spec.kv_cache_config.kv_cache_groups[idx].kv_cache_spec
sw = get_sliding_window_size_in_blocks(
kv_spec, gpu_block_size * spec.block_size_factor
)
if sw is None:
full_attn_offloaded_block_sizes.add(
gpu_block_size * spec.block_size_factor
)
# Only apply the optimization if there's a single consistent
# full-attention alignment size.
alignment_tokens: int | None = None
if len(full_attn_offloaded_block_sizes) == 1:
alignment_tokens = full_attn_offloaded_block_sizes.pop()
def _alignment_block_count(
offloaded_block_size: int,
sliding_window_size_in_blocks: int | None,
) -> int | None:
if alignment_tokens is None or sliding_window_size_in_blocks is None:
return None
if alignment_tokens <= offloaded_block_size:
return None
per_segment = alignment_tokens // offloaded_block_size
if sliding_window_size_in_blocks >= per_segment:
return None
return per_segment
return cls(
num_workers=spec.vllm_config.parallel_config.world_size,
kv_group_configs=tuple(
@@ -100,9 +141,14 @@ class SchedulerOffloadConfig(NamedTuple):
(gpu_block_size * spec.block_size_factor)
// spec.hash_block_size
),
sliding_window_size_in_blocks=get_sliding_window_size_in_blocks(
spec.kv_cache_config.kv_cache_groups[idx].kv_cache_spec,
gpu_block_size * spec.block_size_factor,
sliding_window_size_in_blocks=(
sw := get_sliding_window_size_in_blocks(
spec.kv_cache_config.kv_cache_groups[idx].kv_cache_spec,
gpu_block_size * spec.block_size_factor,
)
),
alignment_block_count=_alignment_block_count(
gpu_block_size * spec.block_size_factor, sw
),
)
for idx, gpu_block_size in enumerate(spec.gpu_block_size)
@@ -639,6 +685,7 @@ class OffloadingConnectorScheduler:
num_offloadable_tokens = min(num_tokens_after_batch, req.num_tokens)
# Filter out blocks skipped due to sliding window attention / SSM
# or unreachable by the load path's alignment constraints.
new_offload_keys: list[OffloadKey] = []
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
@@ -663,9 +710,26 @@ class OffloadingConnectorScheduler:
]
assert len(offload_keys) == len(offload_block_ids)
for offload_key, block_id in zip(offload_keys, offload_block_ids):
if block_id != 0:
new_offload_keys.append(offload_key)
alignment_block_count = group_config.alignment_block_count
tail = group_config.sliding_window_size_in_blocks
for key_idx, (offload_key, block_id) in enumerate(
zip(offload_keys, offload_block_ids)
):
if block_id == 0:
continue
# Skip SWA blocks that can never serve a load hit:
# within each full-attention alignment segment, only the
# trailing `tail` blocks are reachable by
# _sliding_window_lookup. For DeepSeek V4 with 100K
# tokens this reduces SWA stores by ~78%.
if alignment_block_count is not None:
assert tail is not None
abs_block_idx = start_block_idx + key_idx
pos_in_segment = abs_block_idx % alignment_block_count
if pos_in_segment < alignment_block_count - tail:
continue
new_offload_keys.append(offload_key)
if not new_offload_keys:
req_status.advance_stored_idx(num_offloadable_tokens)
@@ -136,6 +136,9 @@ class OffloadingConnectorWorker:
page_size_bytes[layer_name] = (
layer_kv_cache_spec.page_size_bytes
)
unpadded_page_size_bytes[layer_name] = (
layer_kv_cache_spec.real_page_size_bytes
)
else:
# Flash Attention case: (2, num_blocks, ...)
assert test_shape[0] == 2
@@ -157,8 +160,9 @@ class OffloadingConnectorWorker:
tensors_per_block[layer_name] = tuple(raw.unbind(0))
page_size_bytes[layer_name] = half_page_size
unpadded_page_size_bytes[layer_name] = page_size_bytes[layer_name]
unpadded_page_size_bytes[layer_name] = (
layer_kv_cache_spec.real_page_size_bytes // 2
)
elif isinstance(layer_kv_cache_spec, MambaSpec):
state_tensors = kv_caches[layer_name]
@@ -191,7 +195,16 @@ class OffloadingConnectorWorker:
block_tensors: list[CanonicalKVCacheTensor] = []
block_data_refs: dict[str, list[CanonicalKVCacheRef]] = defaultdict(list)
for kv_cache_tensor in self.spec.kv_cache_config.kv_cache_tensors:
tensor_layer_names = kv_cache_tensor.shared_by
# Filter to layers that were actually processed above.
# _get_kv_cache_config_deepseek_v4 emits KVCacheTensor entries for
# every (tuple_idx, page_size) slot; slots where no group has a
# layer at that index produce an empty shared_by (reserved memory
# with no corresponding model layer).
tensor_layer_names = [
n for n in kv_cache_tensor.shared_by if n in tensors_per_block
]
if not tensor_layer_names:
continue
# verify all layers in the group reference the exact same tensors
assert len({len(tensors_per_block[n]) for n in tensor_layer_names}) == 1