[KV] Side-cache layout -- fold the PLE state into a GDN group, pack QSA rings
The packed pool strides every block by the widest group, and every group draws whole blocks. The PLE short-conv state now shares a GDN cache group, and the QSA raw-key rings are packed as equal slots inside shared blocks, so neither holds a whole block per request. VLLM_E2_SIDE_CACHE_LAYOUT (default 1) selects the old layout with 0. On its own the layout raises the Flash-Next pool from 806,792 to 853,994 tokens at the same budget. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -458,6 +458,16 @@ if TYPE_CHECKING:
|
|||||||
# for A/B. AOT: no. CAPTURE: no -- scheduler-side block bookkeeping only;
|
# for A/B. AOT: no. CAPTURE: no -- scheduler-side block bookkeeping only;
|
||||||
# the worker sees the same block tables either way.
|
# the worker sees the same block tables either way.
|
||||||
VLLM_E2_MAMBA_RETAIN_CHECKPOINT: bool = True
|
VLLM_E2_MAMBA_RETAIN_CHECKPOINT: bool = True
|
||||||
|
# v2.5 side-cache layout (Flash-Next PLE + QSA ring): stop charging each
|
||||||
|
# tiny side cache a whole packed pool block per request. The TP-replicated
|
||||||
|
# PLE short-conv state group is folded into a GDN group that has a spare
|
||||||
|
# page on every PP stage, and the one-block-per-request QSA ring group packs
|
||||||
|
# several requests' rings into one pool block. Strict "0"/"1", default ON;
|
||||||
|
# "0" restores the old one-group-each layout for A/B. Read in
|
||||||
|
# vllm/v1/core/kv_cache_utils.py (get_kv_cache_configs). AOT: yes, hashed --
|
||||||
|
# it changes the ring views' block stride and which block table the PLE
|
||||||
|
# state is read through, so ON and OFF must not share a compile cache.
|
||||||
|
VLLM_E2_SIDE_CACHE_LAYOUT: bool = True
|
||||||
# Opt-out for the fork startup guards in vllm/v1/e2_guards.py, read at
|
# Opt-out for the fork startup guards in vllm/v1/e2_guards.py, read at
|
||||||
# Qwen4ExpForCausalLM construction. AOT: no. CAPTURE: no -- it changes
|
# Qwen4ExpForCausalLM construction. AOT: no. CAPTURE: no -- it changes
|
||||||
# nothing itself; it only permits a boot whose identity IS different
|
# nothing itself; it only permits a boot whose identity IS different
|
||||||
@@ -2694,6 +2704,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT": lambda: _e2_strict_switch(
|
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT": lambda: _e2_strict_switch(
|
||||||
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT", default=True
|
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT", default=True
|
||||||
),
|
),
|
||||||
|
# Side-cache layout (v2.5; PLE state folded into a GDN group + QSA ring slot
|
||||||
|
# packing), default ON. Strict "0"/"1"; read once per get_kv_cache_configs call.
|
||||||
|
"VLLM_E2_SIDE_CACHE_LAYOUT": lambda: _e2_strict_switch(
|
||||||
|
"VLLM_E2_SIDE_CACHE_LAYOUT", default=True
|
||||||
|
),
|
||||||
# Seconds between `topk-exact-counters` lines from each GPU worker
|
# Seconds between `topk-exact-counters` lines from each GPU worker
|
||||||
# (0, the default, disables the reader entirely). The line reports the
|
# (0, the default, disables the reader entirely). The line reports the
|
||||||
# medium-path exact-restart counters; see
|
# medium-path exact-restart counters; see
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from vllm.v1.core.kv_cache_utils import (
|
|||||||
KVCacheBlock,
|
KVCacheBlock,
|
||||||
)
|
)
|
||||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||||
|
CircularBufferManager,
|
||||||
CrossAttentionManager,
|
CrossAttentionManager,
|
||||||
MambaManager,
|
MambaManager,
|
||||||
SingleTypeKVCacheManager,
|
SingleTypeKVCacheManager,
|
||||||
@@ -147,6 +148,17 @@ class KVCacheCoordinator(ABC):
|
|||||||
)
|
)
|
||||||
for i, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups)
|
for i, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups)
|
||||||
)
|
)
|
||||||
|
# v2.5 side-cache layout: a slot-packed ring group shares each pool
|
||||||
|
# block between several requests (kv_cache_utils._choose_ring_slots).
|
||||||
|
for kv_cache_group, manager in zip(
|
||||||
|
self.kv_cache_config.kv_cache_groups, self.single_type_managers
|
||||||
|
):
|
||||||
|
if kv_cache_group.slots_per_block > 1:
|
||||||
|
assert type(manager) is CircularBufferManager, (
|
||||||
|
f"slot packing is only defined for the QSA ring, got "
|
||||||
|
f"{type(manager).__name__}"
|
||||||
|
)
|
||||||
|
manager.enable_slot_packing(kv_cache_group.slots_per_block)
|
||||||
# Match Mamba checkpoints to Eagle's attention replay boundary.
|
# Match Mamba checkpoints to Eagle's attention replay boundary.
|
||||||
if use_eagle:
|
if use_eagle:
|
||||||
for manager in self.single_type_managers:
|
for manager in self.single_type_managers:
|
||||||
|
|||||||
@@ -1121,7 +1121,19 @@ def get_max_concurrency_for_kv_cache_config(
|
|||||||
"""
|
"""
|
||||||
num_blocks_per_request = 0
|
num_blocks_per_request = 0
|
||||||
host_blocks_per_request = 0
|
host_blocks_per_request = 0
|
||||||
|
# v2.5 side-cache layout: a slot-packed ring group shares each pool block
|
||||||
|
# between `slots_per_block` requests, so it costs cdiv(max_num_seqs, slots)
|
||||||
|
# blocks for the whole pool instead of one block per request.
|
||||||
|
shared_blocks = 0
|
||||||
|
max_num_seqs = getattr(
|
||||||
|
getattr(vllm_config, "scheduler_config", None), "max_num_seqs", 1
|
||||||
|
)
|
||||||
|
if not isinstance(max_num_seqs, int) or max_num_seqs < 1:
|
||||||
|
max_num_seqs = 1
|
||||||
for group in kv_cache_config.kv_cache_groups:
|
for group in kv_cache_config.kv_cache_groups:
|
||||||
|
if group.slots_per_block > 1 and not group.host_resident:
|
||||||
|
shared_blocks += cdiv(max_num_seqs, group.slots_per_block)
|
||||||
|
continue
|
||||||
required = cdiv(
|
required = cdiv(
|
||||||
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
|
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
|
||||||
group.kv_cache_spec.page_size_bytes,
|
group.kv_cache_spec.page_size_bytes,
|
||||||
@@ -1130,7 +1142,9 @@ def get_max_concurrency_for_kv_cache_config(
|
|||||||
host_blocks_per_request += required
|
host_blocks_per_request += required
|
||||||
else:
|
else:
|
||||||
num_blocks_per_request += required
|
num_blocks_per_request += required
|
||||||
limits = [kv_cache_config.num_blocks / num_blocks_per_request]
|
limits = [
|
||||||
|
max(kv_cache_config.num_blocks - shared_blocks, 0) / num_blocks_per_request
|
||||||
|
]
|
||||||
if host_blocks_per_request:
|
if host_blocks_per_request:
|
||||||
assert kv_cache_config.hisparse_host_num_blocks is not None
|
assert kv_cache_config.hisparse_host_num_blocks is not None
|
||||||
limits.append(
|
limits.append(
|
||||||
@@ -1817,14 +1831,36 @@ def get_kv_cache_config_from_groups(
|
|||||||
elif group.layer_names:
|
elif group.layer_names:
|
||||||
layers_by_spec[group_spec].extend(group.layer_names)
|
layers_by_spec[group_spec].extend(group.layer_names)
|
||||||
|
|
||||||
|
# v2.5 side-cache layout: a slot-packed group views every pool block as
|
||||||
|
# `slots` equal slots, so its block stride is one slot and its view has
|
||||||
|
# num_blocks * slots blocks. Pages of its layers must fit in one slot.
|
||||||
|
slots = group.slots_per_block
|
||||||
|
group_block_stride = interleaved_block_stride
|
||||||
|
group_num_blocks = num_blocks
|
||||||
|
if slots > 1:
|
||||||
|
assert layout.is_block_outermost, (
|
||||||
|
"Slot-packed KV cache groups need a block-outermost layout"
|
||||||
|
)
|
||||||
|
assert bytes_per_block % slots == 0, (bytes_per_block, slots)
|
||||||
|
group_block_stride = bytes_per_block // slots
|
||||||
|
group_num_blocks = num_blocks * slots
|
||||||
|
group_bytes = sum(
|
||||||
|
spec.page_size_bytes * len(names)
|
||||||
|
for spec, names in layers_by_spec.items()
|
||||||
|
)
|
||||||
|
assert group_bytes <= group_block_stride, (
|
||||||
|
f"slot-packed group needs {group_bytes} B per slot, slot is "
|
||||||
|
f"{group_block_stride} B"
|
||||||
|
)
|
||||||
|
|
||||||
byte_offset = 0
|
byte_offset = 0
|
||||||
for spec, layer_names in layers_by_spec.items():
|
for spec, layer_names in layers_by_spec.items():
|
||||||
layer_stride, block_stride, _, _, _ = compute_layout_strides(
|
layer_stride, block_stride, _, _, _ = compute_layout_strides(
|
||||||
spec,
|
spec,
|
||||||
num_blocks,
|
group_num_blocks,
|
||||||
len(layer_names),
|
len(layer_names),
|
||||||
layout,
|
layout,
|
||||||
fixed_strides=(None, interleaved_block_stride, None, None, None),
|
fixed_strides=(None, group_block_stride, None, None, None),
|
||||||
)
|
)
|
||||||
offset = (
|
offset = (
|
||||||
byte_offset
|
byte_offset
|
||||||
@@ -1838,6 +1874,7 @@ def get_kv_cache_config_from_groups(
|
|||||||
layer_stride=layer_stride,
|
layer_stride=layer_stride,
|
||||||
block_stride=block_stride,
|
block_stride=block_stride,
|
||||||
offset=offset,
|
offset=offset,
|
||||||
|
slots_per_block=slots,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
byte_offset += len(layer_names) * spec.page_size_bytes
|
byte_offset += len(layer_names) * spec.page_size_bytes
|
||||||
@@ -2705,6 +2742,289 @@ def _project_kv_cache_groups_to_worker(
|
|||||||
return projected_groups
|
return projected_groups
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# v2.5 side-cache layout (VLLM_E2_SIDE_CACHE_LAYOUT, default ON).
|
||||||
|
#
|
||||||
|
# The packed block-outermost layout strides every pool block by the widest
|
||||||
|
# group, and every group draws whole blocks from the one pool. Flash-Next has
|
||||||
|
# two side caches whose real state is tiny next to that stride (12.9 MB on the
|
||||||
|
# binding PP stage): the TP-replicated PLE short-conv state (one layer, 2 + K
|
||||||
|
# blocks per request in align mode, a 1.6 MB padded page) and the QSA raw-key
|
||||||
|
# ring (one block per request, 16 KB for all of a stage's layers). Together
|
||||||
|
# they hold 6 of the 103 blocks a 262K request needs. Two layout-only changes
|
||||||
|
# remove that cost without changing any value a kernel reads:
|
||||||
|
#
|
||||||
|
# 1. PLE fold. The replicated Mamba group joins a sharded Mamba group with the
|
||||||
|
# same scheduling parameters whose block has a spare page on every worker
|
||||||
|
# (Flash-Next: a GDN group holds 6 of PP0's 7 pages), so the PLE state rides
|
||||||
|
# in that group's blocks under the block table it would have had on its
|
||||||
|
# own. No worker's stride changes, so the pool block count does not change.
|
||||||
|
# 2. Ring slot packing. The ring group splits every pool block into S equal
|
||||||
|
# slots and the ring manager hands out slots; a pool block goes back to the
|
||||||
|
# pool when its last slot is released. The ring then costs cdiv(requests, S)
|
||||||
|
# blocks in all instead of one per request.
|
||||||
|
#
|
||||||
|
# Off (=0) restores one group each, block for block.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Slot stride alignment for packed ring views (bytes) and the largest S tried.
|
||||||
|
_RING_SLOT_ALIGN = 256
|
||||||
|
_RING_MAX_SLOTS = 1024
|
||||||
|
|
||||||
|
|
||||||
|
def _side_cache_layout_enabled(vllm_config: VllmConfig) -> bool:
|
||||||
|
"""Whether the v2.5 side-cache layout may apply to this engine.
|
||||||
|
|
||||||
|
KV connectors (NIXL, offload, P/D) address groups and blocks themselves
|
||||||
|
(the NIXL worker requires exactly one single-layer replicated group), and
|
||||||
|
HiSparse has its own pool layout, so those engines keep the old layout.
|
||||||
|
"""
|
||||||
|
if not envs.VLLM_E2_SIDE_CACHE_LAYOUT:
|
||||||
|
return False
|
||||||
|
if getattr(vllm_config, "kv_transfer_config", None) is not None:
|
||||||
|
return False
|
||||||
|
attention_config = getattr(vllm_config, "attention_config", None)
|
||||||
|
if getattr(attention_config, "hisparse_config", None) is not None:
|
||||||
|
return False
|
||||||
|
parallel_config = getattr(vllm_config, "parallel_config", None)
|
||||||
|
if getattr(parallel_config, "decode_context_parallel_size", 1) not in (1, None):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||||
|
except ValueError:
|
||||||
|
# Unresolved layout: only reachable from callers that bypass the
|
||||||
|
# engine core; nothing to pack without a block-outermost layout.
|
||||||
|
return False
|
||||||
|
return bool(layout.is_block_outermost)
|
||||||
|
|
||||||
|
|
||||||
|
def _group_layer_specs(group: KVCacheGroupSpec) -> list[KVCacheSpec]:
|
||||||
|
spec = group.kv_cache_spec
|
||||||
|
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||||
|
return [spec.kv_cache_specs[name] for name in group.layer_names]
|
||||||
|
return [spec] * len(group.layer_names)
|
||||||
|
|
||||||
|
|
||||||
|
def _mamba_schedule_key(spec: MambaSpec) -> tuple:
|
||||||
|
"""Everything the scheduler's MambaManager and the worker's align-mode
|
||||||
|
bookkeeping read from a group's spec. Two Mamba groups with equal keys get
|
||||||
|
identical block tables for every request."""
|
||||||
|
return (
|
||||||
|
spec.block_size,
|
||||||
|
spec.page_size_bytes,
|
||||||
|
spec.num_speculative_blocks,
|
||||||
|
spec.num_prefill_checkpoint_blocks,
|
||||||
|
spec.prefill_checkpoint_alignment,
|
||||||
|
spec.mamba_cache_mode,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _worker_strides(
|
||||||
|
groups: list[KVCacheGroupSpec], worker_specs: list[dict[str, KVCacheSpec]]
|
||||||
|
) -> list[int]:
|
||||||
|
"""Pool block bytes of every worker for this global grouping."""
|
||||||
|
strides = []
|
||||||
|
for spec in worker_specs:
|
||||||
|
projected = _project_kv_cache_groups_to_worker(groups, spec)
|
||||||
|
if not any(g.layer_names for g in projected):
|
||||||
|
strides.append(0)
|
||||||
|
continue
|
||||||
|
strides.append(_get_kv_cache_bytes_per_block(projected))
|
||||||
|
return strides
|
||||||
|
|
||||||
|
|
||||||
|
def _fold_replicated_mamba_groups(
|
||||||
|
groups: list[KVCacheGroupSpec],
|
||||||
|
worker_specs: list[dict[str, KVCacheSpec]],
|
||||||
|
) -> tuple[list[KVCacheGroupSpec], list[str]]:
|
||||||
|
"""Fold each TP-replicated Mamba group into a compatible sharded one.
|
||||||
|
|
||||||
|
A target holds only sharded Mamba layers with the same scheduling key and
|
||||||
|
EAGLE flag, and the merged group must not widen the block of any worker
|
||||||
|
(checked on every worker's projection, so PP stages are covered). Among
|
||||||
|
valid targets the lowest index wins. Returns the new group list and one
|
||||||
|
log note per replicated group.
|
||||||
|
"""
|
||||||
|
groups = list(groups)
|
||||||
|
notes: list[str] = []
|
||||||
|
base_strides = _worker_strides(groups, worker_specs)
|
||||||
|
index = 0
|
||||||
|
while index < len(groups):
|
||||||
|
source = groups[index]
|
||||||
|
layer_specs = _group_layer_specs(source)
|
||||||
|
if not (
|
||||||
|
layer_specs
|
||||||
|
and all(
|
||||||
|
isinstance(spec, MambaSpec) and spec.tp_replicated
|
||||||
|
for spec in layer_specs
|
||||||
|
)
|
||||||
|
and len({_mamba_schedule_key(spec) for spec in layer_specs}) == 1
|
||||||
|
):
|
||||||
|
index += 1
|
||||||
|
continue
|
||||||
|
key = _mamba_schedule_key(cast(MambaSpec, layer_specs[0]))
|
||||||
|
chosen = None
|
||||||
|
for target_index, target in enumerate(groups):
|
||||||
|
if target_index == index or target.is_eagle_group != source.is_eagle_group:
|
||||||
|
continue
|
||||||
|
target_specs = _group_layer_specs(target)
|
||||||
|
if not target_specs or not all(
|
||||||
|
isinstance(spec, MambaSpec)
|
||||||
|
and not spec.tp_replicated
|
||||||
|
and _mamba_schedule_key(spec) == key
|
||||||
|
for spec in target_specs
|
||||||
|
):
|
||||||
|
continue
|
||||||
|
merged_specs: dict[str, KVCacheSpec] = dict(
|
||||||
|
zip(target.layer_names, target_specs)
|
||||||
|
)
|
||||||
|
merged_specs.update(zip(source.layer_names, layer_specs))
|
||||||
|
# Built directly: is_uniform_type rejects mixed tp_replicated on
|
||||||
|
# purpose (NIXL), and connectors never reach this path.
|
||||||
|
merged = KVCacheGroupSpec(
|
||||||
|
target.layer_names + source.layer_names,
|
||||||
|
UniformTypeKVCacheSpecs(
|
||||||
|
block_size=target_specs[0].block_size,
|
||||||
|
kv_cache_specs=merged_specs,
|
||||||
|
),
|
||||||
|
is_eagle_group=target.is_eagle_group,
|
||||||
|
)
|
||||||
|
trial = [
|
||||||
|
merged if i == target_index else g
|
||||||
|
for i, g in enumerate(groups)
|
||||||
|
if i != index
|
||||||
|
]
|
||||||
|
strides = _worker_strides(trial, worker_specs)
|
||||||
|
if all(new <= old for new, old in zip(strides, base_strides)):
|
||||||
|
chosen = (target_index, merged, trial)
|
||||||
|
break
|
||||||
|
if chosen is None:
|
||||||
|
notes.append(
|
||||||
|
f"replicated Mamba group {index} ({len(source.layer_names)} "
|
||||||
|
"layer(s)) kept separate: no sharded group has a spare page on "
|
||||||
|
"every worker"
|
||||||
|
)
|
||||||
|
index += 1
|
||||||
|
continue
|
||||||
|
target_index, merged, trial = chosen
|
||||||
|
notes.append(
|
||||||
|
f"folded replicated Mamba group {index} "
|
||||||
|
f"({', '.join(source.layer_names)}) into Mamba group "
|
||||||
|
f"{target_index} (now {len(merged.layer_names)} layers); per-worker "
|
||||||
|
f"block bytes unchanged {sorted(set(base_strides))}"
|
||||||
|
)
|
||||||
|
groups = trial
|
||||||
|
# The group after the folded one now sits at `index`.
|
||||||
|
return groups, notes
|
||||||
|
|
||||||
|
|
||||||
|
def _is_ring_group(group: KVCacheGroupSpec) -> bool:
|
||||||
|
spec = group.kv_cache_spec
|
||||||
|
layer_specs = (
|
||||||
|
list(spec.kv_cache_specs.values())
|
||||||
|
if isinstance(spec, UniformTypeKVCacheSpecs)
|
||||||
|
else [spec]
|
||||||
|
)
|
||||||
|
return bool(layer_specs) and all(
|
||||||
|
type(layer_spec) is CircularBufferSpec for layer_spec in layer_specs
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _choose_ring_slots(
|
||||||
|
projected_groups_per_worker: list[list[KVCacheGroupSpec]],
|
||||||
|
) -> dict[int, int]:
|
||||||
|
"""Slots per pool block for every ring group (group index -> S >= 2).
|
||||||
|
|
||||||
|
S must cut every worker's block stride into equal slots that are
|
||||||
|
`_RING_SLOT_ALIGN`-aligned and hold that worker's ring pages; the largest
|
||||||
|
such S up to `_RING_MAX_SLOTS` is taken. Block ids are global, so S is the
|
||||||
|
same on every worker. A ring group with no valid S keeps one pool block
|
||||||
|
per request.
|
||||||
|
"""
|
||||||
|
chosen: dict[int, int] = {}
|
||||||
|
if not projected_groups_per_worker or not projected_groups_per_worker[0]:
|
||||||
|
return chosen
|
||||||
|
num_groups = len(projected_groups_per_worker[0])
|
||||||
|
strides = [
|
||||||
|
_get_kv_cache_bytes_per_block(groups)
|
||||||
|
if any(g.layer_names for g in groups)
|
||||||
|
else 0
|
||||||
|
for groups in projected_groups_per_worker
|
||||||
|
]
|
||||||
|
for group_index in range(num_groups):
|
||||||
|
if not _is_ring_group(projected_groups_per_worker[0][group_index]):
|
||||||
|
continue
|
||||||
|
needs = []
|
||||||
|
for groups, stride in zip(projected_groups_per_worker, strides):
|
||||||
|
group = groups[group_index]
|
||||||
|
if not group.layer_names:
|
||||||
|
continue
|
||||||
|
ring_bytes = sum(spec.page_size_bytes for spec in _group_layer_specs(group))
|
||||||
|
needs.append((stride, ring_bytes))
|
||||||
|
if not needs:
|
||||||
|
continue
|
||||||
|
for slots in range(_RING_MAX_SLOTS, 1, -1):
|
||||||
|
if all(
|
||||||
|
stride % slots == 0
|
||||||
|
and (stride // slots) % _RING_SLOT_ALIGN == 0
|
||||||
|
and stride // slots >= ring_bytes
|
||||||
|
for stride, ring_bytes in needs
|
||||||
|
):
|
||||||
|
chosen[group_index] = slots
|
||||||
|
break
|
||||||
|
return chosen
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_side_cache_layout(
|
||||||
|
vllm_config: VllmConfig,
|
||||||
|
global_kv_cache_groups: list[KVCacheGroupSpec],
|
||||||
|
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
||||||
|
) -> tuple[list[KVCacheGroupSpec], list[list[KVCacheGroupSpec]]]:
|
||||||
|
"""Fold side Mamba groups, project per worker, then pick ring slots.
|
||||||
|
|
||||||
|
Returns the (possibly folded) global groups and every worker's projection,
|
||||||
|
with `slots_per_block` set on packed ring groups.
|
||||||
|
"""
|
||||||
|
relevant = any(
|
||||||
|
(isinstance(spec, MambaSpec) and spec.tp_replicated)
|
||||||
|
or type(spec) is CircularBufferSpec
|
||||||
|
for worker_spec in kv_cache_specs
|
||||||
|
for spec in worker_spec.values()
|
||||||
|
)
|
||||||
|
enabled = relevant and _side_cache_layout_enabled(vllm_config)
|
||||||
|
notes: list[str] = []
|
||||||
|
if enabled:
|
||||||
|
global_kv_cache_groups, notes = _fold_replicated_mamba_groups(
|
||||||
|
global_kv_cache_groups, kv_cache_specs
|
||||||
|
)
|
||||||
|
projected = [
|
||||||
|
_project_kv_cache_groups_to_worker(global_kv_cache_groups, worker_spec)
|
||||||
|
for worker_spec in kv_cache_specs
|
||||||
|
]
|
||||||
|
if enabled and all(
|
||||||
|
_glm5_next_tensor_layout(groups) is None for groups in projected if groups
|
||||||
|
):
|
||||||
|
for group_index, slots in _choose_ring_slots(projected).items():
|
||||||
|
for groups in projected:
|
||||||
|
groups[group_index].slots_per_block = slots
|
||||||
|
notes.append(
|
||||||
|
f"QSA ring group {group_index} packs {slots} requests per pool block"
|
||||||
|
)
|
||||||
|
if relevant and envs.VLLM_E2_SIDE_CACHE_LAYOUT and not enabled:
|
||||||
|
notes.append(
|
||||||
|
"not applicable to this engine (KV connector, HiSparse, DCP or a "
|
||||||
|
"layer-outermost layout)"
|
||||||
|
)
|
||||||
|
if relevant:
|
||||||
|
# The A/B evidence line: boot logs must show which arm ran.
|
||||||
|
logger.info(
|
||||||
|
"Side-cache layout: %s (VLLM_E2_SIDE_CACHE_LAYOUT)%s",
|
||||||
|
"on" if enabled else "off",
|
||||||
|
"".join(f"; {note}" for note in notes),
|
||||||
|
)
|
||||||
|
return global_kv_cache_groups, projected
|
||||||
|
|
||||||
|
|
||||||
def get_kv_cache_configs(
|
def get_kv_cache_configs(
|
||||||
vllm_config: VllmConfig,
|
vllm_config: VllmConfig,
|
||||||
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
||||||
@@ -2774,10 +3094,11 @@ def get_kv_cache_configs(
|
|||||||
# If original_max_model_len was -1, automatically
|
# If original_max_model_len was -1, automatically
|
||||||
# determine the maximum model length that fits in available GPU memory.
|
# determine the maximum model length that fits in available GPU memory.
|
||||||
# We use per-worker projected groups to account for PP sharding.
|
# We use per-worker projected groups to account for PP sharding.
|
||||||
projected_groups_per_worker = [
|
# v2.5: the side-cache layout (PLE fold, ring slots) is applied here, where
|
||||||
_project_kv_cache_groups_to_worker(global_kv_cache_groups, worker_spec)
|
# every worker's layers are known, so no PP stage's block can widen.
|
||||||
for worker_spec in kv_cache_specs
|
global_kv_cache_groups, projected_groups_per_worker = _apply_side_cache_layout(
|
||||||
]
|
vllm_config, global_kv_cache_groups, kv_cache_specs
|
||||||
|
)
|
||||||
|
|
||||||
# If `num_gpu_blocks_override` is set, the cache size that will actually
|
# If `num_gpu_blocks_override` is set, the cache size that will actually
|
||||||
# be allocated is decoupled from the profiled `available_memory`:
|
# be allocated is decoupled from the profiled `available_memory`:
|
||||||
|
|||||||
@@ -1619,6 +1619,10 @@ class Scheduler(SchedulerInterface):
|
|||||||
pending = set(block_ids)
|
pending = set(block_ids)
|
||||||
owners: dict[int, set[str]] = {}
|
owners: dict[int, set[str]] = {}
|
||||||
for mgr in self.kv_cache_manager.coordinator.single_type_managers:
|
for mgr in self.kv_cache_manager.coordinator.single_type_managers:
|
||||||
|
if getattr(mgr, "slot_pool", None) is not None:
|
||||||
|
# v2.5 slot-packed ring: its block ids are slot ids, not pool
|
||||||
|
# block ids, and its pages are never zeroed.
|
||||||
|
continue
|
||||||
for req_id, blocks in mgr.req_to_blocks.items():
|
for req_id, blocks in mgr.req_to_blocks.items():
|
||||||
for blk in blocks:
|
for blk in blocks:
|
||||||
if blk.block_id in pending:
|
if blk.block_id in pending:
|
||||||
|
|||||||
@@ -1236,18 +1236,126 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
|||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
class _RingSlotPage:
|
||||||
|
"""One pool block shared by up to ``slots`` requests' rings."""
|
||||||
|
|
||||||
|
__slots__ = ("block", "free_slots", "live")
|
||||||
|
|
||||||
|
def __init__(self, block: KVCacheBlock, slots: int) -> None:
|
||||||
|
self.block = block
|
||||||
|
# Highest first, so pop() hands out the lowest free slot.
|
||||||
|
self.free_slots = list(range(slots - 1, -1, -1))
|
||||||
|
self.live = 0
|
||||||
|
|
||||||
|
|
||||||
|
class RingSlotPool:
|
||||||
|
"""v2.5 side-cache layout: ring blocks as slots of shared pool blocks.
|
||||||
|
|
||||||
|
The QSA ring needs one block per request for the request's lifetime, but
|
||||||
|
its pages for every layer of a stage are ~16 KB against a 12.9 MB pool
|
||||||
|
block. This pool takes whole blocks from the main ``BlockPool`` and cuts
|
||||||
|
each into ``slots_per_block`` slots; a slot's block id is
|
||||||
|
``pool_block_id * slots_per_block + slot``, which is exactly the index of
|
||||||
|
the slot in the worker's ring view (``KVCacheTensor.slots_per_block``).
|
||||||
|
|
||||||
|
The slot blocks it hands out carry ``pool=self``, so every existing free
|
||||||
|
path -- ``SingleTypeKVCacheManager.free``, the coordinator's deferred
|
||||||
|
``pop_blocks_for_free`` and the scheduler's fenced drain, which all call
|
||||||
|
``BlockPool.free_blocks`` -- routes them back here, at the time the old
|
||||||
|
per-request block would have gone back (the deferred free is kept). A pool
|
||||||
|
block returns to the main pool when its last slot is freed. Rings are never
|
||||||
|
prefix-cached, so a slot block carries no hash.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, block_pool: BlockPool, slots_per_block: int) -> None:
|
||||||
|
assert slots_per_block > 1
|
||||||
|
self.block_pool = block_pool
|
||||||
|
self.slots_per_block = slots_per_block
|
||||||
|
# Pool block id -> page, in claim order (the oldest page fills first).
|
||||||
|
self._pages: dict[int, _RingSlotPage] = {}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def num_pages(self) -> int:
|
||||||
|
"""Pool blocks currently held for rings."""
|
||||||
|
return len(self._pages)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def num_live_slots(self) -> int:
|
||||||
|
return sum(page.live for page in self._pages.values())
|
||||||
|
|
||||||
|
def has_free_slot(self) -> bool:
|
||||||
|
return any(page.free_slots for page in self._pages.values())
|
||||||
|
|
||||||
|
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
|
||||||
|
out: list[KVCacheBlock] = []
|
||||||
|
for _ in range(num_blocks):
|
||||||
|
page = next((p for p in self._pages.values() if p.free_slots), None)
|
||||||
|
if page is None:
|
||||||
|
(pool_block,) = self.block_pool.get_new_blocks(1)
|
||||||
|
page = _RingSlotPage(pool_block, self.slots_per_block)
|
||||||
|
self._pages[pool_block.block_id] = page
|
||||||
|
slot = page.free_slots.pop()
|
||||||
|
page.live += 1
|
||||||
|
out.append(
|
||||||
|
KVCacheBlock(
|
||||||
|
page.block.block_id * self.slots_per_block + slot,
|
||||||
|
ref_cnt=1,
|
||||||
|
pool=self, # type: ignore[arg-type]
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
def free_blocks(self, ordered_blocks) -> None:
|
||||||
|
released: list[KVCacheBlock] = []
|
||||||
|
for block in ordered_blocks:
|
||||||
|
assert block.pool is self, "not a ring slot of this pool"
|
||||||
|
block.ref_cnt -= 1
|
||||||
|
assert block.ref_cnt == 0, "ring slots have a single owner"
|
||||||
|
page_id, slot = divmod(block.block_id, self.slots_per_block)
|
||||||
|
page = self._pages[page_id]
|
||||||
|
assert slot not in page.free_slots, f"slot {block.block_id} freed twice"
|
||||||
|
page.free_slots.append(slot)
|
||||||
|
page.free_slots.sort(reverse=True)
|
||||||
|
page.live -= 1
|
||||||
|
if page.live == 0:
|
||||||
|
del self._pages[page_id]
|
||||||
|
released.append(page.block)
|
||||||
|
if released:
|
||||||
|
self.block_pool.free_blocks(released)
|
||||||
|
|
||||||
|
|
||||||
class CircularBufferManager(FullAttentionManager):
|
class CircularBufferManager(FullAttentionManager):
|
||||||
"""Claims the ring's single block per request; prefix caching disabled."""
|
"""Claims the ring's single block per request; prefix caching disabled."""
|
||||||
|
|
||||||
supports_fine_grained_hash_lookup: ClassVar[bool] = False
|
supports_fine_grained_hash_lookup: ClassVar[bool] = False
|
||||||
|
|
||||||
|
# v2.5 side-cache layout: set by the coordinator for a slot-packed group.
|
||||||
|
_slot_pool: RingSlotPool | None = None
|
||||||
|
|
||||||
|
def enable_slot_packing(self, slots_per_block: int) -> None:
|
||||||
|
"""Claim ring slots of shared pool blocks instead of whole blocks."""
|
||||||
|
assert not self.req_to_blocks, "enable slot packing before any claim"
|
||||||
|
self._slot_pool = RingSlotPool(self.block_pool, slots_per_block)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def slot_pool(self) -> RingSlotPool | None:
|
||||||
|
return self._slot_pool
|
||||||
|
|
||||||
|
if self._slot_pool is not None:
|
||||||
|
new_blocks = self._slot_pool.get_new_blocks(1)
|
||||||
|
else:
|
||||||
|
new_blocks = self.block_pool.get_new_blocks(1)
|
||||||
|
|
||||||
def _claim_ring_block(
|
def _claim_ring_block(
|
||||||
self, request_id: str, record_for_zeroing: bool = True
|
self, request_id: str, record_for_zeroing: bool = True
|
||||||
) -> list[KVCacheBlock]:
|
) -> list[KVCacheBlock]:
|
||||||
req_blocks = self.req_to_blocks[request_id]
|
req_blocks = self.req_to_blocks[request_id]
|
||||||
if req_blocks:
|
if req_blocks:
|
||||||
return []
|
return []
|
||||||
new_blocks = self.block_pool.get_new_blocks(1)
|
if self._slot_pool is not None:
|
||||||
|
new_blocks = self._slot_pool.get_new_blocks(1)
|
||||||
|
else:
|
||||||
|
new_blocks = self.block_pool.get_new_blocks(1)
|
||||||
req_blocks.extend(new_blocks)
|
req_blocks.extend(new_blocks)
|
||||||
if self._record_new_block_ids and record_for_zeroing:
|
if self._record_new_block_ids and record_for_zeroing:
|
||||||
self.new_block_ids.extend(block.block_id for block in new_blocks)
|
self.new_block_ids.extend(block.block_id for block in new_blocks)
|
||||||
@@ -1293,7 +1401,12 @@ class CircularBufferManager(FullAttentionManager):
|
|||||||
apply_admission_cap: bool = False,
|
apply_admission_cap: bool = False,
|
||||||
prefill_end: int = 0,
|
prefill_end: int = 0,
|
||||||
) -> int:
|
) -> int:
|
||||||
return 0 if self.req_to_blocks.get(request_id) else 1
|
if self.req_to_blocks.get(request_id):
|
||||||
|
return 0
|
||||||
|
# A free slot in a page the ring already holds costs no pool block.
|
||||||
|
if self._slot_pool is not None and self._slot_pool.has_free_slot():
|
||||||
|
return 0
|
||||||
|
return 1
|
||||||
|
|
||||||
def allocate_new_blocks(
|
def allocate_new_blocks(
|
||||||
self, request_id: str, num_tokens: int, num_tokens_main_model: int
|
self, request_id: str, num_tokens: int, num_tokens_main_model: int
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1431,6 +1431,11 @@ class KVCacheTensor:
|
|||||||
block_stride: int
|
block_stride: int
|
||||||
offset: int = 0 # byte offset of layers[0]'s block 0
|
offset: int = 0 # byte offset of layers[0]'s block 0
|
||||||
host_resident: bool = False
|
host_resident: bool = False
|
||||||
|
# v2.5 side-cache layout: a slot-packed ring tensor views every pool block
|
||||||
|
# as ``slots_per_block`` equal slots (``block_stride`` is then one slot), so
|
||||||
|
# its block ids are ``pool_block * slots_per_block + slot``. 1 = one view
|
||||||
|
# block per pool block, the ordinary case.
|
||||||
|
slots_per_block: int = 1
|
||||||
|
|
||||||
|
|
||||||
class KVCacheGroupRole(str, Enum):
|
class KVCacheGroupRole(str, Enum):
|
||||||
@@ -1458,6 +1463,10 @@ class KVCacheGroupSpec:
|
|||||||
# source group after a connector load.
|
# source group after a connector load.
|
||||||
enable_kv_transfer: bool = True
|
enable_kv_transfer: bool = True
|
||||||
role: KVCacheGroupRole = KVCacheGroupRole.DEFAULT
|
role: KVCacheGroupRole = KVCacheGroupRole.DEFAULT
|
||||||
|
# v2.5 side-cache layout: > 1 when this one-block-per-request ring group
|
||||||
|
# shares each pool block between that many requests (see
|
||||||
|
# ``CircularBufferManager.enable_slot_packing``); 1 = one pool block each.
|
||||||
|
slots_per_block: int = 1
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -1545,6 +1554,18 @@ class KVCacheConfig:
|
|||||||
)
|
)
|
||||||
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
|
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def slot_packed_group_ids(self) -> tuple[int, ...]:
|
||||||
|
"""Groups whose blocks are slots of shared pool blocks (v2.5 side-cache
|
||||||
|
layout). Their views are not one block per pool block, so whole-block
|
||||||
|
zeroing must skip them; their bytes lie inside pool blocks the ring
|
||||||
|
manager owns and that no other group writes."""
|
||||||
|
return tuple(
|
||||||
|
group_id
|
||||||
|
for group_id, group in enumerate(self.kv_cache_groups)
|
||||||
|
if group.slots_per_block > 1
|
||||||
|
)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def host_group_ids(self) -> tuple[int, ...]:
|
def host_group_ids(self) -> tuple[int, ...]:
|
||||||
return tuple(
|
return tuple(
|
||||||
@@ -1556,7 +1577,7 @@ class KVCacheConfig:
|
|||||||
def num_blocks_of(self, tensor: KVCacheTensor) -> int:
|
def num_blocks_of(self, tensor: KVCacheTensor) -> int:
|
||||||
"""Number of blocks addressable by the pool backing ``tensor``."""
|
"""Number of blocks addressable by the pool backing ``tensor``."""
|
||||||
if not tensor.host_resident:
|
if not tensor.host_resident:
|
||||||
return self.num_blocks
|
return self.num_blocks * tensor.slots_per_block
|
||||||
assert self.hisparse_host_num_blocks is not None
|
assert self.hisparse_host_num_blocks is not None
|
||||||
return self.hisparse_host_num_blocks
|
return self.hisparse_host_num_blocks
|
||||||
|
|
||||||
|
|||||||
@@ -866,6 +866,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
|||||||
kernel_block_sizes=self.kernel_block_sizes,
|
kernel_block_sizes=self.kernel_block_sizes,
|
||||||
static_forward_context=self.compilation_config.static_forward_context,
|
static_forward_context=self.compilation_config.static_forward_context,
|
||||||
num_blocks=self.kv_cache_config.num_blocks,
|
num_blocks=self.kv_cache_config.num_blocks,
|
||||||
|
skip_kv_cache_group_ids=self.kv_cache_config.slot_packed_group_ids,
|
||||||
)
|
)
|
||||||
|
|
||||||
@torch.inference_mode()
|
@torch.inference_mode()
|
||||||
|
|||||||
@@ -1149,6 +1149,7 @@ class GPUModelRunner(
|
|||||||
runner_only_attn_layers=self.runner_only_attn_layers,
|
runner_only_attn_layers=self.runner_only_attn_layers,
|
||||||
static_forward_context=self.compilation_config.static_forward_context,
|
static_forward_context=self.compilation_config.static_forward_context,
|
||||||
num_blocks=self.kv_cache_config.num_blocks,
|
num_blocks=self.kv_cache_config.num_blocks,
|
||||||
|
skip_kv_cache_group_ids=self.kv_cache_config.slot_packed_group_ids,
|
||||||
)
|
)
|
||||||
|
|
||||||
def _zero_block_ids(self, block_ids: list[int]) -> None:
|
def _zero_block_ids(self, block_ids: list[int]) -> None:
|
||||||
|
|||||||
@@ -138,6 +138,7 @@ class KVBlockZeroer:
|
|||||||
static_forward_context: dict[str, Any],
|
static_forward_context: dict[str, Any],
|
||||||
num_blocks: int,
|
num_blocks: int,
|
||||||
runner_only_attn_layers: set[str] | None = None,
|
runner_only_attn_layers: set[str] | None = None,
|
||||||
|
skip_kv_cache_group_ids: Iterable[int] = (),
|
||||||
) -> None:
|
) -> None:
|
||||||
"""Precompute the absolute-address table for the Triton zeroing kernel.
|
"""Precompute the absolute-address table for the Triton zeroing kernel.
|
||||||
|
|
||||||
@@ -156,6 +157,13 @@ class KVBlockZeroer:
|
|||||||
physical block stride and zeroed page span remain independent.
|
physical block stride and zeroed page span remain independent.
|
||||||
|
|
||||||
Only AttentionSpec layers are processed; Mamba layers are skipped.
|
Only AttentionSpec layers are processed; Mamba layers are skipped.
|
||||||
|
|
||||||
|
``skip_kv_cache_group_ids`` (v2.5 side-cache layout): slot-packed ring
|
||||||
|
groups, whose views hold several slots per pool block. Their claims
|
||||||
|
never enter whole-block zeroing (the ring-tag path clears their tag words instead),
|
||||||
|
and with one view block per pool block their pages sat inside the
|
||||||
|
first attention page, so skipping them leaves the zeroed bytes of
|
||||||
|
every block unchanged.
|
||||||
"""
|
"""
|
||||||
self.device = device
|
self.device = device
|
||||||
# Bounds-guard instrumentation: pool bound for the guard in _zero_kv_blocks_kernel.
|
# Bounds-guard instrumentation: pool bound for the guard in _zero_kv_blocks_kernel.
|
||||||
@@ -174,12 +182,15 @@ class KVBlockZeroer:
|
|||||||
seg_block_strides: list[int] = []
|
seg_block_strides: list[int] = []
|
||||||
seg_page_sizes: list[int] = []
|
seg_page_sizes: list[int] = []
|
||||||
|
|
||||||
|
skip_group_ids = set(skip_kv_cache_group_ids)
|
||||||
for group in attn_groups_iter:
|
for group in attn_groups_iter:
|
||||||
spec = group.kv_cache_spec
|
spec = group.kv_cache_spec
|
||||||
if not isinstance(spec, AttentionSpec):
|
if not isinstance(spec, AttentionSpec):
|
||||||
continue
|
continue
|
||||||
if group.kv_cache_group_id >= len(kernel_block_sizes):
|
if group.kv_cache_group_id >= len(kernel_block_sizes):
|
||||||
continue
|
continue
|
||||||
|
if group.kv_cache_group_id in skip_group_ids:
|
||||||
|
continue
|
||||||
kernel_bs = kernel_block_sizes[group.kv_cache_group_id]
|
kernel_bs = kernel_block_sizes[group.kv_cache_group_id]
|
||||||
assert spec.block_size % kernel_bs == 0
|
assert spec.block_size % kernel_bs == 0
|
||||||
for layer_name in group.layer_names:
|
for layer_name in group.layer_names:
|
||||||
|
|||||||
Reference in New Issue
Block a user