[KV] Side-cache layout -- fold the PLE state into a GDN group, pack QSA rings
The packed pool strides every block by the widest group, and every group draws whole blocks. The PLE short-conv state now shares a GDN cache group, and the QSA raw-key rings are packed as equal slots inside shared blocks, so neither holds a whole block per request. VLLM_E2_SIDE_CACHE_LAYOUT (default 1) selects the old layout with 0. On its own the layout raises the Flash-Next pool from 806,792 to 853,994 tokens at the same budget. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -458,6 +458,16 @@ if TYPE_CHECKING:
|
||||
# for A/B. AOT: no. CAPTURE: no -- scheduler-side block bookkeeping only;
|
||||
# the worker sees the same block tables either way.
|
||||
VLLM_E2_MAMBA_RETAIN_CHECKPOINT: bool = True
|
||||
# v2.5 side-cache layout (Flash-Next PLE + QSA ring): stop charging each
|
||||
# tiny side cache a whole packed pool block per request. The TP-replicated
|
||||
# PLE short-conv state group is folded into a GDN group that has a spare
|
||||
# page on every PP stage, and the one-block-per-request QSA ring group packs
|
||||
# several requests' rings into one pool block. Strict "0"/"1", default ON;
|
||||
# "0" restores the old one-group-each layout for A/B. Read in
|
||||
# vllm/v1/core/kv_cache_utils.py (get_kv_cache_configs). AOT: yes, hashed --
|
||||
# it changes the ring views' block stride and which block table the PLE
|
||||
# state is read through, so ON and OFF must not share a compile cache.
|
||||
VLLM_E2_SIDE_CACHE_LAYOUT: bool = True
|
||||
# Opt-out for the fork startup guards in vllm/v1/e2_guards.py, read at
|
||||
# Qwen4ExpForCausalLM construction. AOT: no. CAPTURE: no -- it changes
|
||||
# nothing itself; it only permits a boot whose identity IS different
|
||||
@@ -2694,6 +2704,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT": lambda: _e2_strict_switch(
|
||||
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT", default=True
|
||||
),
|
||||
# Side-cache layout (v2.5; PLE state folded into a GDN group + QSA ring slot
|
||||
# packing), default ON. Strict "0"/"1"; read once per get_kv_cache_configs call.
|
||||
"VLLM_E2_SIDE_CACHE_LAYOUT": lambda: _e2_strict_switch(
|
||||
"VLLM_E2_SIDE_CACHE_LAYOUT", default=True
|
||||
),
|
||||
# Seconds between `topk-exact-counters` lines from each GPU worker
|
||||
# (0, the default, disables the reader entirely). The line reports the
|
||||
# medium-path exact-restart counters; see
|
||||
|
||||
@@ -13,6 +13,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
KVCacheBlock,
|
||||
)
|
||||
from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
CircularBufferManager,
|
||||
CrossAttentionManager,
|
||||
MambaManager,
|
||||
SingleTypeKVCacheManager,
|
||||
@@ -147,6 +148,17 @@ class KVCacheCoordinator(ABC):
|
||||
)
|
||||
for i, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups)
|
||||
)
|
||||
# v2.5 side-cache layout: a slot-packed ring group shares each pool
|
||||
# block between several requests (kv_cache_utils._choose_ring_slots).
|
||||
for kv_cache_group, manager in zip(
|
||||
self.kv_cache_config.kv_cache_groups, self.single_type_managers
|
||||
):
|
||||
if kv_cache_group.slots_per_block > 1:
|
||||
assert type(manager) is CircularBufferManager, (
|
||||
f"slot packing is only defined for the QSA ring, got "
|
||||
f"{type(manager).__name__}"
|
||||
)
|
||||
manager.enable_slot_packing(kv_cache_group.slots_per_block)
|
||||
# Match Mamba checkpoints to Eagle's attention replay boundary.
|
||||
if use_eagle:
|
||||
for manager in self.single_type_managers:
|
||||
|
||||
@@ -1121,7 +1121,19 @@ def get_max_concurrency_for_kv_cache_config(
|
||||
"""
|
||||
num_blocks_per_request = 0
|
||||
host_blocks_per_request = 0
|
||||
# v2.5 side-cache layout: a slot-packed ring group shares each pool block
|
||||
# between `slots_per_block` requests, so it costs cdiv(max_num_seqs, slots)
|
||||
# blocks for the whole pool instead of one block per request.
|
||||
shared_blocks = 0
|
||||
max_num_seqs = getattr(
|
||||
getattr(vllm_config, "scheduler_config", None), "max_num_seqs", 1
|
||||
)
|
||||
if not isinstance(max_num_seqs, int) or max_num_seqs < 1:
|
||||
max_num_seqs = 1
|
||||
for group in kv_cache_config.kv_cache_groups:
|
||||
if group.slots_per_block > 1 and not group.host_resident:
|
||||
shared_blocks += cdiv(max_num_seqs, group.slots_per_block)
|
||||
continue
|
||||
required = cdiv(
|
||||
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
|
||||
group.kv_cache_spec.page_size_bytes,
|
||||
@@ -1130,7 +1142,9 @@ def get_max_concurrency_for_kv_cache_config(
|
||||
host_blocks_per_request += required
|
||||
else:
|
||||
num_blocks_per_request += required
|
||||
limits = [kv_cache_config.num_blocks / num_blocks_per_request]
|
||||
limits = [
|
||||
max(kv_cache_config.num_blocks - shared_blocks, 0) / num_blocks_per_request
|
||||
]
|
||||
if host_blocks_per_request:
|
||||
assert kv_cache_config.hisparse_host_num_blocks is not None
|
||||
limits.append(
|
||||
@@ -1817,14 +1831,36 @@ def get_kv_cache_config_from_groups(
|
||||
elif group.layer_names:
|
||||
layers_by_spec[group_spec].extend(group.layer_names)
|
||||
|
||||
# v2.5 side-cache layout: a slot-packed group views every pool block as
|
||||
# `slots` equal slots, so its block stride is one slot and its view has
|
||||
# num_blocks * slots blocks. Pages of its layers must fit in one slot.
|
||||
slots = group.slots_per_block
|
||||
group_block_stride = interleaved_block_stride
|
||||
group_num_blocks = num_blocks
|
||||
if slots > 1:
|
||||
assert layout.is_block_outermost, (
|
||||
"Slot-packed KV cache groups need a block-outermost layout"
|
||||
)
|
||||
assert bytes_per_block % slots == 0, (bytes_per_block, slots)
|
||||
group_block_stride = bytes_per_block // slots
|
||||
group_num_blocks = num_blocks * slots
|
||||
group_bytes = sum(
|
||||
spec.page_size_bytes * len(names)
|
||||
for spec, names in layers_by_spec.items()
|
||||
)
|
||||
assert group_bytes <= group_block_stride, (
|
||||
f"slot-packed group needs {group_bytes} B per slot, slot is "
|
||||
f"{group_block_stride} B"
|
||||
)
|
||||
|
||||
byte_offset = 0
|
||||
for spec, layer_names in layers_by_spec.items():
|
||||
layer_stride, block_stride, _, _, _ = compute_layout_strides(
|
||||
spec,
|
||||
num_blocks,
|
||||
group_num_blocks,
|
||||
len(layer_names),
|
||||
layout,
|
||||
fixed_strides=(None, interleaved_block_stride, None, None, None),
|
||||
fixed_strides=(None, group_block_stride, None, None, None),
|
||||
)
|
||||
offset = (
|
||||
byte_offset
|
||||
@@ -1838,6 +1874,7 @@ def get_kv_cache_config_from_groups(
|
||||
layer_stride=layer_stride,
|
||||
block_stride=block_stride,
|
||||
offset=offset,
|
||||
slots_per_block=slots,
|
||||
)
|
||||
)
|
||||
byte_offset += len(layer_names) * spec.page_size_bytes
|
||||
@@ -2705,6 +2742,289 @@ def _project_kv_cache_groups_to_worker(
|
||||
return projected_groups
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# v2.5 side-cache layout (VLLM_E2_SIDE_CACHE_LAYOUT, default ON).
|
||||
#
|
||||
# The packed block-outermost layout strides every pool block by the widest
|
||||
# group, and every group draws whole blocks from the one pool. Flash-Next has
|
||||
# two side caches whose real state is tiny next to that stride (12.9 MB on the
|
||||
# binding PP stage): the TP-replicated PLE short-conv state (one layer, 2 + K
|
||||
# blocks per request in align mode, a 1.6 MB padded page) and the QSA raw-key
|
||||
# ring (one block per request, 16 KB for all of a stage's layers). Together
|
||||
# they hold 6 of the 103 blocks a 262K request needs. Two layout-only changes
|
||||
# remove that cost without changing any value a kernel reads:
|
||||
#
|
||||
# 1. PLE fold. The replicated Mamba group joins a sharded Mamba group with the
|
||||
# same scheduling parameters whose block has a spare page on every worker
|
||||
# (Flash-Next: a GDN group holds 6 of PP0's 7 pages), so the PLE state rides
|
||||
# in that group's blocks under the block table it would have had on its
|
||||
# own. No worker's stride changes, so the pool block count does not change.
|
||||
# 2. Ring slot packing. The ring group splits every pool block into S equal
|
||||
# slots and the ring manager hands out slots; a pool block goes back to the
|
||||
# pool when its last slot is released. The ring then costs cdiv(requests, S)
|
||||
# blocks in all instead of one per request.
|
||||
#
|
||||
# Off (=0) restores one group each, block for block.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Slot stride alignment for packed ring views (bytes) and the largest S tried.
|
||||
_RING_SLOT_ALIGN = 256
|
||||
_RING_MAX_SLOTS = 1024
|
||||
|
||||
|
||||
def _side_cache_layout_enabled(vllm_config: VllmConfig) -> bool:
|
||||
"""Whether the v2.5 side-cache layout may apply to this engine.
|
||||
|
||||
KV connectors (NIXL, offload, P/D) address groups and blocks themselves
|
||||
(the NIXL worker requires exactly one single-layer replicated group), and
|
||||
HiSparse has its own pool layout, so those engines keep the old layout.
|
||||
"""
|
||||
if not envs.VLLM_E2_SIDE_CACHE_LAYOUT:
|
||||
return False
|
||||
if getattr(vllm_config, "kv_transfer_config", None) is not None:
|
||||
return False
|
||||
attention_config = getattr(vllm_config, "attention_config", None)
|
||||
if getattr(attention_config, "hisparse_config", None) is not None:
|
||||
return False
|
||||
parallel_config = getattr(vllm_config, "parallel_config", None)
|
||||
if getattr(parallel_config, "decode_context_parallel_size", 1) not in (1, None):
|
||||
return False
|
||||
try:
|
||||
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||
except ValueError:
|
||||
# Unresolved layout: only reachable from callers that bypass the
|
||||
# engine core; nothing to pack without a block-outermost layout.
|
||||
return False
|
||||
return bool(layout.is_block_outermost)
|
||||
|
||||
|
||||
def _group_layer_specs(group: KVCacheGroupSpec) -> list[KVCacheSpec]:
|
||||
spec = group.kv_cache_spec
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs):
|
||||
return [spec.kv_cache_specs[name] for name in group.layer_names]
|
||||
return [spec] * len(group.layer_names)
|
||||
|
||||
|
||||
def _mamba_schedule_key(spec: MambaSpec) -> tuple:
|
||||
"""Everything the scheduler's MambaManager and the worker's align-mode
|
||||
bookkeeping read from a group's spec. Two Mamba groups with equal keys get
|
||||
identical block tables for every request."""
|
||||
return (
|
||||
spec.block_size,
|
||||
spec.page_size_bytes,
|
||||
spec.num_speculative_blocks,
|
||||
spec.num_prefill_checkpoint_blocks,
|
||||
spec.prefill_checkpoint_alignment,
|
||||
spec.mamba_cache_mode,
|
||||
)
|
||||
|
||||
|
||||
def _worker_strides(
|
||||
groups: list[KVCacheGroupSpec], worker_specs: list[dict[str, KVCacheSpec]]
|
||||
) -> list[int]:
|
||||
"""Pool block bytes of every worker for this global grouping."""
|
||||
strides = []
|
||||
for spec in worker_specs:
|
||||
projected = _project_kv_cache_groups_to_worker(groups, spec)
|
||||
if not any(g.layer_names for g in projected):
|
||||
strides.append(0)
|
||||
continue
|
||||
strides.append(_get_kv_cache_bytes_per_block(projected))
|
||||
return strides
|
||||
|
||||
|
||||
def _fold_replicated_mamba_groups(
|
||||
groups: list[KVCacheGroupSpec],
|
||||
worker_specs: list[dict[str, KVCacheSpec]],
|
||||
) -> tuple[list[KVCacheGroupSpec], list[str]]:
|
||||
"""Fold each TP-replicated Mamba group into a compatible sharded one.
|
||||
|
||||
A target holds only sharded Mamba layers with the same scheduling key and
|
||||
EAGLE flag, and the merged group must not widen the block of any worker
|
||||
(checked on every worker's projection, so PP stages are covered). Among
|
||||
valid targets the lowest index wins. Returns the new group list and one
|
||||
log note per replicated group.
|
||||
"""
|
||||
groups = list(groups)
|
||||
notes: list[str] = []
|
||||
base_strides = _worker_strides(groups, worker_specs)
|
||||
index = 0
|
||||
while index < len(groups):
|
||||
source = groups[index]
|
||||
layer_specs = _group_layer_specs(source)
|
||||
if not (
|
||||
layer_specs
|
||||
and all(
|
||||
isinstance(spec, MambaSpec) and spec.tp_replicated
|
||||
for spec in layer_specs
|
||||
)
|
||||
and len({_mamba_schedule_key(spec) for spec in layer_specs}) == 1
|
||||
):
|
||||
index += 1
|
||||
continue
|
||||
key = _mamba_schedule_key(cast(MambaSpec, layer_specs[0]))
|
||||
chosen = None
|
||||
for target_index, target in enumerate(groups):
|
||||
if target_index == index or target.is_eagle_group != source.is_eagle_group:
|
||||
continue
|
||||
target_specs = _group_layer_specs(target)
|
||||
if not target_specs or not all(
|
||||
isinstance(spec, MambaSpec)
|
||||
and not spec.tp_replicated
|
||||
and _mamba_schedule_key(spec) == key
|
||||
for spec in target_specs
|
||||
):
|
||||
continue
|
||||
merged_specs: dict[str, KVCacheSpec] = dict(
|
||||
zip(target.layer_names, target_specs)
|
||||
)
|
||||
merged_specs.update(zip(source.layer_names, layer_specs))
|
||||
# Built directly: is_uniform_type rejects mixed tp_replicated on
|
||||
# purpose (NIXL), and connectors never reach this path.
|
||||
merged = KVCacheGroupSpec(
|
||||
target.layer_names + source.layer_names,
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=target_specs[0].block_size,
|
||||
kv_cache_specs=merged_specs,
|
||||
),
|
||||
is_eagle_group=target.is_eagle_group,
|
||||
)
|
||||
trial = [
|
||||
merged if i == target_index else g
|
||||
for i, g in enumerate(groups)
|
||||
if i != index
|
||||
]
|
||||
strides = _worker_strides(trial, worker_specs)
|
||||
if all(new <= old for new, old in zip(strides, base_strides)):
|
||||
chosen = (target_index, merged, trial)
|
||||
break
|
||||
if chosen is None:
|
||||
notes.append(
|
||||
f"replicated Mamba group {index} ({len(source.layer_names)} "
|
||||
"layer(s)) kept separate: no sharded group has a spare page on "
|
||||
"every worker"
|
||||
)
|
||||
index += 1
|
||||
continue
|
||||
target_index, merged, trial = chosen
|
||||
notes.append(
|
||||
f"folded replicated Mamba group {index} "
|
||||
f"({', '.join(source.layer_names)}) into Mamba group "
|
||||
f"{target_index} (now {len(merged.layer_names)} layers); per-worker "
|
||||
f"block bytes unchanged {sorted(set(base_strides))}"
|
||||
)
|
||||
groups = trial
|
||||
# The group after the folded one now sits at `index`.
|
||||
return groups, notes
|
||||
|
||||
|
||||
def _is_ring_group(group: KVCacheGroupSpec) -> bool:
|
||||
spec = group.kv_cache_spec
|
||||
layer_specs = (
|
||||
list(spec.kv_cache_specs.values())
|
||||
if isinstance(spec, UniformTypeKVCacheSpecs)
|
||||
else [spec]
|
||||
)
|
||||
return bool(layer_specs) and all(
|
||||
type(layer_spec) is CircularBufferSpec for layer_spec in layer_specs
|
||||
)
|
||||
|
||||
|
||||
def _choose_ring_slots(
|
||||
projected_groups_per_worker: list[list[KVCacheGroupSpec]],
|
||||
) -> dict[int, int]:
|
||||
"""Slots per pool block for every ring group (group index -> S >= 2).
|
||||
|
||||
S must cut every worker's block stride into equal slots that are
|
||||
`_RING_SLOT_ALIGN`-aligned and hold that worker's ring pages; the largest
|
||||
such S up to `_RING_MAX_SLOTS` is taken. Block ids are global, so S is the
|
||||
same on every worker. A ring group with no valid S keeps one pool block
|
||||
per request.
|
||||
"""
|
||||
chosen: dict[int, int] = {}
|
||||
if not projected_groups_per_worker or not projected_groups_per_worker[0]:
|
||||
return chosen
|
||||
num_groups = len(projected_groups_per_worker[0])
|
||||
strides = [
|
||||
_get_kv_cache_bytes_per_block(groups)
|
||||
if any(g.layer_names for g in groups)
|
||||
else 0
|
||||
for groups in projected_groups_per_worker
|
||||
]
|
||||
for group_index in range(num_groups):
|
||||
if not _is_ring_group(projected_groups_per_worker[0][group_index]):
|
||||
continue
|
||||
needs = []
|
||||
for groups, stride in zip(projected_groups_per_worker, strides):
|
||||
group = groups[group_index]
|
||||
if not group.layer_names:
|
||||
continue
|
||||
ring_bytes = sum(spec.page_size_bytes for spec in _group_layer_specs(group))
|
||||
needs.append((stride, ring_bytes))
|
||||
if not needs:
|
||||
continue
|
||||
for slots in range(_RING_MAX_SLOTS, 1, -1):
|
||||
if all(
|
||||
stride % slots == 0
|
||||
and (stride // slots) % _RING_SLOT_ALIGN == 0
|
||||
and stride // slots >= ring_bytes
|
||||
for stride, ring_bytes in needs
|
||||
):
|
||||
chosen[group_index] = slots
|
||||
break
|
||||
return chosen
|
||||
|
||||
|
||||
def _apply_side_cache_layout(
|
||||
vllm_config: VllmConfig,
|
||||
global_kv_cache_groups: list[KVCacheGroupSpec],
|
||||
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
||||
) -> tuple[list[KVCacheGroupSpec], list[list[KVCacheGroupSpec]]]:
|
||||
"""Fold side Mamba groups, project per worker, then pick ring slots.
|
||||
|
||||
Returns the (possibly folded) global groups and every worker's projection,
|
||||
with `slots_per_block` set on packed ring groups.
|
||||
"""
|
||||
relevant = any(
|
||||
(isinstance(spec, MambaSpec) and spec.tp_replicated)
|
||||
or type(spec) is CircularBufferSpec
|
||||
for worker_spec in kv_cache_specs
|
||||
for spec in worker_spec.values()
|
||||
)
|
||||
enabled = relevant and _side_cache_layout_enabled(vllm_config)
|
||||
notes: list[str] = []
|
||||
if enabled:
|
||||
global_kv_cache_groups, notes = _fold_replicated_mamba_groups(
|
||||
global_kv_cache_groups, kv_cache_specs
|
||||
)
|
||||
projected = [
|
||||
_project_kv_cache_groups_to_worker(global_kv_cache_groups, worker_spec)
|
||||
for worker_spec in kv_cache_specs
|
||||
]
|
||||
if enabled and all(
|
||||
_glm5_next_tensor_layout(groups) is None for groups in projected if groups
|
||||
):
|
||||
for group_index, slots in _choose_ring_slots(projected).items():
|
||||
for groups in projected:
|
||||
groups[group_index].slots_per_block = slots
|
||||
notes.append(
|
||||
f"QSA ring group {group_index} packs {slots} requests per pool block"
|
||||
)
|
||||
if relevant and envs.VLLM_E2_SIDE_CACHE_LAYOUT and not enabled:
|
||||
notes.append(
|
||||
"not applicable to this engine (KV connector, HiSparse, DCP or a "
|
||||
"layer-outermost layout)"
|
||||
)
|
||||
if relevant:
|
||||
# The A/B evidence line: boot logs must show which arm ran.
|
||||
logger.info(
|
||||
"Side-cache layout: %s (VLLM_E2_SIDE_CACHE_LAYOUT)%s",
|
||||
"on" if enabled else "off",
|
||||
"".join(f"; {note}" for note in notes),
|
||||
)
|
||||
return global_kv_cache_groups, projected
|
||||
|
||||
|
||||
def get_kv_cache_configs(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_specs: list[dict[str, KVCacheSpec]],
|
||||
@@ -2774,10 +3094,11 @@ def get_kv_cache_configs(
|
||||
# If original_max_model_len was -1, automatically
|
||||
# determine the maximum model length that fits in available GPU memory.
|
||||
# We use per-worker projected groups to account for PP sharding.
|
||||
projected_groups_per_worker = [
|
||||
_project_kv_cache_groups_to_worker(global_kv_cache_groups, worker_spec)
|
||||
for worker_spec in kv_cache_specs
|
||||
]
|
||||
# v2.5: the side-cache layout (PLE fold, ring slots) is applied here, where
|
||||
# every worker's layers are known, so no PP stage's block can widen.
|
||||
global_kv_cache_groups, projected_groups_per_worker = _apply_side_cache_layout(
|
||||
vllm_config, global_kv_cache_groups, kv_cache_specs
|
||||
)
|
||||
|
||||
# If `num_gpu_blocks_override` is set, the cache size that will actually
|
||||
# be allocated is decoupled from the profiled `available_memory`:
|
||||
|
||||
@@ -1619,6 +1619,10 @@ class Scheduler(SchedulerInterface):
|
||||
pending = set(block_ids)
|
||||
owners: dict[int, set[str]] = {}
|
||||
for mgr in self.kv_cache_manager.coordinator.single_type_managers:
|
||||
if getattr(mgr, "slot_pool", None) is not None:
|
||||
# v2.5 slot-packed ring: its block ids are slot ids, not pool
|
||||
# block ids, and its pages are never zeroed.
|
||||
continue
|
||||
for req_id, blocks in mgr.req_to_blocks.items():
|
||||
for blk in blocks:
|
||||
if blk.block_id in pending:
|
||||
|
||||
@@ -1236,18 +1236,126 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
|
||||
return 0
|
||||
|
||||
|
||||
class _RingSlotPage:
|
||||
"""One pool block shared by up to ``slots`` requests' rings."""
|
||||
|
||||
__slots__ = ("block", "free_slots", "live")
|
||||
|
||||
def __init__(self, block: KVCacheBlock, slots: int) -> None:
|
||||
self.block = block
|
||||
# Highest first, so pop() hands out the lowest free slot.
|
||||
self.free_slots = list(range(slots - 1, -1, -1))
|
||||
self.live = 0
|
||||
|
||||
|
||||
class RingSlotPool:
|
||||
"""v2.5 side-cache layout: ring blocks as slots of shared pool blocks.
|
||||
|
||||
The QSA ring needs one block per request for the request's lifetime, but
|
||||
its pages for every layer of a stage are ~16 KB against a 12.9 MB pool
|
||||
block. This pool takes whole blocks from the main ``BlockPool`` and cuts
|
||||
each into ``slots_per_block`` slots; a slot's block id is
|
||||
``pool_block_id * slots_per_block + slot``, which is exactly the index of
|
||||
the slot in the worker's ring view (``KVCacheTensor.slots_per_block``).
|
||||
|
||||
The slot blocks it hands out carry ``pool=self``, so every existing free
|
||||
path -- ``SingleTypeKVCacheManager.free``, the coordinator's deferred
|
||||
``pop_blocks_for_free`` and the scheduler's fenced drain, which all call
|
||||
``BlockPool.free_blocks`` -- routes them back here, at the time the old
|
||||
per-request block would have gone back (the deferred free is kept). A pool
|
||||
block returns to the main pool when its last slot is freed. Rings are never
|
||||
prefix-cached, so a slot block carries no hash.
|
||||
"""
|
||||
|
||||
def __init__(self, block_pool: BlockPool, slots_per_block: int) -> None:
|
||||
assert slots_per_block > 1
|
||||
self.block_pool = block_pool
|
||||
self.slots_per_block = slots_per_block
|
||||
# Pool block id -> page, in claim order (the oldest page fills first).
|
||||
self._pages: dict[int, _RingSlotPage] = {}
|
||||
|
||||
@property
|
||||
def num_pages(self) -> int:
|
||||
"""Pool blocks currently held for rings."""
|
||||
return len(self._pages)
|
||||
|
||||
@property
|
||||
def num_live_slots(self) -> int:
|
||||
return sum(page.live for page in self._pages.values())
|
||||
|
||||
def has_free_slot(self) -> bool:
|
||||
return any(page.free_slots for page in self._pages.values())
|
||||
|
||||
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
|
||||
out: list[KVCacheBlock] = []
|
||||
for _ in range(num_blocks):
|
||||
page = next((p for p in self._pages.values() if p.free_slots), None)
|
||||
if page is None:
|
||||
(pool_block,) = self.block_pool.get_new_blocks(1)
|
||||
page = _RingSlotPage(pool_block, self.slots_per_block)
|
||||
self._pages[pool_block.block_id] = page
|
||||
slot = page.free_slots.pop()
|
||||
page.live += 1
|
||||
out.append(
|
||||
KVCacheBlock(
|
||||
page.block.block_id * self.slots_per_block + slot,
|
||||
ref_cnt=1,
|
||||
pool=self, # type: ignore[arg-type]
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
def free_blocks(self, ordered_blocks) -> None:
|
||||
released: list[KVCacheBlock] = []
|
||||
for block in ordered_blocks:
|
||||
assert block.pool is self, "not a ring slot of this pool"
|
||||
block.ref_cnt -= 1
|
||||
assert block.ref_cnt == 0, "ring slots have a single owner"
|
||||
page_id, slot = divmod(block.block_id, self.slots_per_block)
|
||||
page = self._pages[page_id]
|
||||
assert slot not in page.free_slots, f"slot {block.block_id} freed twice"
|
||||
page.free_slots.append(slot)
|
||||
page.free_slots.sort(reverse=True)
|
||||
page.live -= 1
|
||||
if page.live == 0:
|
||||
del self._pages[page_id]
|
||||
released.append(page.block)
|
||||
if released:
|
||||
self.block_pool.free_blocks(released)
|
||||
|
||||
|
||||
class CircularBufferManager(FullAttentionManager):
|
||||
"""Claims the ring's single block per request; prefix caching disabled."""
|
||||
|
||||
supports_fine_grained_hash_lookup: ClassVar[bool] = False
|
||||
|
||||
# v2.5 side-cache layout: set by the coordinator for a slot-packed group.
|
||||
_slot_pool: RingSlotPool | None = None
|
||||
|
||||
def enable_slot_packing(self, slots_per_block: int) -> None:
|
||||
"""Claim ring slots of shared pool blocks instead of whole blocks."""
|
||||
assert not self.req_to_blocks, "enable slot packing before any claim"
|
||||
self._slot_pool = RingSlotPool(self.block_pool, slots_per_block)
|
||||
|
||||
@property
|
||||
def slot_pool(self) -> RingSlotPool | None:
|
||||
return self._slot_pool
|
||||
|
||||
if self._slot_pool is not None:
|
||||
new_blocks = self._slot_pool.get_new_blocks(1)
|
||||
else:
|
||||
new_blocks = self.block_pool.get_new_blocks(1)
|
||||
|
||||
def _claim_ring_block(
|
||||
self, request_id: str, record_for_zeroing: bool = True
|
||||
) -> list[KVCacheBlock]:
|
||||
req_blocks = self.req_to_blocks[request_id]
|
||||
if req_blocks:
|
||||
return []
|
||||
new_blocks = self.block_pool.get_new_blocks(1)
|
||||
if self._slot_pool is not None:
|
||||
new_blocks = self._slot_pool.get_new_blocks(1)
|
||||
else:
|
||||
new_blocks = self.block_pool.get_new_blocks(1)
|
||||
req_blocks.extend(new_blocks)
|
||||
if self._record_new_block_ids and record_for_zeroing:
|
||||
self.new_block_ids.extend(block.block_id for block in new_blocks)
|
||||
@@ -1293,7 +1401,12 @@ class CircularBufferManager(FullAttentionManager):
|
||||
apply_admission_cap: bool = False,
|
||||
prefill_end: int = 0,
|
||||
) -> int:
|
||||
return 0 if self.req_to_blocks.get(request_id) else 1
|
||||
if self.req_to_blocks.get(request_id):
|
||||
return 0
|
||||
# A free slot in a page the ring already holds costs no pool block.
|
||||
if self._slot_pool is not None and self._slot_pool.has_free_slot():
|
||||
return 0
|
||||
return 1
|
||||
|
||||
def allocate_new_blocks(
|
||||
self, request_id: str, num_tokens: int, num_tokens_main_model: int
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1431,6 +1431,11 @@ class KVCacheTensor:
|
||||
block_stride: int
|
||||
offset: int = 0 # byte offset of layers[0]'s block 0
|
||||
host_resident: bool = False
|
||||
# v2.5 side-cache layout: a slot-packed ring tensor views every pool block
|
||||
# as ``slots_per_block`` equal slots (``block_stride`` is then one slot), so
|
||||
# its block ids are ``pool_block * slots_per_block + slot``. 1 = one view
|
||||
# block per pool block, the ordinary case.
|
||||
slots_per_block: int = 1
|
||||
|
||||
|
||||
class KVCacheGroupRole(str, Enum):
|
||||
@@ -1458,6 +1463,10 @@ class KVCacheGroupSpec:
|
||||
# source group after a connector load.
|
||||
enable_kv_transfer: bool = True
|
||||
role: KVCacheGroupRole = KVCacheGroupRole.DEFAULT
|
||||
# v2.5 side-cache layout: > 1 when this one-block-per-request ring group
|
||||
# shares each pool block between that many requests (see
|
||||
# ``CircularBufferManager.enable_slot_packing``); 1 = one pool block each.
|
||||
slots_per_block: int = 1
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -1545,6 +1554,18 @@ class KVCacheConfig:
|
||||
)
|
||||
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
|
||||
|
||||
@property
|
||||
def slot_packed_group_ids(self) -> tuple[int, ...]:
|
||||
"""Groups whose blocks are slots of shared pool blocks (v2.5 side-cache
|
||||
layout). Their views are not one block per pool block, so whole-block
|
||||
zeroing must skip them; their bytes lie inside pool blocks the ring
|
||||
manager owns and that no other group writes."""
|
||||
return tuple(
|
||||
group_id
|
||||
for group_id, group in enumerate(self.kv_cache_groups)
|
||||
if group.slots_per_block > 1
|
||||
)
|
||||
|
||||
@property
|
||||
def host_group_ids(self) -> tuple[int, ...]:
|
||||
return tuple(
|
||||
@@ -1556,7 +1577,7 @@ class KVCacheConfig:
|
||||
def num_blocks_of(self, tensor: KVCacheTensor) -> int:
|
||||
"""Number of blocks addressable by the pool backing ``tensor``."""
|
||||
if not tensor.host_resident:
|
||||
return self.num_blocks
|
||||
return self.num_blocks * tensor.slots_per_block
|
||||
assert self.hisparse_host_num_blocks is not None
|
||||
return self.hisparse_host_num_blocks
|
||||
|
||||
|
||||
@@ -866,6 +866,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
kernel_block_sizes=self.kernel_block_sizes,
|
||||
static_forward_context=self.compilation_config.static_forward_context,
|
||||
num_blocks=self.kv_cache_config.num_blocks,
|
||||
skip_kv_cache_group_ids=self.kv_cache_config.slot_packed_group_ids,
|
||||
)
|
||||
|
||||
@torch.inference_mode()
|
||||
|
||||
@@ -1149,6 +1149,7 @@ class GPUModelRunner(
|
||||
runner_only_attn_layers=self.runner_only_attn_layers,
|
||||
static_forward_context=self.compilation_config.static_forward_context,
|
||||
num_blocks=self.kv_cache_config.num_blocks,
|
||||
skip_kv_cache_group_ids=self.kv_cache_config.slot_packed_group_ids,
|
||||
)
|
||||
|
||||
def _zero_block_ids(self, block_ids: list[int]) -> None:
|
||||
|
||||
@@ -138,6 +138,7 @@ class KVBlockZeroer:
|
||||
static_forward_context: dict[str, Any],
|
||||
num_blocks: int,
|
||||
runner_only_attn_layers: set[str] | None = None,
|
||||
skip_kv_cache_group_ids: Iterable[int] = (),
|
||||
) -> None:
|
||||
"""Precompute the absolute-address table for the Triton zeroing kernel.
|
||||
|
||||
@@ -156,6 +157,13 @@ class KVBlockZeroer:
|
||||
physical block stride and zeroed page span remain independent.
|
||||
|
||||
Only AttentionSpec layers are processed; Mamba layers are skipped.
|
||||
|
||||
``skip_kv_cache_group_ids`` (v2.5 side-cache layout): slot-packed ring
|
||||
groups, whose views hold several slots per pool block. Their claims
|
||||
never enter whole-block zeroing (the ring-tag path clears their tag words instead),
|
||||
and with one view block per pool block their pages sat inside the
|
||||
first attention page, so skipping them leaves the zeroed bytes of
|
||||
every block unchanged.
|
||||
"""
|
||||
self.device = device
|
||||
# Bounds-guard instrumentation: pool bound for the guard in _zero_kv_blocks_kernel.
|
||||
@@ -174,12 +182,15 @@ class KVBlockZeroer:
|
||||
seg_block_strides: list[int] = []
|
||||
seg_page_sizes: list[int] = []
|
||||
|
||||
skip_group_ids = set(skip_kv_cache_group_ids)
|
||||
for group in attn_groups_iter:
|
||||
spec = group.kv_cache_spec
|
||||
if not isinstance(spec, AttentionSpec):
|
||||
continue
|
||||
if group.kv_cache_group_id >= len(kernel_block_sizes):
|
||||
continue
|
||||
if group.kv_cache_group_id in skip_group_ids:
|
||||
continue
|
||||
kernel_bs = kernel_block_sizes[group.kv_cache_group_id]
|
||||
assert spec.block_size % kernel_bs == 0
|
||||
for layer_name in group.layer_names:
|
||||
|
||||
Reference in New Issue
Block a user