[KV] Side-cache layout -- fold the PLE state into a GDN group, pack QSA rings

The packed pool strides every block by the widest group, and every group draws whole blocks. The PLE short-conv
state now shares a GDN cache group, and the QSA raw-key rings are packed as equal slots inside shared blocks, so
neither holds a whole block per request. VLLM_E2_SIDE_CACHE_LAYOUT (default 1) selects the old layout with 0.
On its own the layout raises the Flash-Next pool from 806,792 to 853,994 tokens at the same budget.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 20:15:11 +07:00
co-authored by Hinata
parent 21fb7de7bf
commit 72485816d9
10 changed files with 3513 additions and 10 deletions
+15
View File
@@ -458,6 +458,16 @@ if TYPE_CHECKING:
# for A/B. AOT: no. CAPTURE: no -- scheduler-side block bookkeeping only;
# the worker sees the same block tables either way.
VLLM_E2_MAMBA_RETAIN_CHECKPOINT: bool = True
# v2.5 side-cache layout (Flash-Next PLE + QSA ring): stop charging each
# tiny side cache a whole packed pool block per request. The TP-replicated
# PLE short-conv state group is folded into a GDN group that has a spare
# page on every PP stage, and the one-block-per-request QSA ring group packs
# several requests' rings into one pool block. Strict "0"/"1", default ON;
# "0" restores the old one-group-each layout for A/B. Read in
# vllm/v1/core/kv_cache_utils.py (get_kv_cache_configs). AOT: yes, hashed --
# it changes the ring views' block stride and which block table the PLE
# state is read through, so ON and OFF must not share a compile cache.
VLLM_E2_SIDE_CACHE_LAYOUT: bool = True
# Opt-out for the fork startup guards in vllm/v1/e2_guards.py, read at
# Qwen4ExpForCausalLM construction. AOT: no. CAPTURE: no -- it changes
# nothing itself; it only permits a boot whose identity IS different
@@ -2694,6 +2704,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT": lambda: _e2_strict_switch(
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT", default=True
),
# Side-cache layout (v2.5; PLE state folded into a GDN group + QSA ring slot
# packing), default ON. Strict "0"/"1"; read once per get_kv_cache_configs call.
"VLLM_E2_SIDE_CACHE_LAYOUT": lambda: _e2_strict_switch(
"VLLM_E2_SIDE_CACHE_LAYOUT", default=True
),
# Seconds between `topk-exact-counters` lines from each GPU worker
# (0, the default, disables the reader entirely). The line reports the
# medium-path exact-restart counters; see
+12
View File
@@ -13,6 +13,7 @@ from vllm.v1.core.kv_cache_utils import (
KVCacheBlock,
)
from vllm.v1.core.single_type_kv_cache_manager import (
CircularBufferManager,
CrossAttentionManager,
MambaManager,
SingleTypeKVCacheManager,
@@ -147,6 +148,17 @@ class KVCacheCoordinator(ABC):
)
for i, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups)
)
# v2.5 side-cache layout: a slot-packed ring group shares each pool
# block between several requests (kv_cache_utils._choose_ring_slots).
for kv_cache_group, manager in zip(
self.kv_cache_config.kv_cache_groups, self.single_type_managers
):
if kv_cache_group.slots_per_block > 1:
assert type(manager) is CircularBufferManager, (
f"slot packing is only defined for the QSA ring, got "
f"{type(manager).__name__}"
)
manager.enable_slot_packing(kv_cache_group.slots_per_block)
# Match Mamba checkpoints to Eagle's attention replay boundary.
if use_eagle:
for manager in self.single_type_managers:
+328 -7
View File
@@ -1121,7 +1121,19 @@ def get_max_concurrency_for_kv_cache_config(
"""
num_blocks_per_request = 0
host_blocks_per_request = 0
# v2.5 side-cache layout: a slot-packed ring group shares each pool block
# between `slots_per_block` requests, so it costs cdiv(max_num_seqs, slots)
# blocks for the whole pool instead of one block per request.
shared_blocks = 0
max_num_seqs = getattr(
getattr(vllm_config, "scheduler_config", None), "max_num_seqs", 1
)
if not isinstance(max_num_seqs, int) or max_num_seqs < 1:
max_num_seqs = 1
for group in kv_cache_config.kv_cache_groups:
if group.slots_per_block > 1 and not group.host_resident:
shared_blocks += cdiv(max_num_seqs, group.slots_per_block)
continue
required = cdiv(
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
group.kv_cache_spec.page_size_bytes,
@@ -1130,7 +1142,9 @@ def get_max_concurrency_for_kv_cache_config(
host_blocks_per_request += required
else:
num_blocks_per_request += required
limits = [kv_cache_config.num_blocks / num_blocks_per_request]
limits = [
max(kv_cache_config.num_blocks - shared_blocks, 0) / num_blocks_per_request
]
if host_blocks_per_request:
assert kv_cache_config.hisparse_host_num_blocks is not None
limits.append(
@@ -1817,14 +1831,36 @@ def get_kv_cache_config_from_groups(
elif group.layer_names:
layers_by_spec[group_spec].extend(group.layer_names)
# v2.5 side-cache layout: a slot-packed group views every pool block as
# `slots` equal slots, so its block stride is one slot and its view has
# num_blocks * slots blocks. Pages of its layers must fit in one slot.
slots = group.slots_per_block
group_block_stride = interleaved_block_stride
group_num_blocks = num_blocks
if slots > 1:
assert layout.is_block_outermost, (
"Slot-packed KV cache groups need a block-outermost layout"
)
assert bytes_per_block % slots == 0, (bytes_per_block, slots)
group_block_stride = bytes_per_block // slots
group_num_blocks = num_blocks * slots
group_bytes = sum(
spec.page_size_bytes * len(names)
for spec, names in layers_by_spec.items()
)
assert group_bytes <= group_block_stride, (
f"slot-packed group needs {group_bytes} B per slot, slot is "
f"{group_block_stride} B"
)
byte_offset = 0
for spec, layer_names in layers_by_spec.items():
layer_stride, block_stride, _, _, _ = compute_layout_strides(
spec,
num_blocks,
group_num_blocks,
len(layer_names),
layout,
fixed_strides=(None, interleaved_block_stride, None, None, None),
fixed_strides=(None, group_block_stride, None, None, None),
)
offset = (
byte_offset
@@ -1838,6 +1874,7 @@ def get_kv_cache_config_from_groups(
layer_stride=layer_stride,
block_stride=block_stride,
offset=offset,
slots_per_block=slots,
)
)
byte_offset += len(layer_names) * spec.page_size_bytes
@@ -2705,6 +2742,289 @@ def _project_kv_cache_groups_to_worker(
return projected_groups
# ---------------------------------------------------------------------------
# v2.5 side-cache layout (VLLM_E2_SIDE_CACHE_LAYOUT, default ON).
#
# The packed block-outermost layout strides every pool block by the widest
# group, and every group draws whole blocks from the one pool. Flash-Next has
# two side caches whose real state is tiny next to that stride (12.9 MB on the
# binding PP stage): the TP-replicated PLE short-conv state (one layer, 2 + K
# blocks per request in align mode, a 1.6 MB padded page) and the QSA raw-key
# ring (one block per request, 16 KB for all of a stage's layers). Together
# they hold 6 of the 103 blocks a 262K request needs. Two layout-only changes
# remove that cost without changing any value a kernel reads:
#
# 1. PLE fold. The replicated Mamba group joins a sharded Mamba group with the
# same scheduling parameters whose block has a spare page on every worker
# (Flash-Next: a GDN group holds 6 of PP0's 7 pages), so the PLE state rides
# in that group's blocks under the block table it would have had on its
# own. No worker's stride changes, so the pool block count does not change.
# 2. Ring slot packing. The ring group splits every pool block into S equal
# slots and the ring manager hands out slots; a pool block goes back to the
# pool when its last slot is released. The ring then costs cdiv(requests, S)
# blocks in all instead of one per request.
#
# Off (=0) restores one group each, block for block.
# ---------------------------------------------------------------------------
# Slot stride alignment for packed ring views (bytes) and the largest S tried.
_RING_SLOT_ALIGN = 256
_RING_MAX_SLOTS = 1024
def _side_cache_layout_enabled(vllm_config: VllmConfig) -> bool:
"""Whether the v2.5 side-cache layout may apply to this engine.
KV connectors (NIXL, offload, P/D) address groups and blocks themselves
(the NIXL worker requires exactly one single-layer replicated group), and
HiSparse has its own pool layout, so those engines keep the old layout.
"""
if not envs.VLLM_E2_SIDE_CACHE_LAYOUT:
return False
if getattr(vllm_config, "kv_transfer_config", None) is not None:
return False
attention_config = getattr(vllm_config, "attention_config", None)
if getattr(attention_config, "hisparse_config", None) is not None:
return False
parallel_config = getattr(vllm_config, "parallel_config", None)
if getattr(parallel_config, "decode_context_parallel_size", 1) not in (1, None):
return False
try:
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
except ValueError:
# Unresolved layout: only reachable from callers that bypass the
# engine core; nothing to pack without a block-outermost layout.
return False
return bool(layout.is_block_outermost)
def _group_layer_specs(group: KVCacheGroupSpec) -> list[KVCacheSpec]:
spec = group.kv_cache_spec
if isinstance(spec, UniformTypeKVCacheSpecs):
return [spec.kv_cache_specs[name] for name in group.layer_names]
return [spec] * len(group.layer_names)
def _mamba_schedule_key(spec: MambaSpec) -> tuple:
"""Everything the scheduler's MambaManager and the worker's align-mode
bookkeeping read from a group's spec. Two Mamba groups with equal keys get
identical block tables for every request."""
return (
spec.block_size,
spec.page_size_bytes,
spec.num_speculative_blocks,
spec.num_prefill_checkpoint_blocks,
spec.prefill_checkpoint_alignment,
spec.mamba_cache_mode,
)
def _worker_strides(
groups: list[KVCacheGroupSpec], worker_specs: list[dict[str, KVCacheSpec]]
) -> list[int]:
"""Pool block bytes of every worker for this global grouping."""
strides = []
for spec in worker_specs:
projected = _project_kv_cache_groups_to_worker(groups, spec)
if not any(g.layer_names for g in projected):
strides.append(0)
continue
strides.append(_get_kv_cache_bytes_per_block(projected))
return strides
def _fold_replicated_mamba_groups(
groups: list[KVCacheGroupSpec],
worker_specs: list[dict[str, KVCacheSpec]],
) -> tuple[list[KVCacheGroupSpec], list[str]]:
"""Fold each TP-replicated Mamba group into a compatible sharded one.
A target holds only sharded Mamba layers with the same scheduling key and
EAGLE flag, and the merged group must not widen the block of any worker
(checked on every worker's projection, so PP stages are covered). Among
valid targets the lowest index wins. Returns the new group list and one
log note per replicated group.
"""
groups = list(groups)
notes: list[str] = []
base_strides = _worker_strides(groups, worker_specs)
index = 0
while index < len(groups):
source = groups[index]
layer_specs = _group_layer_specs(source)
if not (
layer_specs
and all(
isinstance(spec, MambaSpec) and spec.tp_replicated
for spec in layer_specs
)
and len({_mamba_schedule_key(spec) for spec in layer_specs}) == 1
):
index += 1
continue
key = _mamba_schedule_key(cast(MambaSpec, layer_specs[0]))
chosen = None
for target_index, target in enumerate(groups):
if target_index == index or target.is_eagle_group != source.is_eagle_group:
continue
target_specs = _group_layer_specs(target)
if not target_specs or not all(
isinstance(spec, MambaSpec)
and not spec.tp_replicated
and _mamba_schedule_key(spec) == key
for spec in target_specs
):
continue
merged_specs: dict[str, KVCacheSpec] = dict(
zip(target.layer_names, target_specs)
)
merged_specs.update(zip(source.layer_names, layer_specs))
# Built directly: is_uniform_type rejects mixed tp_replicated on
# purpose (NIXL), and connectors never reach this path.
merged = KVCacheGroupSpec(
target.layer_names + source.layer_names,
UniformTypeKVCacheSpecs(
block_size=target_specs[0].block_size,
kv_cache_specs=merged_specs,
),
is_eagle_group=target.is_eagle_group,
)
trial = [
merged if i == target_index else g
for i, g in enumerate(groups)
if i != index
]
strides = _worker_strides(trial, worker_specs)
if all(new <= old for new, old in zip(strides, base_strides)):
chosen = (target_index, merged, trial)
break
if chosen is None:
notes.append(
f"replicated Mamba group {index} ({len(source.layer_names)} "
"layer(s)) kept separate: no sharded group has a spare page on "
"every worker"
)
index += 1
continue
target_index, merged, trial = chosen
notes.append(
f"folded replicated Mamba group {index} "
f"({', '.join(source.layer_names)}) into Mamba group "
f"{target_index} (now {len(merged.layer_names)} layers); per-worker "
f"block bytes unchanged {sorted(set(base_strides))}"
)
groups = trial
# The group after the folded one now sits at `index`.
return groups, notes
def _is_ring_group(group: KVCacheGroupSpec) -> bool:
spec = group.kv_cache_spec
layer_specs = (
list(spec.kv_cache_specs.values())
if isinstance(spec, UniformTypeKVCacheSpecs)
else [spec]
)
return bool(layer_specs) and all(
type(layer_spec) is CircularBufferSpec for layer_spec in layer_specs
)
def _choose_ring_slots(
projected_groups_per_worker: list[list[KVCacheGroupSpec]],
) -> dict[int, int]:
"""Slots per pool block for every ring group (group index -> S >= 2).
S must cut every worker's block stride into equal slots that are
`_RING_SLOT_ALIGN`-aligned and hold that worker's ring pages; the largest
such S up to `_RING_MAX_SLOTS` is taken. Block ids are global, so S is the
same on every worker. A ring group with no valid S keeps one pool block
per request.
"""
chosen: dict[int, int] = {}
if not projected_groups_per_worker or not projected_groups_per_worker[0]:
return chosen
num_groups = len(projected_groups_per_worker[0])
strides = [
_get_kv_cache_bytes_per_block(groups)
if any(g.layer_names for g in groups)
else 0
for groups in projected_groups_per_worker
]
for group_index in range(num_groups):
if not _is_ring_group(projected_groups_per_worker[0][group_index]):
continue
needs = []
for groups, stride in zip(projected_groups_per_worker, strides):
group = groups[group_index]
if not group.layer_names:
continue
ring_bytes = sum(spec.page_size_bytes for spec in _group_layer_specs(group))
needs.append((stride, ring_bytes))
if not needs:
continue
for slots in range(_RING_MAX_SLOTS, 1, -1):
if all(
stride % slots == 0
and (stride // slots) % _RING_SLOT_ALIGN == 0
and stride // slots >= ring_bytes
for stride, ring_bytes in needs
):
chosen[group_index] = slots
break
return chosen
def _apply_side_cache_layout(
vllm_config: VllmConfig,
global_kv_cache_groups: list[KVCacheGroupSpec],
kv_cache_specs: list[dict[str, KVCacheSpec]],
) -> tuple[list[KVCacheGroupSpec], list[list[KVCacheGroupSpec]]]:
"""Fold side Mamba groups, project per worker, then pick ring slots.
Returns the (possibly folded) global groups and every worker's projection,
with `slots_per_block` set on packed ring groups.
"""
relevant = any(
(isinstance(spec, MambaSpec) and spec.tp_replicated)
or type(spec) is CircularBufferSpec
for worker_spec in kv_cache_specs
for spec in worker_spec.values()
)
enabled = relevant and _side_cache_layout_enabled(vllm_config)
notes: list[str] = []
if enabled:
global_kv_cache_groups, notes = _fold_replicated_mamba_groups(
global_kv_cache_groups, kv_cache_specs
)
projected = [
_project_kv_cache_groups_to_worker(global_kv_cache_groups, worker_spec)
for worker_spec in kv_cache_specs
]
if enabled and all(
_glm5_next_tensor_layout(groups) is None for groups in projected if groups
):
for group_index, slots in _choose_ring_slots(projected).items():
for groups in projected:
groups[group_index].slots_per_block = slots
notes.append(
f"QSA ring group {group_index} packs {slots} requests per pool block"
)
if relevant and envs.VLLM_E2_SIDE_CACHE_LAYOUT and not enabled:
notes.append(
"not applicable to this engine (KV connector, HiSparse, DCP or a "
"layer-outermost layout)"
)
if relevant:
# The A/B evidence line: boot logs must show which arm ran.
logger.info(
"Side-cache layout: %s (VLLM_E2_SIDE_CACHE_LAYOUT)%s",
"on" if enabled else "off",
"".join(f"; {note}" for note in notes),
)
return global_kv_cache_groups, projected
def get_kv_cache_configs(
vllm_config: VllmConfig,
kv_cache_specs: list[dict[str, KVCacheSpec]],
@@ -2774,10 +3094,11 @@ def get_kv_cache_configs(
# If original_max_model_len was -1, automatically
# determine the maximum model length that fits in available GPU memory.
# We use per-worker projected groups to account for PP sharding.
projected_groups_per_worker = [
_project_kv_cache_groups_to_worker(global_kv_cache_groups, worker_spec)
for worker_spec in kv_cache_specs
]
# v2.5: the side-cache layout (PLE fold, ring slots) is applied here, where
# every worker's layers are known, so no PP stage's block can widen.
global_kv_cache_groups, projected_groups_per_worker = _apply_side_cache_layout(
vllm_config, global_kv_cache_groups, kv_cache_specs
)
# If `num_gpu_blocks_override` is set, the cache size that will actually
# be allocated is decoupled from the profiled `available_memory`:
+4
View File
@@ -1619,6 +1619,10 @@ class Scheduler(SchedulerInterface):
pending = set(block_ids)
owners: dict[int, set[str]] = {}
for mgr in self.kv_cache_manager.coordinator.single_type_managers:
if getattr(mgr, "slot_pool", None) is not None:
# v2.5 slot-packed ring: its block ids are slot ids, not pool
# block ids, and its pages are never zeroed.
continue
for req_id, blocks in mgr.req_to_blocks.items():
for blk in blocks:
if blk.block_id in pending:
+115 -2
View File
@@ -1236,18 +1236,126 @@ class SlidingWindowManager(SingleTypeKVCacheManager):
return 0
class _RingSlotPage:
"""One pool block shared by up to ``slots`` requests' rings."""
__slots__ = ("block", "free_slots", "live")
def __init__(self, block: KVCacheBlock, slots: int) -> None:
self.block = block
# Highest first, so pop() hands out the lowest free slot.
self.free_slots = list(range(slots - 1, -1, -1))
self.live = 0
class RingSlotPool:
"""v2.5 side-cache layout: ring blocks as slots of shared pool blocks.
The QSA ring needs one block per request for the request's lifetime, but
its pages for every layer of a stage are ~16 KB against a 12.9 MB pool
block. This pool takes whole blocks from the main ``BlockPool`` and cuts
each into ``slots_per_block`` slots; a slot's block id is
``pool_block_id * slots_per_block + slot``, which is exactly the index of
the slot in the worker's ring view (``KVCacheTensor.slots_per_block``).
The slot blocks it hands out carry ``pool=self``, so every existing free
path -- ``SingleTypeKVCacheManager.free``, the coordinator's deferred
``pop_blocks_for_free`` and the scheduler's fenced drain, which all call
``BlockPool.free_blocks`` -- routes them back here, at the time the old
per-request block would have gone back (the deferred free is kept). A pool
block returns to the main pool when its last slot is freed. Rings are never
prefix-cached, so a slot block carries no hash.
"""
def __init__(self, block_pool: BlockPool, slots_per_block: int) -> None:
assert slots_per_block > 1
self.block_pool = block_pool
self.slots_per_block = slots_per_block
# Pool block id -> page, in claim order (the oldest page fills first).
self._pages: dict[int, _RingSlotPage] = {}
@property
def num_pages(self) -> int:
"""Pool blocks currently held for rings."""
return len(self._pages)
@property
def num_live_slots(self) -> int:
return sum(page.live for page in self._pages.values())
def has_free_slot(self) -> bool:
return any(page.free_slots for page in self._pages.values())
def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]:
out: list[KVCacheBlock] = []
for _ in range(num_blocks):
page = next((p for p in self._pages.values() if p.free_slots), None)
if page is None:
(pool_block,) = self.block_pool.get_new_blocks(1)
page = _RingSlotPage(pool_block, self.slots_per_block)
self._pages[pool_block.block_id] = page
slot = page.free_slots.pop()
page.live += 1
out.append(
KVCacheBlock(
page.block.block_id * self.slots_per_block + slot,
ref_cnt=1,
pool=self, # type: ignore[arg-type]
)
)
return out
def free_blocks(self, ordered_blocks) -> None:
released: list[KVCacheBlock] = []
for block in ordered_blocks:
assert block.pool is self, "not a ring slot of this pool"
block.ref_cnt -= 1
assert block.ref_cnt == 0, "ring slots have a single owner"
page_id, slot = divmod(block.block_id, self.slots_per_block)
page = self._pages[page_id]
assert slot not in page.free_slots, f"slot {block.block_id} freed twice"
page.free_slots.append(slot)
page.free_slots.sort(reverse=True)
page.live -= 1
if page.live == 0:
del self._pages[page_id]
released.append(page.block)
if released:
self.block_pool.free_blocks(released)
class CircularBufferManager(FullAttentionManager):
"""Claims the ring's single block per request; prefix caching disabled."""
supports_fine_grained_hash_lookup: ClassVar[bool] = False
# v2.5 side-cache layout: set by the coordinator for a slot-packed group.
_slot_pool: RingSlotPool | None = None
def enable_slot_packing(self, slots_per_block: int) -> None:
"""Claim ring slots of shared pool blocks instead of whole blocks."""
assert not self.req_to_blocks, "enable slot packing before any claim"
self._slot_pool = RingSlotPool(self.block_pool, slots_per_block)
@property
def slot_pool(self) -> RingSlotPool | None:
return self._slot_pool
if self._slot_pool is not None:
new_blocks = self._slot_pool.get_new_blocks(1)
else:
new_blocks = self.block_pool.get_new_blocks(1)
def _claim_ring_block(
self, request_id: str, record_for_zeroing: bool = True
) -> list[KVCacheBlock]:
req_blocks = self.req_to_blocks[request_id]
if req_blocks:
return []
new_blocks = self.block_pool.get_new_blocks(1)
if self._slot_pool is not None:
new_blocks = self._slot_pool.get_new_blocks(1)
else:
new_blocks = self.block_pool.get_new_blocks(1)
req_blocks.extend(new_blocks)
if self._record_new_block_ids and record_for_zeroing:
self.new_block_ids.extend(block.block_id for block in new_blocks)
@@ -1293,7 +1401,12 @@ class CircularBufferManager(FullAttentionManager):
apply_admission_cap: bool = False,
prefill_end: int = 0,
) -> int:
return 0 if self.req_to_blocks.get(request_id) else 1
if self.req_to_blocks.get(request_id):
return 0
# A free slot in a page the ring already holds costs no pool block.
if self._slot_pool is not None and self._slot_pool.has_free_slot():
return 0
return 1
def allocate_new_blocks(
self, request_id: str, num_tokens: int, num_tokens_main_model: int
File diff suppressed because it is too large Load Diff
+22 -1
View File
@@ -1431,6 +1431,11 @@ class KVCacheTensor:
block_stride: int
offset: int = 0 # byte offset of layers[0]'s block 0
host_resident: bool = False
# v2.5 side-cache layout: a slot-packed ring tensor views every pool block
# as ``slots_per_block`` equal slots (``block_stride`` is then one slot), so
# its block ids are ``pool_block * slots_per_block + slot``. 1 = one view
# block per pool block, the ordinary case.
slots_per_block: int = 1
class KVCacheGroupRole(str, Enum):
@@ -1458,6 +1463,10 @@ class KVCacheGroupSpec:
# source group after a connector load.
enable_kv_transfer: bool = True
role: KVCacheGroupRole = KVCacheGroupRole.DEFAULT
# v2.5 side-cache layout: > 1 when this one-block-per-request ring group
# shares each pool block between that many requests (see
# ``CircularBufferManager.enable_slot_packing``); 1 = one pool block each.
slots_per_block: int = 1
@dataclass
@@ -1545,6 +1554,18 @@ class KVCacheConfig:
)
return tuple(block_ids[group_id] for group_id in self.transfer_group_ids)
@property
def slot_packed_group_ids(self) -> tuple[int, ...]:
"""Groups whose blocks are slots of shared pool blocks (v2.5 side-cache
layout). Their views are not one block per pool block, so whole-block
zeroing must skip them; their bytes lie inside pool blocks the ring
manager owns and that no other group writes."""
return tuple(
group_id
for group_id, group in enumerate(self.kv_cache_groups)
if group.slots_per_block > 1
)
@property
def host_group_ids(self) -> tuple[int, ...]:
return tuple(
@@ -1556,7 +1577,7 @@ class KVCacheConfig:
def num_blocks_of(self, tensor: KVCacheTensor) -> int:
"""Number of blocks addressable by the pool backing ``tensor``."""
if not tensor.host_resident:
return self.num_blocks
return self.num_blocks * tensor.slots_per_block
assert self.hisparse_host_num_blocks is not None
return self.hisparse_host_num_blocks
+1
View File
@@ -866,6 +866,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
kernel_block_sizes=self.kernel_block_sizes,
static_forward_context=self.compilation_config.static_forward_context,
num_blocks=self.kv_cache_config.num_blocks,
skip_kv_cache_group_ids=self.kv_cache_config.slot_packed_group_ids,
)
@torch.inference_mode()
+1
View File
@@ -1149,6 +1149,7 @@ class GPUModelRunner(
runner_only_attn_layers=self.runner_only_attn_layers,
static_forward_context=self.compilation_config.static_forward_context,
num_blocks=self.kv_cache_config.num_blocks,
skip_kv_cache_group_ids=self.kv_cache_config.slot_packed_group_ids,
)
def _zero_block_ids(self, block_ids: list[int]) -> None:
+11
View File
@@ -138,6 +138,7 @@ class KVBlockZeroer:
static_forward_context: dict[str, Any],
num_blocks: int,
runner_only_attn_layers: set[str] | None = None,
skip_kv_cache_group_ids: Iterable[int] = (),
) -> None:
"""Precompute the absolute-address table for the Triton zeroing kernel.
@@ -156,6 +157,13 @@ class KVBlockZeroer:
physical block stride and zeroed page span remain independent.
Only AttentionSpec layers are processed; Mamba layers are skipped.
``skip_kv_cache_group_ids`` (v2.5 side-cache layout): slot-packed ring
groups, whose views hold several slots per pool block. Their claims
never enter whole-block zeroing (the ring-tag path clears their tag words instead),
and with one view block per pool block their pages sat inside the
first attention page, so skipping them leaves the zeroed bytes of
every block unchanged.
"""
self.device = device
# Bounds-guard instrumentation: pool bound for the guard in _zero_kv_blocks_kernel.
@@ -174,12 +182,15 @@ class KVBlockZeroer:
seg_block_strides: list[int] = []
seg_page_sizes: list[int] = []
skip_group_ids = set(skip_kv_cache_group_ids)
for group in attn_groups_iter:
spec = group.kv_cache_spec
if not isinstance(spec, AttentionSpec):
continue
if group.kv_cache_group_id >= len(kernel_block_sizes):
continue
if group.kv_cache_group_id in skip_group_ids:
continue
kernel_bs = kernel_block_sizes[group.kv_cache_group_id]
assert spec.block_size % kernel_bs == 0
for layer_name in group.layer_names: