[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
+10
-119
@@ -29,24 +29,16 @@ logger = init_logger(__name__)
|
||||
def get_block_table_width(
|
||||
max_num_blocks: int,
|
||||
block_size: int,
|
||||
kernel_block_size: int | None = None,
|
||||
*,
|
||||
token_alignment: int | None = 128,
|
||||
) -> int:
|
||||
"""Return the width after optional alignment and virtual block splitting."""
|
||||
if kernel_block_size is None:
|
||||
kernel_block_size = block_size
|
||||
if block_size % kernel_block_size != 0:
|
||||
raise ValueError(
|
||||
f"kernel_block_size {kernel_block_size} must divide "
|
||||
f"block_size {block_size} evenly"
|
||||
)
|
||||
"""Return the width after optional alignment."""
|
||||
if token_alignment is not None:
|
||||
if token_alignment <= 0:
|
||||
raise ValueError("token_alignment must be positive")
|
||||
block_alignment = token_alignment // math.gcd(token_alignment, block_size)
|
||||
max_num_blocks = cdiv(max_num_blocks, block_alignment) * block_alignment
|
||||
return max_num_blocks * block_size // kernel_block_size
|
||||
return max_num_blocks
|
||||
|
||||
|
||||
class SlotMappingMode(Enum):
|
||||
@@ -63,7 +55,6 @@ class BlockTable:
|
||||
max_num_batched_tokens: int,
|
||||
pin_memory: bool,
|
||||
device: torch.device,
|
||||
kernel_block_size: int,
|
||||
cp_kv_cache_interleave_size: int,
|
||||
slot_mapping_mode: SlotMappingMode = SlotMappingMode.TOKEN_TO_KV_SLOT,
|
||||
):
|
||||
@@ -74,9 +65,6 @@ class BlockTable:
|
||||
max_num_batched_tokens: Maximum number of tokens in a batch.
|
||||
pin_memory: Whether to pin memory for faster GPU transfers.
|
||||
device: Target device for the block table.
|
||||
kernel_block_size: The block_size of underlying attention kernel.
|
||||
Will be the same as `block_size` if `block_size` is supported
|
||||
by the attention kernel.
|
||||
slot_mapping_mode: How this cache group maps scheduled tokens to
|
||||
cache slots. Mamba-like state caches do not use token slot
|
||||
mappings and should use SlotMappingMode.NONE.
|
||||
@@ -86,30 +74,8 @@ class BlockTable:
|
||||
self.max_num_batched_tokens = max_num_batched_tokens
|
||||
self.pin_memory = pin_memory
|
||||
self.device = device
|
||||
self.kv_cache_block_size = block_size
|
||||
|
||||
if kernel_block_size == block_size:
|
||||
# Standard case: allocation and computation use same block size
|
||||
# No block splitting needed, direct mapping
|
||||
self.block_size = block_size
|
||||
self.blocks_per_kv_block = 1
|
||||
self.use_hybrid_blocks = False
|
||||
else:
|
||||
# Hybrid case: allocation block size differs from kernel block size
|
||||
# Memory blocks are subdivided to match kernel requirements
|
||||
# Example: 32-token memory blocks with 16-token kernel blocks
|
||||
# → Each memory block corresponds to 2 kernel blocks
|
||||
if block_size % kernel_block_size != 0:
|
||||
raise ValueError(
|
||||
f"kernel_block_size {kernel_block_size} must divide "
|
||||
f"kv_manager_block_size size {block_size} evenly"
|
||||
)
|
||||
|
||||
self.block_size = kernel_block_size
|
||||
self.blocks_per_kv_block = block_size // kernel_block_size
|
||||
self.use_hybrid_blocks = True
|
||||
|
||||
self.max_num_blocks_per_req = max_num_blocks_per_req * self.blocks_per_kv_block
|
||||
self.block_size = block_size
|
||||
self.max_num_blocks_per_req = max_num_blocks_per_req
|
||||
|
||||
self.block_table = self._make_buffer(
|
||||
self.max_num_reqs, self.max_num_blocks_per_req, dtype=torch.int32
|
||||
@@ -120,13 +86,6 @@ class BlockTable:
|
||||
self.max_num_batched_tokens, dtype=torch.int64
|
||||
)
|
||||
|
||||
if self.use_hybrid_blocks:
|
||||
self._kernel_block_arange = np.arange(0, self.blocks_per_kv_block).reshape(
|
||||
1, -1
|
||||
)
|
||||
else:
|
||||
self._kernel_block_arange = None
|
||||
|
||||
try:
|
||||
self.pcp_world_size = get_pcp_group().world_size
|
||||
self.pcp_rank = get_pcp_group().rank_in_group
|
||||
@@ -145,13 +104,11 @@ class BlockTable:
|
||||
self.slot_mapping_mode = slot_mapping_mode
|
||||
if self.slot_mapping_mode == SlotMappingMode.TOKEN_TO_KV_SLOT:
|
||||
_COMPUTE_SLOT_MAPPING_KERNEL.register_warmup(
|
||||
kv_cache_block_size=self.kv_cache_block_size,
|
||||
blocks_per_kv_block=self.blocks_per_kv_block,
|
||||
kv_cache_block_size=self.block_size,
|
||||
total_cp_world_size=self.dcp_world_size,
|
||||
total_cp_rank=self.dcp_rank,
|
||||
cp_kv_cache_interleave_size=self.cp_kv_cache_interleave_size,
|
||||
block_table_stride=self.block_table.gpu.stride(0),
|
||||
block_size=self.block_size,
|
||||
)
|
||||
|
||||
def append_row(
|
||||
@@ -162,11 +119,6 @@ class BlockTable:
|
||||
if not block_ids:
|
||||
return
|
||||
|
||||
if self.use_hybrid_blocks:
|
||||
block_ids = self.map_to_kernel_blocks(
|
||||
np.array(block_ids), self.blocks_per_kv_block, self._kernel_block_arange
|
||||
)
|
||||
|
||||
num_blocks = len(block_ids)
|
||||
start = self.num_blocks_per_row[row_idx]
|
||||
self.num_blocks_per_row[row_idx] += num_blocks
|
||||
@@ -219,10 +171,8 @@ class BlockTable:
|
||||
positions,
|
||||
self.block_table.gpu,
|
||||
self.block_table.gpu.stride(0),
|
||||
self.block_size,
|
||||
self.slot_mapping.gpu,
|
||||
self.kv_cache_block_size,
|
||||
self.blocks_per_kv_block,
|
||||
self.block_size,
|
||||
self.dcp_world_size,
|
||||
self.dcp_rank,
|
||||
self.cp_kv_cache_interleave_size,
|
||||
@@ -235,37 +185,6 @@ class BlockTable:
|
||||
self.block_table.gpu.fill_(0)
|
||||
self.block_table.cpu.fill_(0)
|
||||
|
||||
@staticmethod
|
||||
def map_to_kernel_blocks(
|
||||
kv_manager_block_ids: np.ndarray,
|
||||
blocks_per_kv_block: int,
|
||||
kernel_block_arange: np.ndarray,
|
||||
) -> np.ndarray:
|
||||
"""Convert kv_manager_block_id IDs to kernel block IDs.
|
||||
|
||||
Example:
|
||||
# kv_manager_block_ids: 32 tokens,
|
||||
# Kernel block size: 16 tokens
|
||||
# blocks_per_kv_block = 2
|
||||
>>> kv_manager_block_ids = np.array([0, 1, 2])
|
||||
>>> Result: [0, 1, 2, 3, 4, 5]
|
||||
|
||||
# Each kv_manager_block_id maps to 2 kernel block id:
|
||||
# kv_manager_block_id 0 → kernel block id [0, 1]
|
||||
# kv_manager_block_id 1 → kernel block id [2, 3]
|
||||
# kv_manager_block_id 2 → kernel block id [4, 5]
|
||||
|
||||
"""
|
||||
if blocks_per_kv_block == 1:
|
||||
return kv_manager_block_ids
|
||||
|
||||
kernel_block_ids = (
|
||||
kv_manager_block_ids.reshape(-1, 1) * blocks_per_kv_block
|
||||
+ kernel_block_arange
|
||||
)
|
||||
|
||||
return kernel_block_ids.reshape(-1)
|
||||
|
||||
def get_device_tensor(self, num_reqs: int) -> torch.Tensor:
|
||||
"""Returns the device tensor of the block table."""
|
||||
return self.block_table.gpu[:num_reqs]
|
||||
@@ -296,16 +215,10 @@ class MultiGroupBlockTable:
|
||||
pin_memory: bool,
|
||||
device: torch.device,
|
||||
block_sizes: list[int],
|
||||
kernel_block_sizes: list[int],
|
||||
max_num_blocks: list[int],
|
||||
cp_kv_cache_interleave_size: int = 1,
|
||||
slot_mapping_modes: list[SlotMappingMode] | None = None,
|
||||
) -> None:
|
||||
if len(kernel_block_sizes) != len(block_sizes):
|
||||
raise ValueError(
|
||||
f"kernel_block_sizes length ({len(kernel_block_sizes)}) "
|
||||
f"must match block_sizes length ({len(block_sizes)})"
|
||||
)
|
||||
if slot_mapping_modes is None:
|
||||
slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT] * len(block_sizes)
|
||||
if len(slot_mapping_modes) != len(block_sizes):
|
||||
@@ -339,17 +252,11 @@ class MultiGroupBlockTable:
|
||||
max_num_batched_tokens,
|
||||
pin_memory,
|
||||
device,
|
||||
kernel_block_size,
|
||||
cp_kv_cache_interleave_size,
|
||||
slot_mapping_mode=slot_mapping_mode,
|
||||
)
|
||||
for (
|
||||
block_size,
|
||||
kernel_block_size,
|
||||
max_num_blocks_per_req,
|
||||
slot_mapping_mode,
|
||||
) in zip(
|
||||
block_sizes, kernel_block_sizes, max_num_blocks, slot_mapping_modes
|
||||
for block_size, max_num_blocks_per_req, slot_mapping_mode in zip(
|
||||
block_sizes, max_num_blocks, slot_mapping_modes
|
||||
)
|
||||
]
|
||||
|
||||
@@ -403,12 +310,10 @@ class ComputeSlotMappingKernel(
|
||||
@dataclass(frozen=True)
|
||||
class CompileKey:
|
||||
kv_cache_block_size: int
|
||||
blocks_per_kv_block: int
|
||||
total_cp_world_size: int
|
||||
total_cp_rank: int
|
||||
cp_kv_cache_interleave_size: int
|
||||
block_table_stride: int
|
||||
block_size: int
|
||||
|
||||
@staticmethod
|
||||
@triton.jit(do_not_specialize=["num_tokens", "max_num_tokens"])
|
||||
@@ -419,10 +324,8 @@ class ComputeSlotMappingKernel(
|
||||
positions_ptr, # [num_tokens], int64
|
||||
block_table_ptr, # [max_num_reqs, max_num_blocks_per_req], int32 (flat)
|
||||
block_table_stride, # max_num_blocks_per_req
|
||||
block_size,
|
||||
slot_mapping_ptr, # [max_num_tokens], int64
|
||||
KV_CACHE_BLOCK_SIZE: tl.constexpr,
|
||||
BLOCKS_PER_KV_BLOCK: tl.constexpr,
|
||||
TOTAL_CP_WORLD_SIZE: tl.constexpr,
|
||||
TOTAL_CP_RANK: tl.constexpr,
|
||||
CP_KV_CACHE_INTERLEAVE_SIZE: tl.constexpr,
|
||||
@@ -463,17 +366,12 @@ class ComputeSlotMappingKernel(
|
||||
virtual_block_offsets % CP_KV_CACHE_INTERLEAVE_SIZE
|
||||
)
|
||||
|
||||
block_indices = (
|
||||
virtual_block_indices * BLOCKS_PER_KV_BLOCK
|
||||
+ local_block_offsets // block_size
|
||||
)
|
||||
block_numbers = tl.load(
|
||||
block_table_ptr + row_offset + block_indices,
|
||||
block_table_ptr + row_offset + virtual_block_indices,
|
||||
mask=mask & is_local,
|
||||
other=0,
|
||||
).to(tl.int64)
|
||||
slot_offsets = local_block_offsets % block_size
|
||||
slot_ids = block_numbers * block_size + slot_offsets
|
||||
slot_ids = block_numbers * KV_CACHE_BLOCK_SIZE + local_block_offsets
|
||||
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
|
||||
tl.store(slot_mapping_ptr + offsets, slot_ids, mask=mask)
|
||||
|
||||
@@ -481,13 +379,11 @@ class ComputeSlotMappingKernel(
|
||||
self,
|
||||
*,
|
||||
block_table_stride: int,
|
||||
block_size: int,
|
||||
**compile_key_fields: int,
|
||||
) -> CompileKey:
|
||||
return self.CompileKey(
|
||||
**compile_key_fields,
|
||||
block_table_stride=triton_scalar_specialization_rep(block_table_stride),
|
||||
block_size=triton_scalar_specialization_rep(block_size),
|
||||
)
|
||||
|
||||
def get_warmup_keys(self, **dispatch_kwargs: int) -> list[CompileKey]:
|
||||
@@ -507,10 +403,8 @@ class ComputeSlotMappingKernel(
|
||||
shape=(1, compile_key.block_table_stride),
|
||||
),
|
||||
block_table_stride=compile_key.block_table_stride,
|
||||
block_size=compile_key.block_size,
|
||||
slot_mapping=int64_ptr,
|
||||
kv_cache_block_size=compile_key.kv_cache_block_size,
|
||||
blocks_per_kv_block=compile_key.blocks_per_kv_block,
|
||||
total_cp_world_size=compile_key.total_cp_world_size,
|
||||
total_cp_rank=compile_key.total_cp_rank,
|
||||
cp_kv_cache_interleave_size=compile_key.cp_kv_cache_interleave_size,
|
||||
@@ -526,17 +420,14 @@ class ComputeSlotMappingKernel(
|
||||
positions: torch.Tensor,
|
||||
block_table: torch.Tensor,
|
||||
block_table_stride: int,
|
||||
block_size: int,
|
||||
slot_mapping: torch.Tensor,
|
||||
kv_cache_block_size: int,
|
||||
blocks_per_kv_block: int,
|
||||
total_cp_world_size: int,
|
||||
total_cp_rank: int,
|
||||
cp_kv_cache_interleave_size: int,
|
||||
) -> LaunchSpec:
|
||||
return (num_reqs + 1,), dict(
|
||||
KV_CACHE_BLOCK_SIZE=kv_cache_block_size,
|
||||
BLOCKS_PER_KV_BLOCK=blocks_per_kv_block,
|
||||
TOTAL_CP_WORLD_SIZE=total_cp_world_size,
|
||||
TOTAL_CP_RANK=total_cp_rank,
|
||||
CP_KV_CACHE_INTERLEAVE_SIZE=cp_kv_cache_interleave_size,
|
||||
|
||||
@@ -95,11 +95,9 @@ def prepare_dcp_dummy_context_metadata(
|
||||
max_valid_block_id = kv_cache_config.num_blocks - 1
|
||||
assert max_valid_block_id > 0
|
||||
for blk_table in input_batch.block_table.block_tables:
|
||||
max_row_blocks = (
|
||||
blk_table.max_num_blocks_per_req // blk_table.blocks_per_kv_block
|
||||
)
|
||||
block_ids = [
|
||||
(block_idx % max_valid_block_id) + 1 for block_idx in range(max_row_blocks)
|
||||
(block_idx % max_valid_block_id) + 1
|
||||
for block_idx in range(blk_table.max_num_blocks_per_req)
|
||||
]
|
||||
for req_idx in range(num_reqs):
|
||||
blk_table.add_row(block_ids, req_idx)
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
@@ -40,6 +40,8 @@ from vllm.v1.worker.utils import (
|
||||
add_kv_sharing_layers_to_kv_cache_groups,
|
||||
allocate_kv_cache,
|
||||
bind_kv_cache_to_layers,
|
||||
customize_attention_spec,
|
||||
map_kv_caches_to_kernel_blocks,
|
||||
prepare_kernel_block_sizes,
|
||||
)
|
||||
|
||||
@@ -141,7 +143,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
|
||||
# Skip modules that don't need KV cache (eg encoder-only attention)
|
||||
if spec := attn_module.get_kv_cache_spec(vllm_config):
|
||||
if isinstance(spec, AttentionSpec):
|
||||
spec = attn_module.get_attn_backend().customize_spec(spec)
|
||||
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
|
||||
@@ -382,6 +384,7 @@ def init_kv_cache(
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
*,
|
||||
block_tables: "BlockTables | None" = None,
|
||||
attn_groups: Iterable[AttentionGroup] = (),
|
||||
) -> dict[str, Any]:
|
||||
allocation_context = kv_cache_allocation_context or nullcontext()
|
||||
with allocation_context:
|
||||
@@ -409,9 +412,10 @@ def init_kv_cache(
|
||||
num_attn_module = (
|
||||
2 if vllm_config.model_config.hf_config.model_type == "longcat_flash" else 1
|
||||
)
|
||||
bindable_caches = {
|
||||
name: cache for name, cache in kv_caches.items() if name in forward_context
|
||||
}
|
||||
bindable_caches = map_kv_caches_to_kernel_blocks(
|
||||
{name: cache for name, cache in kv_caches.items() if name in forward_context},
|
||||
attn_groups,
|
||||
)
|
||||
bind_kv_cache_to_layers(
|
||||
bindable_caches,
|
||||
forward_context,
|
||||
@@ -536,8 +540,8 @@ def build_attn_metadata(
|
||||
cached_metadata[reuse_key], block_table, slot_mapping
|
||||
)
|
||||
elif for_cudagraph_capture:
|
||||
metadata = attn_metadata_builder.build_for_cudagraph_capture(
|
||||
common_attn_metadata
|
||||
metadata = attn_group.build_metadata_for_cudagraph_capture(
|
||||
common_attn_metadata, ubatch_idx
|
||||
)
|
||||
else:
|
||||
attn_metadata_extra_kwargs = (
|
||||
@@ -548,10 +552,8 @@ def build_attn_metadata(
|
||||
if model_specific_attn_metadata is not None
|
||||
else {}
|
||||
)
|
||||
metadata = attn_metadata_builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
**attn_metadata_extra_kwargs,
|
||||
metadata = attn_group.build_metadata(
|
||||
common_attn_metadata, ubatch_idx, **attn_metadata_extra_kwargs
|
||||
)
|
||||
if reuse_key is not None and reuse_key not in cached_metadata:
|
||||
cached_metadata[reuse_key] = metadata
|
||||
|
||||
@@ -25,7 +25,6 @@ class BlockTables:
|
||||
max_num_batched_tokens: int,
|
||||
max_num_blocks_per_group: list[int],
|
||||
device: torch.device,
|
||||
kernel_block_sizes: list[int],
|
||||
cp_size: int = 1,
|
||||
cp_rank: int = 0,
|
||||
cp_interleave: int = 1,
|
||||
@@ -33,7 +32,6 @@ class BlockTables:
|
||||
dcp_sharded: list[bool] | None = None,
|
||||
):
|
||||
self.block_sizes = block_sizes
|
||||
self.kernel_block_sizes = kernel_block_sizes
|
||||
self.max_num_reqs = max_num_reqs
|
||||
self.max_num_batched_tokens = max_num_batched_tokens
|
||||
self.device = device
|
||||
@@ -53,14 +51,9 @@ class BlockTables:
|
||||
assert len(dcp_sharded) == self.num_kv_cache_groups
|
||||
self.dcp_sharded = torch.tensor(dcp_sharded, dtype=torch.bool, device=device)
|
||||
|
||||
self.blocks_per_kv_block = [
|
||||
bs // kbs for bs, kbs in zip(block_sizes, kernel_block_sizes)
|
||||
]
|
||||
|
||||
# num_kv_cache_groups x [max_num_reqs, max_num_blocks]
|
||||
self.block_tables: list[StagedWriteTensor] = []
|
||||
for i in range(self.num_kv_cache_groups):
|
||||
max_num_blocks = max_num_blocks_per_group[i] * self.blocks_per_kv_block[i]
|
||||
for max_num_blocks in max_num_blocks_per_group:
|
||||
block_table = StagedWriteTensor(
|
||||
(self.max_num_reqs, max_num_blocks), dtype=torch.int32, device=device
|
||||
)
|
||||
@@ -109,9 +102,6 @@ class BlockTables:
|
||||
self.block_sizes_tensor = torch.tensor(
|
||||
self.block_sizes, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.kernel_block_sizes_tensor = torch.tensor(
|
||||
self.kernel_block_sizes, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.slot_mapping_enabled = torch.tensor(
|
||||
self._slot_mapping_enabled, dtype=torch.bool, device=self.device
|
||||
)
|
||||
@@ -126,9 +116,6 @@ class BlockTables:
|
||||
for i in range(self.num_kv_cache_groups):
|
||||
start = self.num_blocks.np[i, req_index] if not overwrite else 0
|
||||
block_ids = new_block_ids[i]
|
||||
bpk = self.blocks_per_kv_block[i]
|
||||
if bpk > 1:
|
||||
block_ids = [b * bpk + k for b in block_ids for k in range(bpk)]
|
||||
if self.redirect_writes_to_null_block:
|
||||
block_ids = [0] * len(block_ids)
|
||||
end = start + len(block_ids)
|
||||
@@ -218,7 +205,6 @@ class BlockTables:
|
||||
self.block_table_ptrs,
|
||||
self.block_table_strides,
|
||||
self.block_sizes_tensor,
|
||||
self.kernel_block_sizes_tensor,
|
||||
self.slot_mapping_enabled,
|
||||
self.dcp_sharded,
|
||||
slot_mappings,
|
||||
@@ -292,7 +278,6 @@ def _compute_slot_mappings_kernel(
|
||||
block_table_ptrs, # [num_kv_cache_groups]
|
||||
block_table_strides, # [num_kv_cache_groups]
|
||||
block_sizes, # [num_kv_cache_groups]
|
||||
kernel_block_sizes, # [num_kv_cache_groups]
|
||||
slot_mapping_enabled, # [num_kv_cache_groups]
|
||||
dcp_sharded, # [num_kv_cache_groups]
|
||||
slot_mappings_ptr, # [num_kv_cache_groups, max_num_tokens]
|
||||
@@ -322,7 +307,6 @@ def _compute_slot_mappings_kernel(
|
||||
block_table_ptr = _load_ptr(block_table_ptrs + group_id, tl.int32)
|
||||
block_table_stride = tl.load(block_table_strides + group_id)
|
||||
kv_block_size = tl.load(block_sizes + group_id)
|
||||
kernel_block_size = tl.load(kernel_block_sizes + group_id)
|
||||
mapping_enabled = tl.load(slot_mapping_enabled + group_id)
|
||||
if CP_SIZE != 1:
|
||||
sharded = tl.load(dcp_sharded + group_id)
|
||||
@@ -354,16 +338,14 @@ def _compute_slot_mappings_kernel(
|
||||
local_positions = tl.where(sharded, local_positions, positions)
|
||||
is_local = ~sharded | is_local
|
||||
|
||||
block_indices = tl.where(
|
||||
mapping_enabled, local_positions // kernel_block_size, 0
|
||||
)
|
||||
block_offsets = local_positions % kernel_block_size
|
||||
block_indices = tl.where(mapping_enabled, local_positions // kv_block_size, 0)
|
||||
block_offsets = local_positions % kv_block_size
|
||||
block_numbers = tl.load(
|
||||
block_table_ptr + req_state_idx * block_table_stride + block_indices,
|
||||
mask=is_local & is_real_req,
|
||||
other=0,
|
||||
)
|
||||
slot_ids = block_numbers * kernel_block_size + block_offsets
|
||||
slot_ids = block_numbers * kv_block_size + block_offsets
|
||||
if CP_SIZE != 1:
|
||||
slot_ids = tl.where(is_local, slot_ids, PAD_ID)
|
||||
|
||||
|
||||
@@ -252,17 +252,18 @@ def set_dummy_context(
|
||||
input_batch.positions.copy_(torch.from_numpy(local_pos + context_len))
|
||||
|
||||
seq_len = context_len + query_len
|
||||
for block_table, block_size, bpk in zip(
|
||||
input_block_tables,
|
||||
block_tables.kernel_block_sizes,
|
||||
block_tables.blocks_per_kv_block,
|
||||
):
|
||||
for block_table, block_size in zip(input_block_tables, block_tables.block_sizes):
|
||||
num_blocks = min(cdiv(seq_len, block_size), block_table.shape[1])
|
||||
# Spans are disjoint until the pool runs out, then they wrap and share
|
||||
# blocks: profiling only needs the reads to be realistic, not distinct.
|
||||
block_ids = torch.arange(
|
||||
num_reqs * num_blocks, dtype=block_table.dtype, device=block_table.device
|
||||
) % (num_kv_blocks * bpk)
|
||||
block_ids = (
|
||||
torch.arange(
|
||||
num_reqs * num_blocks,
|
||||
dtype=block_table.dtype,
|
||||
device=block_table.device,
|
||||
)
|
||||
% num_kv_blocks
|
||||
)
|
||||
block_table[:num_reqs, :num_blocks] = block_ids.view(num_reqs, num_blocks)
|
||||
|
||||
|
||||
|
||||
@@ -688,7 +688,6 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
max_num_batched_tokens=self.max_num_tokens,
|
||||
max_num_blocks_per_group=max_num_blocks_per_group,
|
||||
device=self.device,
|
||||
kernel_block_sizes=self.kernel_block_sizes,
|
||||
slot_mapping_enabled=slot_mapping_enabled,
|
||||
dcp_sharded=dcp_sharded,
|
||||
cp_size=self.dcp_size,
|
||||
@@ -773,6 +772,13 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.vllm_config,
|
||||
kv_cache_allocation_context=kv_cache_allocation_context,
|
||||
block_tables=self.block_tables,
|
||||
# Draft groups build their own metadata, so they map the views too.
|
||||
attn_groups=(
|
||||
g
|
||||
for groups in self.attn_groups
|
||||
+ getattr(self.speculator, "attn_groups", [])
|
||||
for g in groups
|
||||
),
|
||||
)
|
||||
self.kv_caches = [
|
||||
cache for cache in kv_caches_dict.values() if cache.device == self.device
|
||||
|
||||
@@ -432,7 +432,7 @@ class DFlashSpeculator(DraftModelSpeculator):
|
||||
temperature,
|
||||
seeds,
|
||||
self.block_tables.input_block_tables[gid],
|
||||
self.block_tables.kernel_block_sizes[gid],
|
||||
self.block_tables.block_sizes[gid],
|
||||
self.block_tables.cp_rank,
|
||||
self.dcp_size,
|
||||
self.block_tables.cp_interleave,
|
||||
|
||||
@@ -577,6 +577,11 @@ class TargetDependentARSpeculator(DraftModelSpeculator):
|
||||
idx_mapping = self.idx_mapping[:num_reqs]
|
||||
|
||||
if batch_desc.cg_mode == CUDAGraphMode.FULL:
|
||||
for groups in self.attn_groups:
|
||||
for group in groups:
|
||||
group.map_to_kernel_block_table(
|
||||
self.block_tables.input_block_tables[group.kv_cache_group_id]
|
||||
)
|
||||
assert self.decode_cudagraph_manager is not None
|
||||
self.decode_cudagraph_manager.run_fullgraph(batch_desc)
|
||||
return
|
||||
|
||||
@@ -96,7 +96,6 @@ class InputBatch:
|
||||
device: torch.device,
|
||||
vocab_size: int,
|
||||
block_sizes: list[int], # The block_size of each kv cache group
|
||||
kernel_block_sizes: list[int],
|
||||
max_num_blocks_per_req: list[int],
|
||||
logitsprocs: LogitsProcessors | None = None,
|
||||
logitsprocs_need_output_token_ids: bool = False,
|
||||
@@ -187,7 +186,6 @@ class InputBatch:
|
||||
pin_memory=PIN_MEMORY,
|
||||
device=device,
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
max_num_blocks=max_num_blocks_per_req,
|
||||
cp_kv_cache_interleave_size=cp_kv_cache_interleave_size,
|
||||
slot_mapping_modes=slot_mapping_modes,
|
||||
|
||||
@@ -233,6 +233,8 @@ from .utils import (
|
||||
allocate_kv_cache,
|
||||
bind_kv_cache,
|
||||
copy_kv_cache_blocks_inplace,
|
||||
customize_attention_spec,
|
||||
map_kv_caches_to_kernel_blocks,
|
||||
prepare_kernel_block_sizes,
|
||||
sanity_check_mm_encoder_outputs,
|
||||
)
|
||||
@@ -718,7 +720,6 @@ class GPUModelRunner(
|
||||
max(self.max_model_len, self.max_encoder_len), placeholder_block_size
|
||||
)
|
||||
self._init_block_sizes = [placeholder_block_size]
|
||||
self._init_kernel_block_sizes = [placeholder_block_size]
|
||||
self._init_max_num_blocks = [placeholder_max_num_blocks]
|
||||
self._init_slot_mapping_modes = [SlotMappingMode.TOKEN_TO_KV_SLOT]
|
||||
self.cp_kv_cache_interleave_size = (
|
||||
@@ -734,7 +735,6 @@ class GPUModelRunner(
|
||||
device=self.device,
|
||||
vocab_size=self.model_config.get_vocab_size(),
|
||||
block_sizes=[placeholder_block_size],
|
||||
kernel_block_sizes=[placeholder_block_size],
|
||||
max_num_blocks_per_req=[placeholder_max_num_blocks],
|
||||
num_spec_tokens=self.num_spec_tokens,
|
||||
logitsprocs=build_logitsprocs(
|
||||
@@ -2508,8 +2508,8 @@ class GPUModelRunner(
|
||||
)
|
||||
|
||||
if for_cudagraph_capture:
|
||||
attn_metadata_i = builder.build_for_cudagraph_capture(
|
||||
common_attn_metadata
|
||||
attn_metadata_i = attn_group.build_metadata_for_cudagraph_capture(
|
||||
common_attn_metadata, ubid or 0
|
||||
)
|
||||
elif (
|
||||
cache_key in cached_attn_metadata
|
||||
@@ -2517,13 +2517,16 @@ class GPUModelRunner(
|
||||
):
|
||||
attn_metadata_i = builder.update_block_table(
|
||||
cached_attn_metadata[cache_key],
|
||||
common_attn_metadata.block_table_tensor,
|
||||
attn_group.map_to_kernel_block_table(
|
||||
common_attn_metadata.block_table_tensor, ubid or 0
|
||||
),
|
||||
common_attn_metadata.slot_mapping,
|
||||
)
|
||||
else:
|
||||
attn_metadata_i = builder.build(
|
||||
attn_metadata_i = attn_group.build_metadata(
|
||||
common_attn_metadata,
|
||||
ubid or 0,
|
||||
common_prefix_len=cascade_attn_prefix_len,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
**extra_attn_metadata_args,
|
||||
)
|
||||
if builder.supports_update_block_table:
|
||||
@@ -7143,9 +7146,7 @@ class GPUModelRunner(
|
||||
return
|
||||
self.reorder_batch_threshold = reduce(min_none_high, reorder_batch_thresholds) # type: ignore[assignment]
|
||||
|
||||
def may_reinitialize_input_batch(
|
||||
self, kv_cache_config: KVCacheConfig, kernel_block_sizes: list[int]
|
||||
) -> None:
|
||||
def may_reinitialize_input_batch(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
"""Re-initialize the input batch if the block sizes are different from
|
||||
what it was originally created with. This happens when the final
|
||||
block size (determined after model loading) differs from the
|
||||
@@ -7154,7 +7155,6 @@ class GPUModelRunner(
|
||||
|
||||
Args:
|
||||
kv_cache_config: The KV cache configuration.
|
||||
kernel_block_sizes: The kernel block sizes for each KV cache group.
|
||||
|
||||
"""
|
||||
block_sizes = []
|
||||
@@ -7179,14 +7179,12 @@ class GPUModelRunner(
|
||||
|
||||
if (
|
||||
block_sizes != self._init_block_sizes
|
||||
or kernel_block_sizes != self._init_kernel_block_sizes
|
||||
or max_num_blocks != self._init_max_num_blocks
|
||||
or slot_mapping_modes != self._init_slot_mapping_modes
|
||||
or self.cp_kv_cache_interleave_size
|
||||
!= self.parallel_config.cp_kv_cache_interleave_size
|
||||
):
|
||||
self._init_block_sizes = block_sizes
|
||||
self._init_kernel_block_sizes = kernel_block_sizes
|
||||
self._init_max_num_blocks = max_num_blocks
|
||||
self._init_slot_mapping_modes = slot_mapping_modes
|
||||
self.cp_kv_cache_interleave_size = (
|
||||
@@ -7201,7 +7199,6 @@ class GPUModelRunner(
|
||||
device=self.device,
|
||||
vocab_size=self.model_config.get_vocab_size(),
|
||||
block_sizes=block_sizes,
|
||||
kernel_block_sizes=kernel_block_sizes,
|
||||
max_num_blocks_per_req=max_num_blocks,
|
||||
num_spec_tokens=self.num_spec_tokens,
|
||||
logitsprocs=self.input_batch.logitsprocs,
|
||||
@@ -7217,10 +7214,6 @@ class GPUModelRunner(
|
||||
f"InputBatch block_sizes {self._init_block_sizes} != "
|
||||
f"kv_cache block_sizes {block_sizes}"
|
||||
)
|
||||
assert self._init_kernel_block_sizes == kernel_block_sizes, (
|
||||
f"InputBatch kernel_block_sizes {self._init_kernel_block_sizes} "
|
||||
f"!= kv_cache kernel_block_sizes {kernel_block_sizes}"
|
||||
)
|
||||
|
||||
def _attn_group_iterator(self) -> Iterator[AttentionGroup]:
|
||||
return itertools.chain.from_iterable(self.attn_groups)
|
||||
@@ -7267,12 +7260,16 @@ class GPUModelRunner(
|
||||
num_attn_module = (
|
||||
2 if self.model_config.hf_config.model_type == "longcat_flash" else 1
|
||||
)
|
||||
attn_groups = [g for groups in self.attn_groups for g in groups]
|
||||
if self.speculative_config:
|
||||
attn_groups += getattr(self.drafter, "draft_attn_groups", [])
|
||||
bind_kv_cache(
|
||||
kv_caches,
|
||||
self.compilation_config.static_forward_context,
|
||||
self.kv_caches,
|
||||
num_attn_module,
|
||||
kv_cache_groups=kv_cache_config.kv_cache_groups,
|
||||
layer_kv_caches=map_kv_caches_to_kernel_blocks(kv_caches, attn_groups),
|
||||
)
|
||||
return kv_caches
|
||||
|
||||
@@ -7336,6 +7333,8 @@ class GPUModelRunner(
|
||||
# backends for that group only supports block_size 64, we will return
|
||||
# kernel_block_size 64 and split the 256-token-block to 4 blocks with 64
|
||||
# tokens each.
|
||||
# Block-outermost packed groups are not split; their attention groups
|
||||
# map kernel blocks themselves.
|
||||
kernel_block_sizes = prepare_kernel_block_sizes(
|
||||
kv_cache_config, self.attn_groups
|
||||
)
|
||||
@@ -7345,7 +7344,7 @@ class GPUModelRunner(
|
||||
self.initialize_metadata_builders(kv_cache_config, kernel_block_sizes)
|
||||
|
||||
# Reinitialize need to after initialize_attn_backend
|
||||
self.may_reinitialize_input_batch(kv_cache_config, kernel_block_sizes)
|
||||
self.may_reinitialize_input_batch(kv_cache_config)
|
||||
# Capture warmup providers that depend on allocated KV-cache strides.
|
||||
with self.jit_warmup_registry.activate(): # type: ignore[attr-defined]
|
||||
kv_caches = self.initialize_kv_cache_tensors(
|
||||
@@ -7422,7 +7421,9 @@ class GPUModelRunner(
|
||||
# Skip modules that don't need KV cache (eg encoder-only attention)
|
||||
if spec := attn_module.get_kv_cache_spec(self.vllm_config):
|
||||
if isinstance(spec, AttentionSpec):
|
||||
spec = attn_module.get_attn_backend().customize_spec(spec)
|
||||
spec = customize_attention_spec(
|
||||
attn_module.get_attn_backend(), spec
|
||||
)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
|
||||
return kv_cache_spec
|
||||
|
||||
+207
-45
@@ -3,7 +3,7 @@
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from itertools import product as iprod
|
||||
from typing import Any
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
@@ -35,7 +36,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
create_kv_cache_views,
|
||||
)
|
||||
@@ -166,6 +166,9 @@ class KVBlockZeroer:
|
||||
kv = static_forward_context[layer_name].kv_cache
|
||||
if not isinstance(kv, torch.Tensor):
|
||||
continue
|
||||
if group.kernel_block_stride:
|
||||
# Bound views address kernel blocks; zero by manager block.
|
||||
kv = group.map_kv_cache_to_manager_blocks(kv, num_blocks)
|
||||
if kv.device.type != self.device.type:
|
||||
continue
|
||||
dp = kv.data_ptr()
|
||||
@@ -255,12 +258,22 @@ class KVBlockZeroer:
|
||||
self.zero_block_ids([0])
|
||||
|
||||
|
||||
@dataclass
|
||||
@dataclass(eq=False)
|
||||
class AttentionGroup:
|
||||
backend: type[AttentionBackend]
|
||||
layer_names: list[str]
|
||||
kv_cache_spec: KVCacheSpec
|
||||
kv_cache_group_id: int
|
||||
# Set when the backend needs blocks smaller than the KV cache manager's: its
|
||||
# block tables and KV cache views then address kernel blocks, kernel block j
|
||||
# of block b being b * kernel_block_stride + j. The stride is set when a
|
||||
# packed KV cache is bound (other layers' pages sit between its blocks);
|
||||
# otherwise it stays 0 and the kernel blocks per block are used.
|
||||
kernel_block_size: int | None = None
|
||||
kernel_block_stride: int = 0
|
||||
# Persistent per ubatch: CUDA graphs capture the mapped block table.
|
||||
kernel_block_table: torch.Tensor | None = None
|
||||
kernel_block_offsets: torch.Tensor | None = None
|
||||
# When ubatching is enabled we will have a metadata builder for each ubatch
|
||||
# so that if they use internal persistent buffers for cudagraphs, and they
|
||||
# won't have to worry about conflicting with the other ubatches.
|
||||
@@ -275,27 +288,44 @@ class AttentionGroup:
|
||||
kernel_block_size: int | None = None,
|
||||
num_metadata_builders: int = 1,
|
||||
):
|
||||
spec = self.kv_cache_spec
|
||||
if kernel_block_size is not None and isinstance(spec, AttentionSpec):
|
||||
# A backend in a packed KV cache group may need smaller blocks than
|
||||
# the group is split into.
|
||||
kernel_block_size = select_common_block_size(
|
||||
kernel_block_size, [self.backend], [spec]
|
||||
)
|
||||
if kernel_block_size != spec.block_size:
|
||||
self.kernel_block_size = kernel_block_size
|
||||
if kernel_block_size is None:
|
||||
kv_cache_spec_builder = self.kv_cache_spec
|
||||
elif (
|
||||
isinstance(self.kv_cache_spec, MLAAttentionSpec)
|
||||
and self.kv_cache_spec.storage_block_size is not None
|
||||
):
|
||||
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
|
||||
self.kv_cache_spec.storage_block_size
|
||||
)
|
||||
else:
|
||||
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
|
||||
kernel_block_size
|
||||
)
|
||||
builder_cls = self.backend.get_builder_cls()
|
||||
builder_kwargs = {}
|
||||
if builder_cls.requires_block_table_width:
|
||||
if builder_cls.requires_block_table_width or self.kernel_block_size is not None:
|
||||
max_num_blocks = self.kv_cache_spec.max_num_blocks_per_req(
|
||||
vllm_config, vllm_config.model_config.max_model_len
|
||||
)
|
||||
builder_kwargs["block_table_width"] = get_block_table_width(
|
||||
max_num_blocks, self.kv_cache_spec.block_size, kernel_block_size
|
||||
width = get_block_table_width(max_num_blocks, self.kv_cache_spec.block_size)
|
||||
if kernel_block_size is not None:
|
||||
width *= self.kv_cache_spec.block_size // kernel_block_size
|
||||
if builder_cls.requires_block_table_width:
|
||||
builder_kwargs["block_table_width"] = width
|
||||
if self.kernel_block_size is not None:
|
||||
self.kernel_block_table = torch.zeros(
|
||||
num_metadata_builders,
|
||||
vllm_config.scheduler_config.max_num_seqs,
|
||||
width,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self.kernel_block_offsets = torch.arange(
|
||||
spec.block_size // self.kernel_block_size,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self.metadata_builders = [
|
||||
builder_cls(
|
||||
@@ -307,9 +337,109 @@ class AttentionGroup:
|
||||
)
|
||||
for _ in range(num_metadata_builders)
|
||||
]
|
||||
if kernel_block_size is not None:
|
||||
for builder in self.metadata_builders:
|
||||
builder.set_kernel_block_size(kernel_block_size)
|
||||
|
||||
def build_metadata(
|
||||
self,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
ubatch_idx: int = 0,
|
||||
common_prefix_len: int = 0,
|
||||
**kwargs,
|
||||
) -> Any:
|
||||
return self.get_metadata_builder(ubatch_idx).build(
|
||||
common_prefix_len,
|
||||
self._with_kernel_block_table(common_attn_metadata, ubatch_idx),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def build_metadata_for_cudagraph_capture(
|
||||
self, common_attn_metadata: CommonAttentionMetadata, ubatch_idx: int = 0
|
||||
) -> Any:
|
||||
return self.get_metadata_builder(ubatch_idx).build_for_cudagraph_capture(
|
||||
self._with_kernel_block_table(common_attn_metadata, ubatch_idx)
|
||||
)
|
||||
|
||||
def build_metadata_for_drafting(
|
||||
self, common_attn_metadata: CommonAttentionMetadata, draft_index: int
|
||||
) -> Any:
|
||||
return self.get_metadata_builder().build_for_drafting(
|
||||
self._with_kernel_block_table(common_attn_metadata), draft_index
|
||||
)
|
||||
|
||||
def _with_kernel_block_table(
|
||||
self, common_attn_metadata: CommonAttentionMetadata, ubatch_idx: int = 0
|
||||
) -> CommonAttentionMetadata:
|
||||
if self.kernel_block_size is None:
|
||||
return common_attn_metadata
|
||||
return replace(
|
||||
common_attn_metadata,
|
||||
block_table_tensor=self.map_to_kernel_block_table(
|
||||
common_attn_metadata.block_table_tensor, ubatch_idx
|
||||
),
|
||||
)
|
||||
|
||||
def map_to_kernel_block_table(
|
||||
self, block_table: torch.Tensor, ubatch_idx: int = 0
|
||||
) -> torch.Tensor:
|
||||
if self.kernel_block_size is None:
|
||||
return block_table
|
||||
assert self.kernel_block_table is not None
|
||||
blocks_per_kv_block = self.kv_cache_spec.block_size // self.kernel_block_size
|
||||
rows, cols = block_table.shape
|
||||
out = self.kernel_block_table[ubatch_idx, :rows, : cols * blocks_per_kv_block]
|
||||
torch.add(
|
||||
self.kernel_block_offsets,
|
||||
block_table.unsqueeze(-1),
|
||||
alpha=self.kernel_block_stride or blocks_per_kv_block,
|
||||
out=out.unflatten(1, (cols, -1)),
|
||||
)
|
||||
return out
|
||||
|
||||
def map_kv_cache_to_kernel_blocks(self, kv_cache: torch.Tensor) -> torch.Tensor:
|
||||
"""Views a packed cache in manager blocks as kernel blocks."""
|
||||
assert self.kernel_block_size is not None
|
||||
spec = self.kv_cache_spec
|
||||
kernel_rows = spec.get_num_kernel_states(self.kernel_block_size)
|
||||
if kv_cache.shape[-2] == kernel_rows:
|
||||
return kv_cache
|
||||
# Token and pooled caches are written through manager-block slot
|
||||
# mappings, so only compressed caches (written from their block table)
|
||||
# can be mapped.
|
||||
if (
|
||||
spec.tokens_per_state <= 1
|
||||
or kv_cache.shape[1] != 1
|
||||
or kv_cache.stride(0) % (kernel_rows * kv_cache.stride(-2))
|
||||
):
|
||||
raise ValueError(
|
||||
f"{self.backend.get_name()} needs {self.kernel_block_size}-token "
|
||||
f"kernel blocks, which packed {spec.block_size}-token KV cache "
|
||||
f"blocks cannot be split into. Reduce --block-size to "
|
||||
f"{self.kernel_block_size} or set VLLM_KV_CACHE_LAYOUT to a "
|
||||
"layer-compact layout (e.g. LBNHC)."
|
||||
)
|
||||
# Kernel block j of block b is b * kernel_block_stride + j. The view
|
||||
# spans the other layers' pages between blocks (never addressing them),
|
||||
# which ``view`` cannot express.
|
||||
page_stride = kernel_rows * kv_cache.stride(-2)
|
||||
self.kernel_block_stride = kv_cache.stride(0) // page_stride
|
||||
num_pages = (kv_cache.shape[0] - 1) * self.kernel_block_stride + (
|
||||
kv_cache.shape[-2] // kernel_rows
|
||||
)
|
||||
return kv_cache.as_strided(
|
||||
(num_pages, *kv_cache.shape[1:-2], kernel_rows, kv_cache.shape[-1]),
|
||||
(page_stride, *kv_cache.stride()[1:]),
|
||||
)
|
||||
|
||||
def map_kv_cache_to_manager_blocks(
|
||||
self, kv_cache: torch.Tensor, num_blocks: int
|
||||
) -> torch.Tensor:
|
||||
"""Views a packed cache in kernel blocks as manager blocks."""
|
||||
assert self.kernel_block_stride
|
||||
spec = self.kv_cache_spec
|
||||
rows = spec.get_num_kernel_states(spec.block_size)
|
||||
return kv_cache.as_strided(
|
||||
(num_blocks, *kv_cache.shape[1:-2], rows, kv_cache.shape[-1]),
|
||||
(kv_cache.stride(0) * self.kernel_block_stride, *kv_cache.stride()[1:]),
|
||||
)
|
||||
|
||||
def get_metadata_builder(self, ubatch_id: int = 0) -> AttentionMetadataBuilder:
|
||||
assert len(self.metadata_builders) > ubatch_id
|
||||
@@ -327,17 +457,48 @@ class AttentionGroup:
|
||||
self.get_metadata_builder().update_draft_decode_metadata(metadata)
|
||||
|
||||
|
||||
def map_kv_caches_to_kernel_blocks(
|
||||
kv_caches: dict[str, torch.Tensor], attn_groups: Iterable[AttentionGroup]
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""The views to bind to layers. The manager views in ``kv_caches`` stay for
|
||||
KV connectors."""
|
||||
mapped = dict(kv_caches)
|
||||
for group in attn_groups:
|
||||
if group.kernel_block_size is not None:
|
||||
mapped.update(
|
||||
(name, group.map_kv_cache_to_kernel_blocks(kv_caches[name]))
|
||||
for name in group.layer_names
|
||||
if name in kv_caches
|
||||
)
|
||||
return mapped
|
||||
|
||||
|
||||
def customize_attention_spec(
|
||||
backend: type[AttentionBackend], spec: AttentionSpec
|
||||
) -> AttentionSpec:
|
||||
"""The backend's spec. Blocks larger than every kernel block size a backend
|
||||
takes are split into kernel blocks, which packed blocks must keep whole."""
|
||||
spec = backend.customize_spec(spec)
|
||||
sizes = backend.get_supported_kernel_block_sizes(spec)
|
||||
fixed_sizes = [s for s in sizes if isinstance(s, int)]
|
||||
if spec.block_stride_alignment is None and sizes and fixed_sizes == sizes:
|
||||
spec = replace(spec, block_stride_alignment=MultipleOf(max(fixed_sizes)))
|
||||
return spec
|
||||
|
||||
|
||||
def _block_size_is_supported(
|
||||
backends: list[type[AttentionBackend]], block_size: int
|
||||
backends: list[type[AttentionBackend]],
|
||||
block_size: int,
|
||||
specs: Sequence[KVCacheSpec | None],
|
||||
) -> bool:
|
||||
"""Check if the block size is supported by all backends.
|
||||
|
||||
An exact ``int`` declaration must match exactly; a ``MultipleOf``
|
||||
declaration accepts any multiple of its base.
|
||||
"""
|
||||
for backend in backends:
|
||||
for backend, spec in zip(backends, specs, strict=True):
|
||||
is_supported = False
|
||||
for supported_size in backend.get_supported_kernel_block_sizes():
|
||||
for supported_size in backend.get_supported_kernel_block_sizes(spec):
|
||||
if isinstance(supported_size, int):
|
||||
if block_size == supported_size:
|
||||
is_supported = True
|
||||
@@ -354,6 +515,7 @@ def _block_size_is_supported(
|
||||
def select_common_block_size(
|
||||
kv_manager_block_size: int,
|
||||
backends: list[type[AttentionBackend]],
|
||||
kv_cache_specs: Sequence[KVCacheSpec | None] | None = None,
|
||||
) -> int:
|
||||
"""Select a block size that is supported by all backends and is a factor of
|
||||
kv_manager_block_size.
|
||||
@@ -364,6 +526,7 @@ def select_common_block_size(
|
||||
Args:
|
||||
kv_manager_block_size: Block size of KV cache.
|
||||
backends: List of attention backend classes.
|
||||
kv_cache_specs: Per-backend cache specs for spec-specific constraints.
|
||||
|
||||
Returns:
|
||||
The selected block size.
|
||||
@@ -372,25 +535,27 @@ def select_common_block_size(
|
||||
ValueError: If no valid block size found.
|
||||
|
||||
"""
|
||||
if _block_size_is_supported(backends, kv_manager_block_size):
|
||||
specs = kv_cache_specs or [None] * len(backends)
|
||||
if _block_size_is_supported(backends, kv_manager_block_size, specs):
|
||||
return kv_manager_block_size
|
||||
|
||||
# MultipleOf constraints also accept the manager size if they accept a divisor.
|
||||
# Any remaining candidate must therefore be an explicit size from a backend.
|
||||
candidates = {
|
||||
size
|
||||
for backend in backends
|
||||
for size in backend.get_supported_kernel_block_sizes()
|
||||
for backend, spec in zip(backends, specs, strict=True)
|
||||
for size in backend.get_supported_kernel_block_sizes(spec)
|
||||
if isinstance(size, int) and kv_manager_block_size % size == 0
|
||||
}
|
||||
|
||||
for size in sorted(candidates, reverse=True):
|
||||
if _block_size_is_supported(backends, size):
|
||||
if _block_size_is_supported(backends, size, specs):
|
||||
return size
|
||||
raise ValueError(
|
||||
f"No common block size for {kv_manager_block_size} ("
|
||||
+ "; ".join(
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}" for b in backends
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes(s)}"
|
||||
for b, s in zip(backends, specs, strict=True)
|
||||
)
|
||||
+ ")."
|
||||
)
|
||||
@@ -450,9 +615,6 @@ def allocate_kv_cache(
|
||||
kernel_block_size = None
|
||||
if kernel_block_sizes is not None and group_id < len(kernel_block_sizes):
|
||||
kernel_block_size = kernel_block_sizes[group_id]
|
||||
if isinstance(spec, MLAAttentionSpec) and spec.storage_block_size is not None:
|
||||
kernel_block_size = spec.storage_block_size
|
||||
|
||||
views = create_kv_cache_views(
|
||||
buf,
|
||||
spec,
|
||||
@@ -497,24 +659,20 @@ def prepare_kernel_block_sizes(
|
||||
# This is an attention backend that supports virtual block splitting.
|
||||
kv_manager_block_size = kv_cache_group.kv_cache_spec.block_size
|
||||
group_backends = [g.backend for g in attn_groups[kv_cache_gid]]
|
||||
storage_block_size = (
|
||||
kv_cache_spec.storage_block_size
|
||||
if isinstance(kv_cache_spec, MLAAttentionSpec)
|
||||
else None
|
||||
)
|
||||
if storage_block_size is not None and _block_size_is_supported(
|
||||
group_backends, storage_block_size
|
||||
):
|
||||
# Storage-block specs (e.g. the GLM-5.3-Flash kpool indexer
|
||||
# cache) address the cache in pool pages, and every other
|
||||
# consumer (cache views, metadata builders, hisparse) already
|
||||
# uses storage_block_size as the kernel block. Fall back to the
|
||||
# backend vote when the group's backends do not accept it.
|
||||
selected_kernel_size = storage_block_size
|
||||
else:
|
||||
selected_kernel_size = select_common_block_size(
|
||||
kv_manager_block_size, group_backends
|
||||
group_specs = [g.kv_cache_spec for g in attn_groups[kv_cache_gid]]
|
||||
# Block-outermost packing puts other layers' pages between the
|
||||
# blocks, so they are never split group-wide.
|
||||
selected_kernel_size = (
|
||||
kv_manager_block_size
|
||||
if any(
|
||||
t.block_stride > t.layer_stride
|
||||
for t in kv_cache_config.kv_cache_tensors
|
||||
if not set(t.layers).isdisjoint(kv_cache_group.layer_names)
|
||||
)
|
||||
else select_common_block_size(
|
||||
kv_manager_block_size, group_backends, group_specs
|
||||
)
|
||||
)
|
||||
kernel_block_sizes.append(selected_kernel_size)
|
||||
elif isinstance(kv_cache_spec, MambaSpec):
|
||||
# This is likely Mamba or other non-attention cache, no splitting.
|
||||
@@ -656,6 +814,7 @@ def bind_kv_cache(
|
||||
runner_kv_caches: list[torch.Tensor],
|
||||
num_attn_module: int = 1,
|
||||
kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None,
|
||||
layer_kv_caches: dict[str, torch.Tensor] | None = None,
|
||||
) -> None:
|
||||
"""Bind the allocated KV cache to both ModelRunner and forward context so
|
||||
that the KV cache can be used in the forward pass.
|
||||
@@ -674,6 +833,9 @@ def bind_kv_cache(
|
||||
runner_kv_caches: The kv_cache declared by ModelRunner.
|
||||
kv_cache_groups: The KV cache groups of the model, used to resolve
|
||||
layers that share a KV cache.
|
||||
layer_kv_caches: The views to bind to layers when they differ from
|
||||
kv_caches (e.g. kernel-block views). The runner keeps kv_caches,
|
||||
which are addressed by scheduler block IDs.
|
||||
|
||||
"""
|
||||
# Bind kv_caches to ModelRunner
|
||||
@@ -699,7 +861,7 @@ def bind_kv_cache(
|
||||
runner_kv_caches.append(kv_caches[layer_name])
|
||||
|
||||
bind_kv_cache_to_layers(
|
||||
kv_caches, forward_context, num_attn_module, kv_cache_groups
|
||||
layer_kv_caches or kv_caches, forward_context, num_attn_module, kv_cache_groups
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user