[2/N][Core] support partial prefix cache hit for hybrid model (#46384)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Jiangyun Zhu
2026-07-12 05:37:51 +00:00
committed by GitHub
co-authored by Yifan Qiao
parent 8e981630c9
commit 481e481be7
21 changed files with 1653 additions and 177 deletions
+43 -1
View File
@@ -2,11 +2,12 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections import defaultdict
from collections.abc import Iterable
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from itertools import product as iprod
from typing import Any
import numpy as np
import torch
from vllm.config import CacheConfig, VllmConfig
@@ -18,11 +19,13 @@ from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.math_utils import largest_power_of_2_divisor
from vllm.utils.mem_utils import MemorySnapshot, format_gib
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionMetadataBuilder,
MultipleOf,
)
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.kv_cache_interface import (
AttentionSpec,
EncoderOnlyAttentionSpec,
@@ -535,6 +538,45 @@ def bind_kv_cache(
forward_context[layer_name].kv_cache = kv_cache
def copy_kv_cache_blocks_inplace(
kv_caches: Iterable[torch.Tensor | list[torch.Tensor]],
num_blocks: int,
kv_cache_block_copies: Sequence[KVCacheBlockCopy],
) -> None:
if not kv_cache_block_copies:
return
storage_tensors: list[torch.Tensor] = []
seen_storage: set[int] = set()
for entry in kv_caches:
# Mamba layers hold a list of state tensors; attention layers a single
# tensor. Both alias the shared block-major backing storage.
tensors = entry if isinstance(entry, (list, tuple)) else (entry,)
for tensor in tensors:
ptr = tensor.untyped_storage().data_ptr()
if ptr in seen_storage:
continue
seen_storage.add(ptr)
storage_tensors.append(tensor)
if not storage_tensors:
return
device = storage_tensors[0].device
indices_np = np.array(kv_cache_block_copies, dtype=np.int64)
indices = async_tensor_h2d(indices_np, device=device)
src_indices, dst_indices = indices.unbind(dim=1)
for tensor in storage_tensors:
assert tensor.device == device
blocks = torch.empty(0, dtype=torch.uint8, device=device)
blocks.set_(tensor.untyped_storage())
# Block-major backing storage: block i owns the contiguous byte range
# [i * page_size, (i + 1) * page_size).
assert blocks.numel() % num_blocks == 0
blocks = blocks.view(num_blocks, -1)
blocks[dst_indices] = blocks[src_indices]
def is_residual_scattered_for_sp(
vllm_config: VllmConfig, num_input_tokens: int
) -> bool: