[2/N][Core] support partial prefix cache hit for hybrid model (#46384)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Yifan Qiao
parent
8e981630c9
commit
481e481be7
+43
-1
@@ -2,11 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import product as iprod
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm.config import CacheConfig, VllmConfig
|
||||
@@ -18,11 +19,13 @@ from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.math_utils import largest_power_of_2_divisor
|
||||
from vllm.utils.mem_utils import MemorySnapshot, format_gib
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionMetadataBuilder,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
EncoderOnlyAttentionSpec,
|
||||
@@ -535,6 +538,45 @@ def bind_kv_cache(
|
||||
forward_context[layer_name].kv_cache = kv_cache
|
||||
|
||||
|
||||
def copy_kv_cache_blocks_inplace(
|
||||
kv_caches: Iterable[torch.Tensor | list[torch.Tensor]],
|
||||
num_blocks: int,
|
||||
kv_cache_block_copies: Sequence[KVCacheBlockCopy],
|
||||
) -> None:
|
||||
if not kv_cache_block_copies:
|
||||
return
|
||||
|
||||
storage_tensors: list[torch.Tensor] = []
|
||||
seen_storage: set[int] = set()
|
||||
for entry in kv_caches:
|
||||
# Mamba layers hold a list of state tensors; attention layers a single
|
||||
# tensor. Both alias the shared block-major backing storage.
|
||||
tensors = entry if isinstance(entry, (list, tuple)) else (entry,)
|
||||
for tensor in tensors:
|
||||
ptr = tensor.untyped_storage().data_ptr()
|
||||
if ptr in seen_storage:
|
||||
continue
|
||||
seen_storage.add(ptr)
|
||||
storage_tensors.append(tensor)
|
||||
|
||||
if not storage_tensors:
|
||||
return
|
||||
device = storage_tensors[0].device
|
||||
indices_np = np.array(kv_cache_block_copies, dtype=np.int64)
|
||||
indices = async_tensor_h2d(indices_np, device=device)
|
||||
src_indices, dst_indices = indices.unbind(dim=1)
|
||||
|
||||
for tensor in storage_tensors:
|
||||
assert tensor.device == device
|
||||
blocks = torch.empty(0, dtype=torch.uint8, device=device)
|
||||
blocks.set_(tensor.untyped_storage())
|
||||
# Block-major backing storage: block i owns the contiguous byte range
|
||||
# [i * page_size, (i + 1) * page_size).
|
||||
assert blocks.numel() % num_blocks == 0
|
||||
blocks = blocks.view(num_blocks, -1)
|
||||
blocks[dst_indices] = blocks[src_indices]
|
||||
|
||||
|
||||
def is_residual_scattered_for_sp(
|
||||
vllm_config: VllmConfig, num_input_tokens: int
|
||||
) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user