Allocate kv_cache with stride order (#16605)
Signed-off-by: shuw <shuw@nvidia.com>
This commit is contained in:
@@ -77,6 +77,10 @@ class AttentionBackend(ABC):
|
||||
) -> Tuple[int, ...]:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order() -> Tuple[int, ...]:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def swap_blocks(
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
from collections import defaultdict
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
@@ -48,6 +49,9 @@ if TYPE_CHECKING:
|
||||
from vllm.worker.model_runner import (ModelInputForGPUBuilder,
|
||||
ModelInputForGPUWithSamplingMetadata)
|
||||
|
||||
FLASHINFER_KV_CACHE_LAYOUT: str = os.getenv("FLASHINFER_KV_CACHE_LAYOUT",
|
||||
"NHD").upper()
|
||||
|
||||
|
||||
class FlashInferBackend(AttentionBackend):
|
||||
|
||||
@@ -80,6 +84,14 @@ class FlashInferBackend(AttentionBackend):
|
||||
) -> Tuple[int, ...]:
|
||||
return (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
|
||||
@staticmethod
|
||||
def get_kv_cache_stride_order() -> Tuple[int, ...]:
|
||||
cache_layout = FLASHINFER_KV_CACHE_LAYOUT
|
||||
assert (cache_layout in ("NHD", "HND"))
|
||||
stride_order = (0, 1, 2, 3, 4) if cache_layout == "NHD" else (0, 1, 3,
|
||||
2, 4)
|
||||
return stride_order
|
||||
|
||||
@staticmethod
|
||||
def swap_blocks(
|
||||
src_kv_cache: torch.Tensor,
|
||||
@@ -188,6 +200,7 @@ class FlashInferState(AttentionState):
|
||||
self.global_hyperparameters: Optional[PerLayerParameters] = None
|
||||
|
||||
self.vllm_config = self.runner.vllm_config
|
||||
self._kv_cache_layout = None
|
||||
|
||||
def _get_workspace_buffer(self):
|
||||
if self._workspace_buffer is None:
|
||||
@@ -197,10 +210,15 @@ class FlashInferState(AttentionState):
|
||||
device=self.runner.device)
|
||||
return self._workspace_buffer
|
||||
|
||||
def get_kv_cache_layout(self):
|
||||
if self._kv_cache_layout is None:
|
||||
self._kv_cache_layout = FLASHINFER_KV_CACHE_LAYOUT
|
||||
return self._kv_cache_layout
|
||||
|
||||
def _get_prefill_wrapper(self):
|
||||
if self._prefill_wrapper is None:
|
||||
self._prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper(
|
||||
self._get_workspace_buffer(), "NHD")
|
||||
self._get_workspace_buffer(), self.get_kv_cache_layout())
|
||||
return self._prefill_wrapper
|
||||
|
||||
def _get_decode_wrapper(self):
|
||||
@@ -213,7 +231,7 @@ class FlashInferState(AttentionState):
|
||||
num_qo_heads // num_kv_heads > 4)
|
||||
self._decode_wrapper = BatchDecodeWithPagedKVCacheWrapper(
|
||||
self._get_workspace_buffer(),
|
||||
"NHD",
|
||||
self.get_kv_cache_layout(),
|
||||
use_tensor_cores=use_tensor_cores)
|
||||
return self._decode_wrapper
|
||||
|
||||
@@ -274,7 +292,8 @@ class FlashInferState(AttentionState):
|
||||
self._graph_decode_wrapper = \
|
||||
CUDAGraphBatchDecodeWithPagedKVCacheWrapper(
|
||||
self._graph_decode_workspace_buffer, _indptr_buffer,
|
||||
self._graph_indices_buffer, _last_page_len_buffer, "NHD",
|
||||
self._graph_indices_buffer, _last_page_len_buffer,
|
||||
self.get_kv_cache_layout(),
|
||||
use_tensor_cores)
|
||||
if self.runner.kv_cache_dtype.startswith("fp8"):
|
||||
kv_cache_dtype = FlashInferBackend.get_fp8_dtype_for_flashinfer(
|
||||
@@ -1005,6 +1024,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
|
||||
prefill_output: Optional[torch.Tensor] = None
|
||||
decode_output: Optional[torch.Tensor] = None
|
||||
stride_order = FlashInferBackend.get_kv_cache_stride_order()
|
||||
if prefill_meta := attn_metadata.prefill_metadata:
|
||||
# We will use flash attention for prefill
|
||||
# when kv_cache is not provided.
|
||||
@@ -1036,7 +1056,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
|
||||
prefill_output = prefill_meta.prefill_wrapper.run(
|
||||
query,
|
||||
kv_cache,
|
||||
kv_cache.permute(*stride_order),
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
)
|
||||
@@ -1051,7 +1071,7 @@ class FlashInferImpl(AttentionImpl):
|
||||
|
||||
decode_output = decode_meta.decode_wrapper.run(
|
||||
decode_query,
|
||||
kv_cache,
|
||||
kv_cache.permute(*stride_order),
|
||||
k_scale=layer._k_scale_float,
|
||||
v_scale=layer._v_scale_float,
|
||||
)
|
||||
|
||||
+10
-3
@@ -765,21 +765,28 @@ def create_kv_caches_with_random_flash(
|
||||
model_dtype: Optional[Union[str, torch.dtype]] = None,
|
||||
seed: Optional[int] = None,
|
||||
device: Optional[str] = "cuda",
|
||||
cache_layout: Optional[str] = "NHD",
|
||||
) -> tuple[list[torch.Tensor], list[torch.Tensor]]:
|
||||
from vllm.platforms import current_platform
|
||||
current_platform.seed_everything(seed)
|
||||
|
||||
torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
|
||||
key_value_cache_shape = (num_blocks, 2, block_size, num_heads, head_size)
|
||||
generic_kv_cache_shape = (num_blocks, 2, block_size, num_heads, head_size)
|
||||
assert cache_layout in ("NHD", "HND")
|
||||
stride_order = (0, 1, 2, 3, 4) if cache_layout == "NHD" else (0, 1, 3, 2,
|
||||
4)
|
||||
|
||||
kv_cache_allocation_shape = tuple(generic_kv_cache_shape[i]
|
||||
for i in stride_order)
|
||||
scale = head_size**-0.5
|
||||
|
||||
key_caches: list[torch.Tensor] = []
|
||||
value_caches: list[torch.Tensor] = []
|
||||
|
||||
for _ in range(num_layers):
|
||||
key_value_cache = torch.empty(size=key_value_cache_shape,
|
||||
key_value_cache = torch.empty(size=kv_cache_allocation_shape,
|
||||
dtype=torch_dtype,
|
||||
device=device)
|
||||
device=device).permute(*stride_order)
|
||||
if cache_dtype in ["auto", "half", "bfloat16", "float"]:
|
||||
key_value_cache.uniform_(-scale, scale)
|
||||
elif cache_dtype == 'fp8':
|
||||
|
||||
@@ -71,19 +71,32 @@ class CacheEngine:
|
||||
device: str,
|
||||
) -> List[torch.Tensor]:
|
||||
"""Allocates KV cache on the specified device."""
|
||||
kv_cache_shape = self.attn_backend.get_kv_cache_shape(
|
||||
kv_cache_generic_shape = self.attn_backend.get_kv_cache_shape(
|
||||
num_blocks, self.block_size, self.num_kv_heads, self.head_size)
|
||||
pin_memory = is_pin_memory_available() if device == "cpu" else False
|
||||
kv_cache: List[torch.Tensor] = []
|
||||
try:
|
||||
kv_cache_stride_order = self.attn_backend.get_kv_cache_stride_order(
|
||||
)
|
||||
except (AttributeError, NotImplementedError):
|
||||
kv_cache_stride_order = tuple(range(len(kv_cache_generic_shape)))
|
||||
|
||||
# The allocation respects the backend-defined stride order to ensure
|
||||
# the semantic remains consistent for each backend. We first obtain the
|
||||
# generic kv cache shape and then permute it according to the stride
|
||||
# order which could result in a non-contiguous tensor.
|
||||
kv_cache_allocation_shape = tuple(kv_cache_generic_shape[i]
|
||||
for i in kv_cache_stride_order)
|
||||
|
||||
for _ in range(self.num_attention_layers):
|
||||
# null block in CpuGpuBlockAllocator requires at least that
|
||||
# block to be zeroed-out.
|
||||
# We zero-out everything for simplicity.
|
||||
layer_kv_cache = torch.zeros(kv_cache_shape,
|
||||
dtype=self.dtype,
|
||||
pin_memory=pin_memory,
|
||||
device=device)
|
||||
layer_kv_cache = torch.zeros(
|
||||
kv_cache_allocation_shape,
|
||||
dtype=self.dtype,
|
||||
pin_memory=pin_memory,
|
||||
device=device).permute(*kv_cache_stride_order)
|
||||
|
||||
# view back to (TOTAL_PAGES, PAGE_SIZE, entry_shape...) for cases
|
||||
# when entry_shape is higher than 1D
|
||||
|
||||
Reference in New Issue
Block a user