[Attention] Make FlexAttention and FlashAttention use num-blocks first layouts (#42095)
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Matthew Bonanni
Nicolò Lucchesi
parent
d8eebe6d97
commit
7e33081cee
@@ -420,7 +420,7 @@ class TransferTopology:
|
||||
head_size=1,
|
||||
)
|
||||
logger.debug("Test kv_cache_shape: %s", kv_cache_shape)
|
||||
# Non-MLA backends caches have 5 dims [2, num_blocks, H,N,D],
|
||||
# Non-MLA backends caches have 5 dims [num_blocks, 2, H,N,D],
|
||||
# we just mock num_blocks to 1 for the dimension check below.
|
||||
# Hybrid SSM models assume a single blocks_first layout
|
||||
self._is_kv_layout_blocks_first = self.is_mamba or (
|
||||
@@ -484,6 +484,15 @@ class TransferTopology:
|
||||
def cross_layers_blocks(self) -> bool:
|
||||
return self._cross_layers_blocks
|
||||
|
||||
@property
|
||||
def virtually_split_kv_in_blocks(self) -> bool:
|
||||
# Whether to logically split each block into K and V halves.
|
||||
# Applies when K/V are interleaved within each block (blocks-first),
|
||||
# but NOT when cross-layer blocks are used — cross-layer blocks have
|
||||
# per-layer K/V interleaving (L0_K, L0_V, L1_K, L1_V, ...) so a
|
||||
# simple half-split does not separate K from V.
|
||||
return self._is_kv_layout_blocks_first and not self._cross_layers_blocks
|
||||
|
||||
@property
|
||||
def split_k_and_v(self) -> bool:
|
||||
# Whether to register regions for K and V separately (when present).
|
||||
|
||||
@@ -17,7 +17,6 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
from vllm.utils.hashing import safe_hash
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.attention.backends.triton_attn import TritonAttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -130,33 +129,24 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
|
||||
Args:
|
||||
dst_kv_cache_layer (torch.Tensor): the destination KV cache
|
||||
layer. In shape [2, num_pages, page_size, xxx] if not
|
||||
using MLA, [num_pages, page_size, xxx] otherwise.
|
||||
src_kv_cache (torch.Tensor): the source KV cache. In shape
|
||||
[2, num_tokens, xxx] if not using MLA, [num_tokens, xxx]
|
||||
otherwise.
|
||||
layer. In shape [num_pages, page_size, xxx] for MLA,
|
||||
[num_pages, 2, page_size, xxx] otherwise.
|
||||
src_kv_cache (torch.Tensor): the source KV cache.
|
||||
slot_mapping (torch.Tensor): the slot mapping. In shape
|
||||
[num_tokens].
|
||||
"""
|
||||
dst_kv_cache_layer_shape = dst_kv_cache_layer.shape
|
||||
if isinstance(attn_metadata, MLACommonMetadata):
|
||||
dst_kv_cache_layer_shape = dst_kv_cache_layer.shape
|
||||
num_pages = dst_kv_cache_layer_shape[0]
|
||||
page_size = dst_kv_cache_layer_shape[1]
|
||||
dst_kv_cache_layer = dst_kv_cache_layer.reshape(
|
||||
num_pages * page_size, -1
|
||||
)
|
||||
dst_kv_cache_layer[slot_mapping, ...] = src_kv_cache
|
||||
elif isinstance(attn_metadata, TritonAttentionMetadata):
|
||||
else:
|
||||
block_idxs = slot_mapping // self._block_size
|
||||
offsets = slot_mapping % self._block_size
|
||||
dst_kv_cache_layer[block_idxs, :, offsets] = src_kv_cache
|
||||
else:
|
||||
num_pages = dst_kv_cache_layer_shape[1]
|
||||
page_size = dst_kv_cache_layer_shape[2]
|
||||
dst_kv_cache_layer = dst_kv_cache_layer.reshape(
|
||||
2, num_pages * page_size, -1
|
||||
)
|
||||
dst_kv_cache_layer[:, slot_mapping, ...] = src_kv_cache
|
||||
|
||||
# Get the metadata
|
||||
metadata: KVConnectorMetadata = self._get_connector_metadata()
|
||||
@@ -234,18 +224,15 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
) -> torch.Tensor:
|
||||
"""Extract the KV cache from the layer.
|
||||
|
||||
Assume the shape of the layer is (2, num_pages, page_size, xxx)
|
||||
if MLA is not used, and (num_pages, page_size, xxx) otherwise.
|
||||
Assume the shape of the layer is (num_pages, page_size, xxx)
|
||||
for MLA, and (num_pages, 2, page_size, xxx) otherwise.
|
||||
"""
|
||||
if isinstance(attn_metadata, MLACommonMetadata):
|
||||
num_pages, page_size = layer.shape[0], layer.shape[1]
|
||||
return layer.reshape(num_pages * page_size, -1)[slot_mapping, ...]
|
||||
elif isinstance(attn_metadata, TritonAttentionMetadata):
|
||||
block_idxs = slot_mapping // self._block_size
|
||||
offsets = slot_mapping % self._block_size
|
||||
return layer[block_idxs, :, offsets]
|
||||
num_pages, page_size = layer.shape[1], layer.shape[2]
|
||||
return layer.reshape(2, num_pages * page_size, -1)[:, slot_mapping, ...]
|
||||
block_idxs = slot_mapping // self._block_size
|
||||
offsets = slot_mapping % self._block_size
|
||||
return layer[block_idxs, :, offsets]
|
||||
|
||||
connector_metadata = self._get_connector_metadata()
|
||||
assert isinstance(connector_metadata, ExampleConnectorMetadata)
|
||||
|
||||
@@ -1745,7 +1745,7 @@ class MooncakeConnectorWorker:
|
||||
return _expand_transfer_regions(
|
||||
base_addrs=base_addrs,
|
||||
block_lens=block_lens,
|
||||
is_kv_layout_blocks_first=self.transfer_topo.is_kv_layout_blocks_first,
|
||||
is_kv_layout_blocks_first=self.transfer_topo.virtually_split_kv_in_blocks,
|
||||
)
|
||||
|
||||
def _get_sender_transfer_plan(
|
||||
|
||||
@@ -943,7 +943,7 @@ class NixlConnectorWorker:
|
||||
self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses
|
||||
self.num_regions = len(caches_data)
|
||||
|
||||
if self.transfer_topo.is_kv_layout_blocks_first:
|
||||
if self.transfer_topo.virtually_split_kv_in_blocks:
|
||||
# NOTE (NickLucche) When FlashInfer is used, memory is registered
|
||||
# with joint KV for each block. This minimizes the overhead in
|
||||
# registerMem allowing faster descs queries. In order to be able to
|
||||
@@ -1118,7 +1118,7 @@ class NixlConnectorWorker:
|
||||
addr = base_addr + block_offset
|
||||
result.append((addr, kv_block_len, self.device_id))
|
||||
|
||||
if self.transfer_topo.is_kv_layout_blocks_first:
|
||||
if self.transfer_topo.virtually_split_kv_in_blocks:
|
||||
# Separate and interleave K/V regions to maintain the same
|
||||
# descs ordering. This is needed for selecting contiguous heads
|
||||
# when split across TP ranks.
|
||||
@@ -1167,7 +1167,7 @@ class NixlConnectorWorker:
|
||||
addr = base_addr + block_offset + rank_offset
|
||||
result.append((addr, local_block_len, nixl_agent_meta.device_id))
|
||||
|
||||
if self.transfer_topo.is_kv_layout_blocks_first:
|
||||
if self.transfer_topo.virtually_split_kv_in_blocks:
|
||||
# With FlashInfer index V separately to allow head splitting.
|
||||
second_split = self.get_backend_aware_kv_block_len(
|
||||
layer_idx=i, first_split=False, mamba_view=False
|
||||
@@ -2416,7 +2416,7 @@ class NixlConnectorWorker:
|
||||
|1st_split-2nd_split| |1st_split-2nd_split |
|
||||
"""
|
||||
assert self.transfer_topo is not None
|
||||
if self.transfer_topo.is_kv_layout_blocks_first:
|
||||
if self.transfer_topo.virtually_split_kv_in_blocks:
|
||||
if mamba_view:
|
||||
block_len = self._mamba_ssm_size[not first_split]
|
||||
else:
|
||||
|
||||
@@ -5,7 +5,6 @@ from dataclasses import replace
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config import get_layers_from_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
KVConnectorStats,
|
||||
)
|
||||
@@ -18,7 +17,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
OffloadingConnectorStats,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
AttentionSpec,
|
||||
@@ -59,25 +57,9 @@ class OffloadingConnectorWorker:
|
||||
def register_kv_caches(
|
||||
self, kv_caches: dict[str, torch.Tensor | list[torch.Tensor]]
|
||||
):
|
||||
layer_names = list(kv_caches.keys())
|
||||
layers = get_layers_from_vllm_config(
|
||||
self.spec.vllm_config,
|
||||
AttentionLayerBase, # type: ignore[type-abstract]
|
||||
layer_names,
|
||||
)
|
||||
attn_backends = {
|
||||
layer_name: layers[layer_name].get_attn_backend()
|
||||
for layer_name in layer_names
|
||||
if layer_name in layers
|
||||
}
|
||||
|
||||
num_blocks = self.spec.kv_cache_config.num_blocks
|
||||
|
||||
# layer_name -> list of matching KV cache tensors
|
||||
# such that each tensor starts with the num_blocks dimension.
|
||||
# FlashAttention layers which use the (2, num_blocks, ...) layout
|
||||
# will possibly map to 2 tensors, one per K and one per V.
|
||||
# All other layers will probably map to a single tensor.
|
||||
# layer_name -> (num_blocks, page_size_bytes) tensor
|
||||
tensors_per_block: dict[str, tuple[torch.Tensor, ...]] = {}
|
||||
# layer_name -> size of (un-padded) page in bytes
|
||||
unpadded_page_size_bytes: dict[str, int] = {}
|
||||
@@ -99,71 +81,22 @@ class OffloadingConnectorWorker:
|
||||
assert isinstance(layer_kv_cache, torch.Tensor)
|
||||
assert layer_kv_cache.storage_offset() == 0
|
||||
|
||||
# get the logical dimension for num_blocks
|
||||
test_shape = attn_backends[layer_name].get_kv_cache_shape(
|
||||
num_blocks=1234,
|
||||
block_size=16,
|
||||
num_kv_heads=1,
|
||||
head_size=256,
|
||||
storage = layer_kv_cache.untyped_storage()
|
||||
page = layer_kv_cache_spec.page_size_bytes
|
||||
tensors_per_block[layer_name] = (
|
||||
torch.tensor(
|
||||
[],
|
||||
dtype=torch.int8,
|
||||
device=layer_kv_cache.device,
|
||||
)
|
||||
.set_(storage)
|
||||
.view(num_blocks, page),
|
||||
)
|
||||
num_blocks_logical_dim = test_shape.index(1234)
|
||||
|
||||
# sort the logical dimensions by stride (high to low)
|
||||
# to get a physical-to-logical mapping:
|
||||
# physical_to_logical[physical_pos] = logical_dim
|
||||
logical_strides = layer_kv_cache.stride()
|
||||
physical_to_logical = sorted(
|
||||
range(len(logical_strides)),
|
||||
key=lambda idx: logical_strides[idx],
|
||||
reverse=True,
|
||||
page_size_bytes[layer_name] = layer_kv_cache_spec.page_size_bytes
|
||||
unpadded_page_size_bytes[layer_name] = (
|
||||
layer_kv_cache_spec.real_page_size_bytes
|
||||
)
|
||||
|
||||
num_blocks_physical_dim = physical_to_logical.index(
|
||||
num_blocks_logical_dim
|
||||
)
|
||||
if num_blocks_physical_dim == 0:
|
||||
storage = layer_kv_cache.untyped_storage()
|
||||
page = layer_kv_cache_spec.page_size_bytes
|
||||
tensors_per_block[layer_name] = (
|
||||
torch.tensor(
|
||||
[],
|
||||
dtype=torch.int8,
|
||||
device=layer_kv_cache.device,
|
||||
)
|
||||
.set_(storage)
|
||||
.view(num_blocks, page),
|
||||
)
|
||||
page_size_bytes[layer_name] = (
|
||||
layer_kv_cache_spec.page_size_bytes
|
||||
)
|
||||
unpadded_page_size_bytes[layer_name] = (
|
||||
layer_kv_cache_spec.real_page_size_bytes
|
||||
)
|
||||
else:
|
||||
# Flash Attention case: (2, num_blocks, ...)
|
||||
assert test_shape[0] == 2
|
||||
assert physical_to_logical[0] == 0
|
||||
assert num_blocks_physical_dim == 1
|
||||
|
||||
# unbind the tensor to separate K and V tensors
|
||||
half_page_size = layer_kv_cache_spec.page_size_bytes // 2
|
||||
storage = layer_kv_cache.untyped_storage()
|
||||
raw = (
|
||||
torch.tensor(
|
||||
[],
|
||||
dtype=torch.int8,
|
||||
device=layer_kv_cache.device,
|
||||
)
|
||||
.set_(storage)
|
||||
.view(2, num_blocks, half_page_size)
|
||||
)
|
||||
tensors_per_block[layer_name] = tuple(raw.unbind(0))
|
||||
|
||||
page_size_bytes[layer_name] = half_page_size
|
||||
unpadded_page_size_bytes[layer_name] = (
|
||||
layer_kv_cache_spec.real_page_size_bytes // 2
|
||||
)
|
||||
|
||||
elif isinstance(layer_kv_cache_spec, MambaSpec):
|
||||
state_tensors = kv_caches[layer_name]
|
||||
assert isinstance(state_tensors, list)
|
||||
|
||||
@@ -18,7 +18,6 @@ from vllm.distributed.kv_transfer.kv_connector.v1.p2p.p2p_nccl_engine import (
|
||||
)
|
||||
from vllm.distributed.parallel_state import get_world_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLACommonMetadata
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
|
||||
@@ -140,12 +139,9 @@ class P2pNcclConnector(KVConnectorBase_V1):
|
||||
"""
|
||||
Inject KV cache data into a given attention layer tensor.
|
||||
|
||||
This function updates `layer` in-place with values from `kv_cache`,
|
||||
handling different backend layouts:
|
||||
- MLA (Multi-Linear Attention) or FlashInfer: KV tensors are
|
||||
indexed along the first dimension.
|
||||
- FlashAttention: KV tensors are indexed along the second
|
||||
dimension.
|
||||
This function updates `layer` in-place with values from `kv_cache`.
|
||||
All backends (MLA, FlashAttention, FlashInfer, TritonAttention)
|
||||
are indexed along the first dimension (block index).
|
||||
|
||||
If the number of provided block IDs does not match the number of KV
|
||||
blocks, only the overlapping portion is updated, and a warning is
|
||||
@@ -160,37 +156,19 @@ class P2pNcclConnector(KVConnectorBase_V1):
|
||||
Returns:
|
||||
None. The function modifies `layer` in-place.
|
||||
"""
|
||||
if (
|
||||
isinstance(attn_metadata, MLACommonMetadata) or layer.shape[1] == 2
|
||||
): # MLA or FlashInfer
|
||||
num_block = kv_cache.shape[0]
|
||||
self.check_tensors_except_dim(layer, kv_cache, 0)
|
||||
if len(block_ids) == num_block:
|
||||
layer[block_ids, ...] = kv_cache
|
||||
else:
|
||||
layer[block_ids[:num_block], ...] = kv_cache
|
||||
logger.warning(
|
||||
"🚧kv_cache does not match, block_ids:%d, "
|
||||
"num_block:%d, request_id:%s",
|
||||
len(block_ids),
|
||||
num_block,
|
||||
request_id,
|
||||
)
|
||||
|
||||
elif layer.shape[0] == 2: # FlashAttention
|
||||
num_block = kv_cache.shape[1]
|
||||
self.check_tensors_except_dim(layer, kv_cache, 1)
|
||||
if len(block_ids) == num_block:
|
||||
layer[:, block_ids, ...] = kv_cache
|
||||
else:
|
||||
layer[:, block_ids[:num_block], ...] = kv_cache
|
||||
logger.warning(
|
||||
"🚧kv_cache does not match, block_ids:%d, "
|
||||
"num_block:%d, request_id:%s",
|
||||
len(block_ids),
|
||||
num_block,
|
||||
request_id,
|
||||
)
|
||||
num_block = kv_cache.shape[0]
|
||||
self.check_tensors_except_dim(layer, kv_cache, 0)
|
||||
if len(block_ids) == num_block:
|
||||
layer[block_ids, ...] = kv_cache
|
||||
else:
|
||||
layer[block_ids[:num_block], ...] = kv_cache
|
||||
logger.warning(
|
||||
"🚧kv_cache does not match, block_ids:%d, "
|
||||
"num_block:%d, request_id:%s",
|
||||
len(block_ids),
|
||||
num_block,
|
||||
request_id,
|
||||
)
|
||||
|
||||
# Get the metadata
|
||||
metadata: KVConnectorMetadata = self._get_connector_metadata()
|
||||
@@ -263,37 +241,6 @@ class P2pNcclConnector(KVConnectorBase_V1):
|
||||
|
||||
assert self.p2p_nccl_engine is not None
|
||||
|
||||
def extract_kv_from_layer(
|
||||
layer: torch.Tensor,
|
||||
block_ids: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Extract KV cache slices from a given attention layer tensor.
|
||||
|
||||
This function handles multiple backend layouts:
|
||||
- MLA (Multi-Linear Attention) or FlashInfer: KV tensors are
|
||||
indexed along the first dimension.
|
||||
- FlashAttention: KV tensors are indexed along the second
|
||||
dimension.
|
||||
|
||||
Args:
|
||||
layer (torch.Tensor): The KV cache from the attention layer.
|
||||
block_ids (torch.Tensor): Indices of blocks to extract.
|
||||
|
||||
Returns:
|
||||
torch.Tensor: A tensor containing the extracted KV slices.
|
||||
Returns None if the layout is unsupported.
|
||||
"""
|
||||
if (
|
||||
isinstance(attn_metadata, MLACommonMetadata) or layer.shape[1] == 2
|
||||
): # MLA or FlashInfer
|
||||
return layer[block_ids, ...]
|
||||
|
||||
if layer.shape[0] == 2: # FlashAttention
|
||||
return layer[:, block_ids, ...]
|
||||
|
||||
return None
|
||||
|
||||
connector_metadata = self._get_connector_metadata()
|
||||
assert isinstance(connector_metadata, P2pNcclConnectorMetadata)
|
||||
for request in connector_metadata.requests:
|
||||
@@ -301,7 +248,7 @@ class P2pNcclConnector(KVConnectorBase_V1):
|
||||
ip, port = self.parse_request_id(request_id, True)
|
||||
remote_address = ip + ":" + str(port + self._rank)
|
||||
|
||||
kv_cache = extract_kv_from_layer(kv_layer, request.block_ids)
|
||||
kv_cache = kv_layer[request.block_ids, ...]
|
||||
self.p2p_nccl_engine.send_tensor(
|
||||
request_id + "#" + layer_name, kv_cache, remote_address
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user