[NIXL][1/N] Refactor kernel_block_size detection (#35752)
Signed-off-by: NickLucche <nlucches@redhat.com>
This commit is contained in:
@@ -358,15 +358,6 @@ class TpKVTopology:
|
||||
# stride_order to retrieve physical position of block_size
|
||||
kv_cache_shape = tuple(kv_cache_shape[i] for i in kv_cache_stride_order)
|
||||
|
||||
# In the default non-cross layers layout the block_size position
|
||||
# is logical while in the cross layers case it is the physical
|
||||
# position. This matches the shape of the actual kv cache tensors
|
||||
# passed at register_kv_caches()/register_cross_layers_kv_cache()
|
||||
block_size_position = kv_cache_shape.index(_MOCK_BLOCK_SIZE)
|
||||
|
||||
assert block_size_position is not None
|
||||
self._block_size_position = -(len(kv_cache_shape) - block_size_position)
|
||||
|
||||
@property
|
||||
def is_kv_layout_blocks_first(self) -> bool:
|
||||
return self._is_kv_layout_blocks_first
|
||||
@@ -390,10 +381,6 @@ class TpKVTopology:
|
||||
def cross_layers_blocks(self) -> bool:
|
||||
return self._cross_layers_blocks
|
||||
|
||||
@property
|
||||
def block_size_position(self) -> int:
|
||||
return self._block_size_position
|
||||
|
||||
def tp_ratio(
|
||||
self,
|
||||
remote_tp_size: int,
|
||||
@@ -484,23 +471,46 @@ class TpKVTopology:
|
||||
return self.get_target_remote_ranks(remote_tp_size)
|
||||
|
||||
|
||||
def get_current_attn_backend(vllm_config: VllmConfig):
|
||||
layer_type = cast(type[Any], AttentionLayerBase)
|
||||
layers = get_layers_from_vllm_config(vllm_config, layer_type, None)
|
||||
if layers:
|
||||
backend = next(iter(layers.values())).get_attn_backend()
|
||||
else:
|
||||
# Fallback for tests, when static_forward_context is empty.
|
||||
logger.debug(
|
||||
"No layers found in the vLLM config. "
|
||||
"Falling back to default attention backend."
|
||||
)
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
def get_current_attn_backends(
|
||||
vllm_config: VllmConfig, layer_names: list[str] | None = None
|
||||
) -> list[type[AttentionBackend]]:
|
||||
"""Get all distinct attention backends for the given layers.
|
||||
|
||||
backend = get_attn_backend(
|
||||
Args:
|
||||
vllm_config: The current vLLM configuration.
|
||||
layer_names: Optional list of layer names to scope the lookup.
|
||||
When None, all attention layers are considered.
|
||||
|
||||
Returns:
|
||||
Deduplicated list of attention backend classes.
|
||||
"""
|
||||
layer_type = cast(type[Any], AttentionLayerBase)
|
||||
layers = get_layers_from_vllm_config(vllm_config, layer_type, layer_names)
|
||||
if layers:
|
||||
seen: dict[str, type[AttentionBackend]] = {}
|
||||
for layer in layers.values():
|
||||
backend = layer.get_attn_backend()
|
||||
seen[backend.full_cls_name()] = backend
|
||||
return list(seen.values())
|
||||
|
||||
# Fallback for tests, when static_forward_context is empty.
|
||||
logger.debug(
|
||||
"No layers found in the vLLM config. Falling back to default attention backend."
|
||||
)
|
||||
from vllm.v1.attention.selector import get_attn_backend
|
||||
|
||||
return [
|
||||
get_attn_backend(
|
||||
head_size=vllm_config.model_config.get_head_size(),
|
||||
dtype=vllm_config.model_config.dtype,
|
||||
kv_cache_dtype=vllm_config.cache_config.cache_dtype,
|
||||
use_mla=vllm_config.model_config.use_mla,
|
||||
)
|
||||
return backend
|
||||
]
|
||||
|
||||
|
||||
def get_current_attn_backend(
|
||||
vllm_config: VllmConfig, layer_names: list[str] | None = None
|
||||
) -> type[AttentionBackend]:
|
||||
"""Get the first attention backend for the given layers."""
|
||||
return get_current_attn_backends(vllm_config, layer_names)[0]
|
||||
|
||||
@@ -13,7 +13,7 @@ from collections import defaultdict
|
||||
from collections.abc import Iterator
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
@@ -27,6 +27,7 @@ from vllm.distributed.kv_transfer.kv_connector.utils import (
|
||||
EngineId,
|
||||
TpKVTopology,
|
||||
get_current_attn_backend,
|
||||
get_current_attn_backends,
|
||||
kv_postprocess_blksize_and_layout_on_receive,
|
||||
kv_postprocess_blksize_on_receive,
|
||||
kv_postprocess_layout_on_receive,
|
||||
@@ -61,6 +62,7 @@ from vllm.v1.attention.backends.utils import get_kv_cache_layout
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import FullAttentionSpec, MambaSpec, SlidingWindowSpec
|
||||
from vllm.v1.worker.block_table import BlockTable
|
||||
from vllm.v1.worker.utils import select_common_block_size
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
@@ -945,7 +947,8 @@ class NixlConnectorWorker:
|
||||
|
||||
# Config.
|
||||
self.vllm_config = vllm_config
|
||||
self.block_size = vllm_config.cache_config.block_size
|
||||
# mypy will complain on re-assignment otherwise.
|
||||
self.block_size: int = cast(int, vllm_config.cache_config.block_size)
|
||||
|
||||
if vllm_config.kv_transfer_config is None:
|
||||
raise ValueError("kv_transfer_config must be set for NixlConnector")
|
||||
@@ -993,7 +996,7 @@ class NixlConnectorWorker:
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
self.world_size = get_tensor_model_parallel_world_size()
|
||||
self.tp_group = get_tp_group()
|
||||
self.num_blocks = 0
|
||||
self.num_blocks = kv_cache_config.num_blocks
|
||||
self.enable_permute_local_kv = False
|
||||
|
||||
# KV Caches and nixl tracking data.
|
||||
@@ -1131,11 +1134,30 @@ class NixlConnectorWorker:
|
||||
self.xfer_stats = NixlKVConnectorStats()
|
||||
|
||||
self._physical_blocks_per_logical_kv_block = 1
|
||||
self._sync_block_size_with_kernel()
|
||||
|
||||
self.enforce_compat_hash = self.kv_transfer_config.get_from_extra_config(
|
||||
"enforce_handshake_compat", True
|
||||
)
|
||||
|
||||
def _sync_block_size_with_kernel(self) -> None:
|
||||
backends = get_current_attn_backends(self.vllm_config)
|
||||
kernel_block_size = select_common_block_size(self.block_size, backends)
|
||||
if self.block_size != kernel_block_size:
|
||||
logger.info_once(
|
||||
"User-specified logical block size (%s) does not match"
|
||||
" physical kernel block size (%s). Using the latter.",
|
||||
self.block_size,
|
||||
kernel_block_size,
|
||||
)
|
||||
assert self.block_size > kernel_block_size
|
||||
self._physical_blocks_per_logical_kv_block = (
|
||||
self.block_size // kernel_block_size
|
||||
)
|
||||
self.block_size = kernel_block_size
|
||||
self._block_size[self.engine_id] = kernel_block_size
|
||||
self.num_blocks *= self._physical_blocks_per_logical_kv_block
|
||||
|
||||
def _nixl_handshake(
|
||||
self,
|
||||
host: str,
|
||||
@@ -1469,7 +1491,6 @@ class NixlConnectorWorker:
|
||||
|
||||
# Enable different block lengths for different layers when MLA is used.
|
||||
self.block_len_per_layer = list[int]()
|
||||
self.slot_size_per_layer = list[int]() # HD bytes in kv terms
|
||||
for layer_name, cache_or_caches in xfer_buffers.items():
|
||||
cache_list = (
|
||||
cache_or_caches if self.kv_topo.split_k_and_v else [cache_or_caches]
|
||||
@@ -1486,26 +1507,11 @@ class NixlConnectorWorker:
|
||||
logger.debug(
|
||||
"Registering layer %s with cache shape: %s", layer_name, cache.shape
|
||||
)
|
||||
kernel_block_size = cache.shape[self.kv_topo.block_size_position]
|
||||
if self.block_size != kernel_block_size:
|
||||
logger.info_once(
|
||||
"User-specified logical block size (%s) does not match"
|
||||
" physical kernel block size (%s). Using the latter. ",
|
||||
self.block_size,
|
||||
kernel_block_size,
|
||||
)
|
||||
self._physical_blocks_per_logical_kv_block = (
|
||||
self.block_size // kernel_block_size
|
||||
)
|
||||
self.block_size = kernel_block_size
|
||||
self._block_size[self.engine_id] = kernel_block_size
|
||||
|
||||
seen_base_addresses.append(base_addr)
|
||||
curr_tensor_size_bytes = cache.numel() * cache.element_size()
|
||||
|
||||
if tensor_size_bytes is None:
|
||||
tensor_size_bytes = curr_tensor_size_bytes
|
||||
self.num_blocks = cache.shape[0]
|
||||
|
||||
assert cache.shape[0] == self.num_blocks, (
|
||||
"All kv cache tensors must have the same number of blocks"
|
||||
@@ -1514,9 +1520,6 @@ class NixlConnectorWorker:
|
||||
self.block_len_per_layer.append(
|
||||
curr_tensor_size_bytes // self.num_blocks
|
||||
)
|
||||
self.slot_size_per_layer.append(
|
||||
self.block_len_per_layer[-1] // self.block_size
|
||||
)
|
||||
|
||||
if not self.use_mla:
|
||||
# Different kv cache shape is not supported by HeteroTP
|
||||
@@ -1534,7 +1537,6 @@ class NixlConnectorWorker:
|
||||
"Different block lengths collected: %s", set(self.block_len_per_layer)
|
||||
)
|
||||
assert len(self.block_len_per_layer) == len(seen_base_addresses)
|
||||
assert self.num_blocks != 0
|
||||
|
||||
self.kv_caches_base_addr[self.engine_id][self.tp_rank] = seen_base_addresses
|
||||
self.num_regions = len(caches_data)
|
||||
@@ -1550,10 +1552,6 @@ class NixlConnectorWorker:
|
||||
self.dst_num_blocks[self.engine_id] = self.num_blocks
|
||||
|
||||
if self.kv_topo.is_kv_layout_blocks_first:
|
||||
for i in range(len(self.slot_size_per_layer)):
|
||||
assert self.slot_size_per_layer[i] % 2 == 0
|
||||
self.slot_size_per_layer[i] //= 2
|
||||
|
||||
# NOTE (NickLucche) When FlashInfer is used, memory is registered
|
||||
# with joint KV for each block. This minimizes the overhead in
|
||||
# registerMem allowing faster descs queries. In order to be able to
|
||||
|
||||
Reference in New Issue
Block a user