[KVConnector][NIXL] Support attention-HMA layouts in pipeline-parallel push prefill (#50494)
Signed-off-by: zixi-qi <zixi@inferact.ai> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Yiliu Dong <91178480+qianlihuang@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude
Yiliu Dong
Codex
parent
c6fa1f05d1
commit
0fefffc934
@@ -164,6 +164,10 @@ class NixlBaseConnectorWorker:
|
||||
# Overridden by NixlPushConnectorWorker.
|
||||
_TRANSFER_MODE: str = "pull"
|
||||
|
||||
# Layer-name routing is supported only by NixlPushConnector for HMA KV
|
||||
# caches under PP.
|
||||
_supports_pp_hma = False
|
||||
|
||||
def _compute_desc_ids(
|
||||
self,
|
||||
block_ids: BlockIds,
|
||||
@@ -191,28 +195,42 @@ class NixlBaseConnectorWorker:
|
||||
num_blocks = dst_num_blocks
|
||||
if block_size_ratio is not None:
|
||||
num_blocks = int(num_blocks * block_size_ratio)
|
||||
num_regions = self.num_regions or len(self.block_len_per_layer)
|
||||
use_layer_name_routing = bool(self._transfer_layer_group_ids)
|
||||
num_desc_regions = (
|
||||
len(self._transfer_layer_group_ids)
|
||||
if use_layer_name_routing
|
||||
else self.num_regions or len(self.block_len_per_layer)
|
||||
)
|
||||
if region_num_blocks is None:
|
||||
region_num_blocks = [num_blocks] * num_regions
|
||||
region_num_blocks = [num_blocks] * num_desc_regions
|
||||
elif block_size_ratio is not None:
|
||||
region_num_blocks = [
|
||||
int(count * block_size_ratio) for count in region_num_blocks
|
||||
]
|
||||
if region_num_blocks:
|
||||
num_regions = len(region_num_blocks)
|
||||
assert len(region_num_blocks) == num_regions
|
||||
if region_num_blocks and not use_layer_name_routing:
|
||||
num_desc_regions = len(region_num_blocks)
|
||||
assert len(region_num_blocks) == num_desc_regions
|
||||
region_offsets = np.cumsum([0, *region_num_blocks[:-1]])
|
||||
num_fa_descs = sum(region_num_blocks)
|
||||
|
||||
if use_layer_name_routing:
|
||||
# Descriptors in layer order, using each layer's KV-cache group.
|
||||
return np.concatenate(
|
||||
[
|
||||
np.asarray(block_ids[g], dtype=np.int32) + region_offsets[k]
|
||||
for k, g in enumerate(self._transfer_layer_group_ids)
|
||||
]
|
||||
)
|
||||
|
||||
# All-attention fast path.
|
||||
if num_ssm_regions == 0:
|
||||
if region_group_ids is None:
|
||||
region_group_ids = self.region_group_ids
|
||||
if uses_region_group_mapping is None:
|
||||
uses_region_group_mapping = self._uses_region_group_mapping
|
||||
if not region_group_ids and num_regions == 1:
|
||||
if not region_group_ids and num_desc_regions == 1:
|
||||
region_group_ids = [0]
|
||||
assert len(region_group_ids) == num_regions
|
||||
assert len(region_group_ids) == num_desc_regions
|
||||
if uses_region_group_mapping is None:
|
||||
uses_region_group_mapping = len(set(region_group_ids)) > 1
|
||||
region_group_ids_array = np.asarray(region_group_ids, dtype=np.int32)
|
||||
@@ -392,6 +410,103 @@ class NixlBaseConnectorWorker:
|
||||
"""
|
||||
return region_idx < len(self._region_is_mla) and self._region_is_mla[region_idx]
|
||||
|
||||
def _set_region_layers(self, region_layers: list[list[str]]) -> None:
|
||||
layer_names = [layer_name for region in region_layers for layer_name in region]
|
||||
assert len(layer_names) == len(set(layer_names)), (
|
||||
"A KV cache layer spans multiple NIXL regions"
|
||||
)
|
||||
self.region_members: list[list[str]] = region_layers
|
||||
if not self._requires_layer_name_routing():
|
||||
return
|
||||
group_by_layer = self.kv_cache_config.transfer_group_index_by_layer
|
||||
ungrouped = [name for name in layer_names if name not in group_by_layer]
|
||||
assert not ungrouped, f"KV cache layers outside any local group: {ungrouped}"
|
||||
self._transfer_layer_names = tuple(layer_names)
|
||||
self._transfer_layer_region_indices = tuple(
|
||||
region_idx
|
||||
for region_idx, region in enumerate(region_layers)
|
||||
for _ in region
|
||||
)
|
||||
self._transfer_layer_group_ids = tuple(
|
||||
group_by_layer[name] for name in layer_names
|
||||
)
|
||||
|
||||
def _requires_layer_name_routing(self) -> bool:
|
||||
"""Whether PP push must match HMA layers by name, not region index."""
|
||||
return (
|
||||
self._supports_pp_hma
|
||||
and self.pp_size > 1
|
||||
and self._is_hma_required
|
||||
and not self._has_mamba
|
||||
)
|
||||
|
||||
def _align_remote_regions_by_layer(
|
||||
self, nixl_agent_meta: NixlAgentMetadata
|
||||
) -> None:
|
||||
"""Select remote regions for this PP stage in local layer order."""
|
||||
remote_region_layers = nixl_agent_meta.region_members
|
||||
# Region-index routing here would silently transfer stale KV.
|
||||
assert remote_region_layers, "Remote advertised no region_members"
|
||||
assert (
|
||||
len(nixl_agent_meta.kv_caches_base_addr)
|
||||
== len(nixl_agent_meta.block_lens)
|
||||
== len(nixl_agent_meta.block_strides)
|
||||
== len(remote_region_layers)
|
||||
), "Remote region metadata lengths disagree"
|
||||
assert all(
|
||||
values is None or len(values) == len(remote_region_layers)
|
||||
for values in (
|
||||
nixl_agent_meta.region_num_blocks,
|
||||
nixl_agent_meta.region_group_ids,
|
||||
nixl_agent_meta.region_names,
|
||||
nixl_agent_meta.region_mem_types,
|
||||
)
|
||||
), "Remote region metadata lengths disagree"
|
||||
|
||||
remote_region_by_layer: dict[str, int] = {}
|
||||
for region_idx, layer_names in enumerate(remote_region_layers):
|
||||
for layer_name in layer_names:
|
||||
assert layer_name not in remote_region_by_layer, (
|
||||
f"Remote advertised layer {layer_name!r} in multiple regions"
|
||||
)
|
||||
remote_region_by_layer[layer_name] = region_idx
|
||||
|
||||
missing = [
|
||||
name
|
||||
for name in self._transfer_layer_names
|
||||
if name not in remote_region_by_layer
|
||||
]
|
||||
assert not missing, f"Remote is missing locally owned layers: {missing}"
|
||||
|
||||
remote_regions = [
|
||||
remote_region_by_layer[name] for name in self._transfer_layer_names
|
||||
]
|
||||
nixl_agent_meta.kv_caches_base_addr = [
|
||||
nixl_agent_meta.kv_caches_base_addr[i] for i in remote_regions
|
||||
]
|
||||
nixl_agent_meta.block_lens = [
|
||||
nixl_agent_meta.block_lens[i] for i in remote_regions
|
||||
]
|
||||
nixl_agent_meta.block_strides = [
|
||||
nixl_agent_meta.block_strides[i] for i in remote_regions
|
||||
]
|
||||
if nixl_agent_meta.region_num_blocks is not None:
|
||||
nixl_agent_meta.region_num_blocks = [
|
||||
nixl_agent_meta.region_num_blocks[i] for i in remote_regions
|
||||
]
|
||||
if nixl_agent_meta.region_group_ids is not None:
|
||||
nixl_agent_meta.region_group_ids = [
|
||||
nixl_agent_meta.region_group_ids[i] for i in remote_regions
|
||||
]
|
||||
if nixl_agent_meta.region_mem_types is not None:
|
||||
nixl_agent_meta.region_mem_types = [
|
||||
nixl_agent_meta.region_mem_types[i] for i in remote_regions
|
||||
]
|
||||
if nixl_agent_meta.region_names is not None:
|
||||
nixl_agent_meta.region_names = list(self._transfer_layer_names)
|
||||
# One layer per region keeps a second alignment pass a no-op.
|
||||
nixl_agent_meta.region_members = [[name] for name in self._transfer_layer_names]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
@@ -638,18 +753,26 @@ class NixlBaseConnectorWorker:
|
||||
# transfers into the matching sub-range of a PP=1 remote's regions.
|
||||
self.pp_size = vllm_config.parallel_config.pipeline_parallel_size
|
||||
self._remote_region_offset = 0
|
||||
# PP push slices regions per layer (uniform count); HMA breaks that.
|
||||
if self.pp_size > 1 and self._is_hma_required:
|
||||
raise NotImplementedError(
|
||||
"NixlPushConnector does not support pipeline_parallel_size > 1 "
|
||||
"with hybrid KV cache layouts (HMA) yet."
|
||||
)
|
||||
# Decode-side PP is unsupported (completions counted per consumer rank).
|
||||
if vllm_config.kv_transfer_config.kv_role == "kv_consumer" and self.pp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"NixlPushConnector consumer (decode) does not support "
|
||||
"pipeline_parallel_size > 1."
|
||||
)
|
||||
if self.pp_size > 1:
|
||||
if self._is_hma_required and not self._supports_pp_hma:
|
||||
raise NotImplementedError(
|
||||
"NixlConnector (pull) does not support pipeline_parallel_size "
|
||||
"> 1 with hybrid KV cache layouts (HMA); use NixlPushConnector "
|
||||
"for PP + HMA."
|
||||
)
|
||||
# PP push routes HMA (hybrid) attention layouts by layer name;
|
||||
# Mamba/SSM hybrids are not yet supported under PP.
|
||||
if self._has_mamba:
|
||||
raise NotImplementedError(
|
||||
"NixlPushConnector does not support pipeline_parallel_size > 1 "
|
||||
"with Mamba/SSM hybrid KV cache layouts yet."
|
||||
)
|
||||
# Decode-side PP is unsupported (completions counted per consumer rank).
|
||||
if vllm_config.kv_transfer_config.kv_role == "kv_consumer":
|
||||
raise NotImplementedError(
|
||||
"NixlPushConnector consumer (decode) does not support "
|
||||
"pipeline_parallel_size > 1."
|
||||
)
|
||||
# Keep heartbeat handshakes to a PP-sharded producer notif-only.
|
||||
self._hb_handshake_notif_only = False
|
||||
|
||||
@@ -773,6 +896,13 @@ class NixlBaseConnectorWorker:
|
||||
# within a block (BLHNC/BHLNC), where stride > block_len.
|
||||
self.block_stride_per_layer = list[int]()
|
||||
|
||||
# Region layer membership is populated for HMA layouts.
|
||||
self.region_members = []
|
||||
# Local layer order, set by ``_set_region_layers`` only when
|
||||
# routing by layer name. Empty means region-index routing.
|
||||
self._transfer_layer_names = tuple[str, ...]()
|
||||
self._transfer_layer_region_indices = tuple[int, ...]()
|
||||
self._transfer_layer_group_ids = tuple[int, ...]()
|
||||
# Per-engine TP mappings. Generated during handshake.
|
||||
self.tp_mappings: dict[EngineId, TPMapping] = {}
|
||||
|
||||
@@ -1306,6 +1436,11 @@ class NixlBaseConnectorWorker:
|
||||
):
|
||||
compressed_region_owners.setdefault(cache.data_ptr(), cache)
|
||||
|
||||
track_region_layers = (
|
||||
self._supports_pp_hma and self._is_hma_required and not self._has_mamba
|
||||
)
|
||||
region_layers: list[list[str]] = []
|
||||
|
||||
# K and V are packed into the content dim, so each attention layer is a
|
||||
# single NIXL region whose block transfers as one unit. Mamba layers instead
|
||||
# register separate conv/ssm sub-regions (see `_build_mamba_local`).
|
||||
@@ -1508,6 +1643,12 @@ class NixlBaseConnectorWorker:
|
||||
self._region_is_mla.append(is_mla_region)
|
||||
region_mem_types.append(mem_type)
|
||||
|
||||
if track_region_layers:
|
||||
if region_index == len(region_layers):
|
||||
region_layers.append([layer_name])
|
||||
else:
|
||||
region_layers[region_index].append(layer_name)
|
||||
|
||||
if isinstance(layer_spec, MambaSpec):
|
||||
if layer_spec.tp_replicated:
|
||||
assert self._is_csa_linear
|
||||
@@ -1563,7 +1704,9 @@ class NixlBaseConnectorWorker:
|
||||
if self._has_mamba:
|
||||
self.region_num_blocks = [self.num_blocks] * self.num_regions
|
||||
|
||||
if self.pp_size > 1:
|
||||
self._set_region_layers(region_layers)
|
||||
|
||||
if self.pp_size > 1 and not self._is_hma_required:
|
||||
start_layer, end_layer = self.model_config.get_layers_start_end_indices(
|
||||
self.vllm_config.parallel_config
|
||||
)
|
||||
@@ -1573,7 +1716,12 @@ class NixlBaseConnectorWorker:
|
||||
self._remote_region_offset = regions_per_layer * start_layer
|
||||
|
||||
# Total local FA descriptors (boundary between FA and mamba descs).
|
||||
self.num_descs = sum(self.region_num_blocks)
|
||||
xfer_region_num_blocks = (
|
||||
[self.region_num_blocks[i] for i in self._transfer_layer_region_indices]
|
||||
if self._transfer_layer_region_indices
|
||||
else self.region_num_blocks
|
||||
)
|
||||
self.num_descs = sum(xfer_region_num_blocks)
|
||||
|
||||
self._mixed_mem_types = len(set(region_mem_types)) > 1
|
||||
if self._mixed_mem_types:
|
||||
@@ -1597,7 +1745,7 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
self.device_kv_caches = kv_caches
|
||||
self.dst_num_blocks[self.engine_id] = self.num_blocks
|
||||
self.dst_region_num_blocks[self.engine_id] = self.region_num_blocks
|
||||
self.dst_region_num_blocks[self.engine_id] = xfer_region_num_blocks
|
||||
self.dst_region_group_ids[self.engine_id] = self.region_group_ids
|
||||
self.dst_uses_region_group_mapping[self.engine_id] = (
|
||||
self._uses_region_group_mapping
|
||||
@@ -1647,6 +1795,7 @@ class NixlBaseConnectorWorker:
|
||||
region_mem_types=self.region_mem_types,
|
||||
dcp_size=self.dcp_size,
|
||||
pcp_size=self.pcp_size,
|
||||
region_members=self.region_members,
|
||||
)
|
||||
# Wrap metadata in payload with hash for defensive decoding
|
||||
assert self.compat_hash is not None
|
||||
@@ -1805,12 +1954,16 @@ class NixlBaseConnectorWorker:
|
||||
base_addresses: list[int],
|
||||
block_size_ratio: int,
|
||||
) -> np.ndarray:
|
||||
"""Build local FA descriptors for all layers as an Nx3 uint64 array."""
|
||||
"""Build local FA descriptors as an Nx3 uint64 array."""
|
||||
assert self.transfer_topo is not None
|
||||
assert base_addresses, "Local KV cache base addresses must not be empty."
|
||||
device_id = self.device_id
|
||||
region_indices = self._transfer_layer_region_indices or range(
|
||||
len(base_addresses)
|
||||
)
|
||||
parts: list[np.ndarray] = []
|
||||
for i, base_addr in enumerate(base_addresses):
|
||||
for i in region_indices:
|
||||
base_addr = base_addresses[i]
|
||||
block_len = self.block_len_per_layer[i] // block_size_ratio
|
||||
logical_blocks = np.repeat(
|
||||
np.arange(self.region_num_blocks[i], dtype=np.uint64),
|
||||
@@ -1834,7 +1987,7 @@ class NixlBaseConnectorWorker:
|
||||
nixl_agent_meta: NixlAgentMetadata,
|
||||
block_size_ratio: int,
|
||||
) -> np.ndarray:
|
||||
"""Build remote FA descriptors for all layers as an Nx3 uint64 array."""
|
||||
"""Build remote FA descriptors as an Nx3 uint64 array."""
|
||||
assert self.transfer_topo is not None
|
||||
assert nixl_agent_meta.kv_caches_base_addr, (
|
||||
"Remote KV cache base addresses must not be empty."
|
||||
@@ -1851,10 +2004,14 @@ class NixlBaseConnectorWorker:
|
||||
device_id = nixl_agent_meta.device_id
|
||||
block_strides = nixl_agent_meta.block_strides
|
||||
parts: list[np.ndarray] = []
|
||||
use_layer_name_routing = bool(self._transfer_layer_region_indices)
|
||||
for i, base_addr in enumerate(nixl_agent_meta.kv_caches_base_addr):
|
||||
replicated = self._is_region_replicated(i)
|
||||
local_region = (
|
||||
self._transfer_layer_region_indices[i] if use_layer_name_routing else i
|
||||
)
|
||||
replicated = self._is_region_replicated(local_region)
|
||||
# Read our whole local region size from remote..
|
||||
local_block_len = self.block_len_per_layer[i]
|
||||
local_block_len = self.block_len_per_layer[local_region]
|
||||
remote_kv_block_len = local_block_len // block_size_ratio
|
||||
if block_size_ratio > 1:
|
||||
# ..using remote kv_block_len as transfer unit
|
||||
@@ -2021,9 +2178,22 @@ class NixlBaseConnectorWorker:
|
||||
)
|
||||
return self._remote_agents[engine_id][(0, remote_tp_rank)]
|
||||
|
||||
assert self.transfer_topo is not None
|
||||
transfer_topo = self.transfer_topo
|
||||
# Number of physical regions registered locally (one per layer/tensor).
|
||||
num_local_regions = len(self.block_len_per_layer)
|
||||
if (
|
||||
if self._transfer_layer_region_indices:
|
||||
if self.block_size != nixl_agent_meta.block_size:
|
||||
raise NotImplementedError(
|
||||
"Attention-HMA push requires identical P/D block sizes."
|
||||
)
|
||||
if remote_tp_size > transfer_topo.tp_size and not self.use_mla:
|
||||
raise NotImplementedError(
|
||||
"Attention-HMA push does not support decode TP greater than "
|
||||
"prefill TP yet"
|
||||
)
|
||||
self._align_remote_regions_by_layer(nixl_agent_meta)
|
||||
elif (
|
||||
self.pp_size > 1
|
||||
and len(nixl_agent_meta.kv_caches_base_addr) > num_local_regions
|
||||
):
|
||||
@@ -2038,6 +2208,10 @@ class NixlBaseConnectorWorker:
|
||||
]
|
||||
nixl_agent_meta.block_lens = nixl_agent_meta.block_lens[start:end]
|
||||
nixl_agent_meta.block_strides = nixl_agent_meta.block_strides[start:end]
|
||||
if nixl_agent_meta.region_num_blocks is not None:
|
||||
nixl_agent_meta.region_num_blocks = nixl_agent_meta.region_num_blocks[
|
||||
start:end
|
||||
]
|
||||
if nixl_agent_meta.region_group_ids is not None:
|
||||
nixl_agent_meta.region_group_ids = nixl_agent_meta.region_group_ids[
|
||||
start:end
|
||||
@@ -2048,14 +2222,8 @@ class NixlBaseConnectorWorker:
|
||||
nixl_agent_meta.region_mem_types = nixl_agent_meta.region_mem_types[
|
||||
start:end
|
||||
]
|
||||
if nixl_agent_meta.region_num_blocks is not None:
|
||||
nixl_agent_meta.region_num_blocks = nixl_agent_meta.region_num_blocks[
|
||||
start:end
|
||||
]
|
||||
|
||||
### Register remote engine in TransferTopology (idempotent).
|
||||
assert self.transfer_topo is not None
|
||||
transfer_topo = self.transfer_topo
|
||||
physical_blocks_per_logical = (
|
||||
nixl_agent_meta.physical_blocks_per_logical_kv_block
|
||||
)
|
||||
@@ -2349,6 +2517,9 @@ class NixlBaseConnectorWorker:
|
||||
# the per-rank KV head ratio rather than the raw tp_ratio, because GQA
|
||||
# replication caps per-rank heads at 1 when tp > total_kv_heads
|
||||
# (issue #45330). Mamba uses the ssm_sizes counterpart, so skip here.
|
||||
local_regions = self._transfer_layer_region_indices or range(
|
||||
len(self.block_len_per_layer)
|
||||
)
|
||||
if self._has_mamba and self.use_mla:
|
||||
# Hybrid MLA+SSM (e.g. KimiLinear's KDA+MLA): regions are
|
||||
# kernel-granularity views of the mamba-unified page. The MLA
|
||||
@@ -2365,7 +2536,7 @@ class NixlBaseConnectorWorker:
|
||||
f"remote={nixl_agent_meta.block_lens}."
|
||||
)
|
||||
elif not self._has_mamba:
|
||||
assert len(self.block_len_per_layer) == len(nixl_agent_meta.block_lens), (
|
||||
assert len(local_regions) == len(nixl_agent_meta.block_lens), (
|
||||
"Number of KV layers must match between prefill and decode"
|
||||
)
|
||||
model_replicated = self.use_mla or self.transfer_topo.is_kv_replicated(
|
||||
@@ -2374,8 +2545,9 @@ class NixlBaseConnectorWorker:
|
||||
total_kv_heads = self.transfer_topo.total_num_kv_heads
|
||||
local_heads = self.transfer_topo.local_physical_heads
|
||||
remote_heads = max(1, total_kv_heads // remote_tp_size)
|
||||
for i, local_len in enumerate(self.block_len_per_layer):
|
||||
replicated = model_replicated or self._is_region_replicated(i)
|
||||
for i, region_idx in enumerate(local_regions):
|
||||
local_len = self.block_len_per_layer[region_idx]
|
||||
replicated = model_replicated or self._is_region_replicated(region_idx)
|
||||
remote_len = nixl_agent_meta.block_lens[i]
|
||||
if replicated:
|
||||
assert local_len // block_size_ratio == remote_len, (
|
||||
@@ -2406,8 +2578,7 @@ class NixlBaseConnectorWorker:
|
||||
|
||||
# TP workers that handhshake with same remote have same #blocks.
|
||||
assert self.dst_num_blocks[remote_engine_id] == nixl_agent_meta.num_blocks
|
||||
# Same number of regions/~layers.
|
||||
assert len(nixl_agent_meta.kv_caches_base_addr) == len(self.block_len_per_layer)
|
||||
assert len(nixl_agent_meta.kv_caches_base_addr) == len(local_regions)
|
||||
num_remote_regions = len(nixl_agent_meta.kv_caches_base_addr)
|
||||
if nixl_agent_meta.region_num_blocks is not None:
|
||||
assert len(nixl_agent_meta.region_num_blocks) == num_remote_regions
|
||||
@@ -2423,7 +2594,12 @@ class NixlBaseConnectorWorker:
|
||||
if nixl_agent_meta.region_group_ids is not None:
|
||||
assert len(nixl_agent_meta.region_group_ids) == num_remote_regions
|
||||
if nixl_agent_meta.region_names is not None:
|
||||
assert nixl_agent_meta.region_names == self.region_names, (
|
||||
expected_names = (
|
||||
list(self._transfer_layer_names)
|
||||
if self._transfer_layer_names
|
||||
else self.region_names
|
||||
)
|
||||
assert nixl_agent_meta.region_names == expected_names, (
|
||||
"NIXL transfer regions must name the same model caches in the "
|
||||
"same order"
|
||||
)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Metadata dataclasses and helpers for the NIXL connector."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
@@ -47,8 +47,9 @@ PUSH_REG_NOTIF_PREFIX = b"PUSH_REG:"
|
||||
# 9: Add block_strides
|
||||
# 10: Add dense virtual transfer pages for compressed MLA caches
|
||||
# 11: Add per-region transfer geometry and memory types to NixlAgentMetadata
|
||||
# 12: Add per-region member names for PP push
|
||||
#
|
||||
NIXL_CONNECTOR_VERSION: int = 11
|
||||
NIXL_CONNECTOR_VERSION: int = 12
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -71,6 +72,8 @@ class NixlAgentMetadata:
|
||||
region_mem_types: list[str] | None = None
|
||||
dcp_size: int = 1
|
||||
pcp_size: int = 1
|
||||
# Layer names sharing each advertised region, in region order.
|
||||
region_members: list[list[str]] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -83,6 +83,8 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
# Distinguishes push from pull in the NIXL compatibility hash.
|
||||
_TRANSFER_MODE: str = "push"
|
||||
|
||||
_supports_pp_hma = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: "VllmConfig",
|
||||
@@ -530,7 +532,7 @@ class NixlPushConnectorWorker(NixlBaseConnectorWorker):
|
||||
local_region_groups = self.region_group_ids
|
||||
remote_region_groups = self.dst_region_group_ids[engine_id]
|
||||
groups_differ = local_region_groups != remote_region_groups
|
||||
if groups_differ:
|
||||
if groups_differ and not self._transfer_layer_group_ids:
|
||||
raise NotImplementedError(
|
||||
"NixlPushConnector does not support different producer and "
|
||||
"consumer cache-group layouts"
|
||||
|
||||
Reference in New Issue
Block a user