[KVConnector][1/N] PP-aware handshake aggregation and intermediate-PP output plumbing (#43720)

Signed-off-by: zixi-qi <zixi@inferact.ai>
This commit is contained in:
qizixi
2026-06-04 22:04:19 -07:00
committed by GitHub
parent da1daf40bf
commit 96229fa99e
9 changed files with 459 additions and 26 deletions
@@ -0,0 +1,232 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import Any
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorHandshakeMetadata,
)
from vllm.v1.engine import core as engine_core_module
pytestmark = pytest.mark.cpu_test
class _Metadata(KVConnectorHandshakeMetadata):
pass
class _FakeExecutor:
handshake_metadata_src: (
list[dict[tuple[int, int], KVConnectorHandshakeMetadata] | None] | None
)
last_instance: "_FakeExecutor | None" = None
def __init__(
self,
vllm_config: Any,
) -> None:
del vllm_config
self.handshake_metadata = self.handshake_metadata_src
self.handshake_calls = 0
_FakeExecutor.last_instance = self
def get_kv_connector_handshake_metadata(
self,
) -> list[dict[tuple[int, int], KVConnectorHandshakeMetadata] | None] | None:
self.handshake_calls += 1
return self.handshake_metadata
def init_kv_output_aggregator(self, connector: KVConnectorBase_V1) -> None:
pass
def _run_engine_core_handshake(
monkeypatch: pytest.MonkeyPatch,
connector: KVConnectorBase_V1,
*,
handshake_metadata: (
list[dict[tuple[int, int], KVConnectorHandshakeMetadata] | None] | None
),
) -> _FakeExecutor:
class _FakeScheduler:
def __init__(self, **kwargs: Any) -> None:
self.connector = connector
def get_kv_connector(self) -> KVConnectorBase_V1:
return connector
_FakeExecutor.handshake_metadata_src = handshake_metadata
_FakeExecutor.last_instance = None
monkeypatch.setattr("vllm.plugins.load_general_plugins", lambda: None)
monkeypatch.setattr(
engine_core_module.EngineCore,
"_initialize_kv_caches",
lambda self, vllm_config: SimpleNamespace(kv_cache_groups=[object()]),
)
monkeypatch.setattr(
engine_core_module,
"StructuredOutputManager",
lambda vllm_config: object(),
)
monkeypatch.setattr(
engine_core_module,
"resolve_kv_cache_block_sizes",
lambda kv_cache_config, vllm_config: (16, 16),
)
monkeypatch.setattr(
engine_core_module,
"MULTIMODAL_REGISTRY",
SimpleNamespace(engine_receiver_cache_from_config=lambda vllm_config: None),
)
monkeypatch.setattr(engine_core_module, "freeze_gc_heap", lambda: None)
monkeypatch.setattr(
engine_core_module, "maybe_attach_gc_debug_callback", lambda: None
)
monkeypatch.setattr(engine_core_module, "enable_envs_cache", lambda: None)
monkeypatch.setattr(engine_core_module, "get_hash_fn_by_name", lambda name: None)
monkeypatch.setattr(engine_core_module, "init_none_hash", lambda hash_fn: None)
monkeypatch.setattr(
engine_core_module, "get_request_block_hasher", lambda *args: None
)
vllm_config = SimpleNamespace(
parallel_config=SimpleNamespace(data_parallel_rank_local=0),
scheduler_config=SimpleNamespace(
get_scheduler_cls=lambda: _FakeScheduler,
enable_chunked_prefill=False,
async_scheduling=False,
),
speculative_config=None,
ec_transfer_config=None,
max_concurrent_batches=1,
model_config=SimpleNamespace(runner_type="generate"),
cache_config=SimpleNamespace(
enable_prefix_caching=False,
prefix_caching_hash_algo="builtin",
),
)
engine_core_module.EngineCore(vllm_config, _FakeExecutor, log_stats=False)
assert _FakeExecutor.last_instance is not None
return _FakeExecutor.last_instance
class _LegacyConnector(KVConnectorBase_V1):
def __init__(self) -> None:
self.legacy_metadata: dict[int, KVConnectorHandshakeMetadata] | None = None
def start_load_kv(self, forward_context: Any, **kwargs: Any) -> None:
pass
def wait_for_layer_load(self, layer_name: str) -> None:
pass
def save_kv_layer(
self,
layer_name: str,
kv_layer: Any,
attn_metadata: Any,
**kwargs: Any,
) -> None:
pass
def wait_for_save(self) -> None:
pass
def get_num_new_matched_tokens(
self, request: Any, num_computed_tokens: int
) -> tuple[int | None, bool]:
return 0, False
def update_state_after_alloc(
self, request: Any, blocks: Any, num_external_tokens: int
) -> None:
pass
def build_connector_meta(self, scheduler_output: Any) -> Any:
raise NotImplementedError
def set_xfer_handshake_metadata(
self, metadata: dict[int, KVConnectorHandshakeMetadata]
) -> None:
self.legacy_metadata = metadata
class _PPAwareConnector(_LegacyConnector):
def __init__(self) -> None:
super().__init__()
self.pp_aware_metadata: (
dict[tuple[int, int], KVConnectorHandshakeMetadata] | None
) = None
def set_xfer_handshake_metadata_pp_aware(
self, metadata: dict[tuple[int, int], KVConnectorHandshakeMetadata]
) -> None:
self.pp_aware_metadata = metadata
def test_engine_unwraps_handshake_metadata_for_legacy_connector(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Engine core always asks workers for `(pp_rank, tp_rank)`-keyed metadata,
then unwraps to `{tp_rank: metadata}` for a connector that has not opted
into PP-aware handshake (single-PP producer, all `pp_rank == 0`)."""
metadata_0 = _Metadata()
metadata_1 = _Metadata()
connector = _LegacyConnector()
executor = _run_engine_core_handshake(
monkeypatch,
connector,
handshake_metadata=[
{(0, 0): metadata_0},
None,
{(0, 1): metadata_1},
],
)
assert executor.handshake_calls == 1
assert connector.legacy_metadata == {0: metadata_0, 1: metadata_1}
def test_engine_rejects_pp_producer_for_legacy_connector(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A connector that has not opted into PP-aware handshake must not silently
drop metadata from `pp_rank > 0`; engine core init raises instead."""
connector = _LegacyConnector()
with pytest.raises(ValueError, match="does not support PP-disaggregated"):
_run_engine_core_handshake(
monkeypatch,
connector,
handshake_metadata=[{(0, 0): _Metadata()}, {(1, 0): _Metadata()}],
)
def test_engine_passes_handshake_metadata_through_for_pp_aware_connector(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""A PP-aware connector receives the full `(pp_rank, tp_rank)`-keyed dict
unchanged."""
metadata_0 = _Metadata()
metadata_1 = _Metadata()
connector = _PPAwareConnector()
executor = _run_engine_core_handshake(
monkeypatch,
connector,
handshake_metadata=[{(0, 0): metadata_0}, {(1, 0): metadata_1}],
)
assert executor.handshake_calls == 1
assert connector.legacy_metadata is None
assert connector.pp_aware_metadata == {
(0, 0): metadata_0,
(1, 0): metadata_1,
}
@@ -261,11 +261,12 @@ def test_multi_example_connector_consistency():
storage1_scheduler_events = _ignore_event_collection(events["storage1-SCHEDULER"])
storage2_scheduler_events = _ignore_event_collection(events["storage2-SCHEDULER"])
# First event is bind_gpu_block_pool from initialization, then
# set_xfer_handshake_metadata, then on_new_request when the request is enqueued,
# then get_num_new_matched_tokens and update_state_after_alloc from generate().
# set_xfer_handshake_metadata_pp_aware, then on_new_request when the request is
# enqueued, then get_num_new_matched_tokens and update_state_after_alloc from
# generate().
assert storage1_scheduler_events[:6] == [
"bind_gpu_block_pool",
"set_xfer_handshake_metadata",
"set_xfer_handshake_metadata_pp_aware",
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[0] 0",
@@ -285,7 +286,7 @@ def test_multi_example_connector_consistency():
]
assert storage2_scheduler_events[:6] == [
"bind_gpu_block_pool",
"set_xfer_handshake_metadata",
"set_xfer_handshake_metadata_pp_aware",
"on_new_request",
"get_num_new_matched_tokens 0",
"update_state_after_alloc num_blocks=[0] 0",
@@ -0,0 +1,146 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm.distributed.kv_transfer.kv_connector.utils import (
EngineTransferInfo,
TransferTopology,
)
pytestmark = pytest.mark.cpu_test
class _FakeAttentionBackend:
@staticmethod
def get_kv_cache_shape(
num_blocks: int,
block_size: int,
num_kv_heads: int,
head_size: int,
) -> tuple[int, int, int, int, int]:
return (2, num_blocks, num_kv_heads, block_size, head_size)
def _make_topology(
*,
tp_rank: int = 1,
tp_size: int = 4,
total_num_kv_heads: int = 8,
) -> TransferTopology:
return TransferTopology(
tp_rank=tp_rank,
tp_size=tp_size,
block_size=16,
engine_id="local-engine",
is_mla=False,
is_mamba=False,
total_num_kv_heads=total_num_kv_heads,
attn_backends=[_FakeAttentionBackend],
)
def test_legacy_register_remote_engine_uses_pp_rank_zero() -> None:
topology = _make_topology()
info = EngineTransferInfo(
remote_tp_size=2,
remote_block_len=1024,
remote_block_size=16,
remote_physical_blocks_per_logical=1,
)
registered = topology.register_remote_engine("remote-engine", info)
assert registered == info
assert registered.remote_pp_rank == 0
assert topology.get_engine_info("remote-engine") == info
assert topology._engines[("remote-engine", 0)] == info
assert topology.target_remote_ranks("remote-engine") == [0]
def test_register_remote_engine_stores_pp_ranks_separately() -> None:
topology = _make_topology(tp_rank=0, tp_size=2)
info_0 = EngineTransferInfo(
remote_tp_size=2,
remote_block_len=1024,
remote_block_size=16,
remote_physical_blocks_per_logical=1,
remote_pp_rank=0,
start_layer=0,
end_layer=16,
)
info_1 = EngineTransferInfo(
remote_tp_size=1,
remote_block_len=512,
remote_block_size=8,
remote_physical_blocks_per_logical=2,
remote_pp_rank=1,
start_layer=16,
end_layer=32,
)
registered_0 = topology.register_remote_engine("remote-engine", info_0)
registered_1 = topology.register_remote_engine("remote-engine", info_1)
assert registered_0 == info_0
assert registered_1 == info_1
assert topology.get_engine_info("remote-engine") == info_0
assert topology.get_engine_info("remote-engine", 0) == info_0
assert topology.get_engine_info("remote-engine", 1) == info_1
assert set(topology._engines) == {
("remote-engine", 0),
("remote-engine", 1),
}
def test_helpers_use_requested_pp_rank() -> None:
topology = _make_topology(tp_rank=1, tp_size=2, total_num_kv_heads=2)
topology.register_remote_engine(
"remote-engine",
EngineTransferInfo(
remote_tp_size=1,
remote_block_len=1024,
remote_block_size=16,
remote_physical_blocks_per_logical=1,
remote_pp_rank=0,
start_layer=0,
end_layer=8,
),
)
topology.register_remote_engine(
"remote-engine",
EngineTransferInfo(
remote_tp_size=4,
remote_block_len=1024,
remote_block_size=16,
remote_physical_blocks_per_logical=1,
remote_pp_rank=1,
start_layer=8,
end_layer=16,
),
)
assert not topology.is_kv_replicated("remote-engine", 0)
assert topology.is_kv_replicated("remote-engine", 1)
assert topology.replicates_kv_cache("remote-engine", 1)
assert topology.target_remote_ranks("remote-engine", 0) == [0]
assert topology.target_remote_ranks("remote-engine", 1) == [2, 3]
assert "remote_pp=1" in topology.describe("remote-engine", 1)
def test_engine_info_fields_have_backward_compatible_defaults() -> None:
topology = _make_topology()
info = EngineTransferInfo(
remote_tp_size=2,
remote_block_len=1024,
remote_block_size=16,
remote_physical_blocks_per_logical=1,
)
registered = topology.register_remote_engine("remote-engine", info)
assert topology.get_engine_info("remote-engine") == registered
assert registered.remote_pp_rank == 0
assert registered.start_layer == 0
assert registered.end_layer == 0