[Bugfix][HiSparse][PD] Align region-mapped pulls across logical block sizes (#57077)
Signed-off-by: NickLucche <nicolo.lucchesi@mistral.ai>
This commit is contained in:
@@ -3,6 +3,8 @@
|
||||
"""Unit tests for NixlConnectorScheduler with HMA and Mamba N-1 prefill."""
|
||||
|
||||
import gc
|
||||
import queue
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import msgspec
|
||||
@@ -39,6 +41,173 @@ from .utils import (
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def region_pull_worker():
|
||||
"""Real page mapping and descriptors, with only the transport mocked."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.tp_mapping import TPMapping
|
||||
|
||||
worker = object.__new__(NixlConnectorWorker)
|
||||
worker.engine_id = "D"
|
||||
worker.tp_rank = 0
|
||||
worker.block_size = 64
|
||||
worker._physical_blocks_per_logical_kv_block = 1
|
||||
worker.enable_permute_local_kv = False
|
||||
worker.enable_heterogeneous_attn_post_process = False
|
||||
worker._engine_last_active = {}
|
||||
worker._bidirectional_kv_xfer_enabled = False
|
||||
worker._recving_transfers = {}
|
||||
worker.use_mla, worker._has_mamba = True, False
|
||||
worker.dcp_size = 1
|
||||
spec = MLAAttentionSpec(
|
||||
block_size=64, num_kv_heads=1, head_size=8, dtype=torch.bfloat16
|
||||
)
|
||||
worker.kv_cache_config = KVCacheConfig(
|
||||
num_blocks=100,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["resident"], spec, enable_kv_transfer=False),
|
||||
KVCacheGroupSpec(["host"], spec),
|
||||
KVCacheGroupSpec(["indexer"], spec),
|
||||
],
|
||||
)
|
||||
worker._group_spec_types = (MLAAttentionSpec, MLAAttentionSpec)
|
||||
worker.region_group_ids = [0, 1]
|
||||
worker.dst_region_group_ids = {"P": [0, 0]}
|
||||
worker.num_regions = 2
|
||||
worker.block_len_per_layer = [1024, 1024]
|
||||
worker.transfer_topo = MagicMock()
|
||||
worker.transfer_topo.tp_ratio.return_value = 1
|
||||
worker.transfer_topo.block_size_ratio.return_value = 1
|
||||
worker.transfer_topo.get_engine_info.return_value = SimpleNamespace(
|
||||
remote_tp_size=4,
|
||||
remote_dcp_size=1,
|
||||
remote_block_size=64,
|
||||
remote_physical_blocks_per_logical=2,
|
||||
)
|
||||
worker.tp_mappings = {"P": TPMapping(((0,), (0,)), (0,), {0: 0}, 0)}
|
||||
worker._mixed_mem_types = True
|
||||
worker.src_xfer_handles_by_block_size = {64: 1}
|
||||
worker._dram_src_handles_by_block_size = {64: 2}
|
||||
worker.dst_xfer_side_handles = {"P": {0: 3}}
|
||||
worker.dst_num_blocks = {"P": 100, "D": 100}
|
||||
worker.dst_region_num_blocks = {"P": [100, 100], "D": [100, 100]}
|
||||
worker._remote_agents = {"P": {(0, 0): "P-rank0"}}
|
||||
worker._read_blocks_mixed = MagicMock()
|
||||
worker.nixl_wrapper = MagicMock()
|
||||
return worker
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("local_ratio,remote_ratio", [(1, 2), (2, 1), (2, 2)])
|
||||
@pytest.mark.parametrize("num_tokens", [160, 224])
|
||||
@pytest.mark.parametrize("cached", [0, 1])
|
||||
def test_region_pull_ignores_allocation_padding(
|
||||
region_pull_worker, local_ratio, remote_ratio, num_tokens, cached
|
||||
):
|
||||
"""Padded capacity must not shift either the host or indexer token range."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
NixlConnectorMetadata,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
|
||||
worker = region_pull_worker
|
||||
worker._physical_blocks_per_logical_kv_block = local_ratio
|
||||
worker.transfer_topo.get_engine_info.return_value = SimpleNamespace(
|
||||
remote_tp_size=4,
|
||||
remote_dcp_size=1,
|
||||
remote_block_size=64,
|
||||
remote_physical_blocks_per_logical=remote_ratio,
|
||||
)
|
||||
local_count = cdiv(num_tokens, 64 * local_ratio)
|
||||
remote_count = cdiv(num_tokens, 64 * remote_ratio)
|
||||
# The host has a cached prefix, while the indexer still needs the full range.
|
||||
local = [
|
||||
list(range(30 + cached, 30 + local_count)),
|
||||
list(range(40, 40 + local_count)),
|
||||
]
|
||||
metadata = NixlConnectorMetadata()
|
||||
metadata.add_new_req_to_recv(
|
||||
request_id="request",
|
||||
local_block_ids=local,
|
||||
local_num_computed_blocks=(0, cached, 0),
|
||||
awaiting_kvs=True,
|
||||
kv_transfer_params={
|
||||
"remote_engine_id": "P",
|
||||
"remote_request_id": "request-P",
|
||||
"remote_host": "localhost",
|
||||
"remote_port": 1,
|
||||
"remote_num_tokens": num_tokens,
|
||||
"remote_block_ids": [list(range(10, 10 + remote_count))],
|
||||
},
|
||||
)
|
||||
meta = metadata.reqs_to_recv["request"]
|
||||
meta.local_physical_block_ids = worker._logical_to_kernel_block_ids(
|
||||
local, local_ratio
|
||||
)
|
||||
worker._read_blocks_for_req("request", meta)
|
||||
read = worker._read_blocks_mixed.call_args.kwargs
|
||||
valid_pages = cdiv(num_tokens, 64)
|
||||
source_start = 10 * remote_ratio
|
||||
assert read["remote_block_descs_ids"].tolist() == (
|
||||
list(range(source_start + cached * local_ratio, source_start + valid_pages))
|
||||
+ list(range(100 + source_start, 100 + source_start + valid_pages))
|
||||
)
|
||||
assert read["local_block_descs_ids"].tolist() == (
|
||||
list(range((30 + cached) * local_ratio, 30 * local_ratio + valid_pages))
|
||||
+ list(range(100 + 40 * local_ratio, 100 + 40 * local_ratio + valid_pages))
|
||||
)
|
||||
assert meta.region_blocks_to_zero == [
|
||||
list(
|
||||
range(base * local_ratio + valid_pages, (base + local_count) * local_ratio)
|
||||
)
|
||||
for base in (30, 40)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_region_pull_completion_zeros_only_own_padding(region_pull_worker):
|
||||
"""Completion must preserve neighboring layers and already imported pages."""
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.metadata import (
|
||||
RemoteMeta,
|
||||
ReqMeta,
|
||||
)
|
||||
|
||||
worker = region_pull_worker
|
||||
# Two strided layer regions in one backing, with nonzero layer offsets.
|
||||
backing = torch.full((4, 3, 8), 7, dtype=torch.uint8)
|
||||
caches = {"host": backing[:, 1], "indexer": backing[:, 2]}
|
||||
worker.device_kv_caches = caches
|
||||
worker.region_names = list(caches)
|
||||
worker.kv_caches_base_addr = {"D": {0: [c.data_ptr() for c in caches.values()]}}
|
||||
worker.region_num_blocks = [4, 4]
|
||||
worker.block_len_per_layer = [8, 8]
|
||||
worker.block_stride_per_layer = [24, 24]
|
||||
worker._recving_metadata = {
|
||||
"request": ReqMeta(
|
||||
local_block_ids=[[1, 2], [0, 1]],
|
||||
local_physical_block_ids=[[1, 2], [0, 1]],
|
||||
tp_size=1,
|
||||
remote=RemoteMeta([[3]], "localhost", 1, "P", "request-P"),
|
||||
region_blocks_to_zero=[[2], [1]],
|
||||
)
|
||||
}
|
||||
worker._get_new_notifs = MagicMock(return_value=set())
|
||||
worker._pop_done_transfers = MagicMock(return_value=({"request"}, set()))
|
||||
worker._replicated_pcp_done_sending = set()
|
||||
worker._failed_recv_reqs = queue.Queue()
|
||||
worker._recv_failures = set()
|
||||
worker._send_pending_recv_notifs = MagicMock()
|
||||
worker._sync_device_after_direct_recv = MagicMock()
|
||||
worker.use_host_buffer = False
|
||||
worker._reqs_to_send = {}
|
||||
result = worker.get_transfer_results()
|
||||
expected = torch.full_like(backing, 7)
|
||||
expected[2, 1] = 0
|
||||
expected[1, 2] = 0
|
||||
assert torch.equal(backing, expected)
|
||||
assert result.finished_recving == {"request"}
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize(
|
||||
"swa_enabled,expected_sw_sizes",
|
||||
|
||||
Reference in New Issue
Block a user