[ROCm][P/D] Fix MoRIIO WRITE mode for mixed KV layouts (#46290)
Signed-off-by: Tan Pin Siang <tanpinsiang@gmail.com> Co-authored-by: vllmellm <vllm.ellm@embeddedllm.com> Co-authored-by: Hongxia Yang <hongxia.yang@amd.com> Co-authored-by: Jun Kang Chow <junkangchow@gmail.com> Co-authored-by: Chun Fang <chun.fang@amd.com> Co-authored-by: TianDi101 <ditian12@amd.com> Co-authored-by: functionstackx <47992694+functionstackx@users.noreply.github.com>
This commit is contained in:
co-authored by
vllmellm
Hongxia Yang
Jun Kang Chow
Chun Fang
TianDi101
functionstackx
parent
a8481be7a9
commit
7e47fb72b5
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import importlib.util
|
||||
import socket
|
||||
import uuid
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -9,7 +10,6 @@ import pytest
|
||||
import torch
|
||||
import zmq
|
||||
|
||||
from tests.conftest import _find_free_port
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
DeviceConfig,
|
||||
@@ -23,12 +23,14 @@ from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common import (
|
||||
MoRIIOAgentMetadata,
|
||||
MoRIIOConnectorMetadata,
|
||||
MoRIIOConstants,
|
||||
MoRIIOMode,
|
||||
resolve_host_ip,
|
||||
zmq_ctx,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_connector import (
|
||||
KVConnectorRole,
|
||||
MoRIIOConnector,
|
||||
MoRIIOConnectorScheduler,
|
||||
MoRIIOConnectorWorker,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
@@ -46,6 +48,12 @@ from vllm.v1.kv_cache_interface import (
|
||||
from .utils import create_request, create_scheduler
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.bind(("", 0))
|
||||
return sock.getsockname()[1]
|
||||
|
||||
|
||||
def _make_test_kv_cache_config() -> KVCacheConfig:
|
||||
layer_names = ["layer0", "layer1", "layer2"]
|
||||
return KVCacheConfig(
|
||||
@@ -121,6 +129,16 @@ def _setup_kv_transfer_request(
|
||||
return request
|
||||
|
||||
|
||||
def _write_consumer_scheduler_for_finished_request(tp_size: int = 2):
|
||||
scheduler = MoRIIOConnectorScheduler.__new__(MoRIIOConnectorScheduler)
|
||||
scheduler.is_producer = False
|
||||
scheduler.mode = MoRIIOMode.WRITE
|
||||
scheduler.tp_size = tp_size
|
||||
scheduler._reqs_need_recv = {}
|
||||
scheduler.unmap_request_id = MagicMock()
|
||||
return scheduler
|
||||
|
||||
|
||||
class FakeMoRIIOWrapper:
|
||||
# A fake MoRIIOWrapper for testing purposes
|
||||
def __init__(self, *args, **kwargs):
|
||||
@@ -177,7 +195,7 @@ class FakeMoRIIOWrapper:
|
||||
def _handle_completion_message(self, msg: str):
|
||||
pass
|
||||
|
||||
def send_notify(self, req_ids, remote_ip, remote_port):
|
||||
def send_notify(self, req_ids, remote_ip, remote_port, message_type=None):
|
||||
pass
|
||||
|
||||
def pop_finished_req_ids(self):
|
||||
@@ -434,6 +452,61 @@ def test_read_mode_loads_remote_block_ids():
|
||||
assert block_id == block.block_id, f"{block_id} != {block.block_id}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("transfer_id", "extra_params", "expected_notifications"),
|
||||
[
|
||||
pytest.param(
|
||||
"xfer-7",
|
||||
{"remote_host": "127.0.0.1", "remote_notify_port": 7000},
|
||||
[
|
||||
("xfer-7", "127.0.0.1", 7000),
|
||||
("xfer-7", "127.0.0.1", 7001),
|
||||
],
|
||||
id="address-available",
|
||||
),
|
||||
pytest.param("xfer-8", {}, [], id="address-unavailable-plain-id"),
|
||||
],
|
||||
)
|
||||
def test_write_mode_finished_before_alloc_releases_prefill_blocks(
|
||||
transfer_id, extra_params, expected_notifications
|
||||
):
|
||||
scheduler = _write_consumer_scheduler_for_finished_request(tp_size=2)
|
||||
notifications = []
|
||||
scheduler._send_transfer_release = lambda transfer_id, host, port: (
|
||||
notifications.append((transfer_id, host, port))
|
||||
)
|
||||
request = create_request(request_id=7, do_remote_prefill=True)
|
||||
request.request_id = "plain-decode-id"
|
||||
request.kv_transfer_params = {
|
||||
"do_remote_prefill": True,
|
||||
"do_remote_decode": False,
|
||||
"transfer_id": transfer_id,
|
||||
} | extra_params
|
||||
|
||||
delay_free, new_params = scheduler.request_finished(request, block_ids=[])
|
||||
|
||||
assert not delay_free
|
||||
assert new_params is None
|
||||
assert request.kv_transfer_params["do_remote_prefill"] is False
|
||||
assert scheduler._reqs_need_recv == {}
|
||||
assert notifications == expected_notifications
|
||||
|
||||
|
||||
def test_send_transfer_release_sends_structured_release_message():
|
||||
scheduler = _write_consumer_scheduler_for_finished_request()
|
||||
path = make_zmq_path("tcp", "127.0.0.1", 7000)
|
||||
sock = MagicMock()
|
||||
scheduler.paths = {path: sock}
|
||||
|
||||
scheduler._send_transfer_release("xfer-7", "127.0.0.1", 7000)
|
||||
|
||||
payload = sock.send.call_args.args[0]
|
||||
assert msgspec.msgpack.decode(payload) == {
|
||||
"type": "release",
|
||||
"transfer_id": "xfer-7",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not aiter_available, reason="Requires aiter package for ROCm FlashAttention backend"
|
||||
)
|
||||
|
||||
@@ -2,7 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import importlib.util
|
||||
import threading
|
||||
from collections import OrderedDict, defaultdict
|
||||
from queue import Queue
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -19,16 +23,33 @@ if not (current_platform.is_rocm() and mori_available):
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
moriio_common = importlib.import_module(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_common"
|
||||
)
|
||||
moriio_engine = importlib.import_module(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_engine"
|
||||
)
|
||||
moriio_layout = importlib.import_module(
|
||||
"vllm.distributed.kv_transfer.kv_connector.v1.moriio.moriio_layout"
|
||||
)
|
||||
msgpack = importlib.import_module("msgpack")
|
||||
|
||||
ROLE = moriio_common.ROLE
|
||||
MoRIIOError = moriio_common.MoRIIOError
|
||||
RemoteAllocInfo = moriio_common.RemoteAllocInfo
|
||||
WriteTask = moriio_common.WriteTask
|
||||
set_role = moriio_common.set_role
|
||||
MoRIIOWrapper = moriio_engine.MoRIIOWrapper
|
||||
MoRIIOWriter = moriio_engine.MoRIIOWriter
|
||||
|
||||
|
||||
def _full_spec(block_size: int = 4) -> FullAttentionSpec:
|
||||
def _full_spec(
|
||||
block_size: int = 4, num_kv_heads: int = 2, head_size: int = 3
|
||||
) -> FullAttentionSpec:
|
||||
return FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=2,
|
||||
head_size=3,
|
||||
num_kv_heads=num_kv_heads,
|
||||
head_size=head_size,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
@@ -59,55 +80,216 @@ def _remote_meta(num_blocks: int = 16) -> SimpleNamespace:
|
||||
return SimpleNamespace(num_blocks=num_blocks)
|
||||
|
||||
|
||||
def test_separated_kv_layout_uses_kv_axis_zero_and_block_axis_one():
|
||||
cache = torch.empty((2, 8, 4, 2, 3), dtype=torch.bfloat16)
|
||||
worker = _worker({"layer": cache}, {"layer": _full_spec()})
|
||||
def _writer_with_fake_worker(fake_worker: Any) -> Any:
|
||||
writer = MoRIIOWriter.__new__(MoRIIOWriter)
|
||||
writer._worker_ref = lambda: fake_worker
|
||||
writer._write_task_q = Queue()
|
||||
writer._write_state_lock = threading.Lock()
|
||||
writer._scheduled_writes = defaultdict(int)
|
||||
writer._scheduled_layers = defaultdict(set)
|
||||
writer._sealed_writes = {}
|
||||
writer.ensure_worker_started = lambda: None
|
||||
return writer
|
||||
|
||||
|
||||
def _wrapper_for_messages() -> Any:
|
||||
wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper)
|
||||
wrapper.lock = threading.Lock()
|
||||
wrapper.done_remote_allocate_req_dict = {}
|
||||
wrapper.done_req_ids = []
|
||||
wrapper.done_write_cache_req_ids = []
|
||||
wrapper._terminal_transfer_ids = OrderedDict()
|
||||
return wrapper
|
||||
|
||||
|
||||
def _write_task(layer_name: str, transfer_id: str = "xfer") -> Any:
|
||||
return WriteTask(
|
||||
request_id="req",
|
||||
transfer_id=transfer_id,
|
||||
dst_engine_id="remote-engine",
|
||||
local_block_ids=[1, 3],
|
||||
remote_block_ids_hint=None,
|
||||
layer_name=layer_name,
|
||||
event=None,
|
||||
remote_notify_port=7000,
|
||||
remote_ip="127.0.0.1",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("shape", "spec", "remote_num_blocks", "expected_geometry", "expected_offsets"),
|
||||
[
|
||||
pytest.param(
|
||||
(2, 8, 4, 2, 3),
|
||||
_full_spec(),
|
||||
16,
|
||||
{
|
||||
"block_stride": 24,
|
||||
"local_kv_stride": 192,
|
||||
"remote_kv_stride": 384,
|
||||
"split_kv_regions": True,
|
||||
},
|
||||
([48, 144, 432, 528], [192, 240, 960, 1008], [48, 48, 48, 48]),
|
||||
id="separated",
|
||||
),
|
||||
pytest.param(
|
||||
(8, 2, 4, 2, 3),
|
||||
_full_spec(),
|
||||
16,
|
||||
{
|
||||
"block_stride": 48,
|
||||
"local_kv_stride": 24,
|
||||
"remote_kv_stride": 24,
|
||||
"split_kv_regions": False,
|
||||
},
|
||||
([96, 288], [384, 480], [96, 96]),
|
||||
id="interleaved",
|
||||
),
|
||||
pytest.param(
|
||||
(2, 8, 2, 4, 3),
|
||||
_full_spec(),
|
||||
16,
|
||||
{
|
||||
"block_size": 4,
|
||||
"block_stride": 24,
|
||||
"local_kv_stride": 192,
|
||||
"remote_kv_stride": 384,
|
||||
"split_kv_regions": True,
|
||||
},
|
||||
([48, 144, 432, 528], [192, 240, 960, 1008], [48, 48, 48, 48]),
|
||||
id="shuffled-separated",
|
||||
),
|
||||
pytest.param(
|
||||
(8, 2, 2, 4, 3),
|
||||
_full_spec(),
|
||||
16,
|
||||
{
|
||||
"block_size": 4,
|
||||
"block_stride": 48,
|
||||
"local_kv_stride": 24,
|
||||
"remote_kv_stride": 24,
|
||||
"split_kv_regions": False,
|
||||
},
|
||||
([96, 288], [384, 480], [96, 96]),
|
||||
id="shuffled-interleaved",
|
||||
),
|
||||
pytest.param(
|
||||
(2, 16, 2, 2, 3),
|
||||
_full_spec(),
|
||||
16,
|
||||
{
|
||||
"num_blocks": 8,
|
||||
"block_size": 4,
|
||||
"block_stride": 24,
|
||||
"local_kv_stride": 192,
|
||||
"remote_kv_stride": 384,
|
||||
"split_kv_regions": True,
|
||||
},
|
||||
([48, 144, 432, 528], [192, 240, 960, 1008], [48, 48, 48, 48]),
|
||||
id="separated-kernel-blocks",
|
||||
),
|
||||
pytest.param(
|
||||
(16, 2, 2, 2, 3),
|
||||
_full_spec(),
|
||||
16,
|
||||
{
|
||||
"num_blocks": 8,
|
||||
"block_size": 4,
|
||||
"block_stride": 48,
|
||||
"local_kv_stride": None,
|
||||
"remote_kv_stride": None,
|
||||
"transfers_per_block": 1,
|
||||
},
|
||||
([96, 288], [384, 480], [96, 96]),
|
||||
id="interleaved-kernel-blocks",
|
||||
),
|
||||
pytest.param(
|
||||
(2, 32, 8, 2, 3),
|
||||
_full_spec(block_size=16, num_kv_heads=8),
|
||||
8,
|
||||
{
|
||||
"num_blocks": 4,
|
||||
"block_size": 16,
|
||||
"block_len": 768,
|
||||
"block_stride": 384,
|
||||
"local_kv_stride": 1536,
|
||||
"remote_kv_stride": 3072,
|
||||
"split_kv_regions": True,
|
||||
},
|
||||
(
|
||||
[768, 2304, 3840, 5376],
|
||||
[3072, 3840, 9216, 9984],
|
||||
[768, 768, 768, 768],
|
||||
),
|
||||
id="separated-kernel-axis-from-spec",
|
||||
),
|
||||
pytest.param(
|
||||
(32, 2, 8, 2, 3),
|
||||
_full_spec(block_size=16, num_kv_heads=8),
|
||||
8,
|
||||
{
|
||||
"num_blocks": 4,
|
||||
"block_size": 16,
|
||||
"block_len": 1536,
|
||||
"block_stride": 768,
|
||||
"local_kv_stride": None,
|
||||
"remote_kv_stride": None,
|
||||
"transfers_per_block": 1,
|
||||
},
|
||||
([1536, 4608], [6144, 7680], [1536, 1536]),
|
||||
id="interleaved-kernel-axis-from-spec",
|
||||
),
|
||||
pytest.param(
|
||||
(8, 4, 3),
|
||||
_mla_spec(),
|
||||
16,
|
||||
{
|
||||
"block_stride": 12,
|
||||
"local_kv_stride": None,
|
||||
"remote_kv_stride": None,
|
||||
"transfers_per_block": 1,
|
||||
},
|
||||
([24, 72], [96, 120], [24, 24]),
|
||||
id="mla-key-only",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_supported_layouts_compute_expected_geometry_and_offsets(
|
||||
shape, spec, remote_num_blocks, expected_geometry, expected_offsets
|
||||
):
|
||||
cache = torch.empty(shape, dtype=torch.bfloat16)
|
||||
worker = _worker({"layer": cache}, {"layer": spec})
|
||||
|
||||
geometry = moriio_layout.get_layer_transfer_geometry(
|
||||
"layer", cache, worker.layer_to_spec, remote_num_blocks=16
|
||||
"layer", cache, worker.layer_to_spec, remote_num_blocks=remote_num_blocks
|
||||
)
|
||||
assert geometry.block_stride == 24
|
||||
assert geometry.local_kv_stride == 192
|
||||
assert geometry.remote_kv_stride == 384
|
||||
assert geometry.split_kv_regions
|
||||
for field, expected in expected_geometry.items():
|
||||
assert getattr(geometry, field) == expected
|
||||
|
||||
assert moriio_layout.compute_block_transfer_offsets(
|
||||
"layer", cache, worker.layer_to_spec, [1, 3], [4, 5], _remote_meta().num_blocks
|
||||
) == ([48, 144, 432, 528], [192, 240, 960, 1008], [48, 48, 48, 48])
|
||||
|
||||
|
||||
def test_interleaved_kv_layout_uses_block_axis_zero_and_kv_axis_one():
|
||||
cache = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16)
|
||||
worker = _worker({"layer": cache}, {"layer": _full_spec()})
|
||||
|
||||
geometry = moriio_layout.get_layer_transfer_geometry(
|
||||
"layer", cache, worker.layer_to_spec, remote_num_blocks=16
|
||||
assert (
|
||||
moriio_layout.compute_block_transfer_offsets(
|
||||
"layer",
|
||||
cache,
|
||||
worker.layer_to_spec,
|
||||
[1, 3],
|
||||
[4, 5],
|
||||
remote_num_blocks,
|
||||
)
|
||||
== expected_offsets
|
||||
)
|
||||
assert geometry.block_stride == 48
|
||||
assert geometry.local_kv_stride == 24
|
||||
assert geometry.remote_kv_stride == 24
|
||||
assert not geometry.split_kv_regions
|
||||
|
||||
assert moriio_layout.compute_block_transfer_offsets(
|
||||
"layer", cache, worker.layer_to_spec, [1, 3], [4, 5], _remote_meta().num_blocks
|
||||
) == ([96, 288], [384, 480], [96, 96])
|
||||
|
||||
|
||||
def test_mla_key_only_layout_transfers_one_slab_per_block():
|
||||
cache = torch.empty((8, 4, 3), dtype=torch.bfloat16)
|
||||
worker = _worker({"layer": cache}, {"layer": _mla_spec()})
|
||||
|
||||
geometry = moriio_layout.get_layer_transfer_geometry(
|
||||
"layer", cache, worker.layer_to_spec, remote_num_blocks=16
|
||||
def test_kernel_block_layout_without_spec_dimensions_rejects_ambiguous_axes():
|
||||
cache = torch.empty((2, 32, 8, 2, 3), dtype=torch.bfloat16)
|
||||
worker = _worker(
|
||||
{"layer": cache},
|
||||
{"layer": SimpleNamespace(block_size=16)},
|
||||
)
|
||||
assert geometry.block_stride == 12
|
||||
assert geometry.local_kv_stride is None
|
||||
assert geometry.remote_kv_stride is None
|
||||
assert geometry.transfers_per_block == 1
|
||||
|
||||
assert moriio_layout.compute_block_transfer_offsets(
|
||||
"layer", cache, worker.layer_to_spec, [1, 3], [4, 5], _remote_meta().num_blocks
|
||||
) == ([24, 72], [96, 120], [24, 24])
|
||||
with pytest.raises(ValueError, match="Ambiguous MoRIIO kernel-block"):
|
||||
moriio_layout.get_layer_transfer_geometry(
|
||||
"layer", cache, worker.layer_to_spec, remote_num_blocks=8
|
||||
)
|
||||
|
||||
|
||||
def test_mixed_layers_compute_distinct_offsets_per_layer():
|
||||
@@ -155,6 +337,319 @@ def test_mixed_layers_compute_distinct_offsets_per_layer():
|
||||
assert interleaved != indexer
|
||||
|
||||
|
||||
def test_write_transfer_plan_caches_offsets_per_geometry():
|
||||
kv_caches = {
|
||||
"dense0": torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16),
|
||||
"dense1": torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16),
|
||||
"indexer": torch.empty((8, 4, 3), dtype=torch.bfloat16),
|
||||
}
|
||||
calls: list[str] = []
|
||||
|
||||
class FakeWorker:
|
||||
kv_caches: dict[str, torch.Tensor]
|
||||
layer_name_to_local_kv_cache_metadata: dict[str, list[Any]]
|
||||
|
||||
def _compute_block_transfer_offsets(
|
||||
self, layer_name, local_block_ids, remote_block_ids, remote_moriio_meta
|
||||
):
|
||||
calls.append(layer_name)
|
||||
call_id = len(calls)
|
||||
return ([call_id], [call_id + 10], [call_id + 20])
|
||||
|
||||
fake_worker = FakeWorker()
|
||||
fake_worker.kv_caches = kv_caches
|
||||
fake_worker.layer_name_to_local_kv_cache_metadata = {name: [] for name in kv_caches}
|
||||
writer = MoRIIOWriter.__new__(MoRIIOWriter)
|
||||
writer._worker_ref = lambda: fake_worker
|
||||
request_info = RemoteAllocInfo(block_ids=[4, 5])
|
||||
remote_meta = _remote_meta()
|
||||
|
||||
dense0_plan = writer._prepare_transfer_plan(
|
||||
SimpleNamespace(
|
||||
layer_name="dense0",
|
||||
local_block_ids=[1, 3],
|
||||
request_id="req",
|
||||
transfer_id="xfer",
|
||||
),
|
||||
request_info,
|
||||
remote_meta,
|
||||
)
|
||||
dense1_plan = writer._prepare_transfer_plan(
|
||||
SimpleNamespace(
|
||||
layer_name="dense1",
|
||||
local_block_ids=[1, 3],
|
||||
request_id="req",
|
||||
transfer_id="xfer",
|
||||
),
|
||||
request_info,
|
||||
remote_meta,
|
||||
)
|
||||
indexer_plan = writer._prepare_transfer_plan(
|
||||
SimpleNamespace(
|
||||
layer_name="indexer",
|
||||
local_block_ids=[1, 3],
|
||||
request_id="req",
|
||||
transfer_id="xfer",
|
||||
),
|
||||
request_info,
|
||||
remote_meta,
|
||||
)
|
||||
|
||||
assert calls == ["dense0", "indexer"]
|
||||
assert dense0_plan.transfer_local_offsets == [1]
|
||||
assert dense1_plan.transfer_local_offsets == [1]
|
||||
assert indexer_plan.transfer_local_offsets == [2]
|
||||
assert len(request_info.transfer_offsets) == 2
|
||||
|
||||
|
||||
def test_write_scheduler_deduplicates_layers_and_seals_expected_count():
|
||||
request_info = RemoteAllocInfo(block_ids=[4, 5])
|
||||
wrapper = _wrapper_for_messages()
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = request_info
|
||||
writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
|
||||
assert writer.schedule_write(_write_task("dense0"))
|
||||
assert not writer.schedule_write(_write_task("dense0"))
|
||||
assert writer.schedule_write(_write_task("indexer"))
|
||||
|
||||
assert writer._write_task_q.qsize() == 2
|
||||
writer.seal_pending_transfers()
|
||||
|
||||
assert request_info.writes_expected == 2
|
||||
assert writer._sealed_writes["xfer"] == 2
|
||||
|
||||
|
||||
def test_write_completion_notifies_once_after_all_sealed_writes_finish():
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.done_remote_allocate_req_dict = {}
|
||||
self.done_req_ids = []
|
||||
self.lock = threading.Lock()
|
||||
self.notifications = []
|
||||
self.wait_count = 0
|
||||
self.waited_statuses = []
|
||||
self._terminal_transfer_ids = OrderedDict()
|
||||
|
||||
def waiting_for_transfer_complete(self, transfer_statuses=None):
|
||||
self.wait_count += 1
|
||||
self.waited_statuses.append(list(transfer_statuses or []))
|
||||
|
||||
def _is_transfer_terminal_locked(self, transfer_id):
|
||||
return transfer_id in self._terminal_transfer_ids
|
||||
|
||||
def _mark_transfer_terminal_locked(self, transfer_id):
|
||||
self._terminal_transfer_ids[transfer_id] = None
|
||||
|
||||
def send_notify(self, transfer_id, remote_ip, remote_port, message_type=None):
|
||||
self.notifications.append(
|
||||
(transfer_id, remote_ip, remote_port, message_type)
|
||||
)
|
||||
|
||||
wrapper = FakeWrapper()
|
||||
request_info = RemoteAllocInfo(block_ids=[4, 5], writes_expected=2)
|
||||
request_info.transfer_statuses.extend(["status-a", "status-b"])
|
||||
request_info.completion_request_id = "req"
|
||||
request_info.completion_remote_notify_port = 7000
|
||||
request_info.completion_remote_ip = "127.0.0.1"
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = request_info
|
||||
writer = _writer_with_fake_worker(
|
||||
SimpleNamespace(moriio_wrapper=wrapper, tp_rank=2)
|
||||
)
|
||||
writer._scheduled_writes["xfer"] = 2
|
||||
writer._scheduled_layers["xfer"] = {"dense0", "indexer"}
|
||||
writer._sealed_writes["xfer"] = 2
|
||||
|
||||
writer._mark_write_done("xfer", request_info)
|
||||
assert wrapper.notifications == []
|
||||
writer._mark_write_done("xfer", request_info)
|
||||
writer._finalize_if_complete("xfer", request_info)
|
||||
|
||||
assert wrapper.notifications == [("xfer", "127.0.0.1", 7002, "write_done")]
|
||||
assert wrapper.done_req_ids == ["xfer"]
|
||||
assert wrapper.done_remote_allocate_req_dict == {}
|
||||
assert wrapper.wait_count == 1
|
||||
assert wrapper.waited_statuses == [["status-a", "status-b"]]
|
||||
assert request_info.transfer_statuses == []
|
||||
assert wrapper._is_transfer_terminal_locked("xfer")
|
||||
|
||||
|
||||
def test_moriio_wrapper_waits_scoped_statuses_without_global_drain():
|
||||
class FakeStatus:
|
||||
def __init__(self):
|
||||
self.checked = 0
|
||||
|
||||
def Succeeded(self):
|
||||
self.checked += 1
|
||||
return True
|
||||
|
||||
def Failed(self):
|
||||
return False
|
||||
|
||||
wrapper = MoRIIOWrapper.__new__(MoRIIOWrapper)
|
||||
wrapper.lock = threading.Lock()
|
||||
wrapper._transfer_timeout = 1
|
||||
global_status = FakeStatus()
|
||||
scoped_status = FakeStatus()
|
||||
wrapper.transfer_status = [global_status]
|
||||
|
||||
wrapper.waiting_for_transfer_complete([scoped_status])
|
||||
|
||||
assert scoped_status.checked == 1
|
||||
assert global_status.checked == 0
|
||||
assert wrapper.transfer_status == [global_status]
|
||||
|
||||
|
||||
def test_write_failure_marks_terminal_and_clears_scheduled_state():
|
||||
wrapper = _wrapper_for_messages()
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
|
||||
writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
writer._scheduled_writes["xfer"] = 2
|
||||
writer._scheduled_layers["xfer"] = {"dense0", "indexer"}
|
||||
writer._sealed_writes["xfer"] = 2
|
||||
|
||||
writer._mark_request_done("xfer")
|
||||
|
||||
assert wrapper.done_req_ids == ["xfer"]
|
||||
assert wrapper.done_remote_allocate_req_dict == {}
|
||||
assert wrapper._is_transfer_terminal_locked("xfer")
|
||||
assert "xfer" not in writer._scheduled_writes
|
||||
assert "xfer" not in writer._scheduled_layers
|
||||
assert "xfer" not in writer._sealed_writes
|
||||
|
||||
|
||||
def test_schedule_write_rejects_terminal_transfer_without_recreating_state():
|
||||
wrapper = _wrapper_for_messages()
|
||||
wrapper.done_remote_allocate_req_dict["xfer"] = RemoteAllocInfo(block_ids=[4, 5])
|
||||
writer = _writer_with_fake_worker(SimpleNamespace(moriio_wrapper=wrapper))
|
||||
writer._scheduled_writes["xfer"] = 1
|
||||
writer._scheduled_layers["xfer"] = {"dense0"}
|
||||
writer._sealed_writes["xfer"] = 1
|
||||
|
||||
writer._mark_request_done("xfer")
|
||||
|
||||
assert not writer.schedule_write(_write_task("indexer"))
|
||||
assert writer._write_task_q.empty()
|
||||
assert "xfer" not in writer._scheduled_writes
|
||||
assert "xfer" not in writer._scheduled_layers
|
||||
assert "xfer" not in writer._sealed_writes
|
||||
|
||||
|
||||
def test_late_remote_blocks_message_is_ignored_after_transfer_done():
|
||||
set_role(ROLE.PRODUCER)
|
||||
wrapper = _wrapper_for_messages()
|
||||
with wrapper.lock:
|
||||
wrapper._mark_transfer_terminal_locked("xfer")
|
||||
|
||||
wrapper._handle_message(
|
||||
msgpack.dumps(
|
||||
{
|
||||
"type": "remote_blocks",
|
||||
"req_id": "req",
|
||||
"transfer_id": "xfer",
|
||||
"block_notify_list": [4, 5],
|
||||
"decode_rank": 3,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert "xfer" not in wrapper.done_remote_allocate_req_dict
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("role", "payload", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
ROLE.PRODUCER,
|
||||
msgpack.dumps(
|
||||
{
|
||||
"type": "remote_blocks",
|
||||
"req_id": "req",
|
||||
"transfer_id": "xfer",
|
||||
"block_notify_list": [4, 5],
|
||||
"decode_rank": 3,
|
||||
}
|
||||
),
|
||||
"remote_blocks",
|
||||
id="remote-blocks",
|
||||
),
|
||||
pytest.param(
|
||||
ROLE.CONSUMER,
|
||||
msgpack.dumps({"type": "write_done", "transfer_id": "xfer"}),
|
||||
"write_done",
|
||||
id="write-done",
|
||||
),
|
||||
pytest.param(
|
||||
ROLE.PRODUCER,
|
||||
msgpack.dumps({"type": "release", "transfer_id": "xfer"}),
|
||||
"release",
|
||||
id="release",
|
||||
),
|
||||
pytest.param(None, b"xfer", "plain", id="plain-string"),
|
||||
],
|
||||
)
|
||||
def test_moriio_wrapper_routes_valid_messages(role, payload, expected):
|
||||
wrapper = _wrapper_for_messages()
|
||||
completions: list[str] = []
|
||||
if role is not None:
|
||||
set_role(role)
|
||||
if expected == "plain":
|
||||
wrapper._handle_completion_message = completions.append
|
||||
|
||||
wrapper._handle_message(payload)
|
||||
|
||||
if expected == "remote_blocks":
|
||||
request_info = wrapper.done_remote_allocate_req_dict["xfer"]
|
||||
assert request_info.block_ids == [4, 5]
|
||||
assert request_info.decode_dp_rank == 3
|
||||
elif expected == "write_done":
|
||||
assert wrapper.done_write_cache_req_ids == ["xfer"]
|
||||
elif expected == "release":
|
||||
assert wrapper.done_req_ids == ["xfer"]
|
||||
assert wrapper._is_transfer_terminal_locked("xfer")
|
||||
else:
|
||||
assert completions == ["xfer"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("role", "payload", "match"),
|
||||
[
|
||||
pytest.param(
|
||||
None,
|
||||
msgpack.dumps({"type": "unknown", "transfer_id": "xfer"}),
|
||||
"Unhandled structured message type",
|
||||
id="unknown-structured-type",
|
||||
),
|
||||
pytest.param(
|
||||
ROLE.PRODUCER,
|
||||
msgpack.dumps(
|
||||
{
|
||||
"type": "remote_blocks",
|
||||
"req_id": "req",
|
||||
"transfer_id": "xfer",
|
||||
"block_notify_list": [],
|
||||
}
|
||||
),
|
||||
"block_notify_list cannot be empty",
|
||||
id="empty-remote-blocks",
|
||||
),
|
||||
pytest.param(
|
||||
None,
|
||||
b"",
|
||||
"Unhandled message format",
|
||||
id="empty-completion",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_moriio_wrapper_rejects_invalid_messages(role, payload, match):
|
||||
wrapper = _wrapper_for_messages()
|
||||
if role is not None:
|
||||
set_role(role)
|
||||
wrapper._handle_completion_message = lambda msg: None
|
||||
|
||||
with pytest.raises(MoRIIOError, match=match):
|
||||
wrapper._handle_message(payload)
|
||||
|
||||
|
||||
def test_block_id_length_mismatch_raises_value_error():
|
||||
cache = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16)
|
||||
worker = _worker({"layer": cache}, {"layer": _full_spec()})
|
||||
|
||||
Reference in New Issue
Block a user