[Bugfix][KV-transfer] MoRIIO: retry RDMA send-queue-full backpressure instead of failing the read (#47495)

Signed-off-by: Edwin Lim <edwin.lim@mangoboost.io>
Signed-off-by: Edwin Lim <edwinlim0919@gmail.com>
Signed-off-by: harishk-mangoboost <harish.kambhampaty@mangoboost.io>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: harishk-mangoboost <harish.kambhampaty@mangoboost.io>
This commit is contained in:
limeward
2026-07-16 20:02:27 +00:00
committed by GitHub
co-authored by Claude harishk-mangoboost
parent efa2e424f6
commit 971dac2caa
4 changed files with 91 additions and 14 deletions
@@ -670,16 +670,25 @@ def test_moriio_wrapper_rejects_invalid_messages(role, payload, match):
wrapper._handle_message(payload)
def test_block_id_length_mismatch_raises_value_error():
def test_local_block_ids_longer_than_remote_raises_value_error():
cache = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16)
worker = _worker({"layer": cache}, {"layer": _full_spec()})
with pytest.raises(ValueError, match="must have the same length"):
with pytest.raises(ValueError, match="longer than remote_block_ids"):
moriio_layout.compute_block_transfer_offsets(
"layer", cache, worker.layer_to_spec, [1, 3], [4], _remote_meta().num_blocks
)
def test_empty_local_block_ids_is_free_only_noop():
cache = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16)
worker = _worker({"layer": cache}, {"layer": _full_spec()})
assert moriio_layout.compute_block_transfer_offsets(
"layer", cache, worker.layer_to_spec, [], [4, 5], _remote_meta().num_blocks
) == ([], [], [])
def test_registration_regions_do_not_split_interleaved_or_mla_cache():
separated = torch.empty((2, 8, 4, 2, 3), dtype=torch.bfloat16)
interleaved = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16)
@@ -253,6 +253,7 @@ def test_worker_get_finished_counts_structured_release_fan_in():
worker.transfer_id_to_request_id = {"tx-fanin": "req-fanin"}
worker._consumer_notification_counts = {}
worker._completed_consumer_notifications = set()
worker._pending_unmapped_acks = []
assert worker.get_finished() == (set(), set())
assert worker._consumer_notification_counts == {"tx-fanin": 1}