[Bugfix][KV-transfer] MoRIIO: retry RDMA send-queue-full backpressure instead of failing the read (#47495)
Signed-off-by: Edwin Lim <edwin.lim@mangoboost.io> Signed-off-by: Edwin Lim <edwinlim0919@gmail.com> Signed-off-by: harishk-mangoboost <harish.kambhampaty@mangoboost.io> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: harishk-mangoboost <harish.kambhampaty@mangoboost.io>
This commit is contained in:
co-authored by
Claude
harishk-mangoboost
parent
efa2e424f6
commit
971dac2caa
@@ -670,16 +670,25 @@ def test_moriio_wrapper_rejects_invalid_messages(role, payload, match):
|
||||
wrapper._handle_message(payload)
|
||||
|
||||
|
||||
def test_block_id_length_mismatch_raises_value_error():
|
||||
def test_local_block_ids_longer_than_remote_raises_value_error():
|
||||
cache = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16)
|
||||
worker = _worker({"layer": cache}, {"layer": _full_spec()})
|
||||
|
||||
with pytest.raises(ValueError, match="must have the same length"):
|
||||
with pytest.raises(ValueError, match="longer than remote_block_ids"):
|
||||
moriio_layout.compute_block_transfer_offsets(
|
||||
"layer", cache, worker.layer_to_spec, [1, 3], [4], _remote_meta().num_blocks
|
||||
)
|
||||
|
||||
|
||||
def test_empty_local_block_ids_is_free_only_noop():
|
||||
cache = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16)
|
||||
worker = _worker({"layer": cache}, {"layer": _full_spec()})
|
||||
|
||||
assert moriio_layout.compute_block_transfer_offsets(
|
||||
"layer", cache, worker.layer_to_spec, [], [4, 5], _remote_meta().num_blocks
|
||||
) == ([], [], [])
|
||||
|
||||
|
||||
def test_registration_regions_do_not_split_interleaved_or_mla_cache():
|
||||
separated = torch.empty((2, 8, 4, 2, 3), dtype=torch.bfloat16)
|
||||
interleaved = torch.empty((8, 2, 4, 2, 3), dtype=torch.bfloat16)
|
||||
|
||||
@@ -253,6 +253,7 @@ def test_worker_get_finished_counts_structured_release_fan_in():
|
||||
worker.transfer_id_to_request_id = {"tx-fanin": "req-fanin"}
|
||||
worker._consumer_notification_counts = {}
|
||||
worker._completed_consumer_notifications = set()
|
||||
worker._pending_unmapped_acks = []
|
||||
|
||||
assert worker.get_finished() == (set(), set())
|
||||
assert worker._consumer_notification_counts == {"tx-fanin": 1}
|
||||
|
||||
Reference in New Issue
Block a user