[Bugfix][HiSparse] Fix TP group hang on steps without scheduled tokens (#60941)
Signed-off-by: Maroon Ayoub <mayoub@redhat.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Robert Shaw
Matthew Bonanni
Claude Opus 5.5
parent
a7b7bc1fbe
commit
5b0b8f251e
@@ -401,6 +401,53 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
|
||||
assert worker_meta.completed_transfer_counts == {transfer_id: 1}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("shared", [True, False])
|
||||
@pytest.mark.parametrize("num_tokens", [0, 4])
|
||||
def test_zero_token_step_holds_tp_ranks_after_host_write_wait(
|
||||
monkeypatch, shared, num_tokens
|
||||
):
|
||||
"""Steps without tokens hold every TP rank after its shared event wait."""
|
||||
calls: list[tuple[str, object]] = []
|
||||
compute_stream = MagicMock()
|
||||
compute_stream.wait_event.side_effect = lambda event: calls.append(("wait", event))
|
||||
tp_group = MagicMock()
|
||||
tp_group.barrier.side_effect = lambda: calls.append(("barrier", None))
|
||||
monkeypatch.setattr(worker_module, "current_stream", lambda: compute_stream)
|
||||
monkeypatch.setattr(worker_module, "get_tp_group", lambda: tp_group)
|
||||
|
||||
worker = object.__new__(HiSparseConnectorWorker)
|
||||
worker.shared_host_region = MagicMock() if shared else None
|
||||
worker.host_write_events = (MagicMock(), MagicMock())
|
||||
worker.host_write_event = worker.host_write_events[1]
|
||||
worker._next_host_write_event = 0
|
||||
worker._slot_mapping_staging = None
|
||||
worker.cache_handles = []
|
||||
worker._pending_invalid_block_ids = []
|
||||
for name in (
|
||||
"_stage_row_mirror_mapping",
|
||||
"_finish_previous_step",
|
||||
"_release_completed_dma_descriptors",
|
||||
"_set_row_mirrors",
|
||||
"_clear_forward_mirror_state",
|
||||
"_copy_host_blocks",
|
||||
"_restore_pages",
|
||||
"_submit_transfers",
|
||||
):
|
||||
setattr(worker, name, MagicMock())
|
||||
|
||||
worker.start_step(
|
||||
HiSparseConnectorMetadata(None, (), (), {}, True, {}),
|
||||
None,
|
||||
[],
|
||||
num_tokens=num_tokens,
|
||||
)
|
||||
|
||||
expected = [("wait", worker.host_write_events[1])]
|
||||
if shared and not num_tokens:
|
||||
expected.append(("barrier", None))
|
||||
assert calls == expected
|
||||
|
||||
|
||||
def test_scheduled_prefix_hit_publishes_adopted_copies():
|
||||
"""Copies adopted after scheduling reach the worker as a residency update,
|
||||
leaving the block-table row the scheduler output carries untouched."""
|
||||
|
||||
@@ -416,6 +416,10 @@ class HiSparseConnectorWorker:
|
||||
self.host_write_event = self.host_write_events[self._next_host_write_event]
|
||||
self._next_host_write_event ^= 1
|
||||
current_stream().wait_event(previous_host_write_event)
|
||||
if self.shared_host_region is not None and not num_tokens:
|
||||
# Without a forward, no TP collective keeps rank 0 from recording
|
||||
# a shared event again before every rank has issued its wait.
|
||||
get_tp_group().barrier()
|
||||
self._release_completed_dma_descriptors()
|
||||
self._dma_submitted = False
|
||||
mirrors = _flatten_row_mirrors(metadata.row_mirrors, request_ids)
|
||||
|
||||
Reference in New Issue
Block a user