[Bugfix][HiSparse] Fix TP group hang on steps without scheduled tokens (#60941)

Signed-off-by: Maroon Ayoub <mayoub@redhat.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Maroon Ayoub
2026-10-12 17:38:12 +00:00
committed by GitHub
co-authored by Robert Shaw Matthew Bonanni Claude Opus 5.5
parent a7b7bc1fbe
commit 5b0b8f251e
2 changed files with 51 additions and 0 deletions
@@ -401,6 +401,53 @@ def test_draft_layer_rows_mirrored_after_drafter(monkeypatch, cg_mode):
assert worker_meta.completed_transfer_counts == {transfer_id: 1}
@pytest.mark.parametrize("shared", [True, False])
@pytest.mark.parametrize("num_tokens", [0, 4])
def test_zero_token_step_holds_tp_ranks_after_host_write_wait(
monkeypatch, shared, num_tokens
):
"""Steps without tokens hold every TP rank after its shared event wait."""
calls: list[tuple[str, object]] = []
compute_stream = MagicMock()
compute_stream.wait_event.side_effect = lambda event: calls.append(("wait", event))
tp_group = MagicMock()
tp_group.barrier.side_effect = lambda: calls.append(("barrier", None))
monkeypatch.setattr(worker_module, "current_stream", lambda: compute_stream)
monkeypatch.setattr(worker_module, "get_tp_group", lambda: tp_group)
worker = object.__new__(HiSparseConnectorWorker)
worker.shared_host_region = MagicMock() if shared else None
worker.host_write_events = (MagicMock(), MagicMock())
worker.host_write_event = worker.host_write_events[1]
worker._next_host_write_event = 0
worker._slot_mapping_staging = None
worker.cache_handles = []
worker._pending_invalid_block_ids = []
for name in (
"_stage_row_mirror_mapping",
"_finish_previous_step",
"_release_completed_dma_descriptors",
"_set_row_mirrors",
"_clear_forward_mirror_state",
"_copy_host_blocks",
"_restore_pages",
"_submit_transfers",
):
setattr(worker, name, MagicMock())
worker.start_step(
HiSparseConnectorMetadata(None, (), (), {}, True, {}),
None,
[],
num_tokens=num_tokens,
)
expected = [("wait", worker.host_write_events[1])]
if shared and not num_tokens:
expected.append(("barrier", None))
assert calls == expected
def test_scheduled_prefix_hit_publishes_adopted_copies():
"""Copies adopted after scheduling reach the worker as a residency update,
leaving the block-table row the scheduler output carries untouched."""
@@ -416,6 +416,10 @@ class HiSparseConnectorWorker:
self.host_write_event = self.host_write_events[self._next_host_write_event]
self._next_host_write_event ^= 1
current_stream().wait_event(previous_host_write_event)
if self.shared_host_region is not None and not num_tokens:
# Without a forward, no TP collective keeps rank 0 from recording
# a shared event again before every rank has issued its wait.
get_tp_group().barrier()
self._release_completed_dma_descriptors()
self._dma_submitted = False
mirrors = _flatten_row_mirrors(metadata.row_mirrors, request_ids)