[Bugfix][HiSparse] Fix MTP acceptance collapse under FULL graphs with a saturated GPU pool (#59309)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-10-01 23:04:21 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Matthew Bonanni
parent 0cbac6cd13
commit 4056c8ac1f
11 changed files with 426 additions and 122 deletions
@@ -301,6 +301,10 @@ class HiSparseConnectorWorker:
self._submitted_mirror_layers: set[int] = set()
self._layer_ready_events = tuple(torch.Event() for _ in cache_handles)
self._forward_ready_event = torch.Event()
self._draft_layers = tuple(
index for index, cache in enumerate(cache_handles) if cache.draft_layer
)
self._draft_mirror_pending = False
self._set_row_mirrors(())
self.cache_layer_names = cache_layer_names
self._group_leaders = tuple(
@@ -344,9 +348,11 @@ class HiSparseConnectorWorker:
self._init_dma()
if self.is_host_writer:
for layer_index, handle in enumerate(cache_handles):
handle.submit_layer_mirror = partial(
self._enqueue_layer_mirror, layer_index
)
# Draft layers are mirrored once in the next start_step instead.
if not handle.draft_layer:
handle.submit_layer_mirror = partial(
self._enqueue_layer_mirror, layer_index
)
self._initialized = True
def set_request_state_indices(self, indices: torch.Tensor) -> None:
@@ -397,16 +403,17 @@ class HiSparseConnectorWorker:
num_tokens: int = 0,
) -> None:
self._stage_row_mirror_mapping(num_tokens)
self._finish_previous_step()
previous_host_write_event = self.host_write_event
self.host_write_event = self.host_write_events[self._next_host_write_event]
self._next_host_write_event ^= 1
current_stream().wait_event(previous_host_write_event)
self._release_completed_dma_descriptors()
self._dma_submitted = False
mirrors = _flatten_row_mirrors(metadata.row_mirrors, request_ids)
if self._slot_mapping_staging is not None:
self._slot_mapping_staging.candidates = mirrors
self._set_row_mirrors(mirrors)
self._dma_submitted = False
self._clear_forward_mirror_state()
for handle in self.cache_handles:
handle.all_context_pages_resident = metadata.all_context_pages_resident
@@ -849,7 +856,12 @@ class HiSparseConnectorWorker:
if num_rows == 0:
return
if self.is_host_writer:
expected_layers = {index for index, _ in active}
# The drafter has not written its layers' rows yet; the next
# start_step mirrors them after it runs.
self._draft_mirror_pending = bool(self._draft_layers)
expected_layers = {index for index, _ in active}.difference(
self._draft_layers
)
if self._per_layer_mirrored and self._per_layer_mirrored != expected_layers:
raise RuntimeError(
"HiSparse per-layer DMA did not mirror every active layer: "
@@ -884,9 +896,6 @@ class HiSparseConnectorWorker:
compute_stream = current_stream()
self._forward_ready_event.record()
self._finish_mirror_phase(self._forward_ready_event)
transfers = self._post_forward_transfers
self._post_forward_transfers = []
self._submit_transfers(transfers)
if self.is_host_writer:
if self._dma_submitted:
compute_stream.wait_event(self.host_write_event)
@@ -895,6 +904,24 @@ class HiSparseConnectorWorker:
self.host_write_event.record(compute_stream)
self._release_completed_dma_descriptors()
def _finish_previous_step(self) -> None:
"""Mirror the previous step's draft-layer rows, then hand its pages over.
Runs before this step's work, so after the previous step's drafter.
Post-forward transfers are ordered behind the draft mirror, so a page is
never reported clean, and its resident block never released, before
its draft-layer rows reach the host. Both land in the previous step's
host write event, which this step waits on before its forward and its
host copies, since the scheduler may already have reused the
finished requests' blocks they read and write.
"""
if self._draft_mirror_pending:
self._draft_mirror_pending = False
self._enqueue_row_dma(self._draft_layers)
transfers = self._post_forward_transfers
self._post_forward_transfers = []
self._submit_transfers(transfers)
def take_completed_host_copies(self) -> list[int]:
"""Drain host copies this worker has enqueued for this step."""
completed = self._completed_host_copy_dst_ids