[Bugfix][HiSparse] Fix MTP acceptance collapse under FULL graphs with a saturated GPU pool (#59309)
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com> Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Matthew Bonanni
parent
0cbac6cd13
commit
4056c8ac1f
@@ -301,6 +301,10 @@ class HiSparseConnectorWorker:
|
||||
self._submitted_mirror_layers: set[int] = set()
|
||||
self._layer_ready_events = tuple(torch.Event() for _ in cache_handles)
|
||||
self._forward_ready_event = torch.Event()
|
||||
self._draft_layers = tuple(
|
||||
index for index, cache in enumerate(cache_handles) if cache.draft_layer
|
||||
)
|
||||
self._draft_mirror_pending = False
|
||||
self._set_row_mirrors(())
|
||||
self.cache_layer_names = cache_layer_names
|
||||
self._group_leaders = tuple(
|
||||
@@ -344,9 +348,11 @@ class HiSparseConnectorWorker:
|
||||
self._init_dma()
|
||||
if self.is_host_writer:
|
||||
for layer_index, handle in enumerate(cache_handles):
|
||||
handle.submit_layer_mirror = partial(
|
||||
self._enqueue_layer_mirror, layer_index
|
||||
)
|
||||
# Draft layers are mirrored once in the next start_step instead.
|
||||
if not handle.draft_layer:
|
||||
handle.submit_layer_mirror = partial(
|
||||
self._enqueue_layer_mirror, layer_index
|
||||
)
|
||||
self._initialized = True
|
||||
|
||||
def set_request_state_indices(self, indices: torch.Tensor) -> None:
|
||||
@@ -397,16 +403,17 @@ class HiSparseConnectorWorker:
|
||||
num_tokens: int = 0,
|
||||
) -> None:
|
||||
self._stage_row_mirror_mapping(num_tokens)
|
||||
self._finish_previous_step()
|
||||
previous_host_write_event = self.host_write_event
|
||||
self.host_write_event = self.host_write_events[self._next_host_write_event]
|
||||
self._next_host_write_event ^= 1
|
||||
current_stream().wait_event(previous_host_write_event)
|
||||
self._release_completed_dma_descriptors()
|
||||
self._dma_submitted = False
|
||||
mirrors = _flatten_row_mirrors(metadata.row_mirrors, request_ids)
|
||||
if self._slot_mapping_staging is not None:
|
||||
self._slot_mapping_staging.candidates = mirrors
|
||||
self._set_row_mirrors(mirrors)
|
||||
self._dma_submitted = False
|
||||
self._clear_forward_mirror_state()
|
||||
for handle in self.cache_handles:
|
||||
handle.all_context_pages_resident = metadata.all_context_pages_resident
|
||||
@@ -849,7 +856,12 @@ class HiSparseConnectorWorker:
|
||||
if num_rows == 0:
|
||||
return
|
||||
if self.is_host_writer:
|
||||
expected_layers = {index for index, _ in active}
|
||||
# The drafter has not written its layers' rows yet; the next
|
||||
# start_step mirrors them after it runs.
|
||||
self._draft_mirror_pending = bool(self._draft_layers)
|
||||
expected_layers = {index for index, _ in active}.difference(
|
||||
self._draft_layers
|
||||
)
|
||||
if self._per_layer_mirrored and self._per_layer_mirrored != expected_layers:
|
||||
raise RuntimeError(
|
||||
"HiSparse per-layer DMA did not mirror every active layer: "
|
||||
@@ -884,9 +896,6 @@ class HiSparseConnectorWorker:
|
||||
compute_stream = current_stream()
|
||||
self._forward_ready_event.record()
|
||||
self._finish_mirror_phase(self._forward_ready_event)
|
||||
transfers = self._post_forward_transfers
|
||||
self._post_forward_transfers = []
|
||||
self._submit_transfers(transfers)
|
||||
if self.is_host_writer:
|
||||
if self._dma_submitted:
|
||||
compute_stream.wait_event(self.host_write_event)
|
||||
@@ -895,6 +904,24 @@ class HiSparseConnectorWorker:
|
||||
self.host_write_event.record(compute_stream)
|
||||
self._release_completed_dma_descriptors()
|
||||
|
||||
def _finish_previous_step(self) -> None:
|
||||
"""Mirror the previous step's draft-layer rows, then hand its pages over.
|
||||
|
||||
Runs before this step's work, so after the previous step's drafter.
|
||||
Post-forward transfers are ordered behind the draft mirror, so a page is
|
||||
never reported clean, and its resident block never released, before
|
||||
its draft-layer rows reach the host. Both land in the previous step's
|
||||
host write event, which this step waits on before its forward and its
|
||||
host copies, since the scheduler may already have reused the
|
||||
finished requests' blocks they read and write.
|
||||
"""
|
||||
if self._draft_mirror_pending:
|
||||
self._draft_mirror_pending = False
|
||||
self._enqueue_row_dma(self._draft_layers)
|
||||
transfers = self._post_forward_transfers
|
||||
self._post_forward_transfers = []
|
||||
self._submit_transfers(transfers)
|
||||
|
||||
def take_completed_host_copies(self) -> list[int]:
|
||||
"""Drain host copies this worker has enqueued for this step."""
|
||||
completed = self._completed_host_copy_dst_ids
|
||||
|
||||
Reference in New Issue
Block a user