[Bugfix][HiSparse] Fix MTP acceptance collapse under FULL graphs with a saturated GPU pool (#59309)

Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Matthew Bonanni <mbonanni@redhat.com>
This commit is contained in:
Lucas Wilkinson
2026-10-01 23:04:21 -07:00
committed by GitHub
co-authored by Claude Opus 5.5 Matthew Bonanni
parent 0cbac6cd13
commit 4056c8ac1f
11 changed files with 426 additions and 122 deletions
+50 -27
View File
@@ -95,6 +95,8 @@ def test_hisparse_spill_and_prefix_restore(
A regression omitted attention metadata before FULL graph replay, so decode
completed normally while its newly written KV rows were never copied to host.
MTP layers write after the target forward; their rows used to be mirrored
before the drafter wrote them, so the drafter later read stale host rows.
"""
capability = current_platform.get_device_capability()
if capability is None or capability.major < 9:
@@ -171,49 +173,70 @@ def test_hisparse_spill_and_prefix_restore(
model_runner.cudagraph_manager, "run_fullgraph", record_fullgraph
)
# The MTP layer writes its rows after the target forward, so it is
# mirrored at the start of the next step instead of in finish_forward.
draft_layers = set(worker._draft_layers)
assert draft_layers
original_finish_forward = worker.finish_forward
verified_full_replay_rows = 0
original_finish_previous_step = worker._finish_previous_step
verified_rows = {"target": 0, "draft": 0}
copied_nonzero_kv = False
draft_check_pending = False
def check_host_rows(layer_indices: set[int], kind: str) -> None:
nonlocal copied_nonzero_kv
# Synchronize only the test's inspection, not the worker itself.
with gpu_sync_allowed():
torch.accelerator.synchronize()
assert worker._row_mirrors
for layer_index in layer_indices:
cache = worker.cache_handles[layer_index]
source_index = cache.runtime.resident_source_index
for mirror in worker._row_mirrors:
source_row = mirror.source_starts[source_index]
destination_row = mirror.destination_start
num_rows = mirror.num_rows
source_block, source_offset = divmod(
source_row, worker.kernel_block_size
)
with gpu_sync_allowed():
gpu_rows = worker.resident_caches[layer_index][
source_block, source_offset : source_offset + num_rows
].cpu()
host_rows = worker.host_caches[layer_index][
destination_row : destination_row + num_rows
]
assert torch.equal(host_rows, gpu_rows)
copied_nonzero_kv |= bool(torch.count_nonzero(gpu_rows).item())
verified_rows[kind] += num_rows
def finish_forward():
nonlocal full_replay_pending, verified_full_replay_rows
nonlocal copied_nonzero_kv
nonlocal full_replay_pending, draft_check_pending
try:
original_finish_forward()
if not full_replay_pending:
return
# Synchronize only the test's inspection, not finish_forward itself.
with gpu_sync_allowed():
torch.accelerator.synchronize()
assert worker._row_mirrors
for layer_index, cache in enumerate(worker.cache_handles):
source_index = cache.runtime.resident_source_index
for mirror in worker._row_mirrors:
source_row = mirror.source_starts[source_index]
destination_row = mirror.destination_start
num_rows = mirror.num_rows
source_block, source_offset = divmod(
source_row, worker.kernel_block_size
)
with gpu_sync_allowed():
gpu_rows = worker.resident_caches[layer_index][
source_block, source_offset : source_offset + num_rows
].cpu()
host_rows = worker.host_caches[layer_index][
destination_row : destination_row + num_rows
]
assert torch.equal(host_rows, gpu_rows)
copied_nonzero_kv |= bool(torch.count_nonzero(gpu_rows).item())
verified_full_replay_rows += num_rows
target_layers = set(range(len(worker.cache_handles))) - draft_layers
check_host_rows(target_layers, "target")
draft_check_pending = True
finally:
full_replay_pending = False
def finish_previous_step():
nonlocal draft_check_pending
original_finish_previous_step()
if draft_check_pending:
draft_check_pending = False
check_host_rows(draft_layers, "draft")
monkeypatch.setattr(worker, "finish_forward", finish_forward)
monkeypatch.setattr(worker, "_finish_previous_step", finish_previous_step)
expected = runner.generate_greedy([target], max_tokens=8)
assert full_graph_calls > 0
assert verified_full_replay_rows > 0
assert verified_rows["target"] > 0
assert verified_rows["draft"] > 0
assert copied_nonzero_kv
runner.generate_greedy(pressure, max_tokens=8)