[Spec Decode] Remove eager metadata rebuild during MTP fused multi-step decode (#58463)

Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
Giancarlo Delfin
2026-10-06 12:17:30 -04:00
committed by GitHub
parent 4ea0c28bc4
commit fcf53d240c
8 changed files with 182 additions and 50 deletions
@@ -122,6 +122,7 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
builder.dcp_world_size = 1
builder.compress_ratio = 16
builder.kernel_block_size = None
builder.num_sms = num_sms
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
builder.kv_cache_spec = SimpleNamespace(num_states=64)
@@ -140,6 +141,7 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
requires_padding=False,
schedule_metadata=schedule_metadata,
),
block_table=block_table,
)
builder.update_draft_decode_metadata(metadata)
@@ -167,6 +169,65 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
torch.testing.assert_close(metadata.slot_mapping, expected_slot_mapping)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize(
"block_size,kernel_block_size,compress_ratio", [(64, 64, 1), (256, 64, 4)]
)
def test_indexer_draft_decode_refresh_follows_new_batch(
monkeypatch, block_size, kernel_block_size, compress_ratio
):
"""Without a prior build(), the refresh must not keep the last batch's state."""
monkeypatch.setattr(indexer_module, "has_deep_gemm", lambda: False)
device = torch.device("cuda")
num_reqs, width = 3, 8
factor = block_size // kernel_block_size
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
builder.dcp_world_size = 1
builder.compress_ratio = compress_ratio
builder.kernel_block_size = kernel_block_size
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
builder.kv_cache_spec = SimpleNamespace(block_size=block_size, num_states=64)
def stale(*shape: int) -> torch.Tensor:
return torch.full(shape, -7, dtype=torch.int32, device=device)
decode = DeepSeekV32IndexerDecodeMetadata(
block_table=stale(num_reqs, width // factor),
seq_lens=stale(num_reqs, 1),
decode_lens=stale(num_reqs),
requires_padding=False,
schedule_metadata=stale(1),
indices=stale(num_reqs),
per_req_decode_lens=stale(num_reqs),
)
request_block_table = torch.randint(
0, 1000, (num_reqs, width), dtype=torch.int32, device=device
)
seq_lens = torch.tensor([17, 33, 65], dtype=torch.int32, device=device)
metadata = DeepseekV32IndexerMetadata(
seq_lens=seq_lens,
max_seq_len=128,
slot_mapping=torch.zeros(num_reqs, dtype=torch.int64, device=device),
num_decodes=num_reqs,
num_decode_tokens=num_reqs,
num_prefills=0,
num_prefill_tokens=0,
decode=decode,
block_table=request_block_table,
)
builder.update_draft_decode_metadata(metadata)
expected_block_table = request_block_table[:, ::factor] // factor
torch.testing.assert_close(decode.block_table, expected_block_table)
torch.testing.assert_close(decode.indices, builder.arange_buffer[:num_reqs])
assert torch.all(decode.per_req_decode_lens == 1)
assert torch.all(decode.decode_lens == 1)
expected_seq_lens = torch.div(seq_lens, compress_ratio, rounding_mode="floor")
torch.testing.assert_close(decode.seq_lens.flatten(), expected_seq_lens)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize(
"query_lens", [[1], [0, 257, 1, 0, 3], [6] * 64, [2, 6, 4, 0], [0, 0, 0]]