[Spec Decode] Remove eager metadata rebuild during MTP fused multi-step decode (#58463)
Signed-off-by: Giancarlo Delfin <gdelfin@inferact.ai>
This commit is contained in:
@@ -122,6 +122,7 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
|
||||
builder.dcp_world_size = 1
|
||||
builder.compress_ratio = 16
|
||||
builder.kernel_block_size = None
|
||||
builder.num_sms = num_sms
|
||||
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
builder.kv_cache_spec = SimpleNamespace(num_states=64)
|
||||
@@ -140,6 +141,7 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
requires_padding=False,
|
||||
schedule_metadata=schedule_metadata,
|
||||
),
|
||||
block_table=block_table,
|
||||
)
|
||||
|
||||
builder.update_draft_decode_metadata(metadata)
|
||||
@@ -167,6 +169,65 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
|
||||
torch.testing.assert_close(metadata.slot_mapping, expected_slot_mapping)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"block_size,kernel_block_size,compress_ratio", [(64, 64, 1), (256, 64, 4)]
|
||||
)
|
||||
def test_indexer_draft_decode_refresh_follows_new_batch(
|
||||
monkeypatch, block_size, kernel_block_size, compress_ratio
|
||||
):
|
||||
"""Without a prior build(), the refresh must not keep the last batch's state."""
|
||||
monkeypatch.setattr(indexer_module, "has_deep_gemm", lambda: False)
|
||||
device = torch.device("cuda")
|
||||
num_reqs, width = 3, 8
|
||||
factor = block_size // kernel_block_size
|
||||
|
||||
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
|
||||
builder.dcp_world_size = 1
|
||||
builder.compress_ratio = compress_ratio
|
||||
builder.kernel_block_size = kernel_block_size
|
||||
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=block_size, num_states=64)
|
||||
|
||||
def stale(*shape: int) -> torch.Tensor:
|
||||
return torch.full(shape, -7, dtype=torch.int32, device=device)
|
||||
|
||||
decode = DeepSeekV32IndexerDecodeMetadata(
|
||||
block_table=stale(num_reqs, width // factor),
|
||||
seq_lens=stale(num_reqs, 1),
|
||||
decode_lens=stale(num_reqs),
|
||||
requires_padding=False,
|
||||
schedule_metadata=stale(1),
|
||||
indices=stale(num_reqs),
|
||||
per_req_decode_lens=stale(num_reqs),
|
||||
)
|
||||
request_block_table = torch.randint(
|
||||
0, 1000, (num_reqs, width), dtype=torch.int32, device=device
|
||||
)
|
||||
seq_lens = torch.tensor([17, 33, 65], dtype=torch.int32, device=device)
|
||||
metadata = DeepseekV32IndexerMetadata(
|
||||
seq_lens=seq_lens,
|
||||
max_seq_len=128,
|
||||
slot_mapping=torch.zeros(num_reqs, dtype=torch.int64, device=device),
|
||||
num_decodes=num_reqs,
|
||||
num_decode_tokens=num_reqs,
|
||||
num_prefills=0,
|
||||
num_prefill_tokens=0,
|
||||
decode=decode,
|
||||
block_table=request_block_table,
|
||||
)
|
||||
|
||||
builder.update_draft_decode_metadata(metadata)
|
||||
|
||||
expected_block_table = request_block_table[:, ::factor] // factor
|
||||
torch.testing.assert_close(decode.block_table, expected_block_table)
|
||||
torch.testing.assert_close(decode.indices, builder.arange_buffer[:num_reqs])
|
||||
assert torch.all(decode.per_req_decode_lens == 1)
|
||||
assert torch.all(decode.decode_lens == 1)
|
||||
expected_seq_lens = torch.div(seq_lens, compress_ratio, rounding_mode="floor")
|
||||
torch.testing.assert_close(decode.seq_lens.flatten(), expected_seq_lens)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
|
||||
@pytest.mark.parametrize(
|
||||
"query_lens", [[1], [0, 257, 1, 0, 3], [6] * 64, [2, 6, 4, 0], [0, 0, 0]]
|
||||
|
||||
Reference in New Issue
Block a user