[Speculative decoding 2/9] Multi-step worker for draft model (#2424)

This commit is contained in:
Cade Daniel
2024-01-21 16:31:47 -08:00
committed by GitHub
parent 71d63ed72e
commit 18bfcdd05c
11 changed files with 658 additions and 12 deletions
+2 -2
View File
@@ -277,8 +277,8 @@ class ModelRunner:
input_block_tables[i, :len(block_table)] = block_table
block_tables = torch.tensor(input_block_tables, device="cuda")
else:
max_block_table_len = (max_context_len + self.block_size -
1) // self.block_size
max_block_table_len = max(
len(block_table) for block_table in block_tables)
block_tables = _make_tensor_with_pad(
block_tables,
max_len=max_block_table_len,