[Speculative decoding 2/9] Multi-step worker for draft model (#2424)
This commit is contained in:
@@ -277,8 +277,8 @@ class ModelRunner:
|
||||
input_block_tables[i, :len(block_table)] = block_table
|
||||
block_tables = torch.tensor(input_block_tables, device="cuda")
|
||||
else:
|
||||
max_block_table_len = (max_context_len + self.block_size -
|
||||
1) // self.block_size
|
||||
max_block_table_len = max(
|
||||
len(block_table) for block_table in block_tables)
|
||||
block_tables = _make_tensor_with_pad(
|
||||
block_tables,
|
||||
max_len=max_block_table_len,
|
||||
|
||||
Reference in New Issue
Block a user