[Core][2/N] Model runner refactoring part 2. Combine prepare prefill / decode to a single API (#4681)

This PR combines prepare_prompt and prepare_decode into a single API. This PR also coelsce the attn metadata for prefill/decode to a single class and allow to slice them when running attn backend.

It also refactors subquery_start_loc which was not refactored in the previous PR
This commit is contained in:
SangBin Cho
2024-05-15 14:00:10 +09:00
committed by GitHub
parent 8a7cc254a0
commit 65bf2ac165
18 changed files with 781 additions and 734 deletions
+3 -7
View File
@@ -159,12 +159,10 @@ class CPUModelRunner:
is_prompt=True,
seq_lens=seq_lens,
seq_lens_tensor=None,
max_seq_len=None,
max_decode_seq_len=None,
num_prefills=len(seq_lens),
num_prefill_tokens=num_prompt_tokens,
num_decode_tokens=0,
prefill_metadata=None,
decode_metadata=None,
block_tables=torch.tensor([]),
slot_mapping=slot_mapping,
)
@@ -213,7 +211,7 @@ class CPUModelRunner:
block_table = block_table[-sliding_window_blocks:]
block_tables.append(block_table)
max_seq_len = max(seq_lens)
max_decode_seq_len = max(seq_lens)
input_tokens = torch.tensor(input_tokens,
dtype=torch.long,
@@ -243,12 +241,10 @@ class CPUModelRunner:
slot_mapping=slot_mapping,
seq_lens=seq_lens,
seq_lens_tensor=seq_lens_tensor,
max_seq_len=max_seq_len,
max_decode_seq_len=max_decode_seq_len,
num_prefill_tokens=0,
num_decode_tokens=len(input_tokens),
num_prefills=0,
prefill_metadata=None,
decode_metadata=None,
block_tables=block_tables,
)
return (