[Core] Optimize block_manager_v2 vs block_manager_v1 (to make V2 default) (#5602)

This commit is contained in:
Alexander Matveev
2024-07-01 20:10:37 -07:00
committed by GitHub
parent 54600709b6
commit 3476ed0809
19 changed files with 1182 additions and 525 deletions
+2 -2
View File
@@ -427,8 +427,8 @@ class SamplingTensors:
if seq_group.do_sample:
for seq_id in seq_ids:
seq_data = seq_group.seq_data[seq_id]
prompt_tokens.append(seq_data.prompt_token_ids)
output_tokens.append(seq_data.output_token_ids)
prompt_tokens.append(list(seq_data.prompt_token_ids))
output_tokens.append(list(seq_data.output_token_ids))
sampling_tensors = SamplingTensors.from_lists(
temperatures, top_ps, top_ks, min_ps, presence_penalties,