[SpecDec] Remove Batch Expansion (2/3) (#9298)

This commit is contained in:
Lily Liu
2024-10-12 05:13:37 +00:00
committed by GitHub
parent ec10cb8511
commit 89feb4c84d
8 changed files with 122 additions and 70 deletions
+1 -1
View File
@@ -313,7 +313,7 @@ class CommonAttentionState(AttentionState):
seq_lens=None,
seq_lens_tensor=self._graph_seq_lens[:batch_size],
max_query_len=1,
decode_query_len=1,
max_decode_query_len=1,
max_prefill_seq_len=0,
max_decode_seq_len=self.runner.max_seq_len_to_capture,
query_start_loc=None,