[Dynamic Spec Decoding] Auto-disable by the running queue size (#4592)
Co-authored-by: Cade Daniel <edacih@gmail.com>
This commit is contained in:
@@ -612,6 +612,12 @@ class SequenceGroupMetadata:
|
||||
self._token_chunk_size = token_chunk_size
|
||||
self.do_sample = do_sample
|
||||
|
||||
# The number of speculative tokens adopted in this request.
|
||||
# None means specuative decoding is not used.
|
||||
# Zero means speculative decoding is disabled for some reasons.
|
||||
# TODO: We should maintain this states out of the sequence group.
|
||||
self.num_speculative_tokens = None
|
||||
|
||||
if self._token_chunk_size is None:
|
||||
if is_prompt:
|
||||
self._token_chunk_size = list(seq_data.values())[0].get_len()
|
||||
|
||||
Reference in New Issue
Block a user