[Speculative decoding 4/9] Lookahead scheduling for speculative decoding (#3250)
This commit is contained in:
@@ -53,8 +53,8 @@ class EngineArgs:
|
||||
max_cpu_loras: Optional[int] = None
|
||||
device: str = 'auto'
|
||||
ray_workers_use_nsight: bool = False
|
||||
|
||||
forced_num_gpu_blocks: Optional[int] = None
|
||||
num_lookahead_slots: int = 0
|
||||
|
||||
# Related to Vision-language models such as llava
|
||||
image_input_type: Optional[str] = None
|
||||
@@ -202,6 +202,14 @@ class EngineArgs:
|
||||
parser.add_argument('--use-v2-block-manager',
|
||||
action='store_true',
|
||||
help='Use BlockSpaceMangerV2')
|
||||
parser.add_argument(
|
||||
'--num-lookahead-slots',
|
||||
type=int,
|
||||
default=EngineArgs.num_lookahead_slots,
|
||||
help='Experimental scheduling config necessary for '
|
||||
'speculative decoding. This will be replaced by '
|
||||
'speculative config in the future; it is present '
|
||||
'to enable correctness tests until then.')
|
||||
|
||||
parser.add_argument('--seed',
|
||||
type=int,
|
||||
@@ -406,6 +414,7 @@ class EngineArgs:
|
||||
self.max_num_seqs,
|
||||
model_config.max_model_len,
|
||||
self.use_v2_block_manager,
|
||||
num_lookahead_slots=self.num_lookahead_slots,
|
||||
delay_factor=self.scheduler_delay_factor,
|
||||
enable_chunked_prefill=self.enable_chunked_prefill,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user