[Misc] [Core] Implement RFC "Augment BaseExecutor interfaces to enable hardware-agnostic speculative decoding" (#3837)
This commit is contained in:
@@ -55,7 +55,7 @@ class EngineArgs:
|
||||
max_cpu_loras: Optional[int] = None
|
||||
device: str = 'auto'
|
||||
ray_workers_use_nsight: bool = False
|
||||
forced_num_gpu_blocks: Optional[int] = None
|
||||
num_gpu_blocks_override: Optional[int] = None
|
||||
num_lookahead_slots: int = 0
|
||||
|
||||
# Related to Vision-language models such as llava
|
||||
@@ -246,7 +246,7 @@ class EngineArgs:
|
||||
'the model executor, which can range from 0 to 1.'
|
||||
'If unspecified, will use the default value of 0.9.')
|
||||
parser.add_argument(
|
||||
'--forced-num-gpu-blocks',
|
||||
'--num-gpu-blocks-override',
|
||||
type=int,
|
||||
default=None,
|
||||
help='If specified, ignore GPU profiling result and use this number'
|
||||
@@ -426,7 +426,7 @@ class EngineArgs:
|
||||
cache_config = CacheConfig(self.block_size,
|
||||
self.gpu_memory_utilization,
|
||||
self.swap_space, self.kv_cache_dtype,
|
||||
self.forced_num_gpu_blocks,
|
||||
self.num_gpu_blocks_override,
|
||||
model_config.get_sliding_window(),
|
||||
self.enable_prefix_caching)
|
||||
parallel_config = ParallelConfig(
|
||||
|
||||
Reference in New Issue
Block a user