[core] Multi Step Scheduling (#7000)
Co-authored-by: afeldman-nm <156691304+afeldman-nm@users.noreply.github.com>
This commit is contained in:
co-authored by
afeldman-nm
parent
dad961ef5c
commit
47b65a5508
@@ -853,6 +853,12 @@ class EngineArgs:
|
||||
"in low performance due to small KV cache space. Consider "
|
||||
"setting --max-model-len to a smaller value.", max_model_len)
|
||||
|
||||
if self.num_scheduler_steps > 1 and not self.use_v2_block_manager:
|
||||
self.use_v2_block_manager = True
|
||||
logger.warning(
|
||||
"Enabled BlockSpaceManagerV2 because it is "
|
||||
"required for multi-step (--num-scheduler-steps > 1)")
|
||||
|
||||
speculative_config = SpeculativeConfig.maybe_create_spec_config(
|
||||
target_model_config=model_config,
|
||||
target_parallel_config=parallel_config,
|
||||
@@ -881,7 +887,6 @@ class EngineArgs:
|
||||
)
|
||||
|
||||
if self.num_scheduler_steps > 1:
|
||||
raise NotImplementedError("Multi-step is not yet supported.")
|
||||
if speculative_config is not None:
|
||||
raise ValueError("Speculative decoding is not supported with "
|
||||
"multi-step (--num-scheduler-steps > 1)")
|
||||
|
||||
Reference in New Issue
Block a user