[V1] Support long_prefill_token_threshold in v1 scheduler (#15419)
Signed-off-by: Lu Fang <lufang@fb.com>
This commit is contained in:
@@ -1625,9 +1625,7 @@ class EngineArgs:
|
||||
if (self.max_num_partial_prefills
|
||||
!= EngineArgs.max_num_partial_prefills
|
||||
or self.max_long_partial_prefills
|
||||
!= EngineArgs.max_long_partial_prefills
|
||||
or self.long_prefill_token_threshold
|
||||
!= EngineArgs.long_prefill_token_threshold):
|
||||
!= EngineArgs.max_long_partial_prefills):
|
||||
_raise_or_fallback(feature_name="Concurrent Partial Prefill",
|
||||
recommend_to_remove=False)
|
||||
return False
|
||||
|
||||
@@ -152,6 +152,10 @@ class Scheduler(SchedulerInterface):
|
||||
|
||||
num_new_tokens = (request.num_tokens_with_spec -
|
||||
request.num_computed_tokens)
|
||||
if self.scheduler_config.long_prefill_token_threshold > 0:
|
||||
num_new_tokens = min(
|
||||
num_new_tokens,
|
||||
self.scheduler_config.long_prefill_token_threshold)
|
||||
num_new_tokens = min(num_new_tokens, token_budget)
|
||||
assert num_new_tokens > 0
|
||||
|
||||
@@ -299,6 +303,10 @@ class Scheduler(SchedulerInterface):
|
||||
num_computed_tokens -= self.block_size
|
||||
num_new_tokens = self.block_size
|
||||
computed_blocks.pop()
|
||||
if self.scheduler_config.long_prefill_token_threshold > 0:
|
||||
num_new_tokens = min(
|
||||
num_new_tokens,
|
||||
self.scheduler_config.long_prefill_token_threshold)
|
||||
num_new_tokens = min(num_new_tokens, token_budget)
|
||||
assert num_new_tokens > 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user