[Bugfix][DFlash]allocate the proper number of lookahead slots (#43733)
Signed-off-by: Benjamin Chislett <bchislett@nvidia.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@gmail.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
2c2c966669
commit
7fb9c0197a
@@ -218,6 +218,11 @@ class Scheduler(SchedulerInterface):
|
|||||||
self.num_lookahead_tokens = self.num_spec_tokens
|
self.num_lookahead_tokens = self.num_spec_tokens
|
||||||
if speculative_config.uses_draft_model():
|
if speculative_config.uses_draft_model():
|
||||||
self.num_lookahead_tokens = self.num_spec_tokens
|
self.num_lookahead_tokens = self.num_spec_tokens
|
||||||
|
if speculative_config.use_dflash():
|
||||||
|
# DFlash requires an extra lookahead slot since it uses in-fill-style
|
||||||
|
# decoding instead of standard next-token sampling, so it has a query
|
||||||
|
# for the last sampled token plus queries for each draft token.
|
||||||
|
self.num_lookahead_tokens = self.num_spec_tokens + 1
|
||||||
|
|
||||||
# Create the KV cache manager.
|
# Create the KV cache manager.
|
||||||
if hash_block_size is None:
|
if hash_block_size is None:
|
||||||
@@ -702,8 +707,9 @@ class Scheduler(SchedulerInterface):
|
|||||||
# extra block gets allocated which
|
# extra block gets allocated which
|
||||||
# creates a mismatch between the number
|
# creates a mismatch between the number
|
||||||
# of local and remote blocks.
|
# of local and remote blocks.
|
||||||
|
limit_lookahead_tokens = load_kv_async and self.use_eagle
|
||||||
effective_lookahead_tokens = (
|
effective_lookahead_tokens = (
|
||||||
0 if request.num_computed_tokens == 0 else self.num_lookahead_tokens
|
0 if limit_lookahead_tokens else self.num_lookahead_tokens
|
||||||
)
|
)
|
||||||
|
|
||||||
# Determine if we need to allocate cross-attention blocks.
|
# Determine if we need to allocate cross-attention blocks.
|
||||||
|
|||||||
Reference in New Issue
Block a user