[V1][Spec Decode] Respect prompt_lookup_max (#15348)
Signed-off-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>
This commit is contained in:
@@ -10,7 +10,8 @@ class NgramProposer:
|
||||
def propose(
|
||||
self,
|
||||
context_token_ids: np.ndarray,
|
||||
n: int,
|
||||
min_n: int,
|
||||
max_n: int,
|
||||
k: int,
|
||||
) -> Optional[np.ndarray]:
|
||||
"""Proposes the next sequence of tokens based on n-gram pattern
|
||||
@@ -21,7 +22,8 @@ class NgramProposer:
|
||||
Args:
|
||||
context_token_ids: Numpy array of token IDs representing the
|
||||
context sequence.
|
||||
n: Length of the n-gram to match.
|
||||
min_n: Minimum length of the n-gram to match.
|
||||
max_n: Maximum length of the n-gram to match.
|
||||
k: Number of tokens follow the match. If there are less
|
||||
than k tokens follow the match, we will return
|
||||
the maximum amount of tokens until the end.
|
||||
@@ -32,14 +34,21 @@ class NgramProposer:
|
||||
None: If no matching n-gram pattern is found.
|
||||
|
||||
Example:
|
||||
If context_token_ids = [1,2,3,4,2,3], n = 2, and k = 4:
|
||||
If context_token_ids = [1,2,3,4,2,3], min_n = 2, max_n = 3, and
|
||||
k = 4:
|
||||
- The last 3 (= max_n) tokens [4,2,3] cannot find a match.
|
||||
- The last 2 tokens [2,3] will be matched against the previous
|
||||
4 tokens [1,2,3,4].
|
||||
- Finding a match of [2,3] would return the tokens that
|
||||
followed that pattern. Here we will return [4,2,3] because
|
||||
we only have three tokens after the match.
|
||||
"""
|
||||
return _find_subarray_kmp(context_token_ids, n, k)
|
||||
# TODO(woosuk): Optimize this.
|
||||
for n in range(max_n, min_n - 1, -1):
|
||||
result = _find_subarray_kmp(context_token_ids, n, k)
|
||||
if result is not None:
|
||||
return result
|
||||
return None
|
||||
|
||||
|
||||
@jit(nopython=True)
|
||||
|
||||
@@ -160,6 +160,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
self.drafter.propose(
|
||||
np.zeros(1024, dtype=np.int32),
|
||||
self.speculative_config.prompt_lookup_min,
|
||||
self.speculative_config.prompt_lookup_max,
|
||||
self.speculative_config.num_speculative_tokens,
|
||||
)
|
||||
self.rejection_sampler = RejectionSampler()
|
||||
@@ -1155,6 +1156,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
|
||||
drafter_output = self.drafter.propose(
|
||||
self.input_batch.token_ids_cpu[i, :end_idx],
|
||||
self.speculative_config.prompt_lookup_min,
|
||||
self.speculative_config.prompt_lookup_max,
|
||||
self.speculative_config.num_speculative_tokens,
|
||||
)
|
||||
if drafter_output is None or len(drafter_output) == 0:
|
||||
|
||||
Reference in New Issue
Block a user