[V1][Spec Decode] Add Dynamic SD (#32374)
Signed-off-by: Ekagra Ranjan <3116519+ekagra-ranjan@users.noreply.github.com> Signed-off-by: Benjamin Chislett <chislett.ben@gmail.com> Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Benjamin Chislett <chislett.ben@gmail.com>
This commit is contained in:
co-authored by
gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Harry Mellor
Benjamin Chislett
parent
78e7293bb1
commit
4ef4492e9b
@@ -81,6 +81,7 @@ def test_ngram_proposer():
|
||||
# No match.
|
||||
token_ids_cpu = np.array([[1, 2, 3, 4, 5]])
|
||||
result = get_ngram_proposer(min_n=2, max_n=2, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -90,6 +91,7 @@ def test_ngram_proposer():
|
||||
# No match for 4-gram.
|
||||
token_ids_cpu = np.array([[1, 2, 3, 4, 1, 2, 3]])
|
||||
result = get_ngram_proposer(min_n=4, max_n=4, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -99,6 +101,7 @@ def test_ngram_proposer():
|
||||
# No match for 4-gram but match for 3-gram.
|
||||
token_ids_cpu = np.array([[1, 2, 3, 4, 1, 2, 3]])
|
||||
result = get_ngram_proposer(min_n=3, max_n=4, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -109,6 +112,7 @@ def test_ngram_proposer():
|
||||
# In this case, the proposer should return the 4-gram match.
|
||||
token_ids_cpu = np.array([[2, 3, 4, 5, 1, 2, 3, 4, 1, 2, 3, 4]])
|
||||
result = get_ngram_proposer(min_n=3, max_n=4, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -118,6 +122,7 @@ def test_ngram_proposer():
|
||||
# Match for 2-gram and 3-gram, but not 4-gram.
|
||||
token_ids_cpu = np.array([[3, 4, 5, 2, 3, 4, 1, 2, 3, 4]])
|
||||
result = get_ngram_proposer(min_n=2, max_n=4, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -127,6 +132,7 @@ def test_ngram_proposer():
|
||||
# Multiple 3-gram matched, but always pick the first one.
|
||||
token_ids_cpu = np.array([[1, 2, 3, 100, 1, 2, 3, 200, 1, 2, 3, 300, 1, 2, 3]])
|
||||
result = get_ngram_proposer(min_n=3, max_n=3, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -136,6 +142,7 @@ def test_ngram_proposer():
|
||||
# check empty input
|
||||
token_ids_cpu = np.array([[]])
|
||||
result = get_ngram_proposer(min_n=2, max_n=2, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0]],
|
||||
num_tokens_no_spec=np.array([len(c) for c in token_ids_cpu]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -147,6 +154,7 @@ def test_ngram_proposer():
|
||||
# second request has 3 tokens and no match. Padded with -1 for max len 5
|
||||
token_ids_cpu = np.array([[1, 2, 3, 1, 2], [4, 5, 6, -1, -1]])
|
||||
result = get_ngram_proposer(min_n=2, max_n=2, k=2).propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0], [1]],
|
||||
num_tokens_no_spec=np.array([5, 3]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -166,6 +174,7 @@ def test_ngram_proposer():
|
||||
num_tokens_no_spec = np.array([5, 3, 5], dtype=np.int32)
|
||||
sampled_token_ids = [[2], [], [8]] # Empty list for request 1 simulates prefill
|
||||
result = proposer.propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=sampled_token_ids,
|
||||
num_tokens_no_spec=num_tokens_no_spec,
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
@@ -195,6 +204,7 @@ def test_ngram_proposer():
|
||||
input_2[:3] = [4, 5, 6]
|
||||
token_ids_cpu = np.array([input_1, input_2])
|
||||
result = ngram_proposer.propose(
|
||||
num_speculative_tokens=2,
|
||||
sampled_token_ids=[[0], [1]],
|
||||
num_tokens_no_spec=np.array([len(input_1), 3]),
|
||||
token_ids_cpu=token_ids_cpu,
|
||||
|
||||
Reference in New Issue
Block a user