[Bugfix][Spec Decode] Trim token_ids/logprobs left past a stop string under speculative decoding (#47616)
Signed-off-by: Ting Sun <suntcrick@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Nick Hill
Claude Opus 5.5
parent
35e920d671
commit
51144dd83e
@@ -1,11 +1,14 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.v1.engine import EngineCoreRequest
|
||||
from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
|
||||
from vllm.v1.engine.detokenizer import BaseIncrementalDetokenizer
|
||||
from vllm.v1.engine.output_processor import OutputProcessor
|
||||
from vllm.v1.outputs import LogprobsLists, SamplingMaskLists
|
||||
|
||||
|
||||
@pytest.fixture(params=[True, False])
|
||||
@@ -82,7 +85,7 @@ def test_stop_string_while_stop_token_terminates(include_stop_str_in_output: boo
|
||||
# Simulate that the last token ('Z') is a stop token (stop_terminated=True).
|
||||
result = detok.update(new_token_ids=token_ids, stop_terminated=True)
|
||||
|
||||
# The update should not report a stop string
|
||||
# The update should report the matched stop string.
|
||||
assert result == stop_string
|
||||
|
||||
# Output text should reflect stop-string handling:
|
||||
@@ -91,9 +94,111 @@ def test_stop_string_while_stop_token_terminates(include_stop_str_in_output: boo
|
||||
expected_text = "abcd" if include_stop_str_in_output else "ab"
|
||||
assert detok.output_text == expected_text
|
||||
|
||||
# The skipped final token should still be recorded in token_ids.
|
||||
assert detok.output_token_ids == token_ids
|
||||
# Tokens after the stop string, including the stop token, are dropped.
|
||||
assert detok.output_token_ids == token_ids[:4]
|
||||
|
||||
# get_next_output_text should return the full text when finished=True.
|
||||
# (Buffering only applies during streaming when finished=False.)
|
||||
assert detok.get_next_output_text(finished=True, delta=False) == expected_text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"min_tokens,num_kept",
|
||||
[(0, 2), (1, 2), (2, 4), (3, 4), (4, None)],
|
||||
)
|
||||
@pytest.mark.parametrize("num_prev_tokens", [0, 1])
|
||||
def test_stop_string_min_tokens_within_step(
|
||||
include_stop_str_in_output: bool, min_tokens, num_kept, num_prev_tokens
|
||||
):
|
||||
"""Stop strings completing within min_tokens are ignored, even when the
|
||||
min_tokens boundary falls in the middle of a multi-token step."""
|
||||
token_ids = [ord(c) for c in "cdcdx"]
|
||||
req = _make_request(
|
||||
stop=["cd"],
|
||||
include_stop_str_in_output=include_stop_str_in_output,
|
||||
min_tokens=min_tokens,
|
||||
)
|
||||
detok = _DummyDetokenizer(req)
|
||||
|
||||
if num_prev_tokens:
|
||||
assert detok.update(token_ids[:num_prev_tokens], False) is None
|
||||
result = detok.update(token_ids[num_prev_tokens:], False)
|
||||
|
||||
if num_kept is None:
|
||||
assert result is None
|
||||
assert detok.output_text == "cdcdx"
|
||||
assert detok.output_token_ids == token_ids
|
||||
return
|
||||
assert result == "cd"
|
||||
text_len = num_kept if include_stop_str_in_output else num_kept - 2
|
||||
assert detok.output_text == "cdcdx"[:text_len]
|
||||
assert detok.output_token_ids == token_ids[:num_kept]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"decoded_tokens,keep",
|
||||
[
|
||||
(["a", "b", "c", "d", "e", "f"], 4),
|
||||
(["ab", "cdx", "ef"], 2),
|
||||
(["ab", "", "c", "", "d", "", "ef"], 5),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("stop_terminated", [False, True])
|
||||
def test_stop_string_trims_speculative_overflow(
|
||||
include_stop_str_in_output: bool,
|
||||
decoded_tokens,
|
||||
keep,
|
||||
stop_terminated: bool,
|
||||
monkeypatch,
|
||||
):
|
||||
if stop_terminated:
|
||||
# The engine also stopped on a stop token at the end of the batch.
|
||||
decoded_tokens = decoded_tokens + [""]
|
||||
token_ids = list(range(len(decoded_tokens)))
|
||||
stop_string = "cd"
|
||||
expected_token_ids = token_ids[:keep]
|
||||
|
||||
req = _make_request(
|
||||
stop=[stop_string], include_stop_str_in_output=include_stop_str_in_output
|
||||
)
|
||||
req.external_req_id = req.request_id
|
||||
assert req.sampling_params is not None
|
||||
req.sampling_params.logprobs = 0
|
||||
detok = _DummyDetokenizer(req)
|
||||
monkeypatch.setattr(detok, "decode_next", decoded_tokens.__getitem__)
|
||||
processor = OutputProcessor(tokenizer=None, log_stats=False)
|
||||
processor.add_request(req, prompt=None)
|
||||
processor.request_states[req.request_id].detokenizer = detok
|
||||
|
||||
outputs = processor.process_outputs(
|
||||
[
|
||||
EngineCoreOutput(
|
||||
request_id=req.request_id,
|
||||
new_token_ids=token_ids,
|
||||
finish_reason=FinishReason.STOP if stop_terminated else None,
|
||||
new_logprobs=LogprobsLists(
|
||||
np.array(token_ids).reshape(-1, 1),
|
||||
np.zeros((len(token_ids), 1)),
|
||||
np.ones(len(token_ids), dtype=int),
|
||||
None,
|
||||
),
|
||||
new_sampling_mask=SamplingMaskLists(
|
||||
np.array(token_ids), np.arange(len(token_ids) + 1)
|
||||
),
|
||||
routed_experts=np.array(token_ids).reshape(-1, 1, 1),
|
||||
)
|
||||
]
|
||||
)
|
||||
result = outputs.request_outputs[0].outputs[0]
|
||||
|
||||
assert result.stop_reason == stop_string
|
||||
expected_text = "abcd" if include_stop_str_in_output else "ab"
|
||||
assert detok.output_text == expected_text
|
||||
assert detok.output_token_ids == expected_token_ids
|
||||
assert result.token_ids == expected_token_ids
|
||||
assert result.logprobs is not None
|
||||
assert len(result.logprobs) == len(expected_token_ids)
|
||||
assert result.sampling_mask is not None
|
||||
assert result.sampling_mask.token_ids == [[token] for token in expected_token_ids]
|
||||
assert result.routed_experts is not None
|
||||
assert result.routed_experts.ravel().tolist() == expected_token_ids
|
||||
|
||||
Reference in New Issue
Block a user