[Core][V1] Support trace_decode_token_ids for deterministic decode replay (#46701)

Signed-off-by: ianzxliu <ianzxliu@tencent.com>
Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Zhixia Liu <liuzhixia.nju@gmail.com>
Co-authored-by: ianzxliu <ianzxliu@tencent.com>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Zhixia Liu
2026-08-20 15:12:43 +08:00
committed by GitHub
co-authored by ianzxliu Claude aoshen02 Codex Nick Hill
parent 754e1c3de4
commit a1c5b1fd9f
13 changed files with 931 additions and 9 deletions
@@ -0,0 +1,110 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from types import SimpleNamespace
from unittest.mock import patch
import pytest
from vllm import SamplingParams
from vllm.config import VllmConfig
from vllm.exceptions import VLLMValidationError
from vllm.v1.core.sched.utils import check_stop
from vllm.v1.engine.input_processor import InputProcessor
from vllm.v1.request import Request, RequestStatus
pytestmark = pytest.mark.skip_global_cleanup
def _make_request(params: SamplingParams) -> Request:
return Request(
request_id="req-0",
client_index=0,
prompt_token_ids=[1, 2, 3],
sampling_params=params,
pooling_params=None,
arrival_time=time.time(),
)
def _normalize(params: SamplingParams, prompt_len: int, max_model_len: int) -> None:
processor = SimpleNamespace(
model_config=SimpleNamespace(max_model_len=max_model_len)
)
InputProcessor._normalize_trace_replay_params(processor, params, prompt_len)
def test_normalize_trace_replay_params():
params = SamplingParams(
max_tokens=16,
min_tokens=10,
stop_token_ids=[20],
trace_decode_token_ids=[10, 20, 30],
)
params.update_from_generation_config({}, eos_token_id=20)
_normalize(params, prompt_len=3, max_model_len=128)
assert params.max_tokens == 3
assert params.min_tokens == 0
assert params.ignore_eos is True
assert params.eos_token_id is None
assert params.stop_token_ids == []
assert params.all_stop_token_ids == set()
request = _make_request(params)
assert request.max_tokens == 3
request.append_output_token_ids([10, 20, 30])
assert check_stop(request, max_model_len=128)
assert request.status == RequestStatus.FINISHED_LENGTH_CAPPED
def test_trace_longer_than_remaining_context_is_truncated():
"""The trace is staged into a max_model_len-wide row, so it must be cut.
An explicitly set max_tokens is never clamped to max_model_len, so it cannot
bound the staged write.
"""
params = SamplingParams(max_tokens=100, trace_decode_token_ids=list(range(20)))
_normalize(params, prompt_len=6, max_model_len=10)
assert params.trace_decode_token_ids == [0, 1, 2, 3]
assert params.max_tokens == 4
def _validate(enable_trace_replay: bool) -> None:
"""Run _validate_params' trace gating with the rest of verify() stubbed."""
processor = SimpleNamespace(
model_config=SimpleNamespace(
return_sampling_mask=False,
enable_trace_replay=enable_trace_replay,
),
vllm_config=SimpleNamespace(reasoning_config=None),
speculative_config=None,
structured_outputs_config=None,
tokenizer=None,
)
params = SamplingParams(trace_decode_token_ids=[1, 2, 3])
with patch.object(SamplingParams, "verify"):
InputProcessor._validate_params(processor, params, ("generate",))
def test_trace_request_rejected_when_feature_disabled():
with pytest.raises(VLLMValidationError, match="--enable-trace-replay"):
_validate(enable_trace_replay=False)
def test_trace_request_accepted_when_feature_enabled():
_validate(enable_trace_replay=True)
def test_trace_replay_requires_v2_model_runner():
config = SimpleNamespace(
model_config=SimpleNamespace(enable_trace_replay=True),
use_v2_model_runner=False,
)
with pytest.raises(ValueError, match="trace replay requires Model Runner V2"):
VllmConfig._verify_trace_replay_config(config)
+111
View File
@@ -0,0 +1,111 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
from vllm import SamplingParams
from vllm.exceptions import VLLMValidationError
from vllm.sampling_params import StructuredOutputsParams
pytestmark = pytest.mark.skip_global_cleanup
# ---------------------------------------------------------------------------
# SamplingParams field
# ---------------------------------------------------------------------------
def test_sampling_params_trace_field_defaults_to_none():
params = SamplingParams(max_tokens=10)
assert params.trace_decode_token_ids is None
def test_sampling_params_trace_field_accepts_list():
ids = [100, 200, 300]
params = SamplingParams(trace_decode_token_ids=ids)
assert params.trace_decode_token_ids == ids
def test_sampling_params_trace_field_preserved_by_clone():
ids = [1, 2, 3]
params = SamplingParams(trace_decode_token_ids=ids)
cloned = params.clone()
assert cloned.trace_decode_token_ids == ids
assert cloned.trace_decode_token_ids is not params.trace_decode_token_ids
def test_sampling_params_trace_field_rejects_empty_list():
params = SamplingParams(trace_decode_token_ids=[])
with pytest.raises(ValueError, match="non-empty"):
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=None
)
def test_sampling_params_trace_field_requires_single_output():
params = SamplingParams(n=2, trace_decode_token_ids=[1])
with pytest.raises(ValueError, match="requires n=1"):
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=None
)
@pytest.mark.parametrize("invalid_ids", [[-1, 5], [1, "2"]])
def test_sampling_params_trace_field_rejects_invalid_token_ids(invalid_ids):
params = SamplingParams(trace_decode_token_ids=invalid_ids)
with pytest.raises(ValueError, match="non-negative integers"):
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=None
)
def _make_model_config(vocab_size: int):
from unittest.mock import Mock
model_config = Mock()
model_config.get_vocab_size = lambda: vocab_size
return model_config
def test_validate_trace_replay_accepts_in_vocab():
params = SamplingParams(trace_decode_token_ids=[0, 50, 99])
# Should not raise.
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=None
)
def test_validate_trace_replay_rejects_out_of_vocab():
# The non-negative check passes at construction, but the token id exceeds
# the vocabulary; verify() must reject it before it reaches the sampler.
params = SamplingParams(trace_decode_token_ids=[0, 100])
with pytest.raises(VLLMValidationError, match="out-of-vocab"):
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=None
)
def test_validate_trace_replay_noop_when_unset():
params = SamplingParams(max_tokens=4)
# Should not raise when the field is unset.
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=None
)
def test_trace_decode_token_ids_rejects_speculative_decoding():
params = SamplingParams(trace_decode_token_ids=[1])
with pytest.raises(ValueError, match="not supported with speculative decoding"):
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=object()
)
def test_trace_decode_token_ids_rejects_structured_outputs():
params = SamplingParams(
trace_decode_token_ids=[1],
structured_outputs=StructuredOutputsParams(json_object=True),
)
with pytest.raises(ValueError, match="not supported with structured outputs"):
params._validate_trace_replay(
_make_model_config(vocab_size=100), speculative_config=None
)
+195
View File
@@ -0,0 +1,195 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for Model Runner V2 inference trace-replay.
Trace-replay forces the sampler to emit a predetermined sequence of decode
token IDs. The replay step for a request is derived from GPU state as
``total_len - prompt_len`` (output tokens produced so far), and the trace token
overwrites the sampled token in place before logprobs are computed.
"""
from types import SimpleNamespace
from typing import cast
import pytest
import torch
pytest.importorskip("triton")
if not torch.cuda.is_available():
pytest.skip("CUDA required for trace-replay tests", allow_module_level=True)
from vllm.sampling_params import SamplingParams
from vllm.v1.worker.gpu.sample.trace_replay import (
TraceReplayState,
apply_trace_tokens,
)
from vllm.v1.worker.gpu.states import RequestState
DEVICE = "cuda"
TEST_MAX_MODEL_LEN = 2048
def _i32(x) -> torch.Tensor:
return torch.tensor(x, dtype=torch.int32, device=DEVICE)
def _i64(x) -> torch.Tensor:
return torch.tensor(x, dtype=torch.int64, device=DEVICE)
def _trace_state(max_num_reqs: int) -> TraceReplayState:
"""Build a state whose req_states exposes the buffers apply_trace reads.
total_len/prompt_len are written by the caller to position each request at
the desired replay step.
"""
req_states = cast(
RequestState,
SimpleNamespace(
max_num_reqs=max_num_reqs,
max_model_len=TEST_MAX_MODEL_LEN,
device=torch.device(DEVICE),
total_len=SimpleNamespace(
gpu=torch.zeros(max_num_reqs, dtype=torch.int32, device=DEVICE)
),
prompt_len=SimpleNamespace(
gpu=torch.zeros(max_num_reqs, dtype=torch.int32, device=DEVICE)
),
),
)
return TraceReplayState(req_states)
def _set_lens(state: TraceReplayState, total_len, prompt_len) -> None:
"""Position each request at a replay step via total_len - prompt_len."""
state.req_states.total_len.gpu[: len(total_len)] = _i32(total_len)
state.req_states.prompt_len.gpu[: len(prompt_len)] = _i32(prompt_len)
# ------------------------------ Kernel ------------------------------------
def test_replay_overwrites_sampled_at_each_step():
"""The trace token for the current step replaces the sampled token."""
trace = [[100, 101, 102], [200, 201, 202]]
trace_len = _i32([3, 3])
trace_token_ids = torch.zeros(
2, TEST_MAX_MODEL_LEN, dtype=torch.int32, device=DEVICE
)
for i, t in enumerate(trace):
trace_token_ids[i, : len(t)] = _i32(t)
prompt_len = _i32([5, 8])
idx_mapping = _i32([0, 1])
for step in range(3):
sampled = _i64([-7, -7]) # sentinel that must be overwritten
total_len = _i32([5 + step, 8 + step])
apply_trace_tokens(
sampled, idx_mapping, trace_token_ids, trace_len, total_len, prompt_len
)
assert sampled.tolist() == [trace[0][step], trace[1][step]]
def test_past_end_of_trace_leaves_sampled_untouched():
"""Once step >= trace_len, the sampler's own token is kept."""
trace_token_ids = torch.zeros(
1, TEST_MAX_MODEL_LEN, dtype=torch.int32, device=DEVICE
)
trace_token_ids[0, :2] = _i32([100, 101])
trace_len = _i32([2])
prompt_len = _i32([4])
idx_mapping = _i32([0])
sampled = _i64([999])
total_len = _i32([4 + 2]) # step == 2 == trace_len -> out of range
apply_trace_tokens(
sampled, idx_mapping, trace_token_ids, trace_len, total_len, prompt_len
)
assert sampled.tolist() == [999]
def test_non_trace_request_untouched():
"""trace_len == 0 means the request never uses replay."""
trace_token_ids = torch.zeros(
1, TEST_MAX_MODEL_LEN, dtype=torch.int32, device=DEVICE
)
trace_len = _i32([0])
prompt_len = _i32([3])
idx_mapping = _i32([0])
sampled = _i64([42])
total_len = _i32([3])
apply_trace_tokens(
sampled, idx_mapping, trace_token_ids, trace_len, total_len, prompt_len
)
assert sampled.tolist() == [42]
def test_idx_mapping_indirection_and_negative_skip():
"""batch_idx -> req_state_idx indirection, and negative entries are skipped."""
# req_state 0: no trace; req_state 1: trace [500, 501].
trace_token_ids = torch.zeros(
2, TEST_MAX_MODEL_LEN, dtype=torch.int32, device=DEVICE
)
trace_token_ids[1, :2] = _i32([500, 501])
trace_len = _i32([0, 2])
prompt_len = _i32([10, 6])
# batch: [maps to state 1, masked (-1), maps to state 0].
idx_mapping = _i32([1, -1, 0])
sampled = _i64([1, 2, 3])
# Indexed by req_state_idx: state 0 at total_len=10, state 1 at step 1.
total_len = _i32([10, 6 + 1])
apply_trace_tokens(
sampled, idx_mapping, trace_token_ids, trace_len, total_len, prompt_len
)
# batch 0 -> state 1 step 1 -> 501; batch 1 masked; batch 2 -> state 0 no trace.
assert sampled.tolist() == [501, 2, 3]
# --------------------------- TraceReplayState -----------------------------
def test_state_end_to_end():
"""add_request -> apply_staged_writes -> apply_trace overwrites correctly."""
state = _trace_state(4)
state.add_request(0, SamplingParams(trace_decode_token_ids=[11, 22, 33]))
state.add_request(1, SamplingParams()) # no trace
state.apply_staged_writes()
idx_mapping = _i32([0, 1])
sampled = _i64([-1, -1])
_set_lens(state, [7 + 1, 7, 0, 0], [7, 7, 0, 0]) # req 0 at step 1
state.apply_trace(sampled, idx_mapping)
assert sampled.tolist() == [22, -1]
def test_state_leaves_non_trace_batch_unchanged():
"""Requests with trace_len == 0 remain unchanged after a trace was seen."""
state = _trace_state(4)
state.add_request(0, SamplingParams(trace_decode_token_ids=[11, 22]))
state.add_request(1, SamplingParams())
state.apply_staged_writes()
# Only the non-trace request (state 1) is in this batch.
idx_mapping = _i32([1])
sampled = _i64([555])
_set_lens(state, [0, 7, 0, 0], [7, 7, 0, 0])
state.apply_trace(sampled, idx_mapping)
assert sampled.tolist() == [555]
def test_slot_reuse_clears_trace():
"""Reusing a slot for a non-trace request must not replay stale tokens."""
state = _trace_state(2)
state.add_request(0, SamplingParams(trace_decode_token_ids=[11, 22]))
state.apply_staged_writes()
# Slot 0 reused by a request without a trace.
state.add_request(0, SamplingParams())
state.apply_staged_writes()
idx_mapping = _i32([0])
sampled = _i64([888])
_set_lens(state, [3, 0], [3, 0])
state.apply_trace(sampled, idx_mapping)
assert sampled.tolist() == [888]